@chainlink/external-adapter-framework 0.0.5 → 0.0.7

package/src/examples/coingecko/index.ts

@@ -1,9 +0,0 @@
-import { Adapter } from '../../adapter'
-import { batchEndpoint } from './batch-warming'
-import { restEndpoint } from './rest'
-
-export const adapter: Adapter = {
-  name: 'coingecko',
-  defaultEndpoint: 'batch',
-  endpoints: [restEndpoint, batchEndpoint],
-}

package/src/examples/coingecko/rest.ts

@@ -1,77 +0,0 @@
-import { AxiosRequestConfig, AxiosResponse } from 'axios'
-import { AdapterEndpoint } from '../../adapter'
-import { RestTransport } from '../../transports'
-import { AdapterRequest, AdapterResponse } from '../../util'
-import { InputParameters } from '../../validation'
-
-const DEFAULT_URL = 'https://api.coingecko.com/api/v3'
-
-interface AdapterRequestParams {
-  base: string
-  quote: string
-}
-
-const inputParameters: InputParameters = {
-  coinid: {
-    description:
-      'The CoinGecko id or array of ids of the coin(s) to query (Note: because of current limitations to use a dummy base will need to be supplied)',
-    required: false,
-  },
-  base: {
-    aliases: ['from', 'coin'],
-    description: 'The symbol or array of symbols of the currency to query',
-    required: true,
-  },
-  quote: {
-    aliases: ['to', 'market'],
-    description: 'The symbol of the currency to convert to',
-    required: true,
-  },
-}
-
-interface ProviderRequestBody {
-  ids: string
-  vs_currencies: string
-}
-
-interface ProviderResponseBody {
-  [base: string]: {
-    [quote: string]: number
-  }
-}
-
-const restEndpointTransport = new RestTransport({
-  prepareRequest: (
-    req: AdapterRequest<AdapterRequestParams>,
-  ): AxiosRequestConfig<ProviderRequestBody> => {
-    return {
-      baseURL: DEFAULT_URL,
-      url: '/simple/price',
-      method: 'GET',
-      params: {
-        ids: req.requestContext.data.base,
-        vs_currencies: req.requestContext.data.quote,
-      },
-    }
-  },
-  parseResponse: (
-    req: AdapterRequest<AdapterRequestParams>,
-    res: AxiosResponse<ProviderResponseBody>,
-  ): AdapterResponse<ProviderResponseBody> => {
-    return {
-      data: res.data,
-      statusCode: 200,
-      result: res.data[req.requestContext.data.base]?.[req.requestContext.data.quote],
-    }
-  },
-  options: {
-    coalescing: true,
-  },
-})
-
-export const restEndpoint: AdapterEndpoint = {
-  name: 'rest',
-  aliases: ['qwe'],
-  transport: restEndpointTransport,
-  inputParameters,
-}

package/src/examples/ncfx/config/index.ts

@@ -1,12 +0,0 @@
-export const customSettings = {
-  USERNAME: {
-    description: 'Username for the NCFX API',
-    type: 'string',
-    required: true,
-  },
-  PASSWORD: {
-    description: 'Password for the NCFX API',
-    type: 'string',
-    required: true,
-  },
-} as const

package/src/examples/ncfx/index.ts

@@ -1,9 +0,0 @@
-import { Adapter } from '../../adapter'
-import { customSettings } from './config'
-import { webSocketEndpoint } from './websocket'
-
-export const adapter: Adapter = {
-  name: 'ncfx',
-  endpoints: [webSocketEndpoint],
-  customSettings,
-}

package/src/examples/ncfx/websocket.ts

@@ -1,100 +0,0 @@
-import { RawData, WebSocket } from 'ws'
-import { AdapterEndpoint } from '../../adapter'
-import { AdapterContext } from '../../transports'
-import { WebSocketTransport } from '../../transports/websocket'
-import { ProviderResult, makeLogger } from '../../util'
-import { InputParameters } from '../../validation'
-import { customSettings } from './config'
-
-interface AdapterRequestParams {
-  base: string
-  quote: string
-}
-
-export const inputParameters: InputParameters = {
-  base: {
-    aliases: ['from', 'coin'],
-    description: 'The symbol of the currency to query',
-    required: true,
-  },
-  quote: {
-    aliases: ['to', 'market'],
-    description: 'The symbol of the currency to convert to',
-    required: true,
-  },
-}
-
-interface ProviderMessage {
-  timestamp: string
-  currencyPair: string
-  bid: number
-  offer: number
-  mid: number
-  changes: [
-    {
-      period: string
-      change: number
-      percentage: number
-    },
-  ]
-}
-
-const logger = makeLogger('NcfxWebSocketTransport')
-
-export const websocketTransport = new WebSocketTransport({
-  url: 'wss://feed.newchangefx.com/cryptodata',
-  handlers: {
-    open(connection: WebSocket, context: AdapterContext<typeof customSettings>) {
-      return new Promise((resolve, reject) => {
-        // Set up listener
-        connection.on('message', (data: RawData) => {
-          const parsed = JSON.parse(data.toString())
-          if (parsed.Message?.startsWith('Logged in as user')) {
-            logger.info('Got logged in response, connection is ready')
-            resolve()
-          } else {
-            reject(new Error('Unexpected message after WS connection open'))
-          }
-        })
-        // Send login payload
-        connection.send(
-          JSON.stringify({
-            request: 'login',
-            username: context.adapterConfig.USERNAME,
-            password: context.adapterConfig.PASSWORD,
-          }),
-        )
-      })
-    },
-
-    message(message: ProviderMessage[]): ProviderResult<AdapterRequestParams>[] {
-      if (!Array.isArray(message)) {
-        logger.debug('WS message is not array, skipping')
-        return []
-      }
-      return message.map((m) => {
-        const [base, quote] = m.currencyPair.split('/')
-        return {
-          params: { base, quote },
-          value: m.offer,
-        }
-      })
-    },
-  },
-  builders: {
-    subscribeMessage: (params: AdapterRequestParams) => ({
-      request: 'subscribe',
-      ccy: `${params.base}/${params.quote}`,
-    }),
-    unsubscribeMessage: (params: AdapterRequestParams) => ({
-      request: 'unsubscribe',
-      ccy: `${params.base}/${params.quote}`,
-    }),
-  },
-})
-
-export const webSocketEndpoint: AdapterEndpoint = {
-  name: 'websocket',
-  transport: websocketTransport,
-  inputParameters,
-}

package/src/index.ts

@@ -1,106 +0,0 @@
-import fastify from 'fastify'
-import { Server } from 'http'
-import { AddressInfo } from 'net'
-import { join } from 'path'
-import { Adapter, InitializedAdapter, initializeAdapter } from './adapter'
-import { callBackgroundExecutes } from './background-executor'
-import { buildCacheMiddleware } from './cache'
-import { AdapterConfig, buildAdapterConfig } from './config'
-import { buildMetricsMiddleware, setupMetricsServer } from './metrics'
-import { AdapterDependencies, buildTransportHandler } from './transports'
-import { AdapterRouteGeneric, loggingContextMiddleware, makeLogger } from './util'
-import { errorCatchingMiddleware, validatorMiddleware } from './validation'
-
-const logger = makeLogger('Main')
-
-const VERSION = process.env['npm_package_version']
-
-/**
- * Main function for the framework.
- * Initializes config and dependencies, uses those to initialize Transports, and starts listening for requests.
- *
- * @param adapter - an object describing an External Adapter
- * @param dependencies - an optional object with adapter dependencies to inject
- * @returns a Promise that resolves to the http.Server listening for connections
- */
-export const expose = async (
-  adapter: Adapter,
-  dependencies?: Partial<AdapterDependencies>,
-): Promise<Server | undefined> => {
-  const config = buildAdapterConfig({
-    overrides: adapter.envDefaultOverrides,
-    customSettings: adapter.customSettings,
-  })
-  // Initialize adapter (create dependencies, inject them, build endpoint map, etc.)
-  const initializedAdapter = await initializeAdapter(adapter, config, dependencies)
-
-  let server: Server | undefined = undefined
-
-  if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
-    setupMetricsServer(adapter.name, config)
-  }
-
-  if (config.EA_MODE === 'reader' || config.EA_MODE === 'reader-writer') {
-    // Main REST API server to handle incoming requests
-    const app = await buildRestApi(config, initializedAdapter)
-
-    // Start listening for incoming requests
-    try {
-      await app.listen(config.EA_PORT, config.EA_HOST)
-    } catch (err) {
-      logger.fatal(`There was an error when starting the EA server: ${err}`)
-      process.exit()
-    }
-
-    logger.info(`Listening on port ${(app.server.address() as AddressInfo).port}`)
-    server = app.server
-  } else {
-    logger.info('REST API is disabled; this instance will not process incoming requests.')
-  }
-
-  if (config.EA_MODE === 'writer' || config.EA_MODE === 'reader-writer') {
-    // Start background loop that will take care of calling any async Transports
-    logger.info('Starting background execution loop')
-    callBackgroundExecutes(initializedAdapter, server)
-  } else {
-    logger.info(
-      'Background executor is disabled; this instance will not perform async background executes.',
-    )
-  }
-
-  return server
-}
-
-async function buildRestApi(config: AdapterConfig, initializedAdapter: InitializedAdapter) {
-  const app = fastify()
-
-  // Add healthcheck endpoint before middlewares to bypass them
-  app.get(join(config.BASE_URL, 'health'), (req, res) => {
-    res.status(200).send({ message: 'OK', version: VERSION })
-  })
-
-  // Use global error handling
-  app.setErrorHandler(errorCatchingMiddleware)
-
-  const transportHandler = await buildTransportHandler(initializedAdapter)
-
-  app.register(async (router) => {
-    // Set up "middlewares" (hooks in fastify)
-    router.addHook<AdapterRouteGeneric>('preHandler', validatorMiddleware(initializedAdapter))
-    router.addHook<AdapterRouteGeneric>('preHandler', buildCacheMiddleware(initializedAdapter))
-    if (config['CORRELATION_ID_ENABLED']) {
-      router.addHook<AdapterRouteGeneric>('onRequest', loggingContextMiddleware)
-    }
-
-    router.route<AdapterRouteGeneric>({
-      url: config.BASE_URL,
-      method: 'POST',
-      handler: transportHandler,
-    }) // Pass config maybe? cleaner that multiple instances / dependencies
-
-    if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
-      router.addHook<AdapterRouteGeneric>('onResponse', buildMetricsMiddleware)
-    }
-  })
-  return app
-}

package/src/metrics/constants.ts

@@ -1,23 +0,0 @@
-export enum HttpRequestType {
-  CACHE_HIT = 'cacheHit',
-  DATA_PROVIDER_HIT = 'dataProviderHit',
-  ADAPTER_ERROR = 'adapterError',
-  INPUT_ERROR = 'inputError',
-  RATE_LIMIT_ERROR = 'rateLimitError',
-  // BURST_LIMIT_ERROR = 'burstLimitError',
-  // BACKOFF_ERROR = 'backoffError',
-  DP_ERROR = 'dataProviderError',
-  TIMEOUT_ERROR = 'timeoutError',
-  CONNECTION_ERROR = 'connectionError',
-  // RES_EMPTY_ERROR = 'responseEmptyError',
-  // RES_INVALID_ERROR = 'responseInvalidError',
-  CUSTOM_ERROR = 'customError',
-}
-
-/**
- * Maxiumum number of characters that a feedId can contain.
- */
-export const MAX_FEED_ID_LENGTH = 300
-
-// We should tune these as we collect data, this is the default bucket distribution that prom comes with
-export const requestDurationBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]

package/src/metrics/index.ts

@@ -1,116 +0,0 @@
-import * as client from 'prom-client'
-import { HttpRequestType, requestDurationBuckets } from './constants'
-import { AdapterRequest, makeLogger } from '../util'
-import { AdapterConfig } from '../config'
-import fastify, { FastifyReply, HookHandlerDoneFunction } from 'fastify'
-import { join } from 'path'
-import { AdapterError } from '../validation/error'
-
-const logger = makeLogger('Metrics')
-
-export function setupMetricsServer(name: string, config: AdapterConfig) {
-  const metricsApp = fastify({
-    logger: false,
-  })
-  const metricsPort = config.METRICS_PORT
-  const endpoint = config.METRICS_USE_BASE_URL ? join(config.BASE_URL, 'metrics') : '/metrics'
-  const eaHost = config.EA_HOST
-
-  setupMetrics(name, config)
-
-  metricsApp.get(endpoint, async (_, res) => {
-    res.type('txt')
-    res.send(await client.register.metrics())
-  })
-
-  metricsApp.listen(metricsPort, eaHost, () =>
-    logger.info(`Monitoring listening on port ${metricsPort}!`),
-  )
-}
-
-export const setupMetrics = (name: string, config: AdapterConfig): void => {
-  client.collectDefaultMetrics()
-  client.register.setDefaultLabels({
-    app_name: config.METRICS_NAME || name || 'N/A',
-    app_version: config['npm_package_version'],
-  })
-}
-
-/**
- * Builds metrics middleware that records end to end EA response times
- * and count of requests
- *
- * @returns the cache middleware function
- */
-export const buildMetricsMiddleware = (
-  req: AdapterRequest,
-  res: FastifyReply,
-  done: HookHandlerDoneFunction,
-) => {
-  const feedId = req.requestContext.meta?.metrics?.feedId || 'N/A'
-  const labels = buildHttpRequestMetricsLabel(
-    feedId,
-    req.requestContext.meta?.error,
-    req.requestContext.meta?.metrics?.cacheHit,
-  )
-
-  // Record number of requests sent to EA
-  httpRequestsTotal.labels(labels).inc()
-
-  // Record response time of request through entire EA
-  httpRequestDurationSeconds.observe(res.getResponseTime())
-  logger.debug(`Response time for ${feedId}: ${res.getResponseTime()}ms`)
-  done()
-}
-
-const buildHttpRequestMetricsLabel = (
-  feedId: string,
-  error?: AdapterError | Error,
-  cacheHit?: boolean,
-): Parameters<typeof httpRequestsTotal.labels>[0] => {
-  const labels: Parameters<typeof httpRequestsTotal.labels>[0] = {}
-  labels.method = 'POST'
-  labels.feed_id = feedId
-  if (error instanceof AdapterError) {
-    // If error present and an instace of AdapterError, build label from error info
-    labels.type = error?.metricsLabel || HttpRequestType.ADAPTER_ERROR
-    labels.status_code = error?.statusCode
-    labels.provider_status_code = error?.providerStatusCode
-  } else if (error instanceof Error) {
-    // If error present and not instance of generic Error, unexpected failure occurred
-    // TODO: Build label with 500 error and withold dp status? Status sent as 200 in error handler
-    labels.type = HttpRequestType.ADAPTER_ERROR
-    labels.status_code = 500
-  } else {
-    // If no error present, request went as expected
-    labels.status_code = 200
-    if (cacheHit) {
-      labels.type = HttpRequestType.CACHE_HIT
-    } else {
-      labels.type = HttpRequestType.DATA_PROVIDER_HIT
-      labels.provider_status_code = 200
-    }
-  }
-
-  return labels
-}
-
-export const httpRequestsTotal = new client.Counter({
-  name: 'http_requests_total',
-  help: 'The number of http requests this external adapter has serviced for its entire uptime',
-  labelNames: [
-    'method',
-    'status_code',
-    'retry',
-    'type',
-    'is_cache_warming',
-    'feed_id',
-    'provider_status_code',
-  ] as const,
-})
-
-export const httpRequestDurationSeconds = new client.Histogram({
-  name: 'http_request_duration_seconds',
-  help: 'A histogram bucket of the distribution of http request durations',
-  buckets: requestDurationBuckets,
-})

package/src/metrics/util.ts

@@ -1,11 +0,0 @@
-import { AdapterMetricsMeta, AdapterRequestData } from '../util'
-import { AdapterEndpoint } from '../adapter'
-import { calculateFeedId } from '../cache'
-
-export const getMetricsMeta = (
-  endpoint: AdapterEndpoint,
-  data: AdapterRequestData,
-): AdapterMetricsMeta => {
-  const feedId = calculateFeedId(endpoint, data)
-  return { feedId }
-}

package/src/rate-limiting/background/fixed-frequency.ts

@@ -1,47 +0,0 @@
-import { AdapterRateLimitTier, BackgroundExecuteRateLimiter, consolidateTierLimits } from '..'
-import { AdapterEndpoint } from '../../adapter'
-import { makeLogger } from '../../util'
-
-const logger = makeLogger('FixedFrequencyRateLimiter')
-export const DEFAULT_SHARED_MS_BETWEEN_REQUESTS = 5000
-
-export class FixedFrequencyRateLimiter implements BackgroundExecuteRateLimiter {
-  msBetweenRequestsMap: {
-    [endpointName: string]: number // In ms
-  } = {}
-
-  initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier) {
-    // Translate the hourly limit into reqs per minute
-    let sharedMsBetweenRequests = 1000 / consolidateTierLimits(limits)
-
-    // If there is no limit set, we use some reasonable number
-    if (!limits?.rateLimit1h && !limits?.rateLimit1m && !limits?.rateLimit1s) {
-      // 5s period for all seems good
-      sharedMsBetweenRequests = DEFAULT_SHARED_MS_BETWEEN_REQUESTS
-    }
-
-    logger.debug('Using fixed frequency batch rate limiting')
-    for (const endpoint of endpoints) {
-      // TODO: See if we can remove this runtime check
-      if (endpoint.rateLimiting?.allocationPercentage == null) {
-        throw new Error(`Allocation percentage for endpoint "${endpoint.name}" is null`)
-      }
-
-      // TODO: Implement different strategy where this is not fixed, but rather divided based on whether all warmers are active
-      this.msBetweenRequestsMap[endpoint.name] =
-        (sharedMsBetweenRequests / endpoint.rateLimiting?.allocationPercentage) * 100
-
-      logger.debug(
-        `Endpoint [${endpoint.name}]: ${
-          this.msBetweenRequestsMap[endpoint.name] / 1000
-        }s between requests`,
-      )
-    }
-
-    return this
-  }
-
-  msUntilNextExecution(endpointName: string): number {
-    return this.msBetweenRequestsMap[endpointName]
-  }
-}

package/src/rate-limiting/index.ts

@@ -1,100 +0,0 @@
-import { AdapterEndpoint } from '../adapter'
-
-export * from './request/simple-counting'
-export * from './background/fixed-frequency'
-
-export interface AdapterRateLimitTier {
-  rateLimit1s?: number
-  rateLimit1m?: number
-  rateLimit1h?: number
-  note?: string
-}
-
-/**
- * Common interface for all RateLimiter classes to implement
- */
-export interface RateLimiter {
-  /**
-   * Method to ensure all RateLimiters can be initialized in the same manner.
-   *
-   * @param limits - settings for how much throughput to allow for the Adapter
-   * @param endpoints - list of adapter endpoints
-   */
-  initialize(endpoints: AdapterEndpoint[], limits: AdapterRateLimitTier): this
-}
-
-/**
- * RequestRateLimiters perform checks agains imminent outbound requests for any transport.
- */
-export interface RequestRateLimiter extends RateLimiter {
-  /**
-   * This method will check using whatever strategy is implemented to determine if
-   * this request can be processed. If so, it returns true; if not, returns false.
-   */
-  isUnderLimits(): boolean
-}
-
-/**
- * BackgroundExecuteFrequencyRateLimiters will implement custom logic to calculate
- * the period of time to wait between background executions for a transport.
- */
-export interface BackgroundExecuteRateLimiter extends RateLimiter {
-  msUntilNextExecution(endpointName: string): number
-}
-
-/**
- * This method will convert all possible settings for a rate limit tier and
- * convert them all to requests per second, returning the lowest one
- *
- * @param limits - the rate limit tier set for the adapter
- * @returns the most restrictive of the set options, in requests per second
- */
-export const consolidateTierLimits = (limits?: AdapterRateLimitTier) => {
-  const perHourLimit = (limits?.rateLimit1h || Infinity) / (60 * 60)
-  const perMinuteLimit = (limits?.rateLimit1m || Infinity) / 60
-  const perSecondLimit = limits?.rateLimit1s || Infinity
-  return Math.min(perHourLimit, perMinuteLimit, perSecondLimit)
-}
-
-/**
- * Validates rate limiting tiers specified for the adapter, and returns the one to use.
- *
- * @param tiers - the adapter config listing the different available API tiers
- * @param selectedTier - chosen API tier from settings, if present
- * @returns the specified API tier, or a default one if none are specified
- */
-export const getRateLimitingTier = (
-  tiers?: Record<string, AdapterRateLimitTier>,
-  selectedTier?: string,
-): AdapterRateLimitTier | undefined => {
-  if (!tiers) {
-    return
-  }
-
-  // Check that if the tiers object is defined, it has values
-  if (Object.values(tiers).length === 0) {
-    throw new Error(`The tiers object is defined, but has no entries`)
-  }
-
-  // Check that the tier set in the AdapterConfig is a valid one
-  if (selectedTier && !tiers[selectedTier]) {
-    const validTiersString = Object.keys(tiers)
-      .map((t) => `"${t}"`)
-      .join(', ')
-
-    throw new Error(
-      `The selected rate limit tier "${selectedTier}" is not valid (can be one of ${validTiersString})`,
-    )
-  }
-
-  if (!selectedTier) {
-    // Sort the tiers by most to least restrictive
-    const sortedTiers = Object.values(tiers).sort(
-      (t1, t2) => consolidateTierLimits(t1) - consolidateTierLimits(t2),
-    )
-
-    return sortedTiers[0]
-  }
-
-  return tiers[selectedTier]
-}

package/src/rate-limiting/metrics.ts

@@ -1,18 +0,0 @@
-import * as client from 'prom-client'
-
-// Retrieve cost field from response if exists
-// If not return default cost of 1
-export const retrieveCost = <ProviderResponseBody>(data: ProviderResponseBody): number => {
-  const cost = (data as Record<string, unknown>)['cost']
-  if (typeof cost === 'number' || typeof cost === 'string') {
-    return Number(cost)
-  } else {
-    return 1
-  }
-}
-
-export const rateLimitCreditsSpentTotal = new client.Counter({
-  name: 'rate_limit_credits_spent_total',
-  help: 'The number of data provider credits the adapter is consuming',
-  labelNames: ['participant_id', 'feed_id'] as const,
-})