@chainlink/external-adapter-framework 0.0.15 → 0.0.16

package/src/examples/coingecko/test/integration/fixtures.ts

@@ -1,42 +0,0 @@
-import nock from 'nock'
-import capturedRequests from './capturedRequests.json'
-
-type RecordedRequest = Record<
-  string,
-  { statusCode: number; response: unknown; method: string; path: string }[]
->
-
-// This is a generic fixture file that can be copy-pasted to generate fixtures from a
-// capturedRequests.json file.  To generate a capturedRequests.json file,
-// set env var RECORD=true and run e2e tests.
-export const createMocks = (): nock.Scope[] => {
-  const mocks: nock.Scope[] = []
-  for (const baseUrl in capturedRequests) {
-    const mock = nock(baseUrl, { encodedQueryParams: true })
-    for (const request of (capturedRequests as RecordedRequest)[baseUrl]) {
-      if (request.method === 'GET') {
-        mock
-          .get((uri: string) => {
-            const deconstructedRequestUrl = uri.split(/[?=,&]+/)
-            const deconstructedMockUrl = request.path.split(/[?=,&]+/)
-            return (
-              deconstructedRequestUrl.sort().join(',') === deconstructedMockUrl.sort().join(',')
-            )
-          })
-          .times(100)
-          .reply(request.statusCode, () => request.response, [
-            'Content-Type',
-            'application/json',
-            'Connection',
-            'close',
-            'Vary',
-            'Accept-Encoding',
-            'Vary',
-            'Origin',
-          ])
-      }
-    }
-    mocks.push(mock)
-  }
-  return mocks
-}

package/src/examples/coingecko-old/batch-warming.ts

@@ -1,79 +0,0 @@
-import { AxiosRequestConfig, AxiosResponse } from 'axios'
-import { AdapterContext, AdapterEndpoint } from '../../adapter'
-import { BatchWarmingTransport } from '../../transports/batch-warming'
-import { ProviderResult } from '../../util'
-import { InputParameters } from '../../validation'
-
-const DEFAULT_URL = 'https://pro-api.coingecko.com/api/v3'
-
-interface AdapterRequestParams {
-  base: string
-  quote: string
-}
-
-const inputParameters: InputParameters = {
-  coinid: {
-    description:
-      'The CoinGecko id or array of ids of the coin(s) to query (Note: because of current limitations to use a dummy base will need to be supplied)',
-    required: false,
-  },
-  base: {
-    aliases: ['from', 'coin'],
-    description: 'The symbol or array of symbols of the currency to query',
-    required: true,
-  },
-  quote: {
-    aliases: ['to', 'market'],
-    description: 'The symbol of the currency to convert to',
-    required: true,
-  },
-}
-interface ProviderRequestBody {
-  ids: string
-  vs_currencies: string
-}
-
-interface ProviderResponseBody {
-  [base: string]: {
-    [quote: string]: number
-  }
-}
-
-const batchEndpointTransport = new BatchWarmingTransport({
-  prepareRequest: (
-    params: AdapterRequestParams[],
-    context: AdapterContext,
-  ): AxiosRequestConfig<ProviderRequestBody> => {
-    return {
-      baseURL: DEFAULT_URL,
-      url: '/simple/price',
-      method: 'GET',
-      params: {
-        ids: [...new Set(params.map((p) => p.base))].join(','),
-        vs_currencies: [...new Set(params.map((p) => p.quote))].join(','),
-        x_cg_pro_api_key: context.adapterConfig.API_KEY,
-      },
-    }
-  },
-  parseResponse: (
-    params: AdapterRequestParams[],
-    res: AxiosResponse<ProviderResponseBody>,
-  ): ProviderResult<AdapterRequestParams>[] => {
-    const entries = [] as ProviderResult<AdapterRequestParams>[]
-    for (const [base, entry] of Object.entries(res.data)) {
-      for (const [quote, price] of Object.entries(entry)) {
-        entries.push({
-          params: { base, quote },
-          value: price,
-        })
-      }
-    }
-    return entries
-  },
-})
-
-export const batchEndpoint: AdapterEndpoint = {
-  name: 'batch',
-  transport: batchEndpointTransport,
-  inputParameters,
-}

package/src/examples/coingecko-old/index.ts

@@ -1,9 +0,0 @@
-import { Adapter } from '../../adapter'
-import { batchEndpoint } from './batch-warming'
-import { restEndpoint } from './rest'
-
-export const adapter: Adapter = {
-  name: 'coingecko',
-  defaultEndpoint: 'batch',
-  endpoints: [restEndpoint, batchEndpoint],
-}

package/src/examples/coingecko-old/rest.ts

@@ -1,77 +0,0 @@
-import { AxiosRequestConfig, AxiosResponse } from 'axios'
-import { AdapterEndpoint } from '../../adapter'
-import { RestTransport } from '../../transports'
-import { AdapterRequest, AdapterResponse } from '../../util'
-import { InputParameters } from '../../validation'
-
-const DEFAULT_URL = 'https://api.coingecko.com/api/v3'
-
-interface AdapterRequestParams {
-  base: string
-  quote: string
-}
-
-const inputParameters: InputParameters = {
-  coinid: {
-    description:
-      'The CoinGecko id or array of ids of the coin(s) to query (Note: because of current limitations to use a dummy base will need to be supplied)',
-    required: false,
-  },
-  base: {
-    aliases: ['from', 'coin'],
-    description: 'The symbol or array of symbols of the currency to query',
-    required: true,
-  },
-  quote: {
-    aliases: ['to', 'market'],
-    description: 'The symbol of the currency to convert to',
-    required: true,
-  },
-}
-
-interface ProviderRequestBody {
-  ids: string
-  vs_currencies: string
-}
-
-interface ProviderResponseBody {
-  [base: string]: {
-    [quote: string]: number
-  }
-}
-
-const restEndpointTransport = new RestTransport({
-  prepareRequest: (
-    req: AdapterRequest<AdapterRequestParams>,
-  ): AxiosRequestConfig<ProviderRequestBody> => {
-    return {
-      baseURL: DEFAULT_URL,
-      url: '/simple/price',
-      method: 'GET',
-      params: {
-        ids: req.requestContext.data.base,
-        vs_currencies: req.requestContext.data.quote,
-      },
-    }
-  },
-  parseResponse: (
-    req: AdapterRequest<AdapterRequestParams>,
-    res: AxiosResponse<ProviderResponseBody>,
-  ): AdapterResponse<ProviderResponseBody> => {
-    return {
-      data: res.data,
-      statusCode: 200,
-      result: res.data[req.requestContext.data.base]?.[req.requestContext.data.quote],
-    }
-  },
-  options: {
-    coalescing: true,
-  },
-})
-
-export const restEndpoint: AdapterEndpoint = {
-  name: 'rest',
-  aliases: ['qwe'],
-  transport: restEndpointTransport,
-  inputParameters,
-}

package/src/examples/ncfx/config/index.ts

@@ -1,12 +0,0 @@
-export const customSettings = {
-  USERNAME: {
-    description: 'Username for the NCFX API',
-    type: 'string',
-    required: true,
-  },
-  PASSWORD: {
-    description: 'Password for the NCFX API',
-    type: 'string',
-    required: true,
-  },
-} as const

package/src/examples/ncfx/index.ts

@@ -1,9 +0,0 @@
-import { Adapter } from '../../adapter'
-import { customSettings } from './config'
-import { webSocketEndpoint } from './websocket'
-
-export const adapter: Adapter = {
-  name: 'ncfx',
-  endpoints: [webSocketEndpoint],
-  customSettings,
-}

package/src/examples/ncfx/websocket.ts

@@ -1,99 +0,0 @@
-import { RawData, WebSocket } from 'ws'
-import { AdapterContext, AdapterEndpoint } from '../../adapter'
-import { WebSocketTransport } from '../../transports/websocket'
-import { ProviderResult, makeLogger } from '../../util'
-import { InputParameters } from '../../validation'
-import { customSettings } from './config'
-
-interface AdapterRequestParams {
-  base: string
-  quote: string
-}
-
-export const inputParameters: InputParameters = {
-  base: {
-    aliases: ['from', 'coin'],
-    description: 'The symbol of the currency to query',
-    required: true,
-  },
-  quote: {
-    aliases: ['to', 'market'],
-    description: 'The symbol of the currency to convert to',
-    required: true,
-  },
-}
-
-interface ProviderMessage {
-  timestamp: string
-  currencyPair: string
-  bid: number
-  offer: number
-  mid: number
-  changes: [
-    {
-      period: string
-      change: number
-      percentage: number
-    },
-  ]
-}
-
-const logger = makeLogger('NcfxWebSocketTransport')
-
-export const websocketTransport = new WebSocketTransport({
-  url: 'wss://feed.newchangefx.com/cryptodata',
-  handlers: {
-    open(connection: WebSocket, context: AdapterContext<typeof customSettings>) {
-      return new Promise((resolve, reject) => {
-        // Set up listener
-        connection.on('message', (data: RawData) => {
-          const parsed = JSON.parse(data.toString())
-          if (parsed.Message?.startsWith('Logged in as user')) {
-            logger.info('Got logged in response, connection is ready')
-            resolve()
-          } else {
-            reject(new Error('Unexpected message after WS connection open'))
-          }
-        })
-        // Send login payload
-        connection.send(
-          JSON.stringify({
-            request: 'login',
-            username: context.adapterConfig.USERNAME,
-            password: context.adapterConfig.PASSWORD,
-          }),
-        )
-      })
-    },
-
-    message(message: ProviderMessage[]): ProviderResult<AdapterRequestParams>[] {
-      if (!Array.isArray(message)) {
-        logger.debug('WS message is not array, skipping')
-        return []
-      }
-      return message.map((m) => {
-        const [base, quote] = m.currencyPair.split('/')
-        return {
-          params: { base, quote },
-          value: m.offer,
-        }
-      })
-    },
-  },
-  builders: {
-    subscribeMessage: (params: AdapterRequestParams) => ({
-      request: 'subscribe',
-      ccy: `${params.base}/${params.quote}`,
-    }),
-    unsubscribeMessage: (params: AdapterRequestParams) => ({
-      request: 'unsubscribe',
-      ccy: `${params.base}/${params.quote}`,
-    }),
-  },
-})
-
-export const webSocketEndpoint: AdapterEndpoint = {
-  name: 'websocket',
-  transport: websocketTransport,
-  inputParameters,
-}

package/src/index.ts

@@ -1,149 +0,0 @@
-import fastify, { FastifyInstance } from 'fastify'
-import { Server } from 'http'
-import { AddressInfo } from 'net'
-import { join } from 'path'
-import { Adapter, InitializedAdapter, initializeAdapter, AdapterDependencies } from './adapter'
-import { callBackgroundExecutes } from './background-executor'
-import { buildCacheMiddleware } from './cache'
-import { AdapterConfig, buildAdapterConfig } from './config'
-import { buildMetricsMiddleware, setupMetricsServer } from './metrics'
-import { buildTransportHandler } from './transports'
-import { AdapterRouteGeneric, loggingContextMiddleware, makeLogger } from './util'
-import { loadTestPayload } from './util/test-payload-loader'
-import { errorCatchingMiddleware, validatorMiddleware } from './validation'
-
-const logger = makeLogger('Main')
-
-const VERSION = process.env['npm_package_version']
-
-/**
- * Main function for the framework.
- * Initializes config and dependencies, uses those to initialize Transports, and starts listening for requests.
- *
- * @param adapter - an object describing an External Adapter
- * @param dependencies - an optional object with adapter dependencies to inject
- * @returns a Promise that resolves to the http.Server listening for connections
- */
-export const expose = async (
-  adapter: Adapter,
-  dependencies?: Partial<AdapterDependencies>,
-): Promise<Server | undefined> => {
-  const config = buildAdapterConfig({
-    overrides: adapter.envDefaultOverrides,
-    customSettings: adapter.customSettings,
-  })
-  // Initialize adapter (create dependencies, inject them, build endpoint map, etc.)
-  const initializedAdapter = await initializeAdapter(adapter, config, dependencies)
-
-  let server: Server | undefined = undefined
-
-  if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
-    setupMetricsServer(adapter.name, config)
-  }
-
-  if (config.EA_MODE === 'reader' || config.EA_MODE === 'reader-writer') {
-    // Main REST API server to handle incoming requests
-    const app = await buildRestApi(config, initializedAdapter)
-
-    // Start listening for incoming requests
-    try {
-      await app.listen(config.EA_PORT, config.EA_HOST)
-    } catch (err) {
-      logger.fatal(`There was an error when starting the EA server: ${err}`)
-      process.exit()
-    }
-
-    logger.info(`Listening on port ${(app.server.address() as AddressInfo).port}`)
-    server = app.server
-  } else {
-    logger.info('REST API is disabled; this instance will not process incoming requests.')
-  }
-
-  if (config.EA_MODE === 'writer' || config.EA_MODE === 'reader-writer') {
-    // Start background loop that will take care of calling any async Transports
-    logger.info('Starting background execution loop')
-    callBackgroundExecutes(initializedAdapter, server)
-  } else {
-    logger.info(
-      'Background executor is disabled; this instance will not perform async background executes.',
-    )
-  }
-
-  return server
-}
-
-async function buildRestApi(config: AdapterConfig, initializedAdapter: InitializedAdapter) {
-  const app = fastify()
-
-  // Add healthcheck endpoint before middlewares to bypass them
-  app.get(join(config.BASE_URL, 'health'), (req, res) => {
-    res.status(200).send({ message: 'OK', version: VERSION })
-  })
-
-  // Use global error handling
-  app.setErrorHandler(errorCatchingMiddleware)
-
-  const transportHandler = await buildTransportHandler(initializedAdapter)
-
-  app.register(async (router) => {
-    // Set up "middlewares" (hooks in fastify)
-    router.addHook<AdapterRouteGeneric>('preHandler', validatorMiddleware(initializedAdapter))
-    router.addHook<AdapterRouteGeneric>('preHandler', buildCacheMiddleware(initializedAdapter))
-    if (config['CORRELATION_ID_ENABLED']) {
-      router.addHook<AdapterRouteGeneric>('onRequest', loggingContextMiddleware)
-    }
-
-    router.route<AdapterRouteGeneric>({
-      url: config.BASE_URL,
-      method: 'POST',
-      handler: transportHandler,
-    })
-
-    if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
-      router.addHook<AdapterRouteGeneric>('onResponse', buildMetricsMiddleware)
-    }
-  })
-
-  // Add smoke endpoint after middleware which are needed for tests
-  buildSmokeEndpoint(app, config)
-  return app
-}
-
-/**
- * Adds the /smoke endpoint to the API for smoke testing the adapter
- *
- * @param app - the Fastify instance
- * @param config - the initialized adapter config
- */
-function buildSmokeEndpoint(app: FastifyInstance, config: AdapterConfig) {
-  const testPayload = loadTestPayload(config.SMOKE_TEST_PAYLOAD_FILE_NAME)
-  app.get(join(config.BASE_URL, 'smoke'), async (_, res) => {
-    if (testPayload.isDefault) {
-      return res.status(200).send('OK')
-    }
-
-    const errors = []
-    for (const index in testPayload.requests) {
-      try {
-        const request = { id: index, data: testPayload.requests[index] }
-        // Use Fastify's app inject to pass smoke requests internally
-        const response = await app.inject({
-          method: 'POST',
-          url: '/',
-          payload: request,
-        })
-        const parsedResponse = JSON.parse(response.body)
-        // Throw error if not 2xx status code
-        if (parsedResponse.statusCode < 200 || parsedResponse.statusCode > 299) {
-          throw Error('Smoke test request failed')
-        }
-      } catch (e: unknown) {
-        errors.push(e)
-      }
-    }
-    if (errors.length > 0) {
-      return res.status(500).send(errors)
-    }
-    return res.status(200).send('OK')
-  })
-}

package/src/metrics/constants.ts

@@ -1,23 +0,0 @@
-export enum HttpRequestType {
-  CACHE_HIT = 'cacheHit',
-  DATA_PROVIDER_HIT = 'dataProviderHit',
-  ADAPTER_ERROR = 'adapterError',
-  INPUT_ERROR = 'inputError',
-  RATE_LIMIT_ERROR = 'rateLimitError',
-  // BURST_LIMIT_ERROR = 'burstLimitError',
-  // BACKOFF_ERROR = 'backoffError',
-  DP_ERROR = 'dataProviderError',
-  TIMEOUT_ERROR = 'timeoutError',
-  CONNECTION_ERROR = 'connectionError',
-  // RES_EMPTY_ERROR = 'responseEmptyError',
-  // RES_INVALID_ERROR = 'responseInvalidError',
-  CUSTOM_ERROR = 'customError',
-}
-
-/**
- * Maxiumum number of characters that a feedId can contain.
- */
-export const MAX_FEED_ID_LENGTH = 300
-
-// We should tune these as we collect data, this is the default bucket distribution that prom comes with
-export const requestDurationBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]

package/src/metrics/index.ts

@@ -1,115 +0,0 @@
-import * as client from 'prom-client'
-import { HttpRequestType, requestDurationBuckets } from './constants'
-import { AdapterRequest, makeLogger } from '../util'
-import { AdapterConfig } from '../config'
-import fastify, { FastifyReply, HookHandlerDoneFunction } from 'fastify'
-import { join } from 'path'
-import { AdapterError } from '../validation/error'
-
-const logger = makeLogger('Metrics')
-
-export function setupMetricsServer(name: string, config: AdapterConfig) {
-  const metricsApp = fastify({
-    logger: false,
-  })
-  const metricsPort = config.METRICS_PORT
-  const endpoint = config.METRICS_USE_BASE_URL ? join(config.BASE_URL, 'metrics') : '/metrics'
-  const eaHost = config.EA_HOST
-
-  setupMetrics(name, config)
-
-  metricsApp.get(endpoint, async (_, res) => {
-    res.type('txt')
-    res.send(await client.register.metrics())
-  })
-
-  metricsApp.listen(metricsPort, eaHost, () =>
-    logger.info(`Monitoring listening on port ${metricsPort}!`),
-  )
-}
-
-export const setupMetrics = (name: string, config: AdapterConfig): void => {
-  client.collectDefaultMetrics()
-  client.register.setDefaultLabels({
-    app_name: config.METRICS_NAME || name || 'N/A',
-    app_version: config['npm_package_version'],
-  })
-}
-
-/**
- * Builds metrics middleware that records end to end EA response times
- * and count of requests
- *
- * @returns the cache middleware function
- */
-export const buildMetricsMiddleware = (
-  req: AdapterRequest,
-  res: FastifyReply,
-  done: HookHandlerDoneFunction,
-) => {
-  const feedId = req.requestContext.meta?.metrics?.feedId || 'N/A'
-  const labels = buildHttpRequestMetricsLabel(
-    feedId,
-    req.requestContext.meta?.error,
-    req.requestContext.meta?.metrics?.cacheHit,
-  )
-
-  // Record number of requests sent to EA
-  httpRequestsTotal.labels(labels).inc()
-
-  // Record response time of request through entire EA
-  httpRequestDurationSeconds.observe(res.getResponseTime())
-  logger.debug(`Response time for ${feedId}: ${res.getResponseTime()}ms`)
-  done()
-}
-
-const buildHttpRequestMetricsLabel = (
-  feedId: string,
-  error?: AdapterError | Error,
-  cacheHit?: boolean,
-): Parameters<typeof httpRequestsTotal.labels>[0] => {
-  const labels: Parameters<typeof httpRequestsTotal.labels>[0] = {}
-  labels.method = 'POST'
-  labels.feed_id = feedId
-  if (error instanceof AdapterError) {
-    // If error present and an instace of AdapterError, build label from error info
-    labels.type = error?.metricsLabel || HttpRequestType.ADAPTER_ERROR
-    labels.status_code = error?.statusCode
-    labels.provider_status_code = error?.providerStatusCode
-  } else if (error instanceof Error) {
-    // If error present and not instance of generic Error, unexpected failure occurred
-    labels.type = HttpRequestType.ADAPTER_ERROR
-    labels.status_code = 500
-  } else {
-    // If no error present, request went as expected
-    labels.status_code = 200
-    if (cacheHit) {
-      labels.type = HttpRequestType.CACHE_HIT
-    } else {
-      labels.type = HttpRequestType.DATA_PROVIDER_HIT
-      labels.provider_status_code = 200
-    }
-  }
-
-  return labels
-}
-
-export const httpRequestsTotal = new client.Counter({
-  name: 'http_requests_total',
-  help: 'The number of http requests this external adapter has serviced for its entire uptime',
-  labelNames: [
-    'method',
-    'status_code',
-    'retry',
-    'type',
-    'is_cache_warming',
-    'feed_id',
-    'provider_status_code',
-  ] as const,
-})
-
-export const httpRequestDurationSeconds = new client.Histogram({
-  name: 'http_request_duration_seconds',
-  help: 'A histogram bucket of the distribution of http request durations',
-  buckets: requestDurationBuckets,
-})

package/src/metrics/util.ts

@@ -1,18 +0,0 @@
-import { AdapterMetricsMeta, AdapterRequestData } from '../util'
-import { AdapterEndpoint } from '../adapter'
-import { calculateFeedId } from '../cache'
-import { AdapterConfig } from '../config'
-
-export const getMetricsMeta = (
-  {
-    adapterEndpoint,
-    adapterConfig,
-  }: {
-    adapterEndpoint: AdapterEndpoint
-    adapterConfig: AdapterConfig
-  },
-  data: AdapterRequestData,
-): AdapterMetricsMeta => {
-  const feedId = calculateFeedId({ adapterEndpoint, adapterConfig }, data)
-  return { feedId }
-}

package/src/rate-limiting/background/fixed-frequency.ts

@@ -1,45 +0,0 @@
-import { AdapterRateLimitTier, BackgroundExecuteRateLimiter, consolidateTierLimits } from '..'
-import { AdapterEndpoint } from '../../adapter'
-import { makeLogger } from '../../util'
-
-const logger = makeLogger('FixedFrequencyRateLimiter')
-export const DEFAULT_SHARED_MS_BETWEEN_REQUESTS = 5000
-
-export class FixedFrequencyRateLimiter implements BackgroundExecuteRateLimiter {
-  msBetweenRequestsMap: {
-    [endpointName: string]: number // In ms
-  } = {}
-
-  initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier) {
-    // Translate the hourly limit into reqs per minute
-    let sharedMsBetweenRequests = 1000 / consolidateTierLimits(limits)
-
-    // If there is no limit set, we use some reasonable number
-    if (!limits?.rateLimit1h && !limits?.rateLimit1m && !limits?.rateLimit1s) {
-      // 5s period for all seems good
-      sharedMsBetweenRequests = DEFAULT_SHARED_MS_BETWEEN_REQUESTS
-    }
-
-    logger.debug('Using fixed frequency batch rate limiting')
-    for (const endpoint of endpoints) {
-      if (endpoint.rateLimiting?.allocationPercentage == null) {
-        throw new Error(`Allocation percentage for endpoint "${endpoint.name}" is null`)
-      }
-
-      this.msBetweenRequestsMap[endpoint.name] =
-        (sharedMsBetweenRequests / endpoint.rateLimiting?.allocationPercentage) * 100
-
-      logger.debug(
-        `Endpoint [${endpoint.name}]: ${
-          this.msBetweenRequestsMap[endpoint.name] / 1000
-        }s between requests`,
-      )
-    }
-
-    return this
-  }
-
-  msUntilNextExecution(endpointName: string): number {
-    return this.msBetweenRequestsMap[endpointName]
-  }
-}