@chainlink/external-adapter-framework 0.0.14 → 0.0.15

package/src/examples/coingecko/test/integration/fixtures.ts

@@ -0,0 +1,42 @@
+import nock from 'nock'
+import capturedRequests from './capturedRequests.json'
+
+type RecordedRequest = Record<
+  string,
+  { statusCode: number; response: unknown; method: string; path: string }[]
+>
+
+// This is a generic fixture file that can be copy-pasted to generate fixtures from a
+// capturedRequests.json file.  To generate a capturedRequests.json file,
+// set env var RECORD=true and run e2e tests.
+export const createMocks = (): nock.Scope[] => {
+  const mocks: nock.Scope[] = []
+  for (const baseUrl in capturedRequests) {
+    const mock = nock(baseUrl, { encodedQueryParams: true })
+    for (const request of (capturedRequests as RecordedRequest)[baseUrl]) {
+      if (request.method === 'GET') {
+        mock
+          .get((uri: string) => {
+            const deconstructedRequestUrl = uri.split(/[?=,&]+/)
+            const deconstructedMockUrl = request.path.split(/[?=,&]+/)
+            return (
+              deconstructedRequestUrl.sort().join(',') === deconstructedMockUrl.sort().join(',')
+            )
+          })
+          .times(100)
+          .reply(request.statusCode, () => request.response, [
+            'Content-Type',
+            'application/json',
+            'Connection',
+            'close',
+            'Vary',
+            'Accept-Encoding',
+            'Vary',
+            'Origin',
+          ])
+      }
+    }
+    mocks.push(mock)
+  }
+  return mocks
+}

package/src/examples/coingecko-old/batch-warming.ts

@@ -0,0 +1,79 @@
+import { AxiosRequestConfig, AxiosResponse } from 'axios'
+import { AdapterContext, AdapterEndpoint } from '../../adapter'
+import { BatchWarmingTransport } from '../../transports/batch-warming'
+import { ProviderResult } from '../../util'
+import { InputParameters } from '../../validation'
+
+const DEFAULT_URL = 'https://pro-api.coingecko.com/api/v3'
+
+interface AdapterRequestParams {
+  base: string
+  quote: string
+}
+
+const inputParameters: InputParameters = {
+  coinid: {
+    description:
+      'The CoinGecko id or array of ids of the coin(s) to query (Note: because of current limitations to use a dummy base will need to be supplied)',
+    required: false,
+  },
+  base: {
+    aliases: ['from', 'coin'],
+    description: 'The symbol or array of symbols of the currency to query',
+    required: true,
+  },
+  quote: {
+    aliases: ['to', 'market'],
+    description: 'The symbol of the currency to convert to',
+    required: true,
+  },
+}
+interface ProviderRequestBody {
+  ids: string
+  vs_currencies: string
+}
+
+interface ProviderResponseBody {
+  [base: string]: {
+    [quote: string]: number
+  }
+}
+
+const batchEndpointTransport = new BatchWarmingTransport({
+  prepareRequest: (
+    params: AdapterRequestParams[],
+    context: AdapterContext,
+  ): AxiosRequestConfig<ProviderRequestBody> => {
+    return {
+      baseURL: DEFAULT_URL,
+      url: '/simple/price',
+      method: 'GET',
+      params: {
+        ids: [...new Set(params.map((p) => p.base))].join(','),
+        vs_currencies: [...new Set(params.map((p) => p.quote))].join(','),
+        x_cg_pro_api_key: context.adapterConfig.API_KEY,
+      },
+    }
+  },
+  parseResponse: (
+    params: AdapterRequestParams[],
+    res: AxiosResponse<ProviderResponseBody>,
+  ): ProviderResult<AdapterRequestParams>[] => {
+    const entries = [] as ProviderResult<AdapterRequestParams>[]
+    for (const [base, entry] of Object.entries(res.data)) {
+      for (const [quote, price] of Object.entries(entry)) {
+        entries.push({
+          params: { base, quote },
+          value: price,
+        })
+      }
+    }
+    return entries
+  },
+})
+
+export const batchEndpoint: AdapterEndpoint = {
+  name: 'batch',
+  transport: batchEndpointTransport,
+  inputParameters,
+}

package/src/examples/coingecko-old/index.ts

@@ -0,0 +1,9 @@
+import { Adapter } from '../../adapter'
+import { batchEndpoint } from './batch-warming'
+import { restEndpoint } from './rest'
+
+export const adapter: Adapter = {
+  name: 'coingecko',
+  defaultEndpoint: 'batch',
+  endpoints: [restEndpoint, batchEndpoint],
+}

package/src/examples/coingecko-old/rest.ts

@@ -0,0 +1,77 @@
+import { AxiosRequestConfig, AxiosResponse } from 'axios'
+import { AdapterEndpoint } from '../../adapter'
+import { RestTransport } from '../../transports'
+import { AdapterRequest, AdapterResponse } from '../../util'
+import { InputParameters } from '../../validation'
+
+const DEFAULT_URL = 'https://api.coingecko.com/api/v3'
+
+interface AdapterRequestParams {
+  base: string
+  quote: string
+}
+
+const inputParameters: InputParameters = {
+  coinid: {
+    description:
+      'The CoinGecko id or array of ids of the coin(s) to query (Note: because of current limitations to use a dummy base will need to be supplied)',
+    required: false,
+  },
+  base: {
+    aliases: ['from', 'coin'],
+    description: 'The symbol or array of symbols of the currency to query',
+    required: true,
+  },
+  quote: {
+    aliases: ['to', 'market'],
+    description: 'The symbol of the currency to convert to',
+    required: true,
+  },
+}
+
+interface ProviderRequestBody {
+  ids: string
+  vs_currencies: string
+}
+
+interface ProviderResponseBody {
+  [base: string]: {
+    [quote: string]: number
+  }
+}
+
+const restEndpointTransport = new RestTransport({
+  prepareRequest: (
+    req: AdapterRequest<AdapterRequestParams>,
+  ): AxiosRequestConfig<ProviderRequestBody> => {
+    return {
+      baseURL: DEFAULT_URL,
+      url: '/simple/price',
+      method: 'GET',
+      params: {
+        ids: req.requestContext.data.base,
+        vs_currencies: req.requestContext.data.quote,
+      },
+    }
+  },
+  parseResponse: (
+    req: AdapterRequest<AdapterRequestParams>,
+    res: AxiosResponse<ProviderResponseBody>,
+  ): AdapterResponse<ProviderResponseBody> => {
+    return {
+      data: res.data,
+      statusCode: 200,
+      result: res.data[req.requestContext.data.base]?.[req.requestContext.data.quote],
+    }
+  },
+  options: {
+    coalescing: true,
+  },
+})
+
+export const restEndpoint: AdapterEndpoint = {
+  name: 'rest',
+  aliases: ['qwe'],
+  transport: restEndpointTransport,
+  inputParameters,
+}

package/src/examples/ncfx/config/index.ts

@@ -0,0 +1,12 @@
+export const customSettings = {
+  USERNAME: {
+    description: 'Username for the NCFX API',
+    type: 'string',
+    required: true,
+  },
+  PASSWORD: {
+    description: 'Password for the NCFX API',
+    type: 'string',
+    required: true,
+  },
+} as const

package/src/examples/ncfx/index.ts

@@ -0,0 +1,9 @@
+import { Adapter } from '../../adapter'
+import { customSettings } from './config'
+import { webSocketEndpoint } from './websocket'
+
+export const adapter: Adapter = {
+  name: 'ncfx',
+  endpoints: [webSocketEndpoint],
+  customSettings,
+}

package/src/examples/ncfx/websocket.ts

@@ -0,0 +1,99 @@
+import { RawData, WebSocket } from 'ws'
+import { AdapterContext, AdapterEndpoint } from '../../adapter'
+import { WebSocketTransport } from '../../transports/websocket'
+import { ProviderResult, makeLogger } from '../../util'
+import { InputParameters } from '../../validation'
+import { customSettings } from './config'
+
+interface AdapterRequestParams {
+  base: string
+  quote: string
+}
+
+export const inputParameters: InputParameters = {
+  base: {
+    aliases: ['from', 'coin'],
+    description: 'The symbol of the currency to query',
+    required: true,
+  },
+  quote: {
+    aliases: ['to', 'market'],
+    description: 'The symbol of the currency to convert to',
+    required: true,
+  },
+}
+
+interface ProviderMessage {
+  timestamp: string
+  currencyPair: string
+  bid: number
+  offer: number
+  mid: number
+  changes: [
+    {
+      period: string
+      change: number
+      percentage: number
+    },
+  ]
+}
+
+const logger = makeLogger('NcfxWebSocketTransport')
+
+export const websocketTransport = new WebSocketTransport({
+  url: 'wss://feed.newchangefx.com/cryptodata',
+  handlers: {
+    open(connection: WebSocket, context: AdapterContext<typeof customSettings>) {
+      return new Promise((resolve, reject) => {
+        // Set up listener
+        connection.on('message', (data: RawData) => {
+          const parsed = JSON.parse(data.toString())
+          if (parsed.Message?.startsWith('Logged in as user')) {
+            logger.info('Got logged in response, connection is ready')
+            resolve()
+          } else {
+            reject(new Error('Unexpected message after WS connection open'))
+          }
+        })
+        // Send login payload
+        connection.send(
+          JSON.stringify({
+            request: 'login',
+            username: context.adapterConfig.USERNAME,
+            password: context.adapterConfig.PASSWORD,
+          }),
+        )
+      })
+    },
+
+    message(message: ProviderMessage[]): ProviderResult<AdapterRequestParams>[] {
+      if (!Array.isArray(message)) {
+        logger.debug('WS message is not array, skipping')
+        return []
+      }
+      return message.map((m) => {
+        const [base, quote] = m.currencyPair.split('/')
+        return {
+          params: { base, quote },
+          value: m.offer,
+        }
+      })
+    },
+  },
+  builders: {
+    subscribeMessage: (params: AdapterRequestParams) => ({
+      request: 'subscribe',
+      ccy: `${params.base}/${params.quote}`,
+    }),
+    unsubscribeMessage: (params: AdapterRequestParams) => ({
+      request: 'unsubscribe',
+      ccy: `${params.base}/${params.quote}`,
+    }),
+  },
+})
+
+export const webSocketEndpoint: AdapterEndpoint = {
+  name: 'websocket',
+  transport: websocketTransport,
+  inputParameters,
+}

package/src/index.ts

@@ -0,0 +1,149 @@
+import fastify, { FastifyInstance } from 'fastify'
+import { Server } from 'http'
+import { AddressInfo } from 'net'
+import { join } from 'path'
+import { Adapter, InitializedAdapter, initializeAdapter, AdapterDependencies } from './adapter'
+import { callBackgroundExecutes } from './background-executor'
+import { buildCacheMiddleware } from './cache'
+import { AdapterConfig, buildAdapterConfig } from './config'
+import { buildMetricsMiddleware, setupMetricsServer } from './metrics'
+import { buildTransportHandler } from './transports'
+import { AdapterRouteGeneric, loggingContextMiddleware, makeLogger } from './util'
+import { loadTestPayload } from './util/test-payload-loader'
+import { errorCatchingMiddleware, validatorMiddleware } from './validation'
+
+const logger = makeLogger('Main')
+
+const VERSION = process.env['npm_package_version']
+
+/**
+ * Main function for the framework.
+ * Initializes config and dependencies, uses those to initialize Transports, and starts listening for requests.
+ *
+ * @param adapter - an object describing an External Adapter
+ * @param dependencies - an optional object with adapter dependencies to inject
+ * @returns a Promise that resolves to the http.Server listening for connections
+ */
+export const expose = async (
+  adapter: Adapter,
+  dependencies?: Partial<AdapterDependencies>,
+): Promise<Server | undefined> => {
+  const config = buildAdapterConfig({
+    overrides: adapter.envDefaultOverrides,
+    customSettings: adapter.customSettings,
+  })
+  // Initialize adapter (create dependencies, inject them, build endpoint map, etc.)
+  const initializedAdapter = await initializeAdapter(adapter, config, dependencies)
+
+  let server: Server | undefined = undefined
+
+  if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
+    setupMetricsServer(adapter.name, config)
+  }
+
+  if (config.EA_MODE === 'reader' || config.EA_MODE === 'reader-writer') {
+    // Main REST API server to handle incoming requests
+    const app = await buildRestApi(config, initializedAdapter)
+
+    // Start listening for incoming requests
+    try {
+      await app.listen(config.EA_PORT, config.EA_HOST)
+    } catch (err) {
+      logger.fatal(`There was an error when starting the EA server: ${err}`)
+      process.exit()
+    }
+
+    logger.info(`Listening on port ${(app.server.address() as AddressInfo).port}`)
+    server = app.server
+  } else {
+    logger.info('REST API is disabled; this instance will not process incoming requests.')
+  }
+
+  if (config.EA_MODE === 'writer' || config.EA_MODE === 'reader-writer') {
+    // Start background loop that will take care of calling any async Transports
+    logger.info('Starting background execution loop')
+    callBackgroundExecutes(initializedAdapter, server)
+  } else {
+    logger.info(
+      'Background executor is disabled; this instance will not perform async background executes.',
+    )
+  }
+
+  return server
+}
+
+async function buildRestApi(config: AdapterConfig, initializedAdapter: InitializedAdapter) {
+  const app = fastify()
+
+  // Add healthcheck endpoint before middlewares to bypass them
+  app.get(join(config.BASE_URL, 'health'), (req, res) => {
+    res.status(200).send({ message: 'OK', version: VERSION })
+  })
+
+  // Use global error handling
+  app.setErrorHandler(errorCatchingMiddleware)
+
+  const transportHandler = await buildTransportHandler(initializedAdapter)
+
+  app.register(async (router) => {
+    // Set up "middlewares" (hooks in fastify)
+    router.addHook<AdapterRouteGeneric>('preHandler', validatorMiddleware(initializedAdapter))
+    router.addHook<AdapterRouteGeneric>('preHandler', buildCacheMiddleware(initializedAdapter))
+    if (config['CORRELATION_ID_ENABLED']) {
+      router.addHook<AdapterRouteGeneric>('onRequest', loggingContextMiddleware)
+    }
+
+    router.route<AdapterRouteGeneric>({
+      url: config.BASE_URL,
+      method: 'POST',
+      handler: transportHandler,
+    })
+
+    if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
+      router.addHook<AdapterRouteGeneric>('onResponse', buildMetricsMiddleware)
+    }
+  })
+
+  // Add smoke endpoint after middleware which are needed for tests
+  buildSmokeEndpoint(app, config)
+  return app
+}
+
+/**
+ * Adds the /smoke endpoint to the API for smoke testing the adapter
+ *
+ * @param app - the Fastify instance
+ * @param config - the initialized adapter config
+ */
+function buildSmokeEndpoint(app: FastifyInstance, config: AdapterConfig) {
+  const testPayload = loadTestPayload(config.SMOKE_TEST_PAYLOAD_FILE_NAME)
+  app.get(join(config.BASE_URL, 'smoke'), async (_, res) => {
+    if (testPayload.isDefault) {
+      return res.status(200).send('OK')
+    }
+
+    const errors = []
+    for (const index in testPayload.requests) {
+      try {
+        const request = { id: index, data: testPayload.requests[index] }
+        // Use Fastify's app inject to pass smoke requests internally
+        const response = await app.inject({
+          method: 'POST',
+          url: '/',
+          payload: request,
+        })
+        const parsedResponse = JSON.parse(response.body)
+        // Throw error if not 2xx status code
+        if (parsedResponse.statusCode < 200 || parsedResponse.statusCode > 299) {
+          throw Error('Smoke test request failed')
+        }
+      } catch (e: unknown) {
+        errors.push(e)
+      }
+    }
+    if (errors.length > 0) {
+      return res.status(500).send(errors)
+    }
+    return res.status(200).send('OK')
+  })
+}

package/src/metrics/constants.ts

@@ -0,0 +1,23 @@
+export enum HttpRequestType {
+  CACHE_HIT = 'cacheHit',
+  DATA_PROVIDER_HIT = 'dataProviderHit',
+  ADAPTER_ERROR = 'adapterError',
+  INPUT_ERROR = 'inputError',
+  RATE_LIMIT_ERROR = 'rateLimitError',
+  // BURST_LIMIT_ERROR = 'burstLimitError',
+  // BACKOFF_ERROR = 'backoffError',
+  DP_ERROR = 'dataProviderError',
+  TIMEOUT_ERROR = 'timeoutError',
+  CONNECTION_ERROR = 'connectionError',
+  // RES_EMPTY_ERROR = 'responseEmptyError',
+  // RES_INVALID_ERROR = 'responseInvalidError',
+  CUSTOM_ERROR = 'customError',
+}
+
+/**
+ * Maxiumum number of characters that a feedId can contain.
+ */
+export const MAX_FEED_ID_LENGTH = 300
+
+// We should tune these as we collect data, this is the default bucket distribution that prom comes with
+export const requestDurationBuckets = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]

package/src/metrics/index.ts

@@ -0,0 +1,115 @@
+import * as client from 'prom-client'
+import { HttpRequestType, requestDurationBuckets } from './constants'
+import { AdapterRequest, makeLogger } from '../util'
+import { AdapterConfig } from '../config'
+import fastify, { FastifyReply, HookHandlerDoneFunction } from 'fastify'
+import { join } from 'path'
+import { AdapterError } from '../validation/error'
+
+const logger = makeLogger('Metrics')
+
+export function setupMetricsServer(name: string, config: AdapterConfig) {
+  const metricsApp = fastify({
+    logger: false,
+  })
+  const metricsPort = config.METRICS_PORT
+  const endpoint = config.METRICS_USE_BASE_URL ? join(config.BASE_URL, 'metrics') : '/metrics'
+  const eaHost = config.EA_HOST
+
+  setupMetrics(name, config)
+
+  metricsApp.get(endpoint, async (_, res) => {
+    res.type('txt')
+    res.send(await client.register.metrics())
+  })
+
+  metricsApp.listen(metricsPort, eaHost, () =>
+    logger.info(`Monitoring listening on port ${metricsPort}!`),
+  )
+}
+
+export const setupMetrics = (name: string, config: AdapterConfig): void => {
+  client.collectDefaultMetrics()
+  client.register.setDefaultLabels({
+    app_name: config.METRICS_NAME || name || 'N/A',
+    app_version: config['npm_package_version'],
+  })
+}
+
+/**
+ * Builds metrics middleware that records end to end EA response times
+ * and count of requests
+ *
+ * @returns the cache middleware function
+ */
+export const buildMetricsMiddleware = (
+  req: AdapterRequest,
+  res: FastifyReply,
+  done: HookHandlerDoneFunction,
+) => {
+  const feedId = req.requestContext.meta?.metrics?.feedId || 'N/A'
+  const labels = buildHttpRequestMetricsLabel(
+    feedId,
+    req.requestContext.meta?.error,
+    req.requestContext.meta?.metrics?.cacheHit,
+  )
+
+  // Record number of requests sent to EA
+  httpRequestsTotal.labels(labels).inc()
+
+  // Record response time of request through entire EA
+  httpRequestDurationSeconds.observe(res.getResponseTime())
+  logger.debug(`Response time for ${feedId}: ${res.getResponseTime()}ms`)
+  done()
+}
+
+const buildHttpRequestMetricsLabel = (
+  feedId: string,
+  error?: AdapterError | Error,
+  cacheHit?: boolean,
+): Parameters<typeof httpRequestsTotal.labels>[0] => {
+  const labels: Parameters<typeof httpRequestsTotal.labels>[0] = {}
+  labels.method = 'POST'
+  labels.feed_id = feedId
+  if (error instanceof AdapterError) {
+    // If error present and an instace of AdapterError, build label from error info
+    labels.type = error?.metricsLabel || HttpRequestType.ADAPTER_ERROR
+    labels.status_code = error?.statusCode
+    labels.provider_status_code = error?.providerStatusCode
+  } else if (error instanceof Error) {
+    // If error present and not instance of generic Error, unexpected failure occurred
+    labels.type = HttpRequestType.ADAPTER_ERROR
+    labels.status_code = 500
+  } else {
+    // If no error present, request went as expected
+    labels.status_code = 200
+    if (cacheHit) {
+      labels.type = HttpRequestType.CACHE_HIT
+    } else {
+      labels.type = HttpRequestType.DATA_PROVIDER_HIT
+      labels.provider_status_code = 200
+    }
+  }
+
+  return labels
+}
+
+export const httpRequestsTotal = new client.Counter({
+  name: 'http_requests_total',
+  help: 'The number of http requests this external adapter has serviced for its entire uptime',
+  labelNames: [
+    'method',
+    'status_code',
+    'retry',
+    'type',
+    'is_cache_warming',
+    'feed_id',
+    'provider_status_code',
+  ] as const,
+})
+
+export const httpRequestDurationSeconds = new client.Histogram({
+  name: 'http_request_duration_seconds',
+  help: 'A histogram bucket of the distribution of http request durations',
+  buckets: requestDurationBuckets,
+})

package/src/metrics/util.ts

@@ -0,0 +1,18 @@
+import { AdapterMetricsMeta, AdapterRequestData } from '../util'
+import { AdapterEndpoint } from '../adapter'
+import { calculateFeedId } from '../cache'
+import { AdapterConfig } from '../config'
+
+export const getMetricsMeta = (
+  {
+    adapterEndpoint,
+    adapterConfig,
+  }: {
+    adapterEndpoint: AdapterEndpoint
+    adapterConfig: AdapterConfig
+  },
+  data: AdapterRequestData,
+): AdapterMetricsMeta => {
+  const feedId = calculateFeedId({ adapterEndpoint, adapterConfig }, data)
+  return { feedId }
+}

package/src/rate-limiting/background/fixed-frequency.ts

@@ -0,0 +1,45 @@
+import { AdapterRateLimitTier, BackgroundExecuteRateLimiter, consolidateTierLimits } from '..'
+import { AdapterEndpoint } from '../../adapter'
+import { makeLogger } from '../../util'
+
+const logger = makeLogger('FixedFrequencyRateLimiter')
+export const DEFAULT_SHARED_MS_BETWEEN_REQUESTS = 5000
+
+export class FixedFrequencyRateLimiter implements BackgroundExecuteRateLimiter {
+  msBetweenRequestsMap: {
+    [endpointName: string]: number // In ms
+  } = {}
+
+  initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier) {
+    // Translate the hourly limit into reqs per minute
+    let sharedMsBetweenRequests = 1000 / consolidateTierLimits(limits)
+
+    // If there is no limit set, we use some reasonable number
+    if (!limits?.rateLimit1h && !limits?.rateLimit1m && !limits?.rateLimit1s) {
+      // 5s period for all seems good
+      sharedMsBetweenRequests = DEFAULT_SHARED_MS_BETWEEN_REQUESTS
+    }
+
+    logger.debug('Using fixed frequency batch rate limiting')
+    for (const endpoint of endpoints) {
+      if (endpoint.rateLimiting?.allocationPercentage == null) {
+        throw new Error(`Allocation percentage for endpoint "${endpoint.name}" is null`)
+      }
+
+      this.msBetweenRequestsMap[endpoint.name] =
+        (sharedMsBetweenRequests / endpoint.rateLimiting?.allocationPercentage) * 100
+
+      logger.debug(
+        `Endpoint [${endpoint.name}]: ${
+          this.msBetweenRequestsMap[endpoint.name] / 1000
+        }s between requests`,
+      )
+    }
+
+    return this
+  }
+
+  msUntilNextExecution(endpointName: string): number {
+    return this.msBetweenRequestsMap[endpointName]
+  }
+}