@chainlink/external-adapter-framework 0.0.10 → 0.0.12

package/src/transports/index.ts

@@ -0,0 +1,152 @@
+import { FastifyReply } from 'fastify'
+import { AdapterContext, AdapterDependencies, InitializedAdapter } from '../adapter'
+import {
+  Cache,
+  CacheEntry,
+  calculateCacheKey,
+  calculateFeedId,
+  pollResponseFromCache,
+} from '../cache'
+import { AdapterConfig, SettingsMap } from '../config'
+import { makeLogger } from '../util'
+import { AdapterRequest, AdapterResponse, ProviderResult } from '../util/request'
+
+export * from './batch-warming'
+export * from './rest'
+export * from './websocket'
+
+const logger = makeLogger('Transport')
+
+/**
+ * Generic interface for a Transport.
+ * A Transport defines the way in which an AdapterEndpoint will process incoming requests to
+ * fetch data from a Data Provider. The setup phase will take care of the former, while the
+ * backgroundExecute will be in charge of the latter.
+ * This separation gives us the ability of splitting these concerns, and optionally parallelizing
+ * the reading and writing of data to a centralized Cache.
+ *
+ * @typeParam Params - the structure of the AdapterRequest's body
+ * @typeParam Result - the structure of the AdapterResponse's body
+ */
+export interface Transport<Params, Result, CustomSettings extends SettingsMap> {
+  // TODO: docs / examples to extend dependencies? e.g. JSON rpc
+  /**
+   * Initializes the transport in the Adapter context.
+   *
+   * @param dependencies - Adapter dependencies (e.g. cache instance)
+   * @returns an empty Promise
+   */
+  initialize: (dependencies: AdapterDependencies) => Promise<void>
+
+  /**
+   * Checks if the Transport has already set up the incoming request.
+   * This will likely use the cacheKey generated in the provided request.
+   *
+   * @param req - the incoming AdapterRequest
+   * @returns a Promise that returns true if setup is in place
+   */
+  hasBeenSetUp: (req: AdapterRequest<Params>) => Promise<boolean>
+
+  /**
+   * Sets up the incoming request within the Transport.
+   * In other words, it means that the Adapter has recognized this request, and will begin
+   * fetching the necessary data from a Data Provider.
+   *
+   * @param req - the incoming AdapterRequest
+   * @param config - common configuration for the Adapter as a whole
+   * @returns a Promise that _optionally_ returns an AdapterResponse, if the Transport has the capability of
+   *   immediately fetching data and returning it without the background process.
+   */
+  setup: (
+    req: AdapterRequest<Params>,
+    config: AdapterConfig<CustomSettings>,
+  ) => Promise<AdapterResponse<Result> | void>
+
+  // TODO: Might need a mechanism to know if this bg execute is in flight to avoid multitple simultaneous invocations
+  /**
+   * If the Transport relies on an async mechanism, implementing this function will make it
+   * so the background task executor calls it on Adapter startup.
+   *
+   * @param context - background context for the execution (e.g. endpoint name)
+   */
+  backgroundExecute?: (context: AdapterContext<CustomSettings>) => Promise<number>
+}
+
+/**
+ * Helper method to build cache entries to set after getting a bunch of responses from a DP.
+ *
+ * @param results - a list of results coming from a DataProvider
+ * @param context - context for the Adapter
+ * @returns a list of CacheEntries of AdapterResponses
+ */
+export const buildCacheEntriesFromResults = <Params, CustomSettings extends SettingsMap>(
+  results: ProviderResult<Params>[],
+  context: AdapterContext<CustomSettings>,
+): CacheEntry<AdapterResponse<null>>[] =>
+  results.map((r) => {
+    const cacheEntry = {
+      key: calculateCacheKey(context as AdapterContext, r.params),
+      value: {
+        result: r.value,
+        statusCode: 200,
+        data: null, // TODO: Maybe add data as well?
+      },
+    }
+    if (
+      context.adapterConfig.METRICS_ENABLED &&
+      context.adapterConfig.EXPERIMENTAL_METRICS_ENABLED
+    ) {
+      const metrics = {
+        maxAge: Date.now() + context.adapterConfig.CACHE_MAX_AGE, // TODO: Replace with telemetry structure in future
+        meta: {
+          metrics: {
+            feedId: calculateFeedId(context as AdapterContext, r.params),
+          },
+        },
+      }
+      cacheEntry.value = { ...cacheEntry.value, ...metrics }
+    }
+    return cacheEntry
+  })
+
+/**
+ * Takes an Adapter, its configuration, and its dependencies, and it creates an express middleware
+ * that will pass along the AdapterRequest to the appropriate Transport (acc. to the endpoint in the req.)
+ *
+ * @param adapter - main adapter object, already initialized
+ * @returns the transport handler middleware function
+ */
+export const buildTransportHandler =
+  (adapter: InitializedAdapter) => async (req: AdapterRequest, reply: FastifyReply) => {
+    // Get transport, must be here because it's already checked in the validator
+    const transport = adapter.endpointsMap[req.requestContext.endpointName].transport
+
+    // Set up transport if it hasn't been done already
+    if (!(await transport.hasBeenSetUp(req))) {
+      logger.debug('Transport not set up yet, doing so...')
+      const immediateResponse = await transport.setup(req, adapter.config)
+      if (immediateResponse) {
+        logger.debug('Got immediate response from transport, sending as response')
+        return reply.send(immediateResponse)
+      }
+    }
+    logger.debug('Transport is set up, polling cache for response...')
+    const response = await pollResponseFromCache(
+      adapter.dependencies.cache as Cache<AdapterResponse>,
+      req.requestContext.cacheKey,
+      {
+        maxRetries: adapter.config.CACHE_POLLING_MAX_RETRIES,
+        sleep: adapter.config.CACHE_POLLING_SLEEP_MS,
+      },
+    )
+
+    if (response) {
+      logger.debug('Got a response from the cache, sending that back')
+      return reply.send(response)
+    }
+
+    logger.debug('Ran out of polling attempts, returning timeout')
+    reply.statusCode = 504
+
+    return reply.send()
+  }

package/src/transports/metrics.ts

@@ -0,0 +1,95 @@
+import * as client from 'prom-client'
+import { AdapterContext } from '../adapter'
+import { calculateCacheKey, calculateFeedId } from '../cache'
+import { requestDurationBuckets } from '../metrics/constants'
+
+// Data Provider Metrics
+export const dataProviderMetricsLabel = (providerStatusCode?: number, method = 'get') => ({
+  provider_status_code: providerStatusCode,
+  method: method.toUpperCase(),
+})
+
+export const dataProviderRequests = new client.Counter({
+  name: 'data_provider_requests',
+  help: 'The number of http requests that are made to a data provider',
+  labelNames: ['method', 'provider_status_code'] as const,
+})
+
+export const dataProviderRequestDurationSeconds = new client.Histogram({
+  name: 'data_provider_request_duration_seconds',
+  help: 'A histogram bucket of the distribution of data provider request durations',
+  buckets: requestDurationBuckets,
+})
+
+// Websocket Metrics
+export const connectionErrorLabels = (message: string) => ({
+  // Key,
+  message,
+})
+
+export const messageSubsLabels = (feed_id: string, cache_key: string) => ({
+  feed_id,
+  subscription_key: cache_key,
+})
+
+// Record WS message and subscription metrics
+// Recalculate cacheKey and feedId for metrics
+// since avoiding storing extra info in expiring sorted set
+export const recordWsMessageMetrics = <AdapterParams>(
+  context: AdapterContext,
+  subscribes: AdapterParams[],
+  unsubscrices: AdapterParams[],
+): void => {
+  subscribes.forEach((param) => {
+    const feedId = calculateFeedId(context, param)
+    const cacheKey = calculateCacheKey(context, param)
+    // Record total number of ws messages sent
+    wsMessageTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
+
+    // Record total number of subscriptions made
+    wsSubscriptionTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
+
+    // Record number of active ws subscriptions
+    wsSubscriptionActive.labels(messageSubsLabels(feedId, cacheKey)).inc()
+  })
+  unsubscrices.forEach((param) => {
+    const feedId = calculateFeedId(context, param)
+    const cacheKey = calculateCacheKey(context, param)
+
+    // Record total number of ws messages sent
+    wsMessageTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
+
+    // Record number of active ws subscriptions
+    wsSubscriptionActive.labels(messageSubsLabels(feedId, cacheKey)).dec()
+  })
+}
+
+export const wsConnectionActive = new client.Gauge({
+  name: 'ws_connection_active',
+  help: 'The number of active connections',
+  labelNames: ['url'] as const,
+})
+
+export const wsConnectionErrors = new client.Counter({
+  name: 'ws_connection_errors',
+  help: 'The number of connection errors',
+  labelNames: ['url', 'message'] as const,
+})
+
+export const wsSubscriptionActive = new client.Gauge({
+  name: 'ws_subscription_active',
+  help: 'The number of currently active subscriptions',
+  labelNames: ['connection_url', 'feed_id', 'subscription_key'] as const,
+})
+
+export const wsSubscriptionTotal = new client.Counter({
+  name: 'ws_subscription_total',
+  help: 'The number of subscriptions opened in total',
+  labelNames: ['connection_url', 'feed_id', 'subscription_key'] as const,
+})
+
+export const wsMessageTotal = new client.Counter({
+  name: 'ws_message_total',
+  help: 'The number of messages received in total',
+  labelNames: ['feed_id', 'subscription_key'] as const,
+})

package/src/transports/rest.ts

@@ -0,0 +1,164 @@
+import { AdapterRequest, AdapterResponse } from '../util/request'
+import { Transport } from './'
+import { Cache } from '../cache'
+import { AxiosRequestConfig, AxiosResponse } from 'axios'
+import { makeLogger, sleep } from '../util'
+import { AdapterConfig, SettingsMap } from '../config'
+import { RequestRateLimiter } from '../rate-limiting'
+import { AdapterError } from '../validation/error'
+import { axiosRequest } from './util'
+import * as rateLimitMetrics from '../rate-limiting/metrics'
+import { AdapterDependencies } from '../adapter'
+
+const IN_FLIGHT_PREFIX = 'InFlight'
+
+const logger = makeLogger('RestTransport')
+
+/**
+ * Transport implementation that takes incoming requests, transforms them into a DataProvider request,
+ * and executes that request returning the response immediately from the `setup` function.
+ * Optionally, setting the `coalescing` option to `true` will make it so once a request is in flight,
+ * new adapter requests for the same feed will not trigger a new one, but return an empty promise from
+ * the setup instead so the normal cache polling mechanism is used.
+ *
+ * @typeParam AdapterParams - interface for the adapter request body
+ * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
+ * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
+ */
+export class RestTransport<
+  AdapterParams,
+  ProviderRequestBody,
+  ProviderResponseBody,
+  CustomSettings extends SettingsMap,
+> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings>
+{
+  inFlightPrefix!: string
+  cache!: Cache
+  rateLimiter!: RequestRateLimiter
+
+  constructor(
+    protected config: {
+      prepareRequest: (
+        req: AdapterRequest<AdapterParams>,
+        config: AdapterConfig<CustomSettings>,
+      ) =>
+        | AxiosRequestConfig<ProviderRequestBody>
+        | Promise<AxiosRequestConfig<ProviderRequestBody>>
+      parseResponse: (
+        req: AdapterRequest<AdapterParams>,
+        res: AxiosResponse<ProviderResponseBody>,
+        config: AdapterConfig<CustomSettings>,
+      ) => AdapterResponse<ProviderResponseBody> | Promise<AdapterResponse<ProviderResponseBody>>
+      options: {
+        coalescing: boolean
+      }
+    },
+  ) {}
+
+  async initialize(dependencies: AdapterDependencies): Promise<void> {
+    this.inFlightPrefix = `${IN_FLIGHT_PREFIX}-`
+    this.cache = dependencies.cache
+    this.rateLimiter = dependencies.requestRateLimiter
+  }
+
+  async hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean> {
+    if (!this.config.options.coalescing) {
+      return false
+    }
+
+    // Check if request is in flight
+    const inFlight = await this.cache.get(this.inFlightPrefix + req.requestContext.cacheKey)
+    if (inFlight) {
+      logger.debug('Request is in flight, transport has been set up')
+      return true
+    } else {
+      logger.debug('Request not in flight, transport not set up')
+      return false
+    }
+  }
+
+  protected async waitUntilUnderRateLimit(
+    options: {
+      maxRetries: number
+      msBetweenRetries: number
+    },
+    retry = 0,
+  ) {
+    if (this.rateLimiter.isUnderLimits()) {
+      logger.trace('Incoming request would not be under limits, moving on')
+      return
+    }
+
+    if (retry >= options.maxRetries) {
+      throw new AdapterError({
+        statusCode: 504,
+        message: `REST Transport timed out while waiting for rate limit availability (max retries: ${options.maxRetries})`,
+      })
+    }
+
+    logger.debug(`Request would be over rate limits, sleeping for ${options.msBetweenRetries}`)
+    await sleep(options.msBetweenRetries)
+    await this.waitUntilUnderRateLimit(options, retry + 1)
+  }
+
+  async setup(
+    req: AdapterRequest<AdapterParams>,
+    config: AdapterConfig<CustomSettings>,
+  ): Promise<AdapterResponse<ProviderResponseBody> | undefined> {
+    if (this.config.options.coalescing) {
+      logger.debug('Setting up rest transport, setting request in flight in cache')
+      // TODO: Should this use a separate cache?
+      // TODO: Set viable ttl to approximate timeout from API
+      // TODO: Make this ttl configurable
+      await this.cache.set(this.inFlightPrefix + req.requestContext.cacheKey, true, 2000) // Can't use Infinity for things like Redis
+    }
+
+    const request = await this.config.prepareRequest(req, config)
+
+    logger.trace('Check if we are under rate limits to perform request')
+    await this.waitUntilUnderRateLimit({
+      maxRetries: config.REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES,
+      msBetweenRetries: config.REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES,
+    })
+
+    logger.trace('Sending request to data provider...')
+    const providerResponse = await axiosRequest<ProviderRequestBody, ProviderResponseBody>(request)
+
+    logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`) // TODO: Sensitive data?
+    const parsedResponse = await this.config.parseResponse(req, providerResponse, config)
+
+    if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
+      // TODO: Potentially create function to add all telemetry data
+      parsedResponse.maxAge = Date.now() + config.CACHE_MAX_AGE
+      parsedResponse.meta = {
+        metrics: { feedId: req.requestContext.meta?.metrics?.feedId || 'N/A' },
+      }
+    }
+
+    logger.debug('Setting provider response in cache')
+    await this.cache.set(req.requestContext.cacheKey, parsedResponse, config.CACHE_MAX_AGE)
+
+    // Record cost of data provider call
+    const cost = rateLimitMetrics.retrieveCost(providerResponse.data)
+    rateLimitMetrics.rateLimitCreditsSpentTotal
+      .labels({
+        feed_id: req.requestContext.meta?.metrics?.feedId || 'N/A',
+        participant_id: req.requestContext.cacheKey,
+      })
+      .inc(cost)
+
+    // Update cacheHit flag in request meta for metrics use
+    req.requestContext.meta = {
+      ...req.requestContext.meta,
+      metrics: { ...req.requestContext.meta?.metrics, cacheHit: false },
+    }
+
+    // TODO: move this to a try/catch/finally
+    if (this.config.options.coalescing) {
+      logger.debug('Set provider response in cache, removing in flight from cache')
+      await this.cache.delete(this.inFlightPrefix)
+    }
+
+    return parsedResponse
+  }
+}

package/src/transports/util.ts

@@ -0,0 +1,63 @@
+import axios, { AxiosError, AxiosRequestConfig, AxiosResponse } from 'axios'
+import {
+  AdapterConnectionError,
+  AdapterDataProviderError,
+  AdapterError,
+  AdapterTimeoutError,
+} from '../validation/error'
+import * as transportMetrics from './metrics'
+
+/**
+ * Performs axios request along with metrics recording and error handling
+ *
+ * @param request - axios request config
+ * @returns axios response for the request
+ */
+export async function axiosRequest<ProviderRequestBody, ProviderResponseBody>(
+  request: AxiosRequestConfig<ProviderRequestBody>,
+): Promise<AxiosResponse<ProviderResponseBody>> {
+  const responseTimer = transportMetrics.dataProviderRequestDurationSeconds.startTimer()
+  let providerResponse: AxiosResponse<ProviderResponseBody>
+  try {
+    providerResponse = await axios.request<ProviderResponseBody>(request)
+  } catch (e: unknown) {
+    const error = e as AxiosError
+    // Request error
+    let providerStatusCode: number | undefined
+    let adapterError: AdapterError
+    if (error.code === 'ECONNABORTED') {
+      adapterError = new AdapterTimeoutError({})
+      providerStatusCode = error?.response?.status ?? 504
+      adapterError.name = 'Data Provider Request Timeout error'
+    } else if (error?.response?.status) {
+      adapterError = new AdapterDataProviderError({})
+      providerStatusCode = error?.response?.status
+    } else {
+      adapterError = new AdapterConnectionError({})
+      providerStatusCode = 0 // 0 -> connection error
+    }
+    // Record count of failed data provider request
+    transportMetrics.dataProviderRequests
+      .labels(transportMetrics.dataProviderMetricsLabel(providerStatusCode, request.method))
+      .inc()
+
+    adapterError.statusCode = 200
+    adapterError.providerStatusCode = providerStatusCode
+    adapterError.message = error?.message
+    adapterError.cause = error
+    adapterError.errorResponse = error?.response?.data
+    adapterError.url = request.url
+
+    throw adapterError
+  } finally {
+    // Record time taken for data provider request for success or failure
+    responseTimer()
+  }
+
+  // Record count of successful data provider requests
+  transportMetrics.dataProviderRequests
+    .labels(transportMetrics.dataProviderMetricsLabel(providerResponse.status, request.method))
+    .inc()
+
+  return providerResponse
+}