@chainlink/external-adapter-framework 0.0.12 → 0.0.14

package/src/transports/index.ts

@@ -1,152 +0,0 @@
-import { FastifyReply } from 'fastify'
-import { AdapterContext, AdapterDependencies, InitializedAdapter } from '../adapter'
-import {
-  Cache,
-  CacheEntry,
-  calculateCacheKey,
-  calculateFeedId,
-  pollResponseFromCache,
-} from '../cache'
-import { AdapterConfig, SettingsMap } from '../config'
-import { makeLogger } from '../util'
-import { AdapterRequest, AdapterResponse, ProviderResult } from '../util/request'
-
-export * from './batch-warming'
-export * from './rest'
-export * from './websocket'
-
-const logger = makeLogger('Transport')
-
-/**
- * Generic interface for a Transport.
- * A Transport defines the way in which an AdapterEndpoint will process incoming requests to
- * fetch data from a Data Provider. The setup phase will take care of the former, while the
- * backgroundExecute will be in charge of the latter.
- * This separation gives us the ability of splitting these concerns, and optionally parallelizing
- * the reading and writing of data to a centralized Cache.
- *
- * @typeParam Params - the structure of the AdapterRequest's body
- * @typeParam Result - the structure of the AdapterResponse's body
- */
-export interface Transport<Params, Result, CustomSettings extends SettingsMap> {
-  // TODO: docs / examples to extend dependencies? e.g. JSON rpc
-  /**
-   * Initializes the transport in the Adapter context.
-   *
-   * @param dependencies - Adapter dependencies (e.g. cache instance)
-   * @returns an empty Promise
-   */
-  initialize: (dependencies: AdapterDependencies) => Promise<void>
-
-  /**
-   * Checks if the Transport has already set up the incoming request.
-   * This will likely use the cacheKey generated in the provided request.
-   *
-   * @param req - the incoming AdapterRequest
-   * @returns a Promise that returns true if setup is in place
-   */
-  hasBeenSetUp: (req: AdapterRequest<Params>) => Promise<boolean>
-
-  /**
-   * Sets up the incoming request within the Transport.
-   * In other words, it means that the Adapter has recognized this request, and will begin
-   * fetching the necessary data from a Data Provider.
-   *
-   * @param req - the incoming AdapterRequest
-   * @param config - common configuration for the Adapter as a whole
-   * @returns a Promise that _optionally_ returns an AdapterResponse, if the Transport has the capability of
-   *   immediately fetching data and returning it without the background process.
-   */
-  setup: (
-    req: AdapterRequest<Params>,
-    config: AdapterConfig<CustomSettings>,
-  ) => Promise<AdapterResponse<Result> | void>
-
-  // TODO: Might need a mechanism to know if this bg execute is in flight to avoid multitple simultaneous invocations
-  /**
-   * If the Transport relies on an async mechanism, implementing this function will make it
-   * so the background task executor calls it on Adapter startup.
-   *
-   * @param context - background context for the execution (e.g. endpoint name)
-   */
-  backgroundExecute?: (context: AdapterContext<CustomSettings>) => Promise<number>
-}
-
-/**
- * Helper method to build cache entries to set after getting a bunch of responses from a DP.
- *
- * @param results - a list of results coming from a DataProvider
- * @param context - context for the Adapter
- * @returns a list of CacheEntries of AdapterResponses
- */
-export const buildCacheEntriesFromResults = <Params, CustomSettings extends SettingsMap>(
-  results: ProviderResult<Params>[],
-  context: AdapterContext<CustomSettings>,
-): CacheEntry<AdapterResponse<null>>[] =>
-  results.map((r) => {
-    const cacheEntry = {
-      key: calculateCacheKey(context as AdapterContext, r.params),
-      value: {
-        result: r.value,
-        statusCode: 200,
-        data: null, // TODO: Maybe add data as well?
-      },
-    }
-    if (
-      context.adapterConfig.METRICS_ENABLED &&
-      context.adapterConfig.EXPERIMENTAL_METRICS_ENABLED
-    ) {
-      const metrics = {
-        maxAge: Date.now() + context.adapterConfig.CACHE_MAX_AGE, // TODO: Replace with telemetry structure in future
-        meta: {
-          metrics: {
-            feedId: calculateFeedId(context as AdapterContext, r.params),
-          },
-        },
-      }
-      cacheEntry.value = { ...cacheEntry.value, ...metrics }
-    }
-    return cacheEntry
-  })
-
-/**
- * Takes an Adapter, its configuration, and its dependencies, and it creates an express middleware
- * that will pass along the AdapterRequest to the appropriate Transport (acc. to the endpoint in the req.)
- *
- * @param adapter - main adapter object, already initialized
- * @returns the transport handler middleware function
- */
-export const buildTransportHandler =
-  (adapter: InitializedAdapter) => async (req: AdapterRequest, reply: FastifyReply) => {
-    // Get transport, must be here because it's already checked in the validator
-    const transport = adapter.endpointsMap[req.requestContext.endpointName].transport
-
-    // Set up transport if it hasn't been done already
-    if (!(await transport.hasBeenSetUp(req))) {
-      logger.debug('Transport not set up yet, doing so...')
-      const immediateResponse = await transport.setup(req, adapter.config)
-      if (immediateResponse) {
-        logger.debug('Got immediate response from transport, sending as response')
-        return reply.send(immediateResponse)
-      }
-    }
-    logger.debug('Transport is set up, polling cache for response...')
-    const response = await pollResponseFromCache(
-      adapter.dependencies.cache as Cache<AdapterResponse>,
-      req.requestContext.cacheKey,
-      {
-        maxRetries: adapter.config.CACHE_POLLING_MAX_RETRIES,
-        sleep: adapter.config.CACHE_POLLING_SLEEP_MS,
-      },
-    )
-
-    if (response) {
-      logger.debug('Got a response from the cache, sending that back')
-      return reply.send(response)
-    }
-
-    logger.debug('Ran out of polling attempts, returning timeout')
-    reply.statusCode = 504
-
-    return reply.send()
-  }

package/src/transports/metrics.ts

@@ -1,95 +0,0 @@
-import * as client from 'prom-client'
-import { AdapterContext } from '../adapter'
-import { calculateCacheKey, calculateFeedId } from '../cache'
-import { requestDurationBuckets } from '../metrics/constants'
-
-// Data Provider Metrics
-export const dataProviderMetricsLabel = (providerStatusCode?: number, method = 'get') => ({
-  provider_status_code: providerStatusCode,
-  method: method.toUpperCase(),
-})
-
-export const dataProviderRequests = new client.Counter({
-  name: 'data_provider_requests',
-  help: 'The number of http requests that are made to a data provider',
-  labelNames: ['method', 'provider_status_code'] as const,
-})
-
-export const dataProviderRequestDurationSeconds = new client.Histogram({
-  name: 'data_provider_request_duration_seconds',
-  help: 'A histogram bucket of the distribution of data provider request durations',
-  buckets: requestDurationBuckets,
-})
-
-// Websocket Metrics
-export const connectionErrorLabels = (message: string) => ({
-  // Key,
-  message,
-})
-
-export const messageSubsLabels = (feed_id: string, cache_key: string) => ({
-  feed_id,
-  subscription_key: cache_key,
-})
-
-// Record WS message and subscription metrics
-// Recalculate cacheKey and feedId for metrics
-// since avoiding storing extra info in expiring sorted set
-export const recordWsMessageMetrics = <AdapterParams>(
-  context: AdapterContext,
-  subscribes: AdapterParams[],
-  unsubscrices: AdapterParams[],
-): void => {
-  subscribes.forEach((param) => {
-    const feedId = calculateFeedId(context, param)
-    const cacheKey = calculateCacheKey(context, param)
-    // Record total number of ws messages sent
-    wsMessageTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
-
-    // Record total number of subscriptions made
-    wsSubscriptionTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
-
-    // Record number of active ws subscriptions
-    wsSubscriptionActive.labels(messageSubsLabels(feedId, cacheKey)).inc()
-  })
-  unsubscrices.forEach((param) => {
-    const feedId = calculateFeedId(context, param)
-    const cacheKey = calculateCacheKey(context, param)
-
-    // Record total number of ws messages sent
-    wsMessageTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
-
-    // Record number of active ws subscriptions
-    wsSubscriptionActive.labels(messageSubsLabels(feedId, cacheKey)).dec()
-  })
-}
-
-export const wsConnectionActive = new client.Gauge({
-  name: 'ws_connection_active',
-  help: 'The number of active connections',
-  labelNames: ['url'] as const,
-})
-
-export const wsConnectionErrors = new client.Counter({
-  name: 'ws_connection_errors',
-  help: 'The number of connection errors',
-  labelNames: ['url', 'message'] as const,
-})
-
-export const wsSubscriptionActive = new client.Gauge({
-  name: 'ws_subscription_active',
-  help: 'The number of currently active subscriptions',
-  labelNames: ['connection_url', 'feed_id', 'subscription_key'] as const,
-})
-
-export const wsSubscriptionTotal = new client.Counter({
-  name: 'ws_subscription_total',
-  help: 'The number of subscriptions opened in total',
-  labelNames: ['connection_url', 'feed_id', 'subscription_key'] as const,
-})
-
-export const wsMessageTotal = new client.Counter({
-  name: 'ws_message_total',
-  help: 'The number of messages received in total',
-  labelNames: ['feed_id', 'subscription_key'] as const,
-})

package/src/transports/rest.ts

@@ -1,164 +0,0 @@
-import { AdapterRequest, AdapterResponse } from '../util/request'
-import { Transport } from './'
-import { Cache } from '../cache'
-import { AxiosRequestConfig, AxiosResponse } from 'axios'
-import { makeLogger, sleep } from '../util'
-import { AdapterConfig, SettingsMap } from '../config'
-import { RequestRateLimiter } from '../rate-limiting'
-import { AdapterError } from '../validation/error'
-import { axiosRequest } from './util'
-import * as rateLimitMetrics from '../rate-limiting/metrics'
-import { AdapterDependencies } from '../adapter'
-
-const IN_FLIGHT_PREFIX = 'InFlight'
-
-const logger = makeLogger('RestTransport')
-
-/**
- * Transport implementation that takes incoming requests, transforms them into a DataProvider request,
- * and executes that request returning the response immediately from the `setup` function.
- * Optionally, setting the `coalescing` option to `true` will make it so once a request is in flight,
- * new adapter requests for the same feed will not trigger a new one, but return an empty promise from
- * the setup instead so the normal cache polling mechanism is used.
- *
- * @typeParam AdapterParams - interface for the adapter request body
- * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
- * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
- */
-export class RestTransport<
-  AdapterParams,
-  ProviderRequestBody,
-  ProviderResponseBody,
-  CustomSettings extends SettingsMap,
-> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings>
-{
-  inFlightPrefix!: string
-  cache!: Cache
-  rateLimiter!: RequestRateLimiter
-
-  constructor(
-    protected config: {
-      prepareRequest: (
-        req: AdapterRequest<AdapterParams>,
-        config: AdapterConfig<CustomSettings>,
-      ) =>
-        | AxiosRequestConfig<ProviderRequestBody>
-        | Promise<AxiosRequestConfig<ProviderRequestBody>>
-      parseResponse: (
-        req: AdapterRequest<AdapterParams>,
-        res: AxiosResponse<ProviderResponseBody>,
-        config: AdapterConfig<CustomSettings>,
-      ) => AdapterResponse<ProviderResponseBody> | Promise<AdapterResponse<ProviderResponseBody>>
-      options: {
-        coalescing: boolean
-      }
-    },
-  ) {}
-
-  async initialize(dependencies: AdapterDependencies): Promise<void> {
-    this.inFlightPrefix = `${IN_FLIGHT_PREFIX}-`
-    this.cache = dependencies.cache
-    this.rateLimiter = dependencies.requestRateLimiter
-  }
-
-  async hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean> {
-    if (!this.config.options.coalescing) {
-      return false
-    }
-
-    // Check if request is in flight
-    const inFlight = await this.cache.get(this.inFlightPrefix + req.requestContext.cacheKey)
-    if (inFlight) {
-      logger.debug('Request is in flight, transport has been set up')
-      return true
-    } else {
-      logger.debug('Request not in flight, transport not set up')
-      return false
-    }
-  }
-
-  protected async waitUntilUnderRateLimit(
-    options: {
-      maxRetries: number
-      msBetweenRetries: number
-    },
-    retry = 0,
-  ) {
-    if (this.rateLimiter.isUnderLimits()) {
-      logger.trace('Incoming request would not be under limits, moving on')
-      return
-    }
-
-    if (retry >= options.maxRetries) {
-      throw new AdapterError({
-        statusCode: 504,
-        message: `REST Transport timed out while waiting for rate limit availability (max retries: ${options.maxRetries})`,
-      })
-    }
-
-    logger.debug(`Request would be over rate limits, sleeping for ${options.msBetweenRetries}`)
-    await sleep(options.msBetweenRetries)
-    await this.waitUntilUnderRateLimit(options, retry + 1)
-  }
-
-  async setup(
-    req: AdapterRequest<AdapterParams>,
-    config: AdapterConfig<CustomSettings>,
-  ): Promise<AdapterResponse<ProviderResponseBody> | undefined> {
-    if (this.config.options.coalescing) {
-      logger.debug('Setting up rest transport, setting request in flight in cache')
-      // TODO: Should this use a separate cache?
-      // TODO: Set viable ttl to approximate timeout from API
-      // TODO: Make this ttl configurable
-      await this.cache.set(this.inFlightPrefix + req.requestContext.cacheKey, true, 2000) // Can't use Infinity for things like Redis
-    }
-
-    const request = await this.config.prepareRequest(req, config)
-
-    logger.trace('Check if we are under rate limits to perform request')
-    await this.waitUntilUnderRateLimit({
-      maxRetries: config.REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES,
-      msBetweenRetries: config.REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES,
-    })
-
-    logger.trace('Sending request to data provider...')
-    const providerResponse = await axiosRequest<ProviderRequestBody, ProviderResponseBody>(request)
-
-    logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`) // TODO: Sensitive data?
-    const parsedResponse = await this.config.parseResponse(req, providerResponse, config)
-
-    if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
-      // TODO: Potentially create function to add all telemetry data
-      parsedResponse.maxAge = Date.now() + config.CACHE_MAX_AGE
-      parsedResponse.meta = {
-        metrics: { feedId: req.requestContext.meta?.metrics?.feedId || 'N/A' },
-      }
-    }
-
-    logger.debug('Setting provider response in cache')
-    await this.cache.set(req.requestContext.cacheKey, parsedResponse, config.CACHE_MAX_AGE)
-
-    // Record cost of data provider call
-    const cost = rateLimitMetrics.retrieveCost(providerResponse.data)
-    rateLimitMetrics.rateLimitCreditsSpentTotal
-      .labels({
-        feed_id: req.requestContext.meta?.metrics?.feedId || 'N/A',
-        participant_id: req.requestContext.cacheKey,
-      })
-      .inc(cost)
-
-    // Update cacheHit flag in request meta for metrics use
-    req.requestContext.meta = {
-      ...req.requestContext.meta,
-      metrics: { ...req.requestContext.meta?.metrics, cacheHit: false },
-    }
-
-    // TODO: move this to a try/catch/finally
-    if (this.config.options.coalescing) {
-      logger.debug('Set provider response in cache, removing in flight from cache')
-      await this.cache.delete(this.inFlightPrefix)
-    }
-
-    return parsedResponse
-  }
-}

package/src/transports/util.ts

@@ -1,63 +0,0 @@
-import axios, { AxiosError, AxiosRequestConfig, AxiosResponse } from 'axios'
-import {
-  AdapterConnectionError,
-  AdapterDataProviderError,
-  AdapterError,
-  AdapterTimeoutError,
-} from '../validation/error'
-import * as transportMetrics from './metrics'
-
-/**
- * Performs axios request along with metrics recording and error handling
- *
- * @param request - axios request config
- * @returns axios response for the request
- */
-export async function axiosRequest<ProviderRequestBody, ProviderResponseBody>(
-  request: AxiosRequestConfig<ProviderRequestBody>,
-): Promise<AxiosResponse<ProviderResponseBody>> {
-  const responseTimer = transportMetrics.dataProviderRequestDurationSeconds.startTimer()
-  let providerResponse: AxiosResponse<ProviderResponseBody>
-  try {
-    providerResponse = await axios.request<ProviderResponseBody>(request)
-  } catch (e: unknown) {
-    const error = e as AxiosError
-    // Request error
-    let providerStatusCode: number | undefined
-    let adapterError: AdapterError
-    if (error.code === 'ECONNABORTED') {
-      adapterError = new AdapterTimeoutError({})
-      providerStatusCode = error?.response?.status ?? 504
-      adapterError.name = 'Data Provider Request Timeout error'
-    } else if (error?.response?.status) {
-      adapterError = new AdapterDataProviderError({})
-      providerStatusCode = error?.response?.status
-    } else {
-      adapterError = new AdapterConnectionError({})
-      providerStatusCode = 0 // 0 -> connection error
-    }
-    // Record count of failed data provider request
-    transportMetrics.dataProviderRequests
-      .labels(transportMetrics.dataProviderMetricsLabel(providerStatusCode, request.method))
-      .inc()
-
-    adapterError.statusCode = 200
-    adapterError.providerStatusCode = providerStatusCode
-    adapterError.message = error?.message
-    adapterError.cause = error
-    adapterError.errorResponse = error?.response?.data
-    adapterError.url = request.url
-
-    throw adapterError
-  } finally {
-    // Record time taken for data provider request for success or failure
-    responseTimer()
-  }
-
-  // Record count of successful data provider requests
-  transportMetrics.dataProviderRequests
-    .labels(transportMetrics.dataProviderMetricsLabel(providerResponse.status, request.method))
-    .inc()
-
-  return providerResponse
-}