@chainlink/external-adapter-framework 0.0.14 → 0.0.15

package/src/transports/rest.ts

@@ -0,0 +1,168 @@
+import { AdapterRequest, AdapterResponse } from '../util/request'
+import { Transport } from './'
+import { Cache } from '../cache'
+import { AxiosRequestConfig, AxiosResponse } from 'axios'
+import { makeLogger, recordRequests, sleep } from '../util'
+import { AdapterConfig, SettingsMap } from '../config'
+import { RequestRateLimiter } from '../rate-limiting'
+import { AdapterError } from '../validation/error'
+import { axiosRequest } from './util'
+import * as rateLimitMetrics from '../rate-limiting/metrics'
+import { AdapterDependencies } from '../adapter'
+
+const IN_FLIGHT_PREFIX = 'InFlight'
+
+const logger = makeLogger('RestTransport')
+
+/**
+ * Transport implementation that takes incoming requests, transforms them into a DataProvider request,
+ * and executes that request returning the response immediately from the `setup` function.
+ * Optionally, setting the `coalescing` option to `true` will make it so once a request is in flight,
+ * new adapter requests for the same feed will not trigger a new one, but return an empty promise from
+ * the setup instead so the normal cache polling mechanism is used.
+ *
+ * @typeParam AdapterParams - interface for the adapter request body
+ * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
+ * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
+ */
+export class RestTransport<
+  AdapterParams,
+  ProviderRequestBody,
+  ProviderResponseBody,
+  CustomSettings extends SettingsMap,
+> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings>
+{
+  inFlightPrefix!: string
+  cache!: Cache
+  rateLimiter!: RequestRateLimiter
+
+  constructor(
+    protected config: {
+      prepareRequest: (
+        req: AdapterRequest<AdapterParams>,
+        config: AdapterConfig<CustomSettings>,
+      ) =>
+        | AxiosRequestConfig<ProviderRequestBody>
+        | Promise<AxiosRequestConfig<ProviderRequestBody>>
+      parseResponse: (
+        req: AdapterRequest<AdapterParams>,
+        res: AxiosResponse<ProviderResponseBody>,
+        config: AdapterConfig<CustomSettings>,
+      ) => AdapterResponse<ProviderResponseBody> | Promise<AdapterResponse<ProviderResponseBody>>
+      options: {
+        coalescing: boolean
+      }
+    },
+  ) {}
+
+  async initialize(dependencies: AdapterDependencies): Promise<void> {
+    this.inFlightPrefix = `${IN_FLIGHT_PREFIX}-`
+    this.cache = dependencies.cache
+    this.rateLimiter = dependencies.requestRateLimiter
+  }
+
+  async hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean> {
+    if (!this.config.options.coalescing) {
+      return false
+    }
+
+    // Check if request is in flight
+    const inFlight = await this.cache.get(this.inFlightPrefix + req.requestContext.cacheKey)
+    if (inFlight) {
+      logger.debug('Request is in flight, transport has been set up')
+      return true
+    } else {
+      logger.debug('Request not in flight, transport not set up')
+      return false
+    }
+  }
+
+  protected async waitUntilUnderRateLimit(
+    options: {
+      maxRetries: number
+      msBetweenRetries: number
+    },
+    retry = 0,
+  ) {
+    if (this.rateLimiter.isUnderLimits()) {
+      logger.trace('Incoming request would not be under limits, moving on')
+      return
+    }
+
+    if (retry >= options.maxRetries) {
+      throw new AdapterError({
+        statusCode: 504,
+        message: `REST Transport timed out while waiting for rate limit availability (max retries: ${options.maxRetries})`,
+      })
+    }
+
+    logger.debug(`Request would be over rate limits, sleeping for ${options.msBetweenRetries}`)
+    await sleep(options.msBetweenRetries)
+    await this.waitUntilUnderRateLimit(options, retry + 1)
+  }
+
+  async setup(
+    req: AdapterRequest<AdapterParams>,
+    config: AdapterConfig<CustomSettings>,
+  ): Promise<AdapterResponse<ProviderResponseBody> | undefined> {
+    if (this.config.options.coalescing) {
+      logger.debug('Setting up rest transport, setting request in flight in cache')
+      // TODO: Should this use a separate cache?
+      // TODO: Set viable ttl to approximate timeout from API
+      // TODO: Make this ttl configurable
+      await this.cache.set(this.inFlightPrefix + req.requestContext.cacheKey, true, 2000) // Can't use Infinity for things like Redis
+    }
+
+    const request = await this.config.prepareRequest(req, config)
+
+    logger.trace('Check if we are under rate limits to perform request')
+    await this.waitUntilUnderRateLimit({
+      maxRetries: config.REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES,
+      msBetweenRetries: config.REST_TRANSPORT_MS_BETWEEN_RATE_LIMIT_RETRIES,
+    })
+
+    logger.trace('Sending request to data provider...')
+    const providerResponse = await axiosRequest<ProviderRequestBody, ProviderResponseBody>(request)
+
+    if (process.env['RECORD'] === 'true') {
+      recordRequests(request, providerResponse)
+    }
+
+    logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`) // TODO: Sensitive data?
+    const parsedResponse = await this.config.parseResponse(req, providerResponse, config)
+
+    if (config.METRICS_ENABLED && config.EXPERIMENTAL_METRICS_ENABLED) {
+      // TODO: Potentially create function to add all telemetry data
+      parsedResponse.maxAge = Date.now() + config.CACHE_MAX_AGE
+      parsedResponse.meta = {
+        metrics: { feedId: req.requestContext.meta?.metrics?.feedId || 'N/A' },
+      }
+    }
+
+    logger.debug('Setting provider response in cache')
+    await this.cache.set(req.requestContext.cacheKey, parsedResponse, config.CACHE_MAX_AGE)
+
+    // Record cost of data provider call
+    const cost = rateLimitMetrics.retrieveCost(providerResponse.data)
+    rateLimitMetrics.rateLimitCreditsSpentTotal
+      .labels({
+        feed_id: req.requestContext.meta?.metrics?.feedId || 'N/A',
+        participant_id: req.requestContext.cacheKey,
+      })
+      .inc(cost)
+
+    // Update cacheHit flag in request meta for metrics use
+    req.requestContext.meta = {
+      ...req.requestContext.meta,
+      metrics: { ...req.requestContext.meta?.metrics, cacheHit: false },
+    }
+
+    // TODO: move this to a try/catch/finally
+    if (this.config.options.coalescing) {
+      logger.debug('Set provider response in cache, removing in flight from cache')
+      await this.cache.delete(this.inFlightPrefix)
+    }
+
+    return parsedResponse
+  }
+}

package/src/transports/util.ts

@@ -0,0 +1,63 @@
+import axios, { AxiosError, AxiosRequestConfig, AxiosResponse } from 'axios'
+import {
+  AdapterConnectionError,
+  AdapterDataProviderError,
+  AdapterError,
+  AdapterTimeoutError,
+} from '../validation/error'
+import * as transportMetrics from './metrics'
+
+/**
+ * Performs axios request along with metrics recording and error handling
+ *
+ * @param request - axios request config
+ * @returns axios response for the request
+ */
+export async function axiosRequest<ProviderRequestBody, ProviderResponseBody>(
+  request: AxiosRequestConfig<ProviderRequestBody>,
+): Promise<AxiosResponse<ProviderResponseBody>> {
+  const responseTimer = transportMetrics.dataProviderRequestDurationSeconds.startTimer()
+  let providerResponse: AxiosResponse<ProviderResponseBody>
+  try {
+    providerResponse = await axios.request<ProviderResponseBody>(request)
+  } catch (e: unknown) {
+    const error = e as AxiosError
+    // Request error
+    let providerStatusCode: number | undefined
+    let adapterError: AdapterError
+    if (error.code === 'ECONNABORTED') {
+      adapterError = new AdapterTimeoutError({})
+      providerStatusCode = error?.response?.status ?? 504
+      adapterError.name = 'Data Provider Request Timeout error'
+    } else if (error?.response?.status) {
+      adapterError = new AdapterDataProviderError({})
+      providerStatusCode = error?.response?.status
+    } else {
+      adapterError = new AdapterConnectionError({})
+      providerStatusCode = 0 // 0 -> connection error
+    }
+    // Record count of failed data provider request
+    transportMetrics.dataProviderRequests
+      .labels(transportMetrics.dataProviderMetricsLabel(providerStatusCode, request.method))
+      .inc()
+
+    adapterError.statusCode = 200
+    adapterError.providerStatusCode = providerStatusCode
+    adapterError.message = error?.message
+    adapterError.cause = error
+    adapterError.errorResponse = error?.response?.data
+    adapterError.url = request.url
+
+    throw adapterError
+  } finally {
+    // Record time taken for data provider request for success or failure
+    responseTimer()
+  }
+
+  // Record count of successful data provider requests
+  transportMetrics.dataProviderRequests
+    .labels(transportMetrics.dataProviderMetricsLabel(providerResponse.status, request.method))
+    .inc()
+
+  return providerResponse
+}

package/src/transports/websocket.ts

@@ -0,0 +1,245 @@
+import WebSocket from 'ws'
+import { AdapterContext, AdapterDependencies } from '../adapter'
+import { Cache } from '../cache'
+import { SettingsMap } from '../config'
+import { BackgroundExecuteRateLimiter } from '../rate-limiting'
+import { makeLogger, SubscriptionSet } from '../util'
+import { AdapterRequest, ProviderResult } from '../util/request'
+import { Transport, buildCacheEntriesFromResults } from './'
+import * as transportMetrics from './metrics'
+
+// TODO: Config
+export const DEFAULT_WS_TTL = 10000
+
+const logger = makeLogger('WebSocketTransport')
+
+type WebSocketClass = new (url: string, protocols?: string | string[] | undefined) => WebSocket
+
+export class WebSocketClassProvider {
+  static ctor: WebSocketClass = WebSocket
+
+  static set(ctor: WebSocketClass): void {
+    this.ctor = ctor
+  }
+
+  static get(): WebSocketClass {
+    return this.ctor
+  }
+}
+
+/**
+ * Config object that is provided to the WebSocketTransport constructor.
+ */
+export interface WebSocketTransportConfig<
+  AdapterParams,
+  ProviderDataMessage,
+  CustomSettings extends SettingsMap,
+> {
+  /** Endpoint to which to open the WS connection*/
+  url: string
+
+  /** Map of handlers for different WS lifecycle events */
+  handlers: {
+    /**
+     * Handles when the WS is successfully opened
+     *
+     * @param wsConnection - the WebSocket with an established connection
+     * @returns an empty Promise, or void
+     */
+    open: (wsConnection: WebSocket, context: AdapterContext<CustomSettings>) => Promise<void> | void
+
+    /**
+     * Handles when the WS receives a message
+     *
+     * @param message - the message received by the WS
+     * @param context - the background context for the Adapter
+     * @returns a list of cache entries of adapter responses to set in the cache
+     */
+    message: (
+      message: ProviderDataMessage,
+      context: AdapterContext<CustomSettings>,
+    ) => ProviderResult<AdapterParams>[]
+  }
+
+  /** Map of "builders", functions that will be used to prepare specific WS messages */
+  builders: {
+    /**
+     * Builds a WS message that will be sent to subscribe to a specific feed
+     *
+     * @param params - the body of the adapter request
+     * @returns the WS message (can be any type as long as the [[WebSocket]] doesn't complain)
+     */
+    subscribeMessage: (params: AdapterParams) => unknown
+
+    /**
+     * Builds a WS message that will be sent to unsubscribe to a specific feed
+     *
+     * @param params - the body of the adapter request
+     * @returns the WS message (can be any type as long as the [[WebSocket]] doesn't complain)
+     */
+    unsubscribeMessage: (params: AdapterParams) => unknown
+  }
+}
+
+/**
+ * Transport implementation that takes incoming requests, adds them to an [[subscriptionSet]] and,
+ * through a WebSocket connection, subscribes to the relevant feeds to populate the cache.
+ *
+ * @typeParam AdapterParams - interface for the adapter request body
+ * @typeParam ProviderDataMessage - interface for a WS message containing processable data (i.e. not part of open/close/login/etc)
+ */
+export class WebSocketTransport<
+  AdapterParams,
+  ProviderDataMessage,
+  CustomSettings extends SettingsMap,
+> implements Transport<AdapterParams, null, CustomSettings>
+{
+  cache!: Cache
+  rateLimiter!: BackgroundExecuteRateLimiter
+  subscriptionSet!: SubscriptionSet<AdapterParams>
+
+  // The double sets serve to create a simple polling mechanism instead of needing a subscription
+  // This one would not; this is always local state
+  localSubscriptions: AdapterParams[] = []
+
+  wsConnection!: WebSocket
+
+  constructor(
+    private config: WebSocketTransportConfig<AdapterParams, ProviderDataMessage, CustomSettings>,
+  ) {}
+
+  async initialize(dependencies: AdapterDependencies): Promise<void> {
+    this.cache = dependencies.cache
+    this.rateLimiter = dependencies.backgroundExecuteRateLimiter
+    this.subscriptionSet = dependencies.subscriptionSetFactory.buildSet()
+  }
+
+  async hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean> {
+    return !!(await this.subscriptionSet.get(req.requestContext.cacheKey))
+  }
+
+  async setup(req: AdapterRequest<AdapterParams>): Promise<void> {
+    logger.debug(
+      `Adding entry to subscription set: [${req.requestContext.cacheKey}] = ${req.requestContext.data}`,
+    )
+    await this.subscriptionSet.add(
+      req.requestContext.cacheKey,
+      req.requestContext.data,
+      DEFAULT_WS_TTL,
+    )
+  }
+
+  // TODO: Maybe we don't do this, and leave the preparation on the adapter's side?
+  // TODO: Maybe we store adapter params pre-prepared? That would be more efficient
+  // Assuming always JSON payloads for now, makes it all cleaner
+  serializeMessage(payload: unknown): string {
+    return typeof payload === 'string' ? payload : JSON.stringify(payload)
+  }
+  deserializeMessage(data: WebSocket.Data): ProviderDataMessage {
+    return JSON.parse(data.toString()) as ProviderDataMessage
+  }
+
+  establishWsConnection(context: AdapterContext<CustomSettings>) {
+    return new Promise((resolve) => {
+      const ctor = WebSocketClassProvider.get()
+      this.wsConnection = new ctor(this.config.url)
+      this.wsConnection.addEventListener('open', async (event: WebSocket.Event) => {
+        logger.debug(`Opened websocket connection.`)
+        await this.config.handlers.open(this.wsConnection, context)
+        logger.debug('Successfully executed connection opened handler')
+        // Record active ws connections by incrementing count on open
+        // Using URL in label since connection_key is removed from v3
+        transportMetrics.wsConnectionActive.inc()
+        resolve(true)
+      })
+      this.wsConnection.addEventListener('message', async (event: WebSocket.MessageEvent) => {
+        // TODO: Assuming JSON always, maybe use BSON also?
+        const parsed = this.deserializeMessage(event.data)
+        logger.trace(`Got ws message: ${parsed}`)
+        const results = this.config.handlers.message(parsed, context)
+        if (Array.isArray(results)) {
+          const responses = buildCacheEntriesFromResults(results, context)
+          logger.trace(`Writing ${responses.length} responses to cache`)
+          await this.cache.setMany(responses, context.adapterConfig.CACHE_MAX_AGE)
+        }
+      })
+      this.wsConnection.addEventListener('error', async (event: WebSocket.ErrorEvent) => {
+        // Record connection error count
+        transportMetrics.wsConnectionErrors
+          .labels(transportMetrics.connectionErrorLabels(event.message))
+          .inc()
+      })
+      this.wsConnection.addEventListener('close', (event: WebSocket.CloseEvent) => {
+        logger.debug(
+          `Closed websocket connection. Code: ${event.code} ; reason: ${event.reason?.toString()}`,
+        )
+        // Record active ws connections by decrementing count on close
+        // Using URL in label since connection_key is removed from v3
+        transportMetrics.wsConnectionActive.dec()
+      })
+    })
+  }
+
+  // Unlike cache warming, this execute will manage subscriptions
+  async backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number> {
+    logger.debug('Starting background execute, getting subscriptions from sorted set')
+    const desiredSubs = await this.subscriptionSet.getAll()
+
+    logger.debug('Generating delta (subscribes & unsubscribes)')
+    // TODO: More efficient algorithm, this is really easy to read, but high(er) time complexity
+    const subscribeParams = desiredSubs.filter((s) => !this.localSubscriptions.includes(s))
+    const subscribes = subscribeParams
+      .map(this.config.builders.subscribeMessage)
+      .map(this.serializeMessage)
+
+    const unsubscribeParams = this.localSubscriptions.filter((s) => !desiredSubs.includes(s))
+    const unsubscribes = unsubscribeParams
+      .map(this.config.builders.unsubscribeMessage)
+      .map(this.serializeMessage)
+
+    logger.debug(`${subscribes.length} new subscriptions; ${unsubscribes.length} to unsubscribe`)
+    if (subscribes.length) {
+      logger.trace(`Will subscribe to: ${subscribes}`)
+    }
+    if (unsubscribes.length) {
+      logger.trace(`Will unsubscribe to: ${unsubscribes}`)
+    }
+
+    // New subs && no connection -> connect -> add subs
+    // No new subs && no connection -> skip
+    // New subs && connection -> add subs
+    // No new subs && connection -> unsubs only
+
+    if (!subscribes.length && !this.wsConnection) {
+      logger.debug('No entries in subscription set and no established connection, skipping')
+      return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
+    }
+
+    if (!this.wsConnection && subscribes.length) {
+      logger.debug('No established connection and new subscriptions available, connecting to WS')
+      await this.establishWsConnection(context)
+    }
+
+    // TODO: Close connection at some point?
+
+    logger.debug('Sending subs/unsubs if there are any')
+    const messages = unsubscribes.concat(subscribes)
+    for (const message of messages) {
+      logger.trace(`Sending message: ${JSON.stringify(message)}`)
+      this.wsConnection.send(message)
+    }
+
+    // Record WS message and subscription metrics
+    transportMetrics.recordWsMessageMetrics(
+      context as AdapterContext,
+      subscribeParams,
+      unsubscribeParams,
+    )
+
+    logger.debug('Setting local state to cache value')
+    this.localSubscriptions = desiredSubs
+
+    logger.debug('Background execute complete')
+    return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
+  }
+}

package/src/util/index.ts

@@ -0,0 +1,23 @@
+export * from './request'
+export * from './logger'
+export * from './recordRequests'
+export * from './subscription-set/subscription-set'
+
+/**
+ * Sleeps for the provided number of milliseconds
+ * @param ms - The number of milliseconds to sleep for
+ * @returns a Promise that resolves once the specified time passes
+ */
+export const sleep = (ms: number): Promise<void> => {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms)
+  })
+}
+
+export const isObject = (o: unknown): boolean =>
+  o !== null && typeof o === 'object' && Array.isArray(o) === false
+
+export const isArray = (o: unknown): boolean =>
+  o !== null && typeof o === 'object' && Array.isArray(o)
+
+export type PromiseOrValue<T> = Promise<T> | T

package/src/util/logger.ts

@@ -0,0 +1,69 @@
+import pino from 'pino'
+import { BaseSettings } from '../config'
+import { AdapterRequest } from './request'
+import { FastifyReply, HookHandlerDoneFunction } from 'fastify'
+import { randomUUID } from 'crypto'
+import { AsyncLocalStorage } from 'node:async_hooks'
+
+export const asyncLocalStorage = new AsyncLocalStorage()
+
+export type Store = {
+  correlationId: string
+}
+
+const debugTransport = {
+  target: 'pino-pretty',
+  options: {
+    levelFirst: true,
+    levelLabel: 'level',
+    ignore: 'layer,pid,hostname',
+    messageFormat: '[{correlationId}][{layer}] {msg}',
+    translateTime: 'yyyy-mm-dd HH:MM:ss.l',
+  },
+}
+
+// Base logger, shouldn't be used because we want layers to be specified
+const baseLogger = pino({
+  level: process.env['LOG_LEVEL']?.toLowerCase() || BaseSettings.LOG_LEVEL.default,
+  mixin() {
+    if (process.env['CORRELATION_ID_ENABLED'] === 'true') {
+      const store = asyncLocalStorage.getStore() as Store
+      if (store) {
+        return store
+      }
+    }
+    return {}
+  },
+  transport: process.env['DEBUG'] === 'true' ? debugTransport : undefined,
+})
+
+/**
+ * Instead of using a global logger instance, we want to force using a child logger
+ * with a specific layer set in it, so that we can filter logs by where they're output from.
+ *
+ * Details on what each log level represents:
+ * "trace": Forensic debugging of issues on a local machine.
+ * "debug": Detailed logging level to get more context from users on their environments.
+ * "info": High-level informational messages, to describe at a glance the high level state of the system.
+ * "warn": A mild error occurred that might require non-urgent action.
+ * "error": An unexpected error occurred during the regular operation of a well-maintained EA.
+ * "fatal": The EA encountered an unrecoverable problem and had to exit.
+ *
+ * Full reference this is based on can be found at
+ * https://github.com/smartcontractkit/documentation/blob/main/docs/Node%20Operators/configuration-variables.md#log_level
+ *
+ * @param layer - the layer name to include in the logs (e.g. "SomeMiddleware", "RedisCache", etc.)
+ * @returns a layer specific logger
+ */
+export const makeLogger = (layer: string) => baseLogger.child({ layer })
+
+export const loggingContextMiddleware = (
+  req: AdapterRequest,
+  res: FastifyReply,
+  done: HookHandlerDoneFunction,
+) => {
+  const correlationId = req.headers['x-correlation-id'] || randomUUID()
+  asyncLocalStorage.run({ correlationId: correlationId }, () => {
+    done()
+  })
+}

package/src/util/recordRequests.ts

@@ -0,0 +1,47 @@
+import fs from 'fs'
+import { AxiosRequestConfig, AxiosResponse } from 'axios'
+
+export const recordRequests = (
+  requestConfig: AxiosRequestConfig<unknown>,
+  providerResponse: AxiosResponse<unknown, unknown>,
+) => {
+  let recordedRequests: Record<
+    string,
+    { method: string; path: string; data?: unknown; response: unknown; statusCode: number }[]
+  > = {}
+  try {
+    recordedRequests = JSON.parse(fs.readFileSync('capturedRequests.json').toString())
+  } catch {} // eslint-disable-line no-empty
+  let baseUrl = requestConfig.baseURL as string
+  baseUrl =
+    nthIndexOf(baseUrl, '/', 3) === -1 ? baseUrl : baseUrl.slice(0, nthIndexOf(baseUrl, '/', 3))
+  const recordedRequest = {
+    method: requestConfig.method as string,
+    path: providerResponse.request.path,
+    data: requestConfig.data,
+    response: providerResponse.data,
+    statusCode: providerResponse.status,
+  }
+  if (!recordedRequests[baseUrl]) {
+    recordedRequests[baseUrl] = [recordedRequest]
+  } else if (
+    recordedRequests[baseUrl].every((r) => {
+      return r.path !== recordedRequest.path
+    })
+  ) {
+    recordedRequests[baseUrl].push(recordedRequest)
+  }
+  fs.writeFileSync('capturedRequests.json', JSON.stringify(recordedRequests))
+}
+
+const nthIndexOf = (input: string, match: string, n: number) => {
+  let i = -1
+  let n_ = n
+  while (n_-- && i++ < input.length) {
+    i = input.indexOf(match, i)
+    if (i < 0) {
+      break
+    }
+  }
+  return i
+}