@chainlink/external-adapter-framework 0.0.6 → 0.0.7

package/src/transports/util.ts

@@ -1,63 +0,0 @@
-import axios, { AxiosError, AxiosRequestConfig, AxiosResponse } from 'axios'
-import {
-  AdapterConnectionError,
-  AdapterDataProviderError,
-  AdapterError,
-  AdapterTimeoutError,
-} from '../validation/error'
-import * as transportMetrics from './metrics'
-
-/**
- * Performs axios request along with metrics recording and error handling
- *
- * @param request - axios request config
- * @returns axios response for the request
- */
-export async function axiosRequest<ProviderRequestBody, ProviderResponseBody>(
-  request: AxiosRequestConfig<ProviderRequestBody>,
-): Promise<AxiosResponse<ProviderResponseBody>> {
-  const responseTimer = transportMetrics.dataProviderRequestDurationSeconds.startTimer()
-  let providerResponse: AxiosResponse<ProviderResponseBody>
-  try {
-    providerResponse = await axios.request<ProviderResponseBody>(request)
-  } catch (e: unknown) {
-    const error = e as AxiosError
-    // Request error
-    let providerStatusCode: number | undefined
-    let adapterError: AdapterError
-    if (error.code === 'ECONNABORTED') {
-      adapterError = new AdapterTimeoutError({})
-      providerStatusCode = error?.response?.status ?? 504
-      adapterError.name = 'Data Provider Request Timeout error'
-    } else if (error?.response?.status) {
-      adapterError = new AdapterDataProviderError({})
-      providerStatusCode = error?.response?.status
-    } else {
-      adapterError = new AdapterConnectionError({})
-      providerStatusCode = 0 // 0 -> connection error
-    }
-    // Record count of failed data provider request
-    transportMetrics.dataProviderRequests
-      .labels(transportMetrics.dataProviderMetricsLabel(providerStatusCode, request.method))
-      .inc()
-
-    adapterError.statusCode = 200
-    adapterError.providerStatusCode = providerStatusCode
-    adapterError.message = error?.message
-    adapterError.cause = error
-    adapterError.errorResponse = error?.response?.data
-    adapterError.url = request.url
-
-    throw adapterError
-  } finally {
-    // Record time taken for data provider request for success or failure
-    responseTimer()
-  }
-
-  // Record count of successful data provider requests
-  transportMetrics.dataProviderRequests
-    .labels(transportMetrics.dataProviderMetricsLabel(providerResponse.status, request.method))
-    .inc()
-
-  return providerResponse
-}

package/src/transports/websocket.ts

@@ -1,238 +0,0 @@
-import WebSocket from 'ws'
-import { Cache } from '../cache'
-import { SettingsMap } from '../config'
-import { BackgroundExecuteRateLimiter } from '../rate-limiting'
-import { ExpiringSortedSet, makeLogger } from '../util'
-import { AdapterRequest, ProviderResult } from '../util/request'
-import { AdapterContext, AdapterDependencies, Transport, buildCacheEntriesFromResults } from './'
-import * as transportMetrics from './metrics'
-
-// TODO: Config
-export const DEFAULT_WS_TTL = 10000
-
-const logger = makeLogger('WebSocketTransport')
-
-type WebSocketClass = new (url: string, protocols?: string | string[] | undefined) => WebSocket
-
-export class WebSocketClassProvider {
-  static ctor: WebSocketClass = WebSocket
-
-  static set(ctor: WebSocketClass): void {
-    this.ctor = ctor
-  }
-
-  static get(): WebSocketClass {
-    return this.ctor
-  }
-}
-
-/**
- * Config object that is provided to the WebSocketTransport constructor.
- */
-export interface WebSocketTransportConfig<
-  AdapterParams,
-  ProviderDataMessage,
-  CustomSettings extends SettingsMap,
-> {
-  /** Endpoint to which to open the WS connection*/
-  url: string
-
-  /** Map of handlers for different WS lifecycle events */
-  handlers: {
-    /**
-     * Handles when the WS is successfully opened
-     *
-     * @param wsConnection - the WebSocket with an established connection
-     * @returns an empty Promise, or void
-     */
-    open: (wsConnection: WebSocket, context: AdapterContext<CustomSettings>) => Promise<void> | void
-
-    /**
-     * Handles when the WS receives a message
-     *
-     * @param message - the message received by the WS
-     * @param context - the background context for the Adapter
-     * @returns a list of cache entries of adapter responses to set in the cache
-     */
-    message: (
-      message: ProviderDataMessage,
-      context: AdapterContext<CustomSettings>,
-    ) => ProviderResult<AdapterParams>[]
-  }
-
-  /** Map of "builders", functions that will be used to prepare specific WS messages */
-  builders: {
-    /**
-     * Builds a WS message that will be sent to subscribe to a specific feed
-     *
-     * @param params - the body of the adapter request
-     * @returns the WS message (can be any type as long as the [[WebSocket]] doesn't complain)
-     */
-    subscribeMessage: (params: AdapterParams) => unknown
-
-    /**
-     * Builds a WS message that will be sent to unsubscribe to a specific feed
-     *
-     * @param params - the body of the adapter request
-     * @returns the WS message (can be any type as long as the [[WebSocket]] doesn't complain)
-     */
-    unsubscribeMessage: (params: AdapterParams) => unknown
-  }
-}
-
-/**
- * Transport implementation that takes incoming requests, adds them to an [[ExpiringSortedSet]] and,
- * through a WebSocket connection, subscribes to the relevant feeds to populate the cache.
- *
- * @typeParam AdapterParams - interface for the adapter request body
- * @typeParam ProviderDataMessage - interface for a WS message containing processable data (i.e. not part of open/close/login/etc)
- */
-export class WebSocketTransport<
-  AdapterParams,
-  ProviderDataMessage,
-  CustomSettings extends SettingsMap,
-> implements Transport<AdapterParams, null, CustomSettings>
-{
-  cache!: Cache
-  rateLimiter!: BackgroundExecuteRateLimiter
-
-  // The double sets serve to create a simple polling mechanism instead of needing a subscription
-
-  // This one would be either local, redis, etc
-  expiringSortedSet = new ExpiringSortedSet<AdapterParams>() // TODO: Move to dependencies, inject
-
-  // This one would not; this is always local state
-  localSubscriptions: AdapterParams[] = []
-
-  wsConnection!: WebSocket
-
-  constructor(
-    private config: WebSocketTransportConfig<AdapterParams, ProviderDataMessage, CustomSettings>,
-  ) {}
-
-  async initialize(dependencies: AdapterDependencies): Promise<void> {
-    this.cache = dependencies.cache
-    this.rateLimiter = dependencies.backgroundExecuteRateLimiter
-  }
-
-  async hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean> {
-    return !!this.expiringSortedSet.get(req.requestContext.cacheKey)
-  }
-
-  async setup(req: AdapterRequest<AdapterParams>): Promise<void> {
-    logger.debug(
-      `Adding entry to subscription set: [${req.requestContext.cacheKey}] = ${req.requestContext.data}`,
-    )
-    this.expiringSortedSet.add(req.requestContext.cacheKey, req.requestContext.data, DEFAULT_WS_TTL)
-  }
-
-  // TODO: Maybe we don't do this, and leave the preparation on the adapter's side?
-  // TODO: Maybe we store adapter params pre-prepared? That would be more efficient
-  // Assuming always JSON payloads for now, makes it all cleaner
-  serializeMessage(payload: unknown): string {
-    return typeof payload === 'string' ? payload : JSON.stringify(payload)
-  }
-  deserializeMessage(data: WebSocket.Data): ProviderDataMessage {
-    return JSON.parse(data.toString()) as ProviderDataMessage
-  }
-
-  establishWsConnection(context: AdapterContext<CustomSettings>) {
-    return new Promise((resolve) => {
-      const ctor = WebSocketClassProvider.get()
-      this.wsConnection = new ctor(this.config.url)
-      this.wsConnection.addEventListener('open', async (event: WebSocket.Event) => {
-        logger.debug(`Opened websocket connection.`)
-        await this.config.handlers.open(this.wsConnection, context)
-        logger.debug('Successfully executed connection opened handler')
-        // Record active ws connections by incrementing count on open
-        // Using URL in label since connection_key is removed from v3
-        transportMetrics.wsConnectionActive.inc()
-        resolve(true)
-      })
-      this.wsConnection.addEventListener('message', async (event: WebSocket.MessageEvent) => {
-        // TODO: Assuming JSON always, maybe use BSON also?
-        const parsed = this.deserializeMessage(event.data)
-        logger.trace(`Got ws message: ${parsed}`)
-        const results = this.config.handlers.message(parsed, context)
-        if (Array.isArray(results)) {
-          const responses = buildCacheEntriesFromResults(results, context)
-          logger.trace(`Writing ${responses.length} responses to cache`)
-          await this.cache.setMany(responses, context.adapterConfig.CACHE_MAX_AGE)
-        }
-      })
-      this.wsConnection.addEventListener('error', async (event: WebSocket.ErrorEvent) => {
-        // Record connection error count
-        transportMetrics.wsConnectionErrors
-          .labels(transportMetrics.connectionErrorLabels(event.message))
-          .inc()
-      })
-      this.wsConnection.addEventListener('close', (event: WebSocket.CloseEvent) => {
-        logger.debug(
-          `Closed websocket connection. Code: ${event.code} ; reason: ${event.reason?.toString()}`,
-        )
-        // Record active ws connections by decrementing count on close
-        // Using URL in label since connection_key is removed from v3
-        transportMetrics.wsConnectionActive.dec()
-      })
-    })
-  }
-
-  // Unlike cache warming, this execute will manage subscriptions
-  async backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number> {
-    logger.debug('Starting background execute, getting subscriptions from sorted set')
-    const desiredSubs = this.expiringSortedSet.getAll()
-
-    logger.debug('Generating delta (subscribes & unsubscribes)')
-    // TODO: More efficient algorithm, this is really easy to read, but high(er) time complexity
-    const subscribeParams = desiredSubs.filter((s) => !this.localSubscriptions.includes(s))
-    const subscribes = subscribeParams
-      .map(this.config.builders.subscribeMessage)
-      .map(this.serializeMessage)
-
-    const unsubscribeParams = this.localSubscriptions.filter((s) => !desiredSubs.includes(s))
-    const unsubscribes = unsubscribeParams
-      .map(this.config.builders.unsubscribeMessage)
-      .map(this.serializeMessage)
-
-    logger.debug(`${subscribes.length} new subscriptions; ${unsubscribes.length} to unsubscribe`)
-    if (subscribes.length) {
-      logger.trace(`Will subscribe to: ${subscribes}`)
-    }
-    if (unsubscribes.length) {
-      logger.trace(`Will unsubscribe to: ${unsubscribes}`)
-    }
-
-    // New subs && no connection -> connect -> add subs
-    // No new subs && no connection -> skip
-    // New subs && connection -> add subs
-    // No new subs && connection -> unsubs only
-
-    if (!subscribes.length && !this.wsConnection) {
-      logger.debug('No entries in subscription set and no established connection, skipping')
-      return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
-    }
-
-    if (!this.wsConnection && subscribes.length) {
-      logger.debug('No established connection and new subscriptions available, connecting to WS')
-      await this.establishWsConnection(context)
-    }
-
-    // TODO: Close connection at some point?
-
-    logger.debug('Sending subs/unsubs if there are any')
-    const messages = unsubscribes.concat(subscribes)
-    for (const message of messages) {
-      logger.trace(`Sending message: ${JSON.stringify(message)}`)
-      this.wsConnection.send(message)
-    }
-
-    // Record WS message and subscription metrics
-    transportMetrics.recordWsMessageMetrics(context as AdapterContext, subscribeParams, unsubscribeParams)
-
-    logger.debug('Setting local state to cache value')
-    this.localSubscriptions = desiredSubs
-
-    logger.debug('Background execute complete')
-    return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
-  }
-}

package/src/util/expiring-sorted-set.ts

@@ -1,52 +0,0 @@
-/**
- * An object describing an entry in the expiring sorted set.
- * @typeParam T - the type of the entry's value
- */
-interface ExpiringSortedSetEntry<T> {
-  value: T
-  expirationTimestamp: number
-}
-
-/**
- * This class implements a set of unique items, each of which has an expiration timestamp.
- * On reads, items that have expired will be deleted from the set and not returned.
- *
- * @typeParam T - the type of the set entries' values
- */
-export class ExpiringSortedSet<T> {
-  map = new Map<string, ExpiringSortedSetEntry<T>>()
-
-  add(key: string, value: T, ttl: number) {
-    this.map.set(key, {
-      value,
-      expirationTimestamp: Date.now() + ttl,
-    })
-  }
-
-  get(key: string) {
-    const entry = this.map.get(key)
-    if (!entry) {
-      return
-    } else if (entry.expirationTimestamp < Date.now()) {
-      return entry.value
-    } else {
-      this.map.delete(key)
-    }
-  }
-
-  getAll(): T[] {
-    const results: T[] = []
-    const now = Date.now()
-
-    // Since we're iterating, might as well prune here
-    for (const [key, entry] of this.map.entries()) {
-      if (entry.expirationTimestamp < now) {
-        this.map.delete(key) // In theory, this shouldn't happen frequently for feeds
-      } else {
-        results.push(entry.value)
-      }
-    }
-
-    return results
-  }
-}

package/src/util/index.ts

@@ -1,20 +0,0 @@
-export * from './request'
-export * from './logger'
-export * from './expiring-sorted-set'
-
-/**
- * Sleeps for the provided number of milliseconds
- * @param ms - The number of milliseconds to sleep for
- * @returns a Promise that resolves once the specified time passes
- */
-export const sleep = (ms: number): Promise<void> => {
-  return new Promise((resolve) => {
-    setTimeout(resolve, ms)
-  })
-}
-
-export const isObject = (o: unknown): boolean =>
-  o !== null && typeof o === 'object' && Array.isArray(o) === false
-
-export const isArray = (o: unknown): boolean =>
-  o !== null && typeof o === 'object' && Array.isArray(o)

package/src/util/logger.ts

@@ -1,69 +0,0 @@
-import pino from 'pino'
-import { BaseSettings } from '../config'
-import { AdapterRequest } from './request'
-import { FastifyReply, HookHandlerDoneFunction } from 'fastify'
-import { randomUUID } from 'crypto'
-import { AsyncLocalStorage } from 'node:async_hooks'
-
-export const asyncLocalStorage = new AsyncLocalStorage()
-
-export type Store = {
-  correlationId: string
-}
-
-const debugTransport = {
-  target: 'pino-pretty',
-  options: {
-    levelFirst: true,
-    levelLabel: 'level',
-    ignore: 'layer,pid,hostname',
-    messageFormat: '[{correlationId}][{layer}] {msg}',
-    translateTime: 'yyyy-mm-dd HH:MM:ss.l',
-  },
-}
-
-// Base logger, shouldn't be used because we want layers to be specified
-const baseLogger = pino({
-  level: process.env['LOG_LEVEL']?.toLowerCase() || BaseSettings.LOG_LEVEL.default,
-  mixin() {
-    if (process.env['CORRELATION_ID_ENABLED'] === 'true') {
-      const store = asyncLocalStorage.getStore() as Store
-      if (store) {
-        return store
-      }
-    }
-    return {}
-  },
-  transport: process.env['DEBUG'] === 'true' ? debugTransport : undefined,
-})
-
-/**
- * Instead of using a global logger instance, we want to force using a child logger
- * with a specific layer set in it, so that we can filter logs by where they're output from.
- *
- * Details on what each log level represents:
- * "trace": Forensic debugging of issues on a local machine.
- * "debug": Detailed logging level to get more context from users on their environments.
- * "info": High-level informational messages, to describe at a glance the high level state of the system.
- * "warn": A mild error occurred that might require non-urgent action.
- * "error": An unexpected error occurred during the regular operation of a well-maintained EA.
- * "fatal": The EA encountered an unrecoverable problem and had to exit.
- *
- * Full reference this is based on can be found at
- * https://github.com/smartcontractkit/documentation/blob/main/docs/Node%20Operators/configuration-variables.md#log_level
- *
- * @param layer - the layer name to include in the logs (e.g. "SomeMiddleware", "RedisCache", etc.)
- * @returns a layer specific logger
- */
-export const makeLogger = (layer: string) => baseLogger.child({ layer })
-
-export const loggingContextMiddleware = (
-  req: AdapterRequest,
-  res: FastifyReply,
-  done: HookHandlerDoneFunction,
-) => {
-  const correlationId = req.headers['x-correlation-id'] || randomUUID()
-  asyncLocalStorage.run({ correlationId: correlationId }, () => {
-    done()
-  })
-}

package/src/util/request.ts

@@ -1,115 +0,0 @@
-import { FastifyReply, FastifyRequest, HookHandlerDoneFunction } from 'fastify'
-import { InitializedAdapter } from '../adapter'
-import { AdapterError } from '../validation/error'
-declare module 'fastify' {
-  // eslint-disable-next-line no-shadow
-  export interface FastifyRequest {
-    requestContext: AdapterRequestContext
-  }
-}
-
-export interface AdapterRequestBody<T = AdapterRequestData> {
-  endpoint?: string
-  data: T
-  id?: string
-}
-
-export type AdapterRequestContext<T = AdapterRequestData> = {
-  id: string
-  endpointName: string
-  cacheKey: string
-  data: T // AdapterRequestData
-  meta?: AdapterRequestMeta
-  // MetricsMeta?: AdapterMetricsMeta
-  // Debug?: AdapterDebug
-  // RateLimitMaxAge?: number
-}
-
-export type AdapterRouteGeneric<T = AdapterRequestData> = {
-  Body: AdapterRequestBody<T>
-}
-
-export type AdapterRequest<T = AdapterRequestData> = FastifyRequest<AdapterRouteGeneric<T>> & {
-  // TODO: Uncomment line below & fix compile issues
-  // body: never // So the user does not access the raw request data and uses the Validated one instead
-  requestContext: AdapterRequestContext<T>
-}
-
-// Export type AdapterRequestMeta = {
-//   AvailableFunds?: number
-//   EligibleToSubmit?: boolean
-//   LatestAnswer?: number
-//   OracleCount?: number
-//   PaymentAmount?: number
-//   ReportableRoundID?: number
-//   StartedAt?: number
-//   Timeout?: number
-// }
-
-// Export type AdapterDebug = {
-//   Ws?: boolean
-//   Warmer?: boolean
-//   CacheHit?: boolean
-//   Staleness?: number
-//   Performance?: number
-//   ProviderCost?: number
-//   BatchablePropertyPath?: cacheWarmer.reducer.BatchableProperty[]
-//   NormalizedRequest?: Record<string, unknown>
-// }
-
-/**
- * Meta info that pertains to exposing metrics
- */
- export interface AdapterRequestMeta {
-  metrics?: AdapterMetricsMeta
-  error?: AdapterError | Error
-}
-
-/**
- * Meta info that pertains to exposing metrics
- */
-export interface AdapterMetricsMeta {
-  feedId?: string
-  cacheHit?: boolean
-}
-
-export type AdapterRequestData = Record<string, unknown>
-
-// Export interface AdapterRequest<T = AdapterRequestData> extends Request {
-//   AdapterContext: {
-//     Id: string
-//     Endpoint: string
-//     Data: T //AdapterRequestData
-//     CacheKey: string
-//     // meta?: AdapterRequestMeta
-//     // metricsMeta?: AdapterMetricsMeta
-//     // debug?: AdapterDebug
-//     // rateLimitMaxAge?: number
-//   }
-// }
-
-export interface ProviderResult<Params> {
-  params: Params
-  value: unknown
-}
-
-export type AdapterResponse<T = unknown> = {
-  // JobRunID: string
-  statusCode: number
-  data: T // Response data, holds "result" for Flux Monitor. Correct way.
-  result: unknown // Result for OCR
-  maxAge?: number // TODO: Replace with new telemetry structure
-  meta?: AdapterRequestMeta
-  // Debug?: AdapterDebug
-  providerStatusCode?: number
-}
-
-export type Middleware =
-  | ((
-      req: AdapterRequest,
-      reply: FastifyReply,
-      done: HookHandlerDoneFunction,
-    ) => FastifyReply | void)
-  | ((req: AdapterRequest, reply: FastifyReply) => Promise<FastifyReply | void>)
-
-export type AdapterMiddlewareBuilder = (adapter: InitializedAdapter) => Middleware

package/src/validation/error.ts

@@ -1,116 +0,0 @@
-import { HttpRequestType } from "../metrics/constants"
-
-type ErrorBasic = {
-  name: string
-  message: string
-}
-type ErrorFull = ErrorBasic & {
-  stack: string
-  cause: string
-}
-
-export type AdapterErrorResponse = {
-  jobRunID: string
-  status: string
-  statusCode: number
-  providerStatusCode?: number
-  error: ErrorBasic | ErrorFull
-}
-
-export class AdapterError extends Error {
-  jobRunID: string
-  status: string
-  statusCode: number
-  cause: any
-  url?: string
-  errorResponse: any
-  feedID?: string
-  providerStatusCode?: number
-  metricsLabel?: HttpRequestType
-
-  override name: string
-  override message: string
-
-  constructor({
-    jobRunID = '1',
-    status = 'errored',
-    statusCode = 500,
-    name = 'AdapterError',
-    message = 'An error occurred.',
-    cause,
-    url,
-    errorResponse,
-    feedID,
-    providerStatusCode,
-    metricsLabel = HttpRequestType.ADAPTER_ERROR
-  }: Partial<AdapterError>) {
-    super(message)
-
-    this.jobRunID = jobRunID
-    this.status = status
-    this.statusCode = statusCode
-    this.name = name
-    this.message = message
-    this.cause = cause
-    if (url) {
-      this.url = url
-    }
-    if (feedID) {
-      this.feedID = feedID
-    }
-    this.errorResponse = errorResponse
-    this.providerStatusCode = providerStatusCode
-    this.metricsLabel = metricsLabel
-  }
-
-  toJSONResponse(): AdapterErrorResponse {
-    const showDebugInfo =
-      process.env['NODE_ENV'] === 'development' || process.env['DEBUG'] === 'true'
-    const errorBasic = {
-      name: this.name,
-      message: this.message,
-      url: this.url,
-      errorResponse: this.errorResponse,
-      feedID: this.feedID,
-    }
-    const errorFull = { ...errorBasic, stack: this.stack, cause: this.cause }
-    return {
-      jobRunID: this.jobRunID,
-      status: this.status,
-      statusCode: this.statusCode,
-      providerStatusCode: this.providerStatusCode,
-      error: showDebugInfo ? errorFull : errorBasic,
-    }
-  }
-}
-
-export class AdapterInputError extends AdapterError {
-  constructor(input: Partial<AdapterError>) {
-    super({ ...input, metricsLabel: HttpRequestType.INPUT_ERROR })
-  }
-}
-export class AdapterRateLimitError extends AdapterError {
-  constructor(input: Partial<AdapterError>) {
-    super({ ...input, metricsLabel: HttpRequestType.RATE_LIMIT_ERROR })
-  }
-}
-export class AdapterTimeoutError extends AdapterError {
-  constructor(input: Partial<AdapterError>) {
-    super({ ...input, metricsLabel: HttpRequestType.TIMEOUT_ERROR })
-  }
-}
-export class AdapterDataProviderError extends AdapterError {
-  constructor(input: Partial<AdapterError>) {
-    super({ ...input, metricsLabel: HttpRequestType.DP_ERROR })
-  }
-}
-export class AdapterConnectionError extends AdapterError {
-  constructor(input: Partial<AdapterError>) {
-    super({ ...input, metricsLabel: HttpRequestType.CONNECTION_ERROR })
-  }
-}
-export class AdapterCustomError extends AdapterError {
-  constructor(input: Partial<AdapterError>) {
-    super({ ...input, metricsLabel: HttpRequestType.CUSTOM_ERROR })
-  }
-}