@chainlink/external-adapter-framework 0.0.12 → 0.0.14

package/src/transports/websocket.ts

@@ -1,245 +0,0 @@
-import WebSocket from 'ws'
-import { AdapterContext, AdapterDependencies } from '../adapter'
-import { Cache } from '../cache'
-import { SettingsMap } from '../config'
-import { BackgroundExecuteRateLimiter } from '../rate-limiting'
-import { makeLogger, SubscriptionSet } from '../util'
-import { AdapterRequest, ProviderResult } from '../util/request'
-import { Transport, buildCacheEntriesFromResults } from './'
-import * as transportMetrics from './metrics'
-
-// TODO: Config
-export const DEFAULT_WS_TTL = 10000
-
-const logger = makeLogger('WebSocketTransport')
-
-type WebSocketClass = new (url: string, protocols?: string | string[] | undefined) => WebSocket
-
-export class WebSocketClassProvider {
-  static ctor: WebSocketClass = WebSocket
-
-  static set(ctor: WebSocketClass): void {
-    this.ctor = ctor
-  }
-
-  static get(): WebSocketClass {
-    return this.ctor
-  }
-}
-
-/**
- * Config object that is provided to the WebSocketTransport constructor.
- */
-export interface WebSocketTransportConfig<
-  AdapterParams,
-  ProviderDataMessage,
-  CustomSettings extends SettingsMap,
-> {
-  /** Endpoint to which to open the WS connection*/
-  url: string
-
-  /** Map of handlers for different WS lifecycle events */
-  handlers: {
-    /**
-     * Handles when the WS is successfully opened
-     *
-     * @param wsConnection - the WebSocket with an established connection
-     * @returns an empty Promise, or void
-     */
-    open: (wsConnection: WebSocket, context: AdapterContext<CustomSettings>) => Promise<void> | void
-
-    /**
-     * Handles when the WS receives a message
-     *
-     * @param message - the message received by the WS
-     * @param context - the background context for the Adapter
-     * @returns a list of cache entries of adapter responses to set in the cache
-     */
-    message: (
-      message: ProviderDataMessage,
-      context: AdapterContext<CustomSettings>,
-    ) => ProviderResult<AdapterParams>[]
-  }
-
-  /** Map of "builders", functions that will be used to prepare specific WS messages */
-  builders: {
-    /**
-     * Builds a WS message that will be sent to subscribe to a specific feed
-     *
-     * @param params - the body of the adapter request
-     * @returns the WS message (can be any type as long as the [[WebSocket]] doesn't complain)
-     */
-    subscribeMessage: (params: AdapterParams) => unknown
-
-    /**
-     * Builds a WS message that will be sent to unsubscribe to a specific feed
-     *
-     * @param params - the body of the adapter request
-     * @returns the WS message (can be any type as long as the [[WebSocket]] doesn't complain)
-     */
-    unsubscribeMessage: (params: AdapterParams) => unknown
-  }
-}
-
-/**
- * Transport implementation that takes incoming requests, adds them to an [[subscriptionSet]] and,
- * through a WebSocket connection, subscribes to the relevant feeds to populate the cache.
- *
- * @typeParam AdapterParams - interface for the adapter request body
- * @typeParam ProviderDataMessage - interface for a WS message containing processable data (i.e. not part of open/close/login/etc)
- */
-export class WebSocketTransport<
-  AdapterParams,
-  ProviderDataMessage,
-  CustomSettings extends SettingsMap,
-> implements Transport<AdapterParams, null, CustomSettings>
-{
-  cache!: Cache
-  rateLimiter!: BackgroundExecuteRateLimiter
-  subscriptionSet!: SubscriptionSet<AdapterParams>
-
-  // The double sets serve to create a simple polling mechanism instead of needing a subscription
-  // This one would not; this is always local state
-  localSubscriptions: AdapterParams[] = []
-
-  wsConnection!: WebSocket
-
-  constructor(
-    private config: WebSocketTransportConfig<AdapterParams, ProviderDataMessage, CustomSettings>,
-  ) {}
-
-  async initialize(dependencies: AdapterDependencies): Promise<void> {
-    this.cache = dependencies.cache
-    this.rateLimiter = dependencies.backgroundExecuteRateLimiter
-    this.subscriptionSet = dependencies.subscriptionSetFactory.buildSet()
-  }
-
-  async hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean> {
-    return !!(await this.subscriptionSet.get(req.requestContext.cacheKey))
-  }
-
-  async setup(req: AdapterRequest<AdapterParams>): Promise<void> {
-    logger.debug(
-      `Adding entry to subscription set: [${req.requestContext.cacheKey}] = ${req.requestContext.data}`,
-    )
-    await this.subscriptionSet.add(
-      req.requestContext.cacheKey,
-      req.requestContext.data,
-      DEFAULT_WS_TTL,
-    )
-  }
-
-  // TODO: Maybe we don't do this, and leave the preparation on the adapter's side?
-  // TODO: Maybe we store adapter params pre-prepared? That would be more efficient
-  // Assuming always JSON payloads for now, makes it all cleaner
-  serializeMessage(payload: unknown): string {
-    return typeof payload === 'string' ? payload : JSON.stringify(payload)
-  }
-  deserializeMessage(data: WebSocket.Data): ProviderDataMessage {
-    return JSON.parse(data.toString()) as ProviderDataMessage
-  }
-
-  establishWsConnection(context: AdapterContext<CustomSettings>) {
-    return new Promise((resolve) => {
-      const ctor = WebSocketClassProvider.get()
-      this.wsConnection = new ctor(this.config.url)
-      this.wsConnection.addEventListener('open', async (event: WebSocket.Event) => {
-        logger.debug(`Opened websocket connection.`)
-        await this.config.handlers.open(this.wsConnection, context)
-        logger.debug('Successfully executed connection opened handler')
-        // Record active ws connections by incrementing count on open
-        // Using URL in label since connection_key is removed from v3
-        transportMetrics.wsConnectionActive.inc()
-        resolve(true)
-      })
-      this.wsConnection.addEventListener('message', async (event: WebSocket.MessageEvent) => {
-        // TODO: Assuming JSON always, maybe use BSON also?
-        const parsed = this.deserializeMessage(event.data)
-        logger.trace(`Got ws message: ${parsed}`)
-        const results = this.config.handlers.message(parsed, context)
-        if (Array.isArray(results)) {
-          const responses = buildCacheEntriesFromResults(results, context)
-          logger.trace(`Writing ${responses.length} responses to cache`)
-          await this.cache.setMany(responses, context.adapterConfig.CACHE_MAX_AGE)
-        }
-      })
-      this.wsConnection.addEventListener('error', async (event: WebSocket.ErrorEvent) => {
-        // Record connection error count
-        transportMetrics.wsConnectionErrors
-          .labels(transportMetrics.connectionErrorLabels(event.message))
-          .inc()
-      })
-      this.wsConnection.addEventListener('close', (event: WebSocket.CloseEvent) => {
-        logger.debug(
-          `Closed websocket connection. Code: ${event.code} ; reason: ${event.reason?.toString()}`,
-        )
-        // Record active ws connections by decrementing count on close
-        // Using URL in label since connection_key is removed from v3
-        transportMetrics.wsConnectionActive.dec()
-      })
-    })
-  }
-
-  // Unlike cache warming, this execute will manage subscriptions
-  async backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number> {
-    logger.debug('Starting background execute, getting subscriptions from sorted set')
-    const desiredSubs = await this.subscriptionSet.getAll()
-
-    logger.debug('Generating delta (subscribes & unsubscribes)')
-    // TODO: More efficient algorithm, this is really easy to read, but high(er) time complexity
-    const subscribeParams = desiredSubs.filter((s) => !this.localSubscriptions.includes(s))
-    const subscribes = subscribeParams
-      .map(this.config.builders.subscribeMessage)
-      .map(this.serializeMessage)
-
-    const unsubscribeParams = this.localSubscriptions.filter((s) => !desiredSubs.includes(s))
-    const unsubscribes = unsubscribeParams
-      .map(this.config.builders.unsubscribeMessage)
-      .map(this.serializeMessage)
-
-    logger.debug(`${subscribes.length} new subscriptions; ${unsubscribes.length} to unsubscribe`)
-    if (subscribes.length) {
-      logger.trace(`Will subscribe to: ${subscribes}`)
-    }
-    if (unsubscribes.length) {
-      logger.trace(`Will unsubscribe to: ${unsubscribes}`)
-    }
-
-    // New subs && no connection -> connect -> add subs
-    // No new subs && no connection -> skip
-    // New subs && connection -> add subs
-    // No new subs && connection -> unsubs only
-
-    if (!subscribes.length && !this.wsConnection) {
-      logger.debug('No entries in subscription set and no established connection, skipping')
-      return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
-    }
-
-    if (!this.wsConnection && subscribes.length) {
-      logger.debug('No established connection and new subscriptions available, connecting to WS')
-      await this.establishWsConnection(context)
-    }
-
-    // TODO: Close connection at some point?
-
-    logger.debug('Sending subs/unsubs if there are any')
-    const messages = unsubscribes.concat(subscribes)
-    for (const message of messages) {
-      logger.trace(`Sending message: ${JSON.stringify(message)}`)
-      this.wsConnection.send(message)
-    }
-
-    // Record WS message and subscription metrics
-    transportMetrics.recordWsMessageMetrics(
-      context as AdapterContext,
-      subscribeParams,
-      unsubscribeParams,
-    )
-
-    logger.debug('Setting local state to cache value')
-    this.localSubscriptions = desiredSubs
-
-    logger.debug('Background execute complete')
-    return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
-  }
-}

package/src/util/index.ts

@@ -1,22 +0,0 @@
-export * from './request'
-export * from './logger'
-export * from './subscription-set/subscription-set'
-
-/**
- * Sleeps for the provided number of milliseconds
- * @param ms - The number of milliseconds to sleep for
- * @returns a Promise that resolves once the specified time passes
- */
-export const sleep = (ms: number): Promise<void> => {
-  return new Promise((resolve) => {
-    setTimeout(resolve, ms)
-  })
-}
-
-export const isObject = (o: unknown): boolean =>
-  o !== null && typeof o === 'object' && Array.isArray(o) === false
-
-export const isArray = (o: unknown): boolean =>
-  o !== null && typeof o === 'object' && Array.isArray(o)
-
-export type PromiseOrValue<T> = Promise<T> | T

package/src/util/logger.ts

@@ -1,69 +0,0 @@
-import pino from 'pino'
-import { BaseSettings } from '../config'
-import { AdapterRequest } from './request'
-import { FastifyReply, HookHandlerDoneFunction } from 'fastify'
-import { randomUUID } from 'crypto'
-import { AsyncLocalStorage } from 'node:async_hooks'
-
-export const asyncLocalStorage = new AsyncLocalStorage()
-
-export type Store = {
-  correlationId: string
-}
-
-const debugTransport = {
-  target: 'pino-pretty',
-  options: {
-    levelFirst: true,
-    levelLabel: 'level',
-    ignore: 'layer,pid,hostname',
-    messageFormat: '[{correlationId}][{layer}] {msg}',
-    translateTime: 'yyyy-mm-dd HH:MM:ss.l',
-  },
-}
-
-// Base logger, shouldn't be used because we want layers to be specified
-const baseLogger = pino({
-  level: process.env['LOG_LEVEL']?.toLowerCase() || BaseSettings.LOG_LEVEL.default,
-  mixin() {
-    if (process.env['CORRELATION_ID_ENABLED'] === 'true') {
-      const store = asyncLocalStorage.getStore() as Store
-      if (store) {
-        return store
-      }
-    }
-    return {}
-  },
-  transport: process.env['DEBUG'] === 'true' ? debugTransport : undefined,
-})
-
-/**
- * Instead of using a global logger instance, we want to force using a child logger
- * with a specific layer set in it, so that we can filter logs by where they're output from.
- *
- * Details on what each log level represents:
- * "trace": Forensic debugging of issues on a local machine.
- * "debug": Detailed logging level to get more context from users on their environments.
- * "info": High-level informational messages, to describe at a glance the high level state of the system.
- * "warn": A mild error occurred that might require non-urgent action.
- * "error": An unexpected error occurred during the regular operation of a well-maintained EA.
- * "fatal": The EA encountered an unrecoverable problem and had to exit.
- *
- * Full reference this is based on can be found at
- * https://github.com/smartcontractkit/documentation/blob/main/docs/Node%20Operators/configuration-variables.md#log_level
- *
- * @param layer - the layer name to include in the logs (e.g. "SomeMiddleware", "RedisCache", etc.)
- * @returns a layer specific logger
- */
-export const makeLogger = (layer: string) => baseLogger.child({ layer })
-
-export const loggingContextMiddleware = (
-  req: AdapterRequest,
-  res: FastifyReply,
-  done: HookHandlerDoneFunction,
-) => {
-  const correlationId = req.headers['x-correlation-id'] || randomUUID()
-  asyncLocalStorage.run({ correlationId: correlationId }, () => {
-    done()
-  })
-}

package/src/util/request.ts

@@ -1,117 +0,0 @@
-import { FastifyReply, FastifyRequest, HookHandlerDoneFunction } from 'fastify'
-import { InitializedAdapter } from '../adapter'
-import { AdapterError } from '../validation/error'
-declare module 'fastify' {
-  // eslint-disable-next-line no-shadow
-  export interface FastifyRequest {
-    requestContext: AdapterRequestContext
-  }
-}
-
-export interface AdapterRequestBody<T = AdapterRequestData> {
-  endpoint?: string
-  data: T
-  id?: string
-}
-
-export type AdapterRequestContext<T = AdapterRequestData> = {
-  id: string
-  endpointName: string
-  cacheKey: string
-  data: T // AdapterRequestData
-  meta?: AdapterRequestMeta
-  // MetricsMeta?: AdapterMetricsMeta
-  // Debug?: AdapterDebug
-  // RateLimitMaxAge?: number
-}
-
-export type AdapterRouteGeneric<T = AdapterRequestData> = {
-  Body: AdapterRequestBody<T>
-}
-
-export type AdapterRequest<T = AdapterRequestData> = FastifyRequest<AdapterRouteGeneric<T>> & {
-  // TODO: Uncomment line below & fix compile issues
-  // body: never // So the user does not access the raw request data and uses the Validated one instead
-  requestContext: AdapterRequestContext<T>
-}
-
-// Export type AdapterRequestMeta = {
-//   AvailableFunds?: number
-//   EligibleToSubmit?: boolean
-//   LatestAnswer?: number
-//   OracleCount?: number
-//   PaymentAmount?: number
-//   ReportableRoundID?: number
-//   StartedAt?: number
-//   Timeout?: number
-// }
-
-// Export type AdapterDebug = {
-//   Ws?: boolean
-//   Warmer?: boolean
-//   CacheHit?: boolean
-//   Staleness?: number
-//   Performance?: number
-//   ProviderCost?: number
-//   BatchablePropertyPath?: cacheWarmer.reducer.BatchableProperty[]
-//   NormalizedRequest?: Record<string, unknown>
-// }
-
-/**
- * Meta info that pertains to exposing metrics
- */
-export interface AdapterRequestMeta {
-  metrics?: AdapterMetricsMeta
-  error?: AdapterError | Error
-}
-
-/**
- * Meta info that pertains to exposing metrics
- */
-export interface AdapterMetricsMeta {
-  feedId?: string
-  cacheHit?: boolean
-}
-
-export type AdapterRequestData = Record<string, unknown> & {
-  endpoint?: string
-}
-
-// Export interface AdapterRequest<T = AdapterRequestData> extends Request {
-//   AdapterContext: {
-//     Id: string
-//     Endpoint: string
-//     Data: T //AdapterRequestData
-//     CacheKey: string
-//     // meta?: AdapterRequestMeta
-//     // metricsMeta?: AdapterMetricsMeta
-//     // debug?: AdapterDebug
-//     // rateLimitMaxAge?: number
-//   }
-// }
-
-export interface ProviderResult<Params> {
-  params: Params
-  value: unknown
-}
-
-export type AdapterResponse<T = unknown> = {
-  // JobRunID: string
-  statusCode: number
-  data: T // Response data, holds "result" for Flux Monitor. Correct way.
-  result: unknown // Result for OCR
-  maxAge?: number // TODO: Replace with new telemetry structure
-  meta?: AdapterRequestMeta
-  // Debug?: AdapterDebug
-  providerStatusCode?: number
-}
-
-export type Middleware =
-  | ((
-      req: AdapterRequest,
-      reply: FastifyReply,
-      done: HookHandlerDoneFunction,
-    ) => FastifyReply | void)
-  | ((req: AdapterRequest, reply: FastifyReply) => Promise<FastifyReply | void>)
-
-export type AdapterMiddlewareBuilder = (adapter: InitializedAdapter) => Middleware

package/src/util/subscription-set/expiring-sorted-set.ts

@@ -1,54 +0,0 @@
-import { SubscriptionSet } from './subscription-set'
-
-/**
- * An object describing an entry in the expiring sorted set.
- * @typeParam T - the type of the entry's value
- */
-interface ExpiringSortedSetEntry<T> {
-  value: T
-  expirationTimestamp: number
-}
-
-/**
- * This class implements a set of unique items, each of which has an expiration timestamp.
- * On reads, items that have expired will be deleted from the set and not returned.
- *
- * @typeParam T - the type of the set entries' values
- */
-export class ExpiringSortedSet<T> implements SubscriptionSet<T> {
-  map = new Map<string, ExpiringSortedSetEntry<T>>()
-
-  add(key: string, value: T, ttl: number) {
-    this.map.set(key, {
-      value,
-      expirationTimestamp: Date.now() + ttl,
-    })
-  }
-
-  get(key: string) {
-    const entry = this.map.get(key)
-    if (!entry) {
-      return
-    } else if (entry.expirationTimestamp < Date.now()) {
-      return entry.value
-    } else {
-      this.map.delete(key)
-    }
-  }
-
-  getAll(): T[] {
-    const results: T[] = []
-    const now = Date.now()
-
-    // Since we're iterating, might as well prune here
-    for (const [key, entry] of this.map.entries()) {
-      if (entry.expirationTimestamp < now) {
-        this.map.delete(key) // In theory, this shouldn't happen frequently for feeds
-      } else {
-        results.push(entry.value)
-      }
-    }
-
-    return results
-  }
-}

package/src/util/subscription-set/subscription-set.ts

@@ -1,35 +0,0 @@
-import { PromiseOrValue } from '..'
-import { AdapterConfig } from '../../config'
-import { ExpiringSortedSet } from './expiring-sorted-set'
-
-/**
- * Set to hold items to subscribe to from a provider (regardless of protocol)
- */
-export interface SubscriptionSet<T> {
-  /** Add a new subscription to the set */
-  add(key: string, value: T, ttl: number): PromiseOrValue<void>
-
-  /** Get a specific subscription from the set */
-  get(key: string): PromiseOrValue<T | undefined>
-
-  /** Get all subscriptions from the set as a list */
-  getAll(): PromiseOrValue<T[]>
-}
-
-export class SubscriptionSetFactory {
-  private cacheType: AdapterConfig['CACHE_TYPE']
-
-  constructor(config: AdapterConfig) {
-    this.cacheType = config.CACHE_TYPE
-  }
-
-  buildSet<T>(): SubscriptionSet<T> {
-    switch (this.cacheType) {
-      case 'local':
-        return new ExpiringSortedSet<T>()
-      case 'redis':
-        // TODO: Implement redis set
-        return new ExpiringSortedSet<T>()
-    }
-  }
-}

package/src/util/test-payload-loader.ts

@@ -1,87 +0,0 @@
-import Ajv from 'ajv'
-import fs from 'fs'
-import path from 'path'
-import { makeLogger } from './logger'
-import { AdapterRequestData } from './request'
-
-const logger = makeLogger('TestPayloadLoader')
-
-/**
- * The test payload read in from filesystem
- */
-export interface Payload {
-  requests: Array<AdapterRequestData>
-}
-
-/**
- * Test payload with discriminated union so we can tell when we should just do
- * a simple liveness check rather than a sample request
- */
-type TestPayload = (Payload & { isDefault: false }) | { isDefault: true }
-
-/**
- * Load in a JSON file containing a test payload for the current adapter,
- * used in healthchecks to make sample requests
- *
- * @param fileName - name of file that contains the test payload data for the smoke endpoint
- * @returns the parsed payload with individual requests
- */
-export function loadTestPayload(fileName?: string): TestPayload {
-  const ajv = new Ajv()
-  const schema = {
-    type: 'object',
-    required: ['requests'],
-    properties: {
-      requests: {
-        type: 'array',
-        items: {
-          type: 'object',
-          required: [],
-        },
-      },
-    },
-  }
-  const validate = ajv.compile(schema)
-  try {
-    const payload = resolvePayload(fileName)
-    if (!validate(payload) || !payload?.requests) {
-      throw Error(JSON.stringify(validate?.errors || 'Could not validate schema for test payload'))
-    }
-    return { ...payload, isDefault: false }
-  } catch (e: unknown) {
-    logger.warn(`Could not load payload: ${(e as Error).message}`)
-    logger.warn('Falling back to default empty payload')
-    return { isDefault: true }
-  }
-}
-
-function resolvePayload(fileName?: string): Payload | null {
-  try {
-    let payload = null
-    // Find test payload by specified name, fallback to default names if not found
-    if (fileName && fs.existsSync(path.join(process.cwd(), fileName))) {
-      payload = require(path.join(process.cwd(), fileName))
-    } else if (fileName && fs.existsSync(`./${fileName}`)) {
-      payload = require(`./${fileName}`)
-    }
-    // Search for test payload js first if no filename specified or found
-    else if (fs.existsSync(path.resolve('.', 'test-payload.js'))) {
-      payload = require(path.join(process.cwd(), 'test-payload.js'))
-    } else if (fs.existsSync(`./test-payload.js`)) {
-      payload = require(`./test-payload.js`)
-    }
-    // Search for test payload json second if no filename specified or found
-    else if (fs.existsSync(path.join(process.cwd(), 'test-payload.json'))) {
-      payload = require(path.join(process.cwd(), 'test-payload.json'))
-    } else if (fs.existsSync(`./test-payload.json`)) {
-      payload = require(`./test-payload.json`)
-    }
-    if (typeof payload === 'string') {
-      return JSON.parse(payload)
-    } else {
-      return payload
-    }
-  } catch {
-    return null
-  }
-}