@chainlink/external-adapter-framework 0.0.15 → 0.0.16

package/src/rate-limiting/index.ts

@@ -1,100 +0,0 @@
-import { AdapterEndpoint } from '../adapter'
-
-export * from './request/simple-counting'
-export * from './background/fixed-frequency'
-
-export interface AdapterRateLimitTier {
-  rateLimit1s?: number
-  rateLimit1m?: number
-  rateLimit1h?: number
-  note?: string
-}
-
-/**
- * Common interface for all RateLimiter classes to implement
- */
-export interface RateLimiter {
-  /**
-   * Method to ensure all RateLimiters can be initialized in the same manner.
-   *
-   * @param limits - settings for how much throughput to allow for the Adapter
-   * @param endpoints - list of adapter endpoints
-   */
-  initialize(endpoints: AdapterEndpoint[], limits: AdapterRateLimitTier): this
-}
-
-/**
- * RequestRateLimiters perform checks agains imminent outbound requests for any transport.
- */
-export interface RequestRateLimiter extends RateLimiter {
-  /**
-   * This method will check using whatever strategy is implemented to determine if
-   * this request can be processed. If so, it returns true; if not, returns false.
-   */
-  isUnderLimits(): boolean
-}
-
-/**
- * BackgroundExecuteFrequencyRateLimiters will implement custom logic to calculate
- * the period of time to wait between background executions for a transport.
- */
-export interface BackgroundExecuteRateLimiter extends RateLimiter {
-  msUntilNextExecution(endpointName: string): number
-}
-
-/**
- * This method will convert all possible settings for a rate limit tier and
- * convert them all to requests per second, returning the lowest one
- *
- * @param limits - the rate limit tier set for the adapter
- * @returns the most restrictive of the set options, in requests per second
- */
-export const consolidateTierLimits = (limits?: AdapterRateLimitTier) => {
-  const perHourLimit = (limits?.rateLimit1h || Infinity) / (60 * 60)
-  const perMinuteLimit = (limits?.rateLimit1m || Infinity) / 60
-  const perSecondLimit = limits?.rateLimit1s || Infinity
-  return Math.min(perHourLimit, perMinuteLimit, perSecondLimit)
-}
-
-/**
- * Validates rate limiting tiers specified for the adapter, and returns the one to use.
- *
- * @param tiers - the adapter config listing the different available API tiers
- * @param selectedTier - chosen API tier from settings, if present
- * @returns the specified API tier, or a default one if none are specified
- */
-export const getRateLimitingTier = (
-  tiers?: Record<string, AdapterRateLimitTier>,
-  selectedTier?: string,
-): AdapterRateLimitTier | undefined => {
-  if (!tiers) {
-    return
-  }
-
-  // Check that if the tiers object is defined, it has values
-  if (Object.values(tiers).length === 0) {
-    throw new Error(`The tiers object is defined, but has no entries`)
-  }
-
-  // Check that the tier set in the AdapterConfig is a valid one
-  if (selectedTier && !tiers[selectedTier]) {
-    const validTiersString = Object.keys(tiers)
-      .map((t) => `"${t}"`)
-      .join(', ')
-
-    throw new Error(
-      `The selected rate limit tier "${selectedTier}" is not valid (can be one of ${validTiersString})`,
-    )
-  }
-
-  if (!selectedTier) {
-    // Sort the tiers by most to least restrictive
-    const sortedTiers = Object.values(tiers).sort(
-      (t1, t2) => consolidateTierLimits(t1) - consolidateTierLimits(t2),
-    )
-
-    return sortedTiers[0]
-  }
-
-  return tiers[selectedTier]
-}

package/src/rate-limiting/metrics.ts

@@ -1,18 +0,0 @@
-import * as client from 'prom-client'
-
-// Retrieve cost field from response if exists
-// If not return default cost of 1
-export const retrieveCost = <ProviderResponseBody>(data: ProviderResponseBody): number => {
-  const cost = (data as Record<string, unknown>)['cost']
-  if (typeof cost === 'number' || typeof cost === 'string') {
-    return Number(cost)
-  } else {
-    return 1
-  }
-}
-
-export const rateLimitCreditsSpentTotal = new client.Counter({
-  name: 'rate_limit_credits_spent_total',
-  help: 'The number of data provider credits the adapter is consuming',
-  labelNames: ['participant_id', 'feed_id'] as const,
-})

package/src/rate-limiting/request/simple-counting.ts

@@ -1,76 +0,0 @@
-import { AdapterRateLimitTier, RateLimiter } from '..'
-import { AdapterEndpoint } from '../../adapter'
-import { makeLogger } from '../../util'
-
-const logger = makeLogger('SimpleCountingRateLimiter')
-
-/**
- * This rate limiter is the simplest stateful option.
- * On startup, it'll compare the different thresholds for each tier, calculate them all
- * in the finest window we'll use (seconds), and use the most restrictive one.
- * This is so if the EA were to restart, we don't need to worry about persisting state
- * for things like daily quotas. The downside is that this does not work well for bursty
- * loads or spikes, in cases where e.g. the per second limit is high but daily quotas low.
- */
-export class SimpleCountingRateLimiter implements RateLimiter {
-  latestSecondInterval = 0
-  requestsThisSecond = 0
-  latestMinuteInterval = 0
-  requestsThisMinute = 0
-  perSecondLimit!: number
-  perMinuteLimit!: number
-
-  initialize(endpoints: AdapterEndpoint[], limits?: AdapterRateLimitTier) {
-    // Translate the hourly limit into reqs per minute
-    const perHourLimit = (limits?.rateLimit1h || Infinity) / 60
-    this.perMinuteLimit = Math.min(limits?.rateLimit1m || Infinity, perHourLimit)
-    this.perSecondLimit = limits?.rateLimit1s || Infinity
-    logger.debug(
-      `Using rate limiting settings: perMinute = ${this.perMinuteLimit} | perSecond: = ${this.perSecondLimit}`,
-    )
-
-    return this
-  }
-
-  isUnderLimits() {
-    // If the limit is set to infinity, there was no tier limit specified
-    if (this.perSecondLimit === Infinity && this.perMinuteLimit === Infinity) {
-      return true
-    }
-
-    const now = Date.now()
-    const nearestSecondInterval = Math.floor(now / 1000)
-    const nearestMinuteInterval = Math.floor(now / (1000 * 60))
-
-    // This should always run to completion, even if it doesn't look atomic; therefore the
-    // Ops should be "thread safe". Thank JS and its infinite single threaded dumbness.
-    if (nearestSecondInterval !== this.latestSecondInterval) {
-      logger.trace(
-        `Clearing latest second interval, # of requests logged was: ${this.requestsThisSecond} `,
-      )
-      this.latestSecondInterval = nearestSecondInterval
-      this.requestsThisSecond = 0
-    }
-
-    if (nearestMinuteInterval !== this.latestMinuteInterval) {
-      logger.trace(
-        `Clearing latest second minute, # of requests logged was: ${this.requestsThisMinute} `,
-      )
-      this.latestMinuteInterval = nearestMinuteInterval
-      this.requestsThisMinute = 0
-    }
-
-    if (
-      this.requestsThisSecond < this.perSecondLimit &&
-      this.requestsThisMinute < this.perMinuteLimit
-    ) {
-      logger.trace('Request under limits, counting +1')
-      this.requestsThisSecond++
-      this.requestsThisMinute++
-      return true
-    } else {
-      logger.trace('Requests seen this interval are above limits')
-      return false
-    }
-  }
-}

package/src/transports/batch-warming.ts

@@ -1,127 +0,0 @@
-import { AxiosRequestConfig, AxiosResponse } from 'axios'
-import { Cache } from '../cache'
-import { AdapterConfig, SettingsMap } from '../config'
-import { BackgroundExecuteRateLimiter } from '../rate-limiting'
-import { makeLogger, recordRequests, SubscriptionSet } from '../util'
-import { AdapterRequest, ProviderResult } from '../util/request'
-import { Transport, buildCacheEntriesFromResults } from './'
-import { axiosRequest } from './util'
-import * as rateLimitMetrics from '../rate-limiting/metrics'
-import * as cacheMetrics from '../cache/metrics'
-import { AdapterContext, AdapterDependencies } from '../adapter'
-
-const WARMUP_BATCH_REQUEST_ID = '9002'
-
-const logger = makeLogger('BatchWarmingTransport')
-
-/**
- * Transport implementation that takes incoming batches requests and keeps a warm cache of values.
- * Within the setup function, adapter params are added to an set that also keeps track and expires values.
- * In the background execute, the list of non-expired items in the set is fetched.
- * Then, the list is passed through the `prepareRequest` function, that returns an AxiosRequestConfig.
- * The Data Provider response is, they are passed through the `parseResponse` function to create a [[CacheEntry]] list.
- * Finally, the items in that [[CacheEntry]] list are set in the Cache so the Adapter can fetch values from there.
- *
- * @typeParam AdapterParams - interface for the adapter request body
- * @typeParam ProviderRequestBody - interface for the body of the request to the Data Provider
- * @typeParam ProviderResponseBody - interface for the body of the Data Provider's response
- */
-export class BatchWarmingTransport<
-  AdapterParams,
-  ProviderRequestBody,
-  ProviderResponseBody,
-  CustomSettings extends SettingsMap,
-> implements Transport<AdapterParams, ProviderResponseBody, CustomSettings>
-{
-  cache!: Cache
-  rateLimiter!: BackgroundExecuteRateLimiter
-  subscriptionSet!: SubscriptionSet<AdapterParams>
-
-  // Flag used to track whether the warmer has moved from having no entries to having some and vice versa
-  // Used for recording the cache warmer active metrics accurately
-  WARMER_ACTIVE = false
-
-  constructor(
-    private config: {
-      prepareRequest: (
-        params: AdapterParams[],
-        context: AdapterContext<CustomSettings>,
-      ) => AxiosRequestConfig<ProviderRequestBody>
-      parseResponse: (
-        params: AdapterParams[],
-        res: AxiosResponse<ProviderResponseBody>,
-        context: AdapterContext<CustomSettings>,
-      ) => ProviderResult<AdapterParams>[]
-    },
-  ) {}
-
-  async initialize(dependencies: AdapterDependencies): Promise<void> {
-    this.cache = dependencies.cache
-    this.rateLimiter = dependencies.backgroundExecuteRateLimiter
-    this.subscriptionSet = dependencies.subscriptionSetFactory.buildSet()
-  }
-
-  async hasBeenSetUp(req: AdapterRequest<AdapterParams>): Promise<boolean> {
-    return !!(await this.subscriptionSet.get(req.requestContext.cacheKey))
-  }
-
-  async setup(
-    req: AdapterRequest<AdapterParams>,
-    config: AdapterConfig<CustomSettings>,
-  ): Promise<void> {
-    logger.debug(
-      `Adding entry to batch warming set: [${req.requestContext.cacheKey}] = ${req.requestContext.data}`,
-    )
-    await this.subscriptionSet.add(
-      req.requestContext.cacheKey,
-      req.requestContext.data,
-      config.WARMUP_SUBSCRIPTION_TTL,
-    )
-  }
-
-  async backgroundExecute(context: AdapterContext<CustomSettings>): Promise<number> {
-    logger.debug('Starting background execute')
-    const entries = await this.subscriptionSet.getAll()
-
-    if (!entries.length) {
-      logger.debug('No entries in batch warming set, skipping')
-      if (this.WARMER_ACTIVE) {
-        // Decrement count when warmer changed from having entries to having none
-        cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).dec()
-        this.WARMER_ACTIVE = false
-      }
-      return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
-    } else if (this.WARMER_ACTIVE === false) {
-      // Increment count when warmer changed from having no entries to having some
-      cacheMetrics.cacheWarmerCount.labels({ isBatched: 'true' }).inc()
-      this.WARMER_ACTIVE = true
-    }
-
-    const request = this.config.prepareRequest(entries, context)
-
-    logger.trace('Sending request to data provider...')
-    const providerResponse = await axiosRequest<ProviderRequestBody, ProviderResponseBody>(request)
-
-    if (process.env['RECORD'] === 'true') {
-      recordRequests(request, providerResponse)
-    }
-
-    logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`)
-    const results = this.config.parseResponse(entries, providerResponse, context)
-    const adapterResponses = buildCacheEntriesFromResults(results, context)
-
-    logger.debug('Setting adapter responses in cache')
-    await this.cache.setMany(adapterResponses, context.adapterConfig.CACHE_MAX_AGE)
-
-    // Record cost of data provider call
-    const cost = rateLimitMetrics.retrieveCost(providerResponse.data)
-    rateLimitMetrics.rateLimitCreditsSpentTotal
-      .labels({
-        feed_id: 'N/A',
-        participant_id: WARMUP_BATCH_REQUEST_ID,
-      })
-      .inc(cost)
-
-    return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
-  }
-}

package/src/transports/index.ts

@@ -1,152 +0,0 @@
-import { FastifyReply } from 'fastify'
-import { AdapterContext, AdapterDependencies, InitializedAdapter } from '../adapter'
-import {
-  Cache,
-  CacheEntry,
-  calculateCacheKey,
-  calculateFeedId,
-  pollResponseFromCache,
-} from '../cache'
-import { AdapterConfig, SettingsMap } from '../config'
-import { makeLogger } from '../util'
-import { AdapterRequest, AdapterResponse, ProviderResult } from '../util/request'
-
-export * from './batch-warming'
-export * from './rest'
-export * from './websocket'
-
-const logger = makeLogger('Transport')
-
-/**
- * Generic interface for a Transport.
- * A Transport defines the way in which an AdapterEndpoint will process incoming requests to
- * fetch data from a Data Provider. The setup phase will take care of the former, while the
- * backgroundExecute will be in charge of the latter.
- * This separation gives us the ability of splitting these concerns, and optionally parallelizing
- * the reading and writing of data to a centralized Cache.
- *
- * @typeParam Params - the structure of the AdapterRequest's body
- * @typeParam Result - the structure of the AdapterResponse's body
- */
-export interface Transport<Params, Result, CustomSettings extends SettingsMap> {
-  // TODO: docs / examples to extend dependencies? e.g. JSON rpc
-  /**
-   * Initializes the transport in the Adapter context.
-   *
-   * @param dependencies - Adapter dependencies (e.g. cache instance)
-   * @returns an empty Promise
-   */
-  initialize: (dependencies: AdapterDependencies) => Promise<void>
-
-  /**
-   * Checks if the Transport has already set up the incoming request.
-   * This will likely use the cacheKey generated in the provided request.
-   *
-   * @param req - the incoming AdapterRequest
-   * @returns a Promise that returns true if setup is in place
-   */
-  hasBeenSetUp: (req: AdapterRequest<Params>) => Promise<boolean>
-
-  /**
-   * Sets up the incoming request within the Transport.
-   * In other words, it means that the Adapter has recognized this request, and will begin
-   * fetching the necessary data from a Data Provider.
-   *
-   * @param req - the incoming AdapterRequest
-   * @param config - common configuration for the Adapter as a whole
-   * @returns a Promise that _optionally_ returns an AdapterResponse, if the Transport has the capability of
-   *   immediately fetching data and returning it without the background process.
-   */
-  setup: (
-    req: AdapterRequest<Params>,
-    config: AdapterConfig<CustomSettings>,
-  ) => Promise<AdapterResponse<Result> | void>
-
-  // TODO: Might need a mechanism to know if this bg execute is in flight to avoid multitple simultaneous invocations
-  /**
-   * If the Transport relies on an async mechanism, implementing this function will make it
-   * so the background task executor calls it on Adapter startup.
-   *
-   * @param context - background context for the execution (e.g. endpoint name)
-   */
-  backgroundExecute?: (context: AdapterContext<CustomSettings>) => Promise<number>
-}
-
-/**
- * Helper method to build cache entries to set after getting a bunch of responses from a DP.
- *
- * @param results - a list of results coming from a DataProvider
- * @param context - context for the Adapter
- * @returns a list of CacheEntries of AdapterResponses
- */
-export const buildCacheEntriesFromResults = <Params, CustomSettings extends SettingsMap>(
-  results: ProviderResult<Params>[],
-  context: AdapterContext<CustomSettings>,
-): CacheEntry<AdapterResponse<null>>[] =>
-  results.map((r) => {
-    const cacheEntry = {
-      key: calculateCacheKey(context as AdapterContext, r.params),
-      value: {
-        result: r.value,
-        statusCode: 200,
-        data: null, // TODO: Maybe add data as well?
-      },
-    }
-    if (
-      context.adapterConfig.METRICS_ENABLED &&
-      context.adapterConfig.EXPERIMENTAL_METRICS_ENABLED
-    ) {
-      const metrics = {
-        maxAge: Date.now() + context.adapterConfig.CACHE_MAX_AGE, // TODO: Replace with telemetry structure in future
-        meta: {
-          metrics: {
-            feedId: calculateFeedId(context as AdapterContext, r.params),
-          },
-        },
-      }
-      cacheEntry.value = { ...cacheEntry.value, ...metrics }
-    }
-    return cacheEntry
-  })
-
-/**
- * Takes an Adapter, its configuration, and its dependencies, and it creates an express middleware
- * that will pass along the AdapterRequest to the appropriate Transport (acc. to the endpoint in the req.)
- *
- * @param adapter - main adapter object, already initialized
- * @returns the transport handler middleware function
- */
-export const buildTransportHandler =
-  (adapter: InitializedAdapter) => async (req: AdapterRequest, reply: FastifyReply) => {
-    // Get transport, must be here because it's already checked in the validator
-    const transport = adapter.endpointsMap[req.requestContext.endpointName].transport
-
-    // Set up transport if it hasn't been done already
-    if (!(await transport.hasBeenSetUp(req))) {
-      logger.debug('Transport not set up yet, doing so...')
-      const immediateResponse = await transport.setup(req, adapter.config)
-      if (immediateResponse) {
-        logger.debug('Got immediate response from transport, sending as response')
-        return reply.send(immediateResponse)
-      }
-    }
-    logger.debug('Transport is set up, polling cache for response...')
-    const response = await pollResponseFromCache(
-      adapter.dependencies.cache as Cache<AdapterResponse>,
-      req.requestContext.cacheKey,
-      {
-        maxRetries: adapter.config.CACHE_POLLING_MAX_RETRIES,
-        sleep: adapter.config.CACHE_POLLING_SLEEP_MS,
-      },
-    )
-
-    if (response) {
-      logger.debug('Got a response from the cache, sending that back')
-      return reply.send(response)
-    }
-
-    logger.debug('Ran out of polling attempts, returning timeout')
-    reply.statusCode = 504
-
-    return reply.send()
-  }

package/src/transports/metrics.ts

@@ -1,95 +0,0 @@
-import * as client from 'prom-client'
-import { AdapterContext } from '../adapter'
-import { calculateCacheKey, calculateFeedId } from '../cache'
-import { requestDurationBuckets } from '../metrics/constants'
-
-// Data Provider Metrics
-export const dataProviderMetricsLabel = (providerStatusCode?: number, method = 'get') => ({
-  provider_status_code: providerStatusCode,
-  method: method.toUpperCase(),
-})
-
-export const dataProviderRequests = new client.Counter({
-  name: 'data_provider_requests',
-  help: 'The number of http requests that are made to a data provider',
-  labelNames: ['method', 'provider_status_code'] as const,
-})
-
-export const dataProviderRequestDurationSeconds = new client.Histogram({
-  name: 'data_provider_request_duration_seconds',
-  help: 'A histogram bucket of the distribution of data provider request durations',
-  buckets: requestDurationBuckets,
-})
-
-// Websocket Metrics
-export const connectionErrorLabels = (message: string) => ({
-  // Key,
-  message,
-})
-
-export const messageSubsLabels = (feed_id: string, cache_key: string) => ({
-  feed_id,
-  subscription_key: cache_key,
-})
-
-// Record WS message and subscription metrics
-// Recalculate cacheKey and feedId for metrics
-// since avoiding storing extra info in expiring sorted set
-export const recordWsMessageMetrics = <AdapterParams>(
-  context: AdapterContext,
-  subscribes: AdapterParams[],
-  unsubscrices: AdapterParams[],
-): void => {
-  subscribes.forEach((param) => {
-    const feedId = calculateFeedId(context, param)
-    const cacheKey = calculateCacheKey(context, param)
-    // Record total number of ws messages sent
-    wsMessageTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
-
-    // Record total number of subscriptions made
-    wsSubscriptionTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
-
-    // Record number of active ws subscriptions
-    wsSubscriptionActive.labels(messageSubsLabels(feedId, cacheKey)).inc()
-  })
-  unsubscrices.forEach((param) => {
-    const feedId = calculateFeedId(context, param)
-    const cacheKey = calculateCacheKey(context, param)
-
-    // Record total number of ws messages sent
-    wsMessageTotal.labels(messageSubsLabels(feedId, cacheKey)).inc()
-
-    // Record number of active ws subscriptions
-    wsSubscriptionActive.labels(messageSubsLabels(feedId, cacheKey)).dec()
-  })
-}
-
-export const wsConnectionActive = new client.Gauge({
-  name: 'ws_connection_active',
-  help: 'The number of active connections',
-  labelNames: ['url'] as const,
-})
-
-export const wsConnectionErrors = new client.Counter({
-  name: 'ws_connection_errors',
-  help: 'The number of connection errors',
-  labelNames: ['url', 'message'] as const,
-})
-
-export const wsSubscriptionActive = new client.Gauge({
-  name: 'ws_subscription_active',
-  help: 'The number of currently active subscriptions',
-  labelNames: ['connection_url', 'feed_id', 'subscription_key'] as const,
-})
-
-export const wsSubscriptionTotal = new client.Counter({
-  name: 'ws_subscription_total',
-  help: 'The number of subscriptions opened in total',
-  labelNames: ['connection_url', 'feed_id', 'subscription_key'] as const,
-})
-
-export const wsMessageTotal = new client.Counter({
-  name: 'ws_message_total',
-  help: 'The number of messages received in total',
-  labelNames: ['feed_id', 'subscription_key'] as const,
-})