@chainlink/external-adapter-framework 0.0.5 → 0.0.7

package/.eslintrc.js

@@ -1,96 +0,0 @@
-// https://github.com/typescript-eslint/typescript-eslint/blob/master/docs/getting-started/linting/README.md
-module.exports = {
-  root: true,
-  parser: '@typescript-eslint/parser',
-  plugins: ['@typescript-eslint', 'eslint-plugin-tsdoc'],
-  extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended', 'prettier'],
-  rules: {
-    'tsdoc/syntax': 'warn',
-
-    // Problems
-    'array-callback-return': [
-      'error',
-      {
-        allowImplicit: false,
-        checkForEach: true,
-      },
-    ],
-    'no-constant-binary-expression': 'error',
-    'no-constructor-return': 'error',
-    'no-duplicate-imports': 'error',
-    'no-promise-executor-return': 'error',
-    'no-self-compare': 'error',
-    'no-template-curly-in-string': 'error',
-    'no-unmodified-loop-condition': 'error',
-    'no-unreachable-loop': 'error',
-    'no-unused-private-class-members': 'error',
-    'require-atomic-updates': 'error',
-
-    // Suggestions
-    'capitalized-comments': ['error', 'always', { ignoreConsecutiveComments: true }],
-    complexity: ['error', 25], // TODO: Should be lower (15?), but validator has complex methods
-    curly: 'error',
-    'default-case-last': 'error',
-    'default-param-last': 'error',
-    eqeqeq: ['error', 'smart'],
-    'func-names': 'error',
-    'func-style': ['error', 'declaration', { allowArrowFunctions: true }],
-    'grouped-accessor-pairs': ['error', 'getBeforeSet'],
-    'max-depth': ['error', 4], // TODO: Should be lower (3), but validator has complex methods
-    'max-nested-callbacks': ['error', 2],
-    'max-params': ['error', 4], // TODO: Should be lower (3), but validator has complex methods,
-    'new-cap': ['error', { newIsCapExceptions: ['ctor'] }],
-    'no-caller': 'error',
-    'no-confusing-arrow': [
-      'error',
-      {
-        allowParens: true,
-      },
-    ],
-    'no-console': ['warn'], // We want this to debug, but the check workflow should accept no warns
-    'no-div-regex': 'error',
-    'no-eval': 'error',
-    'no-extend-native': 'error',
-    'no-extra-bind': 'error',
-    'no-extra-label': 'error',
-    'no-extra-semi': 'error',
-    'no-floating-decimal': 'error',
-    'no-implied-eval': 'error',
-    'no-invalid-this': 'error',
-    'no-labels': 'error',
-    'no-lonely-if': 'error',
-    'no-multi-assign': 'error',
-    'no-multi-str': 'error',
-    'no-nested-ternary': 'error',
-    'no-new': 'error',
-    'no-new-func': 'error',
-    'no-new-object': 'error',
-    'no-new-wrappers': 'error',
-    'no-param-reassign': 'error',
-    'no-proto': 'error',
-    'no-return-assign': 'error',
-    'no-return-await': 'error',
-    'no-sequences': 'error',
-    'no-shadow': 'off',
-    '@typescript-eslint/no-shadow': 'error', // https://stackoverflow.com/questions/63961803/eslint-says-all-enums-in-typescript-app-are-already-declared-in-the-upper-scope
-    'no-unneeded-ternary': 'error',
-    'no-useless-call': 'error',
-    'no-useless-computed-key': 'error',
-    'no-useless-concat': 'error',
-    'no-useless-rename': 'error',
-    'no-var': 'error',
-    'operator-assignment': ['error', 'always'],
-    'prefer-arrow-callback': 'error',
-    'prefer-const': 'error',
-    'prefer-exponentiation-operator': 'error',
-    'prefer-object-spread': 'error',
-    'prefer-promise-reject-errors': 'error',
-    'prefer-regex-literals': 'error',
-    'prefer-rest-params': 'error',
-    'prefer-spread': 'error',
-    'prefer-template': 'error',
-    'spaced-comment': ['error', 'always'],
-    'symbol-description': 'error',
-    yoda: 'error',
-  },
-}

package/.github/README.MD

@@ -1,17 +0,0 @@
-# Github workflows for EAv3 Framework
-
-**Please note that this work is in progress. The README may not be accurate**
-
-## Overview
-This directory contains a basic GitHub workflow that builds, tests, and ultimately publishes the EAv3 Framework to NPM
-1. Setup (caches dependencies to speed up other steps)
-2. Build/Prepare/Preflight (all the following happen at the same time)
-   1. yarn build => publish artifact of build
-   2. yarn lint
-   3. yarn test
-3. Publish (currently unimplemented)
-
-### Some steps that still need to be added in:
-1. Check that changelog has an entry matching the current version in package.json
-2. Check if a package is already published to NPM, and DON'T overwrite it unless a parameter explicitly allowing an overwrite has been passed
-3. npm-check step that produces a report as an artifact

package/.github/actions/setup/action.yaml

@@ -1,13 +0,0 @@
-name: 'Setup'
-description: 'Sets up the project, installs dependencies, and builds'
-runs:
-  using: 'composite'
-  steps:
-    - uses: actions/setup-node@v3
-      with:
-        node-version: 16.x
-        registry-url: https://registry.npmjs.org
-        always-auth: true
-        cache: yarn
-    - run: yarn install --frozen-lockfile
-      shell: bash

package/.github/workflows/main.yaml

@@ -1,39 +0,0 @@
-# This is the entry point for CI. It will setup the application, then run lint, test, and eventually publish if not the master or main branch
-name: 'Main'
-on:
-  push:
-    branches:
-      - master
-  pull_request: ~
-
-jobs:
-  setup: # Load the dependencies cache for all jobs
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup
-  build:
-    needs: setup
-    runs-on: ubuntu-latest
-    concurrency: build-${{ github.ref}}
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup
-      - run: yarn build
-  lint:
-    needs: setup
-    runs-on: ubuntu-latest
-    concurrency: lint-${{ github.ref }}
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup
-      - run: yarn lint
-      - run: yarn prettier --check **/*.{ts,md}
-  test:
-    needs: setup
-    concurrency: test-${{ github.ref }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup
-      - run: yarn test

package/.github/workflows/publish.yaml

@@ -1,18 +0,0 @@
-# This is the entry point for CI. It will setup the application, then run lint, test, and eventually publish if not the master or main branch
-name: 'Publish'
-on:
-  workflow_dispatch:
-  push:
-    #branches:
-    #  - master
-    #  - main
-jobs:
-  publish: # Load the dependencies cache for all jobs
-    runs-on: ubuntu-latest
-    environment: main
-    steps:
-      - uses: actions/checkout@v3
-      - uses: ./.github/actions/setup
-      - run: npm publish --access restricted #scoped packages are restricted by default, but this is set because not all branches currently have a scoped package name
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

package/.prettierignore

@@ -1,13 +0,0 @@
-# don't ever lint node_modules
-node_modules
-# don't lint build output (make sure it's set to your correct build folder name)
-dist
-# don't lint nyc coverage output
-coverage
-test-payload.json
-.yarn
-.vscode
-.pnp.cjs
-.pnp.loader.mjs
-
-packages/k6/k8s/templates

package/docker-compose.yaml

@@ -1,35 +0,0 @@
-# Very basic stack to demonstrate horizontal scaling
-services:
-  redis:
-    image: redis
-    container_name: redis
-
-  coingecko-reader:
-    image: node:alpine
-    ports:
-      - 8080
-    deploy:
-      mode: replicated
-      replicas: 2
-    environment:
-      - DISABLE_BACKGROUND_EXECUTOR=true
-    volumes:
-      - ./:/home/node/app
-    command: node /home/node/app/dist/test.js
-
-  coingecko-writer:
-    image: node:alpine
-    volumes:
-      - ./:/home/node/app
-    environment:
-      - DISABLE_REST_API=true
-    command: node /home/node/app/dist/test.js
-
-  lb:
-    image: dockercloud/haproxy
-    links:
-      - coingecko-reader
-    ports:
-      - '80:80'
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock

package/src/adapter.ts

@@ -1,236 +0,0 @@
-import { CacheFactory } from './cache/factory'
-import { AdapterConfig, BaseAdapterConfig, SettingsMap } from './config'
-import {
-  AdapterRateLimitTier,
-  FixedFrequencyRateLimiter,
-  getRateLimitingTier,
-  SimpleCountingRateLimiter,
-} from './rate-limiting'
-import { AdapterDependencies, Transport } from './transports'
-import { makeLogger } from './util'
-import { InputParameters } from './validation/input-params'
-
-const logger = makeLogger('Adapter')
-
-/**
- * Structure to describe rate limits specs for the Adapter
- */
-interface AdapterRateLimitingConfig {
-  /** Adapter rate limits, gotten from the specific tier requested */
-  tiers: Record<string, AdapterRateLimitTier>
-}
-
-/**
- * Main structure of an External Adapter
- */
-export interface Adapter {
-  /** Name of the adapter */
-  name: string
-
-  /** If present, the string that will be used for requests with no specified endpoint */
-  defaultEndpoint?: string
-
-  /** List of [[AdapterEndpoint]]s in the adapter */
-  endpoints: AdapterEndpoint[]
-
-  /** Map of overrides to the default config values for an Adapter */
-  envDefaultOverrides?: Partial<BaseAdapterConfig>
-
-  /** List of custom env vars for this particular adapter (e.g. RPC_URL) */
-  customSettings?: SettingsMap
-
-  /** Configuration relevant to outbound (EA --\> DP) communication rate limiting */
-  rateLimiting?: AdapterRateLimitingConfig
-
-  /** Overrides for converting the 'base' parameter that are hardcoded into the adapter. */
-  // This must be included in the middleware in order to generate deterministing cache keys for hardcoded overrides
-  overrides?: Record<string, string>
-}
-
-/**
- * Structure to describe rate limits specs for a specific adapter endpoint
- */
-export interface EndpointRateLimitingConfig {
-  /**
-   * How much of the total limit for the adapter will be assigned to this specific endpoint.
-   * Should be a non-zero positive number up to 100.
-   * Endpoints in the same adapter without a specific allocation will divide the remaining limits equally.
-   */
-  allocationPercentage: number
-}
-
-/**
- * Structure to describe a specific endpoint in an [[Adapter]]
- */
-export interface AdapterEndpoint {
-  /** Name that will be used to match input params to this endpoint (case insensitive) */
-  name: string
-
-  /** List of alternative endpoint names that will resolve to this same transport (case insensitive) */
-  aliases?: string[]
-
-  /** Transport that will be used to handle data processing and communication for this endpoint */
-  transport: Transport<any, any, any>
-
-  /** Specification of what the body of a request hitting this endpoint should look like (used for validation) */
-  inputParameters: InputParameters
-
-  /** Specific details related to the rate limiting for this endpoint in particular */
-  rateLimiting?: EndpointRateLimitingConfig
-}
-
-/**
- * Structure to describe an adapter that has been initialized
- */
-export interface InitializedAdapter extends Adapter {
-  /** Object containing alias translations for all endpoints */
-  endpointsMap: Record<string, AdapterEndpoint>
-
-  /** Initialized dependencies that the adapter will use */
-  dependencies: AdapterDependencies
-
-  /** Configuration params for various adapter properties */
-  config: AdapterConfig
-}
-
-/**
- * This function will take an adapter structure and go through each endpoint, calculating
- * each one's allocation of the total rate limits that are set for the adapter as a whole.
- *
- * @param adapter - the adapter to initialize rate limiting for
- */
-const calculateRateLimitAllocations = (adapter: Adapter) => {
-  const numberOfEndpoints = adapter.endpoints.length
-  const endpointsWithExplicitAllocations = adapter.endpoints.filter((e) => e.rateLimiting)
-
-  const totalExplicitAllocation = endpointsWithExplicitAllocations
-    .map((e) => e.rateLimiting?.allocationPercentage || 0)
-    .reduce((sum, next) => sum + next, 0)
-
-  if (totalExplicitAllocation > 100) {
-    throw new Error('The total allocation set for all endpoints summed cannot exceed 100%')
-  }
-
-  if (
-    totalExplicitAllocation === 100 &&
-    numberOfEndpoints - endpointsWithExplicitAllocations.length > 0
-  ) {
-    throw new Error(
-      'The explicit allocation is at 100% but there are endpoints with implicit allocation',
-    )
-  }
-
-  const implicitAllocation = 100 - totalExplicitAllocation
-
-  logger.debug('Adapter rate limit allocations:')
-  for (const endpoint of adapter.endpoints) {
-    if (!endpoint.rateLimiting) {
-      endpoint.rateLimiting = {
-        allocationPercentage:
-          implicitAllocation / (numberOfEndpoints - endpointsWithExplicitAllocations.length),
-      }
-    }
-
-    logger.debug(`Endpoint [${endpoint.name}] - ${endpoint.rateLimiting?.allocationPercentage}%`)
-  }
-}
-
-/**
- * This function will process dependencies for an adapter, such as caches or rate limiters,
- * in order to inject them into transports and other relevant places later in the lifecycle.
- *
- * @param config - the configuration for this adapter
- * @param inputDependencies - a partial obj of initialized dependencies to override the created ones
- * @param rateLimitingConfig - details from the adapter regarding rate limiting
- * @returns a set of AdapterDependencies all initialized
- */
-export const initializeDependencies = (
-  adapter: Adapter,
-  config: AdapterConfig,
-  inputDependencies?: Partial<AdapterDependencies>,
-): AdapterDependencies => {
-  const dependencies = inputDependencies || {}
-  if (!dependencies.cache) {
-    dependencies.cache = CacheFactory.buildCache(config)
-  }
-
-  // In the future we might want something more complex, but for now it's better to simplify
-  // and just use the same rate limiting for everything. Once we have a more complex use case we
-  // can think of ways to make this more configurable.
-  const rateLimitingTier = getRateLimitingTier(
-    adapter.rateLimiting?.tiers,
-    config.RATE_LIMIT_API_TIER,
-  )
-  if (!dependencies.requestRateLimiter) {
-    dependencies.requestRateLimiter = new SimpleCountingRateLimiter().initialize(
-      adapter.endpoints,
-      rateLimitingTier,
-    )
-  }
-  if (!dependencies.backgroundExecuteRateLimiter) {
-    dependencies.backgroundExecuteRateLimiter = new FixedFrequencyRateLimiter().initialize(
-      adapter.endpoints,
-      rateLimitingTier,
-    )
-  }
-
-  return dependencies as AdapterDependencies
-}
-
-/**
- * Takes an adapter and normalizes all endpoint names and aliases, as well as the default endpoint.
- * i.e. makes them lowercase for now
- * @param adapter - an instance of an Adapter
- */
-const normalizeEndpointNames = (adapter: Adapter) => {
-  // Make endpoints case insensitive, including default
-  adapter.defaultEndpoint = adapter.defaultEndpoint?.toLowerCase()
-
-  for (const endpoint of adapter.endpoints) {
-    endpoint.name = endpoint.name.toLowerCase()
-    endpoint.aliases = endpoint.aliases?.map((a) => a.toLowerCase())
-  }
-}
-
-/**
- * Initializes all of the [[Transport]]s in the adapter, passing along any [[AdapterDependencies]] and [[AdapterConfig]].
- * Additionally, it builds a map out of all the endpoint names and aliases (checking for duplicates).
- *
- * @param adapter - an instance of an Adapter
- * @param dependencies - dependencies that the adapter will need at initialization
- * @param config - configuration variables already processed and validated
- * @returns - the adapter with all transports initialized and aliases map built
- */
-export const initializeAdapter = async (
-  adapter: Adapter,
-  config: AdapterConfig,
-  dependencies?: Partial<AdapterDependencies>,
-): Promise<InitializedAdapter> => {
-  normalizeEndpointNames(adapter)
-  calculateRateLimitAllocations(adapter)
-  const initializedDependencies = initializeDependencies(adapter, config, dependencies)
-
-  const endpointsMap: Record<string, AdapterEndpoint> = {}
-
-  for (const endpoint of adapter.endpoints) {
-    // Add aliases to map to use in validation
-    const aliases = [endpoint.name, ...(endpoint.aliases || [])]
-    for (const alias of aliases) {
-      if (endpointsMap[alias]) {
-        throw new Error(`Duplicate endpoint / alias: "${alias}"`)
-      }
-      endpointsMap[alias] = endpoint
-    }
-
-    logger.debug(`Initializing transport for endpoint "${endpoint.name}"...`)
-    await endpoint.transport.initialize(initializedDependencies)
-  }
-
-  logger.debug('Adapter initialization complete.')
-  return {
-    ...adapter,
-    endpointsMap,
-    dependencies: initializedDependencies,
-    config,
-  }
-}

package/src/background-executor.ts

@@ -1,53 +0,0 @@
-import { Server } from 'http'
-import { InitializedAdapter } from './adapter'
-import { AdapterContext } from './transports'
-import { makeLogger, sleep } from './util'
-
-const logger = makeLogger('BackgroundExecutor')
-
-/**
- * Very simple background loop that will call the [[Transport.backgroundExecute]] functions in all Transports.
- * It gets the time in ms to wait as the return value from those functions, and sleeps until next execution.
- *
- * @param adapter - an initialized External Adapter
- * @param server - the http server to attach an on close listener to
- */
-export async function callBackgroundExecutes(adapter: InitializedAdapter, server?: Server) {
-  // Set up variable to check later on to see if we need to stop this background "thread"
-  // If no server is provided, the listener won't be set and serverClosed will always be false
-  let serverClosed = false
-  server?.on('close', () => {
-    serverClosed = true
-  })
-
-  for (const endpoint of adapter.endpoints) {
-    const backgroundExecute = endpoint.transport.backgroundExecute?.bind(endpoint.transport)
-    if (!backgroundExecute) {
-      logger.debug(`Endpoint "${endpoint.name}" has no background execute, skipping...`)
-      continue
-    }
-
-    const context: AdapterContext = {
-      adapterEndpoint: endpoint,
-      adapterConfig: adapter.config,
-    }
-
-    const handler = async () => {
-      if (serverClosed) {
-        logger.info('Server closed, stopping recursive backgroundExecute handler chain')
-        return
-      }
-
-      logger.debug(`Calling background execute for endpoint "${endpoint.name}"`)
-      const timeToWait = await backgroundExecute(context)
-      logger.debug(
-        `Finished background execute for endpoint "${endpoint.name}", sleeping for ${timeToWait}ms`,
-      )
-      await sleep(timeToWait)
-      handler()
-    }
-
-    // Start recursive async calls
-    handler()
-  }
-}

package/src/cache/factory.ts

@@ -1,28 +0,0 @@
-import Redis from 'ioredis'
-import { AdapterConfig } from '../config'
-import { makeLogger } from '../util'
-import { LocalCache } from './local'
-import * as cacheMetrics from './metrics'
-import { RedisCache } from './redis'
-
-const logger = makeLogger('CacheFactory')
-export class CacheFactory {
-  static buildCache(config: AdapterConfig) {
-    logger.info(`Using "${config.CACHE_TYPE}" cache.`)
-    if (config.CACHE_TYPE === 'local') {
-      return new LocalCache()
-    } else if (config.CACHE_TYPE === 'redis') {
-      const redis = new Redis({
-        enableAutoPipelining: true, // This will make multiple commands be batch automatically
-        host: config.CACHE_REDIS_HOST,
-        port: config.CACHE_REDIS_PORT,
-      })
-      redis.on('connect', () => {
-        cacheMetrics.redisConnectionsOpen.inc()
-      })
-      // TODO: Maybe track active redis connections instead of just total count of connections created
-      // Use a Gauge and track with a combo of connect/end events listeners
-      return new RedisCache(redis)
-    }
-  }
-}