@chainlink/external-adapter-framework 0.0.14 → 0.0.15

package/.c8rc.json

@@ -0,0 +1,3 @@
+{
+  "reporter": ["lcov", "text"]
+}

package/.eslintignore

@@ -0,0 +1,10 @@
+# don't ever lint node_modules
+node_modules
+# don't lint build output (make sure it's set to your correct build folder name)
+dist
+# don't lint coverage output
+coverage
+
+.yarn
+.vscode
+webpack.config.js

package/.eslintrc.js

@@ -0,0 +1,96 @@
+// https://github.com/typescript-eslint/typescript-eslint/blob/master/docs/getting-started/linting/README.md
+module.exports = {
+  root: true,
+  parser: '@typescript-eslint/parser',
+  plugins: ['@typescript-eslint', 'eslint-plugin-tsdoc'],
+  extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended', 'prettier'],
+  rules: {
+    'tsdoc/syntax': 'warn',
+
+    // Problems
+    'array-callback-return': [
+      'error',
+      {
+        allowImplicit: false,
+        checkForEach: true,
+      },
+    ],
+    'no-constant-binary-expression': 'error',
+    'no-constructor-return': 'error',
+    'no-duplicate-imports': 'error',
+    'no-promise-executor-return': 'error',
+    'no-self-compare': 'error',
+    'no-template-curly-in-string': 'error',
+    'no-unmodified-loop-condition': 'error',
+    'no-unreachable-loop': 'error',
+    'no-unused-private-class-members': 'error',
+    'require-atomic-updates': 'error',
+
+    // Suggestions
+    'capitalized-comments': ['error', 'always', { ignoreConsecutiveComments: true }],
+    complexity: ['error', 25], // TODO: Should be lower (15?), but validator has complex methods
+    curly: 'error',
+    'default-case-last': 'error',
+    'default-param-last': 'error',
+    eqeqeq: ['error', 'smart'],
+    'func-names': 'error',
+    'func-style': ['error', 'declaration', { allowArrowFunctions: true }],
+    'grouped-accessor-pairs': ['error', 'getBeforeSet'],
+    'max-depth': ['error', 4], // TODO: Should be lower (3), but validator has complex methods
+    'max-nested-callbacks': ['error', 2],
+    'max-params': ['error', 4], // TODO: Should be lower (3), but validator has complex methods,
+    'new-cap': ['error', { newIsCapExceptions: ['ctor'] }],
+    'no-caller': 'error',
+    'no-confusing-arrow': [
+      'error',
+      {
+        allowParens: true,
+      },
+    ],
+    'no-console': ['warn'], // We want this to debug, but the check workflow should accept no warns
+    'no-div-regex': 'error',
+    'no-eval': 'error',
+    'no-extend-native': 'error',
+    'no-extra-bind': 'error',
+    'no-extra-label': 'error',
+    'no-extra-semi': 'error',
+    'no-floating-decimal': 'error',
+    'no-implied-eval': 'error',
+    'no-invalid-this': 'error',
+    'no-labels': 'error',
+    'no-lonely-if': 'error',
+    'no-multi-assign': 'error',
+    'no-multi-str': 'error',
+    'no-nested-ternary': 'error',
+    'no-new': 'error',
+    'no-new-func': 'error',
+    'no-new-object': 'error',
+    'no-new-wrappers': 'error',
+    'no-param-reassign': 'error',
+    'no-proto': 'error',
+    'no-return-assign': 'error',
+    'no-return-await': 'error',
+    'no-sequences': 'error',
+    'no-shadow': 'off',
+    '@typescript-eslint/no-shadow': 'error', // https://stackoverflow.com/questions/63961803/eslint-says-all-enums-in-typescript-app-are-already-declared-in-the-upper-scope
+    'no-unneeded-ternary': 'error',
+    'no-useless-call': 'error',
+    'no-useless-computed-key': 'error',
+    'no-useless-concat': 'error',
+    'no-useless-rename': 'error',
+    'no-var': 'error',
+    'operator-assignment': ['error', 'always'],
+    'prefer-arrow-callback': 'error',
+    'prefer-const': 'error',
+    'prefer-exponentiation-operator': 'error',
+    'prefer-object-spread': 'error',
+    'prefer-promise-reject-errors': 'error',
+    'prefer-regex-literals': 'error',
+    'prefer-rest-params': 'error',
+    'prefer-spread': 'error',
+    'prefer-template': 'error',
+    'spaced-comment': ['error', 'always'],
+    'symbol-description': 'error',
+    yoda: 'error',
+  },
+}

package/.github/README.MD

@@ -0,0 +1,42 @@
+# Github workflows for EAv3 Framework
+
+**Please note that this work is in progress. The README may not be kept up to date as the CI process changes. This message will be removed when we're somewhat stable.**
+
+## Actions
+### setup
+Runs yarn install, then yarn build. Split into an action to avoid boilerplate in jobs.
+
+## Workflows
+
+### main
+Builds, tests, and lints the code. Triggered by any pull request action or a push on the master branch.
+
+### label
+
+Adds a label to a PR based on the branch name, and adds a label instructing `npm version` how to bump the version number.
+
+Labels are applied automatically only when a PR is opened. If it applies the wrong label (ex: your improvement is a minor ver, or some PR is a major ver), you can fix it by hand
+
+#### Label mappings:
+
+| Branch Prefix | Resulting label(s)             | Version bump |
+| --- |--------------------------------|--------------|
+| release | release, major | major |
+| feature/ | feature, minor                 | minor        |
+| bugfix/ | bugfix, patch                  | patch        |
+| hotfix/ | hotfix, patch                  | patch        |
+| fix/ | hotpix, patch | patch |
+| chore/ | chore, patch | patch |
+| improvement/ | improvement, patch             | patch        |
+| documentation/ | documentation, no-version-tick | none         |
+| noop/ | no-version-tick                | none         |
+
+### publish
+Bumps the version number based on the label if present, then builds and publishes the package to NPM.  This step is skipped if the change doesn't result in a version bump.
+
+This action **requires** any engineer to approve it.
+
+### Some steps that still need to be added in:
+1. Check that changelog has an entry matching the current version in package.json
+2. Check if a package is already published to NPM, and DON'T overwrite it unless a parameter explicitly allowing an overwrite has been passed
+3. npm-check step that produces a report as an artifact

package/.github/actions/setup/action.yaml

@@ -0,0 +1,13 @@
+name: 'Setup'
+description: 'Sets up the project, installs dependencies, and builds'
+runs:
+  using: 'composite'
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 16.x
+        registry-url: https://registry.npmjs.org
+        always-auth: true
+        cache: yarn
+    - run: yarn install --frozen-lockfile
+      shell: bash

package/.github/workflows/label.yaml

@@ -0,0 +1,39 @@
+# Adds labels to pull requests based on the branch name. Labels are required by the "publish" workflow to determine
+name: 'Labels'
+on:
+  pull_request:
+    # TODO below is commented temporarily for testing
+#    types:
+#    - opened
+
+jobs:
+  publish: # Load the dependencies cache for all jobs
+    runs-on: ubuntu-latest
+    environment: main
+    steps:
+      - uses: actions/checkout@v3
+      - name: Calculate labels
+        id: calc-labels
+        run: | 
+          echo "Calculating labels based on branch: ${{ github.head_ref }}
+          HR=${{ github.head_ref }}
+          if [[ $hr == "improvement/*"  ]]; then
+            run: echo '::set-output name=LABELS::"improvement, patch"'
+          elif [[ $hr == "feature/*"  ]]; then
+            run: echo '::set-output name=LABELS::"feature, minor"'
+          elif [[ $hr == "bugfix/*"  ]]; then
+            run: echo '::set-output name=LABELS::"bugfix, patch"'
+          elif [[ $hr == "hotfix/*"  ]]; then
+            run: echo '::set-output name=LABELS::"hotfix, patch"'
+          elif [[ $hr == "release/*"  ]]; then
+            run: echo '::set-output name=LABELS::"release, major"'
+          elif [[ $hr == "chore/*"  ]]; then
+            run: echo '::set-output name=LABELS::"chore, patch"'
+          elif [[ $hr == "docs/*"  ]]; then
+            run: echo '::set-output name=LABELS::"docs, noop"'
+          else
+            run: echo '::set-output name=LABELS::"FIXME"'
+          fi
+      - uses: andymckay/labeler@master # Used in the official docs
+        with:
+          add-labels: ${{ steps.calc-labels.outputs.LABELS }}

package/.github/workflows/main.yaml

@@ -0,0 +1,39 @@
+# This is the entry point for CI. It will setup the application, then run lint, test, and eventually publish if not the master or main branch
+name: 'Main'
+on:
+  push:
+    branches:
+      - master
+  pull_request: ~
+
+jobs:
+  setup: # Load the dependencies cache for all jobs
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup
+  build:
+    needs: setup
+    runs-on: ubuntu-latest
+    concurrency: build-${{ github.ref}}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup
+      - run: yarn build
+  lint:
+    needs: setup
+    runs-on: ubuntu-latest
+    concurrency: lint-${{ github.ref }}
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup
+      - run: yarn lint
+      - run: yarn prettier --check **/*.{ts,md}
+  test:
+    needs: setup
+    concurrency: test-${{ github.ref }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup
+      - run: yarn test

package/.github/workflows/publish.yaml

@@ -0,0 +1,17 @@
+# This is the entry point for CI. It will setup the application, then run lint, test, and eventually publish if not the master or main branch
+name: 'Publish'
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - master
+jobs:
+  publish: # Load the dependencies cache for all jobs
+    runs-on: ubuntu-latest
+    environment: main
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/setup
+      - run: npm publish --access restricted #scoped packages are restricted by default, but this is set because not all branches currently have a scoped package name
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}

package/.prettierignore

@@ -0,0 +1,13 @@
+# don't ever lint node_modules
+node_modules
+# don't lint build output (make sure it's set to your correct build folder name)
+dist
+# don't lint nyc coverage output
+coverage
+test-payload.json
+.yarn
+.vscode
+.pnp.cjs
+.pnp.loader.mjs
+
+packages/k6/k8s/templates

package/README.md

@@ -0,0 +1,103 @@
+# EA Framework v3
+
+External adapters and its core framework serve as middleware to facilitate connections between Chainlink Nodes and Data Providers (DP). Their main purpose is threefold:
+
+- Abstract provider specific details, specifically:
+  - Transport (REST, WebSockets, RPC, SSE, etc.)
+  - Authentication (login flows, keys)
+  - Accept normalized request payloads and translate into the provider's interface (this also includes things like symbols/tickers)
+  - Parse provider responses into the desired data points (e.g. price from crypto endpoint)
+- Make the overall system more efficient by using as few resources (e.g. API credits, networking traffic, CPU usage) as possible to fetch data, achieved by features like:
+  - Caching: since DPs update data at various times and a request to their API incurs a certain latency, EAs keep a cache of values to
+    - Provide responses to the Chainlink Nodes as quickly as possible
+    - Communicate with DPs only when necessary
+  - Cache Warming: in order to make as many of the requests to EAs be fulfilled from the cache, EAs fetch values from DPs asynchronously to keep the local ones fresh.
+  - Batching: the CL Node requests feeds individually, but it's common for DPs to provide batch endpoints to get many data points at once. With Batching, EAs keep track of incoming requests and consolidate them into one batched request.
+  - Rate limiting: the EA framework additionally checks request rates (frequency) to make sure they fall within acceptable limits (like quotas for the DP's API, or adjusting based on the NOP's API tier).
+- Perform off chain computations (think aggregations, indexing, or any sort of data processing)
+
+By providing a framework that gives users easy access to those features, we reduce the complexity required for Nodes to communicate with DPs since by using EAs there is only one standardized way to do so, while at the same time optimizing said communication so it's as resource efficient as possible. It also makes internal and external development easier and faster, by serving as a strict guideline to implement and add new providers.
+
+## Qs
+
+- Store entire response in cache?
+- Check for valid result in adapter response?
+
+## Env vars
+
+These are all existing env vars, marked DONE if they have been ported to this version, or N/A if the new version disregards them altogether.
+
+| Name                                       | State | Comments                                                     |
+| :----------------------------------------- | :---: | :----------------------------------------------------------- |
+| API_TIMEOUT                                |  N/A  | Only used for WS with handler.noHTtp = true                  |
+| BASE_URL                                   |   √   |                                                              |
+| CACHE_ENABLED                              |  N/A  | Cache is integral to this fw, not including this for now     |
+| CACHE_KEY_IGNORED_PROPS                    |  ??   | Do we want this to be available in v3. Used for feed id      |
+| CACHE_KEY_GROUP                            |  ??   | Dunno if this applies to this fw                             |
+| CACHE_MAX_AGE                              |   √   |                                                              |
+| CACHE_MAX_ITEMS                            |   -   | Should add when replacing obj in local impl with LRU package |
+| CACHE_MIN_AGE                              |   -   | Used for rate limiting                                       |
+| CACHE_REDIS_CONNECTION_TIMEOUT             |   -   | TODO: Redis                                                  |
+| CACHE_REDIS_HOST                           |   -   | TODO: Redis                                                  |
+| CACHE_REDIS_MAX_QUEUED_ITEMS               |   -   | TODO: Redis                                                  |
+| CACHE_REDIS_MAX_RECONNECT_COOLDOWN         |   -   | TODO: Redis                                                  |
+| CACHE_REDIS_PASSWORD                       |   -   | TODO: Redis                                                  |
+| CACHE_REDIS_PATH                           |   -   | TODO: Redis                                                  |
+| CACHE_REDIS_PORT                           |   -   | TODO: Redis                                                  |
+| CACHE_REDIS_TIMEOUT                        |   -   | TODO: Redis                                                  |
+| CACHE_REDIS_URL                            |   -   | TODO: Redis                                                  |
+| CACHE_TYPE                                 |   -   | TODO: Cache factory                                          |
+| CACHE_UPDATE_AGE_ON_GET                    |  N/A  | No longer used in EA currently, apparently                   |
+| DEBUG                                      |       |                                                              |
+| DEFAULT_WS_HEARTBEAT_INTERVAL              |       |                                                              |
+| EA_HOST                                    |       |                                                              |
+| EA_PORT                                    |   √   |                                                              |
+| ENV_ADAPTER_URL                            |       |                                                              |
+| ENV_API_ENDPOINT                           |       |                                                              |
+| ENV_API_KEY                                |       |                                                              |
+| ENV_API_TIMEOUT                            |       |                                                              |
+| ENV_API_VERBOSE                            |       |                                                              |
+| ENV_WS_API_ENDPOINT                        |       |                                                              |
+| ENV_WS_API_KEY                             |       |                                                              |
+| ERROR_CAPACITY                             |       |                                                              |
+| EXPERIMENTAL_METRICS_ENABLED               |   √   | Maintaining for backwards compatibility. Defaults to true    |
+| LEGACY_ENV_ADAPTER_URL                     |       |                                                              |
+| LOG_LEVEL                                  |       |                                                              |
+| METRICS_ENABLED                            |   √   | Defaults to true                                             |
+| METRICS_NAME                               |   √   |                                                              |
+| METRICS_PORT                               |   √   |                                                              |
+| METRICS_USE_BASE_URL                       |   √   |                                                              |
+| NODE_ENV                                   |       |                                                              |
+| npm_package_version                        |       |                                                              |
+| RATE_LIMIT_API_TIER                        |       |                                                              |
+| RATE_LIMIT_CAPACITY                        |       |                                                              |
+| RATE_LIMIT_CAPACITY_MINUTE                 |       |                                                              |
+| RATE_LIMIT_CAPACITY_SECOND                 |       |                                                              |
+| RATE_LIMIT_ENABLED                         |       |                                                              |
+| RECORD                                     |       |                                                              |
+| REQUEST_COALESCING_ENABLED                 |       |                                                              |
+| REQUEST_COALESCING_ENTROPY_MAX             |       |                                                              |
+| REQUEST_COALESCING_INTERVAL                |       |                                                              |
+| REQUEST_COALESCING_INTERVAL_COEFFICIENT    |       |                                                              |
+| REQUEST_COALESCING_INTERVAL_MAX            |       |                                                              |
+| REQUEST_COALESCING_MAX_RETRIES             |       |                                                              |
+| RETRY                                      |       |                                                              |
+| SERVER_RATE_LIMIT_MAX                      |       |                                                              |
+| SERVER_SLOW_DOWN_AFTER_FACTOR              |       |                                                              |
+| SERVER_SLOW_DOWN_DELAY_MS                  |       |                                                              |
+| TIMEOUT                                    |       |                                                              |
+| WARMUP_ENABLED                             |       |                                                              |
+| WARMUP_INTERVAL                            |       |                                                              |
+| WARMUP_SUBSCRIPTION_TTL                    |       |                                                              |
+| WARMUP_UNHEALTHY_THRESHOLD                 |       |                                                              |
+| WS_CONNECTION_KEY                          |       |                                                              |
+| WS_CONNECTION_LIMIT                        |       |                                                              |
+| WS_CONNECTION_RETRY_DELAY                  |       |                                                              |
+| WS_CONNECTION_RETRY_LIMIT                  |       |                                                              |
+| WS_CONNECTION_TTL                          |       |                                                              |
+| WS_ENABLED                                 |  N/A  | Shouldn't be part of this fw                                 |
+| WS_SUBSCRIPTION_LIMIT                      |       |                                                              |
+| WS_SUBSCRIPTION_PRIORITY_LIST              |       |                                                              |
+| WS_SUBSCRIPTION_TTL                        |       |                                                              |
+| WS_SUBSCRIPTION_UNRESPONSIVE_TTL           |       |                                                              |
+| WS_TIME_UNTIL_HANDLE_NEXT_MESSAGE_OVERRIDE |       |                                                              |