@chainlink/external-adapter-framework 0.0.12 → 0.0.15

package/src/examples/coingecko/test/integration/fixtures.ts

@@ -0,0 +1,42 @@
+import nock from 'nock'
+import capturedRequests from './capturedRequests.json'
+
+type RecordedRequest = Record<
+  string,
+  { statusCode: number; response: unknown; method: string; path: string }[]
+>
+
+// This is a generic fixture file that can be copy-pasted to generate fixtures from a
+// capturedRequests.json file.  To generate a capturedRequests.json file,
+// set env var RECORD=true and run e2e tests.
+export const createMocks = (): nock.Scope[] => {
+  const mocks: nock.Scope[] = []
+  for (const baseUrl in capturedRequests) {
+    const mock = nock(baseUrl, { encodedQueryParams: true })
+    for (const request of (capturedRequests as RecordedRequest)[baseUrl]) {
+      if (request.method === 'GET') {
+        mock
+          .get((uri: string) => {
+            const deconstructedRequestUrl = uri.split(/[?=,&]+/)
+            const deconstructedMockUrl = request.path.split(/[?=,&]+/)
+            return (
+              deconstructedRequestUrl.sort().join(',') === deconstructedMockUrl.sort().join(',')
+            )
+          })
+          .times(100)
+          .reply(request.statusCode, () => request.response, [
+            'Content-Type',
+            'application/json',
+            'Connection',
+            'close',
+            'Vary',
+            'Accept-Encoding',
+            'Vary',
+            'Origin',
+          ])
+      }
+    }
+    mocks.push(mock)
+  }
+  return mocks
+}

package/src/examples/{coingecko → coingecko-old}/batch-warming.ts

@@ -56,6 +56,7 @@ const batchEndpointTransport = new BatchWarmingTransport({
     }
   },
   parseResponse: (
+    params: AdapterRequestParams[],
     res: AxiosResponse<ProviderResponseBody>,
   ): ProviderResult<AdapterRequestParams>[] => {
     const entries = [] as ProviderResult<AdapterRequestParams>[]

package/src/transports/batch-warming.ts

@@ -2,7 +2,7 @@ import { AxiosRequestConfig, AxiosResponse } from 'axios'
 import { Cache } from '../cache'
 import { AdapterConfig, SettingsMap } from '../config'
 import { BackgroundExecuteRateLimiter } from '../rate-limiting'
-import { makeLogger, SubscriptionSet } from '../util'
+import { makeLogger, recordRequests, SubscriptionSet } from '../util'
 import { AdapterRequest, ProviderResult } from '../util/request'
 import { Transport, buildCacheEntriesFromResults } from './'
 import { axiosRequest } from './util'
@@ -48,6 +48,7 @@ export class BatchWarmingTransport<
         context: AdapterContext<CustomSettings>,
       ) => AxiosRequestConfig<ProviderRequestBody>
       parseResponse: (
+        params: AdapterParams[],
         res: AxiosResponse<ProviderResponseBody>,
         context: AdapterContext<CustomSettings>,
       ) => ProviderResult<AdapterParams>[]
@@ -101,8 +102,12 @@ export class BatchWarmingTransport<
     logger.trace('Sending request to data provider...')
     const providerResponse = await axiosRequest<ProviderRequestBody, ProviderResponseBody>(request)
 
+    if (process.env['RECORD'] === 'true') {
+      recordRequests(request, providerResponse)
+    }
+
     logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`)
-    const results = this.config.parseResponse(providerResponse, context)
+    const results = this.config.parseResponse(entries, providerResponse, context)
     const adapterResponses = buildCacheEntriesFromResults(results, context)
 
     logger.debug('Setting adapter responses in cache')

package/src/transports/rest.ts

@@ -2,7 +2,7 @@ import { AdapterRequest, AdapterResponse } from '../util/request'
 import { Transport } from './'
 import { Cache } from '../cache'
 import { AxiosRequestConfig, AxiosResponse } from 'axios'
-import { makeLogger, sleep } from '../util'
+import { makeLogger, recordRequests, sleep } from '../util'
 import { AdapterConfig, SettingsMap } from '../config'
 import { RequestRateLimiter } from '../rate-limiting'
 import { AdapterError } from '../validation/error'
@@ -124,6 +124,10 @@ export class RestTransport<
     logger.trace('Sending request to data provider...')
     const providerResponse = await axiosRequest<ProviderRequestBody, ProviderResponseBody>(request)
 
+    if (process.env['RECORD'] === 'true') {
+      recordRequests(request, providerResponse)
+    }
+
     logger.debug(`Got response from provider, parsing (raw body: ${providerResponse.data})`) // TODO: Sensitive data?
     const parsedResponse = await this.config.parseResponse(req, providerResponse, config)
 

package/src/util/index.ts

@@ -1,5 +1,6 @@
 export * from './request'
 export * from './logger'
+export * from './recordRequests'
 export * from './subscription-set/subscription-set'
 
 /**

package/src/util/recordRequests.ts

@@ -0,0 +1,47 @@
+import fs from 'fs'
+import { AxiosRequestConfig, AxiosResponse } from 'axios'
+
+export const recordRequests = (
+  requestConfig: AxiosRequestConfig<unknown>,
+  providerResponse: AxiosResponse<unknown, unknown>,
+) => {
+  let recordedRequests: Record<
+    string,
+    { method: string; path: string; data?: unknown; response: unknown; statusCode: number }[]
+  > = {}
+  try {
+    recordedRequests = JSON.parse(fs.readFileSync('capturedRequests.json').toString())
+  } catch {} // eslint-disable-line no-empty
+  let baseUrl = requestConfig.baseURL as string
+  baseUrl =
+    nthIndexOf(baseUrl, '/', 3) === -1 ? baseUrl : baseUrl.slice(0, nthIndexOf(baseUrl, '/', 3))
+  const recordedRequest = {
+    method: requestConfig.method as string,
+    path: providerResponse.request.path,
+    data: requestConfig.data,
+    response: providerResponse.data,
+    statusCode: providerResponse.status,
+  }
+  if (!recordedRequests[baseUrl]) {
+    recordedRequests[baseUrl] = [recordedRequest]
+  } else if (
+    recordedRequests[baseUrl].every((r) => {
+      return r.path !== recordedRequest.path
+    })
+  ) {
+    recordedRequests[baseUrl].push(recordedRequest)
+  }
+  fs.writeFileSync('capturedRequests.json', JSON.stringify(recordedRequests))
+}
+
+const nthIndexOf = (input: string, match: string, n: number) => {
+  let i = -1
+  let n_ = n
+  while (n_-- && i++ < input.length) {
+    i = input.indexOf(match, i)
+    if (i < 0) {
+      break
+    }
+  }
+  return i
+}

package/src/validation/overrideFunctions.ts

@@ -0,0 +1,44 @@
+import { InitializedAdapter } from '../adapter'
+import { AdapterRequest } from '../util/request'
+
+type Overrides = Record<string, Record<string, string>>
+
+export const performSymbolOverrides = (adapter: InitializedAdapter, req: AdapterRequest) => {
+  let adapterOverrides = {} as Record<string, string>
+  if (
+    req.requestContext.data &&
+    req.requestContext.data['overrides'] &&
+    (req.requestContext.data['overrides'] as Overrides)[adapter.name]
+  ) {
+    adapterOverrides = (req.requestContext.data['overrides'] as Overrides)[adapter.name]
+  }
+  if (!Array.isArray(req.requestContext.data['base'])) {
+    // Perform overrides specified in the request payload
+    req.requestContext.data['base'] =
+      adapterOverrides[req.requestContext.data['base'] as string] ?? req.requestContext.data['base']
+    // Perform hardcoded overrides
+    if (adapter.overrides) {
+      req.requestContext.data['base'] =
+        adapter.overrides[req.requestContext.data['base'] as string] ??
+        req.requestContext.data['base']
+    }
+    return
+  }
+  let requestedSymbols = req.requestContext.data['base']
+  for (let i = 0; i < requestedSymbols.length; i++) {
+    const symbol = requestedSymbols[i]
+    // Perform overrides specified in the request payload
+    let overriddenSymbol =
+      adapterOverrides[symbol.toUpperCase()] ?? adapterOverrides[symbol.toLowerCase()]
+    // Perform hardcoded overrides
+    if (adapter.overrides) {
+      overriddenSymbol =
+        adapter.overrides[symbol.toUpperCase()] ??
+        adapter.overrides[symbol.toLowerCase()] ??
+        overriddenSymbol
+    }
+    requestedSymbols[i] = overriddenSymbol ?? requestedSymbols[i]
+  }
+  requestedSymbols = requestedSymbols.length > 1 ? requestedSymbols : requestedSymbols[1]
+  req.requestContext.data['base'] = requestedSymbols
+}

package/test/metrics/warmer-metrics.test.ts

@@ -96,6 +96,7 @@ class MockBatchWarmingTransport extends BatchWarmingTransport<
         }
       },
       parseResponse: (
+        params: AdapterRequestParams[],
         res: AxiosResponse<ProviderResponseBody>,
       ): ProviderResult<AdapterRequestParams>[] => {
         return res.data.prices.map((p) => {

package/test/{smoke.test.ts → smoke/smoke.test.ts}

@@ -2,12 +2,12 @@ import untypedTest, { TestFn } from 'ava'
 import axios, { AxiosError, AxiosRequestConfig, AxiosResponse } from 'axios'
 import { AddressInfo } from 'net'
 import nock from 'nock'
-import { expose } from '../src'
-import { Adapter, AdapterEndpoint } from '../src/adapter'
-import { EmptySettings } from '../src/config'
-import { RestTransport } from '../src/transports'
-import { AdapterRequest, AdapterResponse } from '../src/util'
-import { loadTestPayload } from '../src/util/test-payload-loader'
+import { expose } from '../../src'
+import { Adapter, AdapterEndpoint } from '../../src/adapter'
+import { EmptySettings } from '../../src/config'
+import { RestTransport } from '../../src/transports'
+import { AdapterRequest, AdapterResponse } from '../../src/util'
+import { loadTestPayload } from '../../src/util/test-payload-loader'
 
 const test = untypedTest as TestFn
 
@@ -99,7 +99,7 @@ const quote = 'USD'
 const price = 1234
 
 test('Test payload loader static', async (t) => {
-  const testPayload = loadTestPayload('test-payload.json')
+  const testPayload = loadTestPayload('test/smoke/test-payload.json')
   if (testPayload.isDefault) {
     t.fail('Test payload loader return isDefault request unexpectedly')
     return
@@ -108,7 +108,7 @@ test('Test payload loader static', async (t) => {
 })
 
 test('Test payload loader dynamic', async (t) => {
-  const testPayload = loadTestPayload('test-payload.js')
+  const testPayload = loadTestPayload('test/smoke/test-payload.js')
   if (testPayload.isDefault) {
     t.fail('Test payload loader return isDefault request unexpectedly')
     return
@@ -119,13 +119,13 @@ test('Test payload loader dynamic', async (t) => {
   t.deepEqual(testPayload.requests[3], { from: 'BTC', to: 'JPY' })
 })
 
-test('Test payload loader bad file name', async (t) => {
-  const testPayload = loadTestPayload('test-payload-bad.json')
-  t.is(testPayload.isDefault, false)
+test.only('Test payload loader bad file name', async (t) => {
+  const testPayload = loadTestPayload('test/smoke/test-payload-bad.json')
+  t.is(testPayload.isDefault, true)
 })
 
 test.serial('Test smoke endpoint success', async (t) => {
-  process.env['SMOKE_TEST_PAYLOAD_FILE_NAME'] = 'test-payload.json'
+  process.env['SMOKE_TEST_PAYLOAD_FILE_NAME'] = 'test/smoke/test-payload.json'
   const server = await expose(adapter)
   if (!server) {
     throw 'Server did not start'
@@ -150,7 +150,7 @@ test.serial('Test smoke endpoint success', async (t) => {
 })
 
 test.serial('Test smoke endpoint failure', async (t) => {
-  process.env['SMOKE_TEST_PAYLOAD_FILE_NAME'] = 'test-payload-fail.json'
+  process.env['SMOKE_TEST_PAYLOAD_FILE_NAME'] = 'test/smoke/test-payload-fail.json'
   const server = await expose(adapter)
   if (!server) {
     throw 'Server did not start'

package/test/transports/batch.test.ts

@@ -75,6 +75,7 @@ class MockBatchWarmingTransport extends BatchWarmingTransport<
         }
       },
       parseResponse: (
+        params: AdapterRequestParams[],
         res: AxiosResponse<ProviderResponseBody>,
       ): ProviderResult<AdapterRequestParams>[] => {
         return res.data.prices.map((p) => {

package/tsconfig.json

@@ -11,6 +11,7 @@
     // Type checking
     "allowUnreachableCode": false,
     "allowUnusedLabels": false,
+    "allowJs": true,
     "alwaysStrict": true,
     "noImplicitAny": true,
     "noImplicitOverride": true,
@@ -19,7 +20,9 @@
     "noPropertyAccessFromIndexSignature": true,
     "noUnusedLocals": true,
     "strict": true,
-    "declaration": true // Emit a d.ts file
+    "sourceMap": true,
+//    "declaration": true
   },
+  "exclude": ["src/**/test/**/*"],
   "include": ["./src/**/*"]
 }

package/webpack.config.js

@@ -1,9 +1,15 @@
 const path = require('path')
+const glob = require('glob')
 
 module.exports = {
-  entry: './src/index.ts',
-
-  devtool: 'inline-source-map',
+  target: 'node',
+  // entry: './src/index.ts',
+  // entry: glob.sync('./src/**/*.test.{ts,tsx}').reduce((acc, file) => {
+  //   acc[file.replace(/^\.\/src\//, '')] = file
+  //   return acc
+  // }, {}),
+  mode: 'development', // This shouldn't be used in production, but is fine for now since the pkg is in active development.
+  devtool: 'inline-source-map', // This shouldn't be used in production, but is fine for now since the pkg is in active development.
   module: {
     rules: [
       {
@@ -11,13 +17,41 @@ module.exports = {
         use: 'ts-loader',
         exclude: /node_modules/,
       },
+      { test: /\.json$/, type: 'json' },
+      // { test: /\.js$/, type: 'js' },
     ],
   },
   resolve: {
-    extensions: ['.tsx', '.ts', '.js'],
+    extensions: ['.tsx', '.ts', '.js', '.json'],
+    fallback: {
+      // buffer: require.resolve('buffer'),
+      // https: require.resolve('https-browserify'),
+      // assert: require.resolve('assert'),
+      // http: require.resolve('stream-http'),
+      // querystring: require.resolve('querystring-es3'),
+      // url: require.resolve('url'),
+      // stream: require.resolve('stream-browserify'),
+      // crypto: require.resolve('crypto-browserify'),
+      // path: require.resolve('path-browserify'),
+      // os: require.resolve('os-browserify/browser'),
+      // process: require.resolve('process/browser'),
+      // fs: false,
+      // tls: false,
+      long: false,
+    },
+    // alias: {
+    //   'node:async_hooks': 'async_hooks',
+    // },
   },
   output: {
-    filename: 'bundle.js',
+    // filename: 'bundle.js',
     path: path.resolve(__dirname, 'dist'),
+    // filename: '[name].[contenthash].js',
+    filename: '[name].js',
+  },
+  optimization: {
+    splitChunks: {
+      chunks: 'all',
+    },
   },
 }