@chainlink/external-adapter-framework 0.0.14 → 0.0.15

package/test/transports/batch.test.ts

@@ -0,0 +1,466 @@
+import FakeTimers, { InstalledClock } from '@sinonjs/fake-timers'
+import untypedTest, { TestFn } from 'ava'
+import axios, { AxiosError, AxiosRequestConfig, AxiosResponse } from 'axios'
+import { AddressInfo } from 'net'
+import nock from 'nock'
+import { expose } from '../../src'
+import { Adapter, AdapterContext } from '../../src/adapter'
+import { SettingsMap } from '../../src/config'
+import { DEFAULT_SHARED_MS_BETWEEN_REQUESTS } from '../../src/rate-limiting'
+import { BatchWarmingTransport } from '../../src/transports'
+import { ProviderResult } from '../../src/util'
+import { MockCache, runAllUntilTime } from '../util'
+
+const test = untypedTest as TestFn<{
+  clock: InstalledClock
+}>
+
+const URL = 'http://test-url.com'
+const endpoint = '/price'
+
+interface AdapterRequestParams {
+  from: string
+  to: string
+}
+
+interface ProviderRequestBody {
+  pairs: Array<{
+    base: string
+    quote: string
+  }>
+}
+
+interface ProviderResponseBody {
+  prices: Array<{
+    pair: string
+    price: number
+  }>
+}
+
+test.before(() => {
+  nock.disableNetConnect()
+  nock.enableNetConnect('localhost')
+})
+
+test.after(() => {
+  nock.restore()
+})
+
+test.beforeEach((t) => {
+  t.context.clock = FakeTimers.install({ shouldAdvanceTime: true, advanceTimeDelta: 100 })
+})
+
+test.afterEach((t) => {
+  t.context.clock.uninstall()
+})
+
+class MockBatchWarmingTransport extends BatchWarmingTransport<
+  AdapterRequestParams,
+  ProviderRequestBody,
+  ProviderResponseBody,
+  SettingsMap
+> {
+  backgroundExecuteCalls = 0
+
+  constructor(private callSuper = false) {
+    super({
+      prepareRequest: (params: AdapterRequestParams[]): AxiosRequestConfig<ProviderRequestBody> => {
+        return {
+          baseURL: URL,
+          url: '/price',
+          method: 'POST',
+          data: {
+            pairs: params.map((p) => ({ base: p.from, quote: p.to })),
+          },
+        }
+      },
+      parseResponse: (
+        params: AdapterRequestParams[],
+        res: AxiosResponse<ProviderResponseBody>,
+      ): ProviderResult<AdapterRequestParams>[] => {
+        return res.data.prices.map((p) => {
+          const [from, to] = p.pair.split('/')
+          return {
+            params: { from, to },
+            value: p.price,
+          }
+        })
+      },
+    })
+  }
+
+  override async backgroundExecute(context: AdapterContext<SettingsMap>): Promise<number> {
+    this.backgroundExecuteCalls++
+    if (this.callSuper) {
+      super.backgroundExecute(context)
+    }
+    return this.rateLimiter.msUntilNextExecution(context.adapterEndpoint.name)
+  }
+}
+
+// Disable retries to make the testing flow easier
+process.env['CACHE_POLLING_MAX_RETRIES'] = '0'
+
+const from = 'ETH'
+const to = 'USD'
+const price = 1234
+
+nock(URL)
+  .post(endpoint, {
+    pairs: [
+      {
+        base: from,
+        quote: to,
+      },
+    ],
+  })
+  .reply(200, {
+    prices: [
+      {
+        pair: `${from}/${to}`,
+        price,
+      },
+    ],
+  })
+  .persist()
+
+const inputParameters = {
+  from: {
+    type: 'string',
+    required: true,
+  },
+  to: {
+    type: 'string',
+    required: true,
+  },
+} as const
+
+test.serial('sends request to DP and returns response', async (t) => {
+  const adapter: Adapter = {
+    name: 'test',
+    defaultEndpoint: 'test',
+    endpoints: [
+      {
+        name: 'test',
+        inputParameters,
+        transport: new MockBatchWarmingTransport(true),
+      },
+    ],
+  }
+
+  // Create mocked cache so we can listen when values are set
+  // This is a more reliable method than expecting precise clock timings
+  const mockCache = new MockCache()
+
+  // Start the adapter
+  const server = await expose(adapter, {
+    cache: mockCache,
+  })
+  const address = `http://localhost:${(server?.address() as AddressInfo)?.port}`
+
+  const makeRequest = () =>
+    axios.post(address, {
+      data: {
+        from,
+        to,
+      },
+    })
+
+  // Expect the first response to time out
+  // The polling behavior is tested in the cache tests, so this is easier here.
+  // Start the request:
+  const errorPromise: Promise<AxiosError | undefined> = t.throwsAsync(makeRequest)
+  // Advance enough time for the initial request async flow
+  t.context.clock.tickAsync(10)
+  // Wait for the failed cache get -> instant 504
+  const error = await errorPromise
+  t.is(error?.response?.status, 504)
+
+  // Advance clock so that the batch warmer executes once again and wait for the cache to be set
+  const cacheValueSetPromise = mockCache.waitForNextSet()
+  await t.context.clock.tickAsync(DEFAULT_SHARED_MS_BETWEEN_REQUESTS + 10)
+  await cacheValueSetPromise
+
+  // Second request should find the response in the cache
+  const response = await makeRequest()
+
+  t.is(response.status, 200)
+  t.deepEqual(response.data, {
+    data: null,
+    result: price,
+    statusCode: 200,
+  })
+})
+
+test.serial(
+  'per minute rate limit of 4 with one batch transport results in a call every 15s',
+  async (t) => {
+    const rateLimit1m = 4
+    const transport = new MockBatchWarmingTransport()
+
+    const adapter: Adapter = {
+      name: 'test',
+      defaultEndpoint: 'test',
+      endpoints: [
+        {
+          name: 'test',
+          inputParameters,
+          transport: transport,
+        },
+      ],
+      rateLimiting: {
+        tiers: {
+          default: {
+            rateLimit1m,
+          },
+        },
+      },
+    }
+
+    // Start the adapter
+    const server = await expose(adapter)
+    const address = `http://localhost:${(server?.address() as AddressInfo)?.port}`
+
+    const makeRequest = () =>
+      axios.post(address, {
+        data: {
+          from,
+          to,
+        },
+      })
+
+    // Expect the first response to time out
+    // The polling behavior is tested in the cache tests, so this is easier here.
+    // Start the request:
+    const errorPromise: Promise<AxiosError | undefined> = t.throwsAsync(makeRequest)
+    // Advance enough time for the initial request async flow
+    t.context.clock.tickAsync(10)
+    // Wait for the failed cache get -> instant 504
+    const error = await errorPromise
+    t.is(error?.response?.status, 504)
+
+    // Wait for the first background execute and check that it's been called
+    await t.context.clock.tickAsync(10)
+    t.is(transport.backgroundExecuteCalls, 1)
+
+    // Advance the clock a few minutes and check that the amount of calls is as expected
+    // +1 because of the previous first
+    await t.context.clock.tickAsync(5 * 60 * 1000) // 5m
+    t.is(transport.backgroundExecuteCalls, 5 * rateLimit1m + 1)
+  },
+)
+
+test.serial(
+  'per second limit of 1 with one batch transport results in a call every 1000ms',
+  async (t) => {
+    const rateLimit1s = 1
+    const transport = new MockBatchWarmingTransport()
+
+    const adapter: Adapter = {
+      name: 'test',
+      defaultEndpoint: 'test',
+      endpoints: [
+        {
+          name: 'test',
+          inputParameters,
+          transport: transport,
+        },
+      ],
+      rateLimiting: {
+        tiers: {
+          default: {
+            rateLimit1s,
+          },
+        },
+      },
+      envDefaultOverrides: {
+        WARMUP_SUBSCRIPTION_TTL: 100000,
+      },
+    }
+
+    // Start the adapter
+    const server = await expose(adapter)
+    const address = `http://localhost:${(server?.address() as AddressInfo)?.port}`
+
+    const makeRequest = () =>
+      axios.post(address, {
+        data: {
+          from,
+          to,
+        },
+      })
+
+    // Expect the first response to time out
+    // The polling behavior is tested in the cache tests, so this is easier here.
+    // Start the request:
+    const errorPromise: Promise<AxiosError | undefined> = t.throwsAsync(makeRequest)
+    // Advance enough time for the initial request async flow
+    t.context.clock.tickAsync(10)
+    // Wait for the failed cache get -> instant 504
+    const error = await errorPromise
+    t.is(error?.response?.status, 504)
+
+    // Wait for the first background execute and check that it's been called
+    await t.context.clock.tickAsync(10)
+    t.is(transport.backgroundExecuteCalls, 1)
+
+    // Run for an entire minute and check that the values are as expected
+    await runAllUntilTime(t.context.clock, 59 * 1000)
+
+    t.is(transport.backgroundExecuteCalls, 60 * rateLimit1s + 1)
+  },
+)
+
+test.serial(
+  'per second limit of 1 with two batch transports results in a call every 2000ms for each',
+  async (t) => {
+    const rateLimit1s = 1
+    const transportA = new MockBatchWarmingTransport()
+    const transportB = new MockBatchWarmingTransport()
+
+    const adapter: Adapter = {
+      name: 'test',
+      endpoints: [
+        {
+          name: 'A',
+          inputParameters,
+          transport: transportA,
+        },
+        {
+          name: 'B',
+          inputParameters,
+          transport: transportB,
+        },
+      ],
+      rateLimiting: {
+        tiers: {
+          default: {
+            rateLimit1s,
+          },
+        },
+      },
+      envDefaultOverrides: {
+        WARMUP_SUBSCRIPTION_TTL: 100000,
+      },
+    }
+
+    // Start the adapter
+    const server = await expose(adapter)
+    const address = `http://localhost:${(server?.address() as AddressInfo)?.port}`
+
+    const makeRequest = (endpointParam: string) =>
+      axios.post(address, {
+        data: {
+          from,
+          to,
+        },
+        endpoint: endpointParam,
+      })
+
+    // Expect the first response to time out
+    // The polling behavior is tested in the cache tests, so this is easier here.
+    // Start the request:
+    const errorPromiseA: Promise<AxiosError | undefined> = t.throwsAsync(() => makeRequest('A'))
+    // Advance enough time for the initial request async flow
+    t.context.clock.tickAsync(10)
+    // Wait for the failed cache get -> instant 504
+    const errorA = await errorPromiseA
+    t.is(errorA?.response?.status, 504)
+
+    // Do the same thing for transport B
+    const errorPromiseB: Promise<AxiosError | undefined> = t.throwsAsync(() => makeRequest('B'))
+    t.context.clock.tickAsync(10)
+    const errorB = await errorPromiseB
+    t.is(errorB?.response?.status, 504)
+
+    // Wait for the first background executes and check that they've been called
+    await t.context.clock.tickAsync(10)
+    t.is(transportA.backgroundExecuteCalls, 1)
+    t.is(transportB.backgroundExecuteCalls, 1)
+
+    // Run for a minute (59s actually, it'll start at 0 and go on regular intervals)
+    await runAllUntilTime(t.context.clock, 59 * 1000 + 10)
+
+    t.is(transportA.backgroundExecuteCalls, 30 * rateLimit1s + 1) // +1 for the first call
+    t.is(transportB.backgroundExecuteCalls, 30 * rateLimit1s)
+  },
+)
+
+test.serial(
+  'per second limit of 1 with two batch transports with different allocations results in correct time distribution',
+  async (t) => {
+    const rateLimit1s = 1
+    const transportA = new MockBatchWarmingTransport()
+    const transportB = new MockBatchWarmingTransport()
+
+    const adapter: Adapter = {
+      name: 'test',
+      endpoints: [
+        {
+          name: 'A',
+          inputParameters,
+          transport: transportA,
+          rateLimiting: {
+            allocationPercentage: 75,
+          },
+        },
+        {
+          // This one should be dynamically allocated
+          name: 'B',
+          inputParameters,
+          transport: transportB,
+        },
+      ],
+      rateLimiting: {
+        tiers: {
+          default: {
+            rateLimit1s,
+          },
+        },
+      },
+      envDefaultOverrides: {
+        WARMUP_SUBSCRIPTION_TTL: 100000,
+      },
+    }
+
+    // Start the adapter
+    const server = await expose(adapter)
+    const address = `http://localhost:${(server?.address() as AddressInfo)?.port}`
+
+    const makeRequest = (endpointParam: string) =>
+      axios.post(address, {
+        data: {
+          from,
+          to,
+        },
+        endpoint: endpointParam,
+      })
+
+    // Expect the first response to time out
+    // The polling behavior is tested in the cache tests, so this is easier here.
+    // Start the request:
+    const errorPromiseA: Promise<AxiosError | undefined> = t.throwsAsync(() => makeRequest('A'))
+    // Advance enough time for the initial request async flow
+    t.context.clock.tickAsync(10)
+    // Wait for the failed cache get -> instant 504
+    const errorA = await errorPromiseA
+    t.is(errorA?.response?.status, 504)
+
+    // Do the same thing for transport B
+    const errorPromiseB: Promise<AxiosError | undefined> = t.throwsAsync(() => makeRequest('B'))
+    t.context.clock.tickAsync(10)
+    const errorB = await errorPromiseB
+    t.is(errorB?.response?.status, 504)
+
+    // Wait for the first background executes and check that they've been called
+    await t.context.clock.tickAsync(10)
+    t.is(transportA.backgroundExecuteCalls, 1)
+    t.is(transportB.backgroundExecuteCalls, 1)
+
+    // Run for a minute (59s actually, it'll start at 0 and go on regular intervals)
+    await runAllUntilTime(t.context.clock, 59 * 1000 + 10)
+
+    t.is(transportA.backgroundExecuteCalls, 45 * rateLimit1s + 1) // +1 for the first call
+    t.is(transportB.backgroundExecuteCalls, 15 * rateLimit1s)
+  },
+)

package/test/transports/rest.test.ts

@@ -0,0 +1,242 @@
+import untypedTest, { TestFn } from 'ava'
+import axios, { AxiosError, AxiosRequestConfig, AxiosResponse } from 'axios'
+import { AddressInfo } from 'net'
+import nock from 'nock'
+import { expose } from '../../src'
+import { Adapter, AdapterEndpoint } from '../../src/adapter'
+import { EmptySettings } from '../../src/config'
+import { RestTransport } from '../../src/transports'
+import { AdapterRequest, AdapterResponse, sleep } from '../../src/util'
+import { deferredPromise } from '../util'
+
+const test = untypedTest as TestFn<{
+  serverAddress: string
+}>
+
+const URL = 'http://test-url.com'
+
+test.before(() => {
+  nock.disableNetConnect()
+  nock.enableNetConnect('localhost')
+})
+
+test.after(() => {
+  nock.restore()
+})
+
+interface AdapterRequestParams {
+  from: string
+  to: string
+}
+
+interface ProviderRequestBody {
+  base: string
+  quote: string
+}
+
+interface ProviderResponseBody {
+  price: number
+}
+
+const endpoint = '/price'
+
+const createAdapterEndpoint = (): AdapterEndpoint => {
+  const restEndpointTransport = new RestTransport<
+    AdapterRequestParams,
+    ProviderRequestBody,
+    ProviderResponseBody,
+    EmptySettings
+  >({
+    prepareRequest: (
+      req: AdapterRequest<AdapterRequestParams>,
+    ): AxiosRequestConfig<ProviderRequestBody> => {
+      return {
+        baseURL: URL,
+        url: endpoint,
+        method: 'GET',
+        params: {
+          base: req.requestContext.data.from,
+          quote: req.requestContext.data.to,
+        },
+      }
+    },
+    parseResponse: (
+      req: AdapterRequest<AdapterRequestParams>,
+      res: AxiosResponse<ProviderResponseBody>,
+    ): AdapterResponse<ProviderResponseBody> => {
+      return {
+        data: res.data,
+        statusCode: 200,
+        result: res.data.price,
+      }
+    },
+    options: {
+      coalescing: true,
+    },
+  })
+
+  return {
+    name: 'test',
+    inputParameters: {
+      from: {
+        type: 'string',
+        required: true,
+      },
+      to: {
+        type: 'string',
+        required: true,
+      },
+    },
+    transport: restEndpointTransport,
+  }
+}
+
+const from = 'ETH'
+const to = 'USD'
+const price = 1234
+
+test('sends request to DP and returns response', async (t) => {
+  const adapter: Adapter = {
+    name: 'test',
+    defaultEndpoint: 'test',
+    endpoints: [createAdapterEndpoint()],
+  }
+
+  const server = await expose(adapter)
+  const address = `http://localhost:${(server?.address() as AddressInfo)?.port}`
+
+  nock(URL)
+    .get(endpoint)
+    .query({
+      base: from,
+      quote: to,
+    })
+    .reply(200, {
+      price,
+    })
+
+  const response = await axios.post(address, {
+    data: {
+      from,
+      to,
+    },
+  })
+
+  t.is(response.status, 200)
+  t.deepEqual(response.data, {
+    data: { price },
+    result: price,
+    statusCode: 200,
+  })
+})
+
+test('identical request to EA is coalesced and returned from cache', async (t) => {
+  const adapter: Adapter = {
+    name: 'test',
+    defaultEndpoint: 'test',
+    endpoints: [createAdapterEndpoint()],
+  }
+
+  const server = await expose(adapter)
+  const address = `http://localhost:${(server?.address() as AddressInfo)?.port}`
+
+  const [promise, resolve] = deferredPromise<ProviderResponseBody>()
+  nock(URL)
+    .get(endpoint)
+    .query({
+      base: from,
+      quote: to,
+    })
+    .reply(200, () => promise)
+
+  // Send first request, that will be blocked until we resolve the promise
+  const request1 = axios.post(address, {
+    data: {
+      from,
+      to,
+    },
+  })
+  await sleep(1) // To ensure all inner ops move along
+
+  // Send second request, and wait for response.
+  // This one will be coalesced with the first one, and made to wait until the cache is filled.
+  // We know it's not going to the DP twice, because Nock would block it.
+  const request2 = axios.post(address, {
+    data: {
+      from,
+      to,
+    },
+  })
+  await sleep(1) // To ensure all inner ops move along
+
+  // Resolve the promise
+  resolve({
+    price,
+  })
+
+  const responses = await Promise.all([request1, request2])
+
+  for (const response of responses) {
+    t.is(response.status, 200)
+    t.deepEqual(response.data, {
+      data: { price },
+      result: price,
+      statusCode: 200,
+    })
+  }
+})
+
+test('rate limits (per second) incoming request (without retries)', async (t) => {
+  const adapter: Adapter = {
+    name: 'test',
+    defaultEndpoint: 'test',
+    endpoints: [createAdapterEndpoint()],
+    rateLimiting: {
+      tiers: {
+        base: {
+          rateLimit1m: 1,
+        },
+      },
+    },
+    envDefaultOverrides: {
+      REST_TRANSPORT_MAX_RATE_LIMIT_RETRIES: 0,
+    },
+  }
+
+  const server = await expose(adapter)
+  const address = `http://localhost:${(server?.address() as AddressInfo)?.port}`
+
+  const payloads = [
+    {
+      base: 'ETH',
+      quote: 'BTC',
+    },
+    {
+      base: 'ETH',
+      quote: 'USD',
+    },
+  ]
+
+  payloads.forEach((p) => {
+    nock(URL)
+      .get(endpoint)
+      .query(p)
+      .reply(200, {
+        price,
+      })
+      .persist()
+  })
+
+  const makeRequest = ({ base, quote }: { base: string; quote: string }) =>
+    axios.post(address, {
+      data: {
+        from: base,
+        to: quote,
+      },
+    })
+
+  const error: AxiosError | undefined = await t.throwsAsync(() =>
+    Promise.all(payloads.map((p) => makeRequest(p))),
+  )
+  t.is(error?.response?.status, 504)
+})