@api3/commons 0.1.0 → 0.3.0

package/src/processing/processing.test.ts

@@ -0,0 +1,272 @@
+/* eslint-disable jest/prefer-strict-equal */ // Because the errors are thrown from the "vm" module (different context), they are not strictly equal.
+import { createEndpoint } from '../../test/fixtures';
+
+import {
+  addReservedParameters,
+  postProcessApiCallResponse,
+  preProcessApiCallParameters,
+  removeReservedParameters,
+} from './processing';
+
+describe(preProcessApiCallParameters.name, () => {
+  it('valid processing code', async () => {
+    const endpoint = createEndpoint({
+      preProcessingSpecifications: [
+        {
+          environment: 'Node',
+          value: 'const output = {...input, from: "ETH"};',
+          timeoutMs: 5000,
+        },
+        {
+          environment: 'Node',
+          value: 'const output = {...input, newProp: "airnode"};',
+          timeoutMs: 5000,
+        },
+      ],
+    });
+    const parameters = { _type: 'int256', _path: 'price' };
+
+    const result = await preProcessApiCallParameters(endpoint, parameters);
+
+    expect(result).toEqual({
+      _path: 'price',
+      _type: 'int256',
+      from: 'ETH',
+      newProp: 'airnode',
+    });
+  });
+
+  it('invalid processing code', async () => {
+    const endpoint = createEndpoint({
+      preProcessingSpecifications: [
+        {
+          environment: 'Node',
+          value: 'something invalid; const output = {...input, from: `ETH`};',
+          timeoutMs: 5000,
+        },
+        {
+          environment: 'Node',
+          value: 'const output = {...input, newProp: "airnode"};',
+          timeoutMs: 5000,
+        },
+      ],
+    });
+    const parameters = { _type: 'int256', _path: 'price', from: 'TBD' };
+
+    const throwingFunc = async () => preProcessApiCallParameters(endpoint, parameters);
+
+    await expect(throwingFunc).rejects.toEqual(new Error('SyntaxError: Unexpected identifier'));
+  });
+
+  it('demonstrates access to endpointParameters, but reserved parameters are inaccessible', async () => {
+    const parameters = { _type: 'int256', _path: 'price', to: 'USD' };
+    const endpoint = createEndpoint({
+      preProcessingSpecifications: [
+        {
+          environment: 'Node',
+          // pretend the user is trying to 1) override _path and 2) set a new parameter based on
+          // the presence of the reserved parameter _type (which is inaccessible)
+          value:
+            'const output = {...input, from: "ETH", _path: "price.newpath", myVal: input._type ? "123" : "456", newTo: endpointParameters.to };',
+          timeoutMs: 5000,
+        },
+      ],
+    });
+
+    const result = await preProcessApiCallParameters(endpoint, parameters);
+
+    expect(result).toEqual({
+      _path: 'price', // is not overridden
+      _type: 'int256',
+      from: 'ETH', // originates from the processing code
+      to: 'USD', // should be unchanged from the original parameters
+      myVal: '456', // is set to "456" because _type is not present in the environment
+      newTo: 'USD', // demonstrates access to endpointParameters
+    });
+  });
+
+  it('uses native modules for processing', async () => {
+    const endpoint = createEndpoint({
+      preProcessingSpecifications: [
+        {
+          environment: 'Node',
+          value: `
+          const randomValue = crypto.randomBytes(4).toString('hex');
+          const output = {...input, randomValue};
+        `,
+          timeoutMs: 5000,
+        },
+      ],
+    });
+    const parameters = { _type: 'int256', _path: 'price' };
+
+    const result = await preProcessApiCallParameters(endpoint, parameters);
+
+    // Check that the result contains the original parameters and a valid 8-character hex random value.
+    expect(result).toMatchObject({
+      _path: 'price',
+      _type: 'int256',
+    });
+    expect(result.randomValue).toHaveLength(8);
+    expect(/^[\da-f]{8}$/i.test(result.randomValue)).toBe(true);
+  });
+
+  it('throws error due to processing timeout', async () => {
+    const endpoint = createEndpoint({
+      preProcessingSpecifications: [
+        {
+          environment: 'Node async',
+          value: `
+          const delay = (ms) => new Promise(resolve => setTimeout(resolve, ms));
+          delay(5000);
+          const output = {...input, from: 'ETH'};
+        `,
+          timeoutMs: 100, // This timeout is shorter than the delay in the processing code.
+        },
+      ],
+    });
+    const parameters = { _type: 'int256', _path: 'price' };
+
+    const throwingFunc = async () => preProcessApiCallParameters(endpoint, parameters);
+
+    await expect(throwingFunc).rejects.toThrow('Timeout exceeded');
+  });
+});
+
+describe(postProcessApiCallResponse.name, () => {
+  it('processes valid code', async () => {
+    const parameters = { _type: 'int256', _path: 'price' };
+    const endpoint = createEndpoint({
+      postProcessingSpecifications: [
+        {
+          environment: 'Node',
+          value: 'const output = parseInt(input.price)*2;',
+          timeoutMs: 5000,
+        },
+        {
+          environment: 'Node',
+          value: 'const output = parseInt(input)*2;',
+          timeoutMs: 5000,
+        },
+      ],
+    });
+
+    const result = await postProcessApiCallResponse({ price: 1000 }, endpoint, parameters);
+
+    expect(result).toBe(4000);
+  });
+
+  it('demonstrates access to endpointParameters, but reserved parameters are inaccessible', async () => {
+    const myMultiplier = 10;
+    const parameters = { _type: 'int256', _path: 'price', myMultiplier };
+    const endpoint = createEndpoint({
+      postProcessingSpecifications: [
+        {
+          environment: 'Node',
+          value: `
+            const reservedMultiplier = endpointParameters._times ? 1 : 2;
+            const output = parseInt(input.price) * endpointParameters.myMultiplier * reservedMultiplier
+          `,
+          timeoutMs: 5000,
+        },
+      ],
+    });
+
+    const price = 1000;
+    const result = await postProcessApiCallResponse({ price }, endpoint, parameters);
+
+    // reserved parameters (_times) should be inaccessible to post-processing for the
+    // http-gateway, hence multiplication by 2 instead of 1
+    expect(result).toEqual(price * myMultiplier * 2);
+  });
+
+  it('throws on invalid code', async () => {
+    const parameters = { _type: 'int256', _path: 'price' };
+    const endpoint = createEndpoint({
+      postProcessingSpecifications: [
+        {
+          environment: 'Node',
+          value: 'const output = parseInt(input.price)*1000;',
+          timeoutMs: 5000,
+        },
+        {
+          environment: 'Node',
+          value: `
+            Something Unexpected;
+            const output = parseInt(input)*2;
+          `,
+          timeoutMs: 5000,
+        },
+      ],
+    });
+
+    const throwingFunc = async () => postProcessApiCallResponse({ price: 1000 }, endpoint, parameters);
+
+    await expect(throwingFunc).rejects.toEqual(new Error('SyntaxError: Unexpected identifier'));
+  });
+});
+
+describe(removeReservedParameters.name, () => {
+  it('removes all reserved parameters', () => {
+    const parameters = {
+      normalParam1: 'value1',
+      _type: 'int256',
+      _path: 'price',
+      normalParam2: 'value2',
+    };
+
+    const result = removeReservedParameters(parameters);
+
+    expect(result).toEqual({
+      normalParam1: 'value1',
+      normalParam2: 'value2',
+    });
+  });
+
+  it('returns same object if no reserved parameters found', () => {
+    const parameters = {
+      normalParam1: 'value1',
+      normalParam2: 'value2',
+    };
+
+    const result = removeReservedParameters(parameters);
+
+    expect(result).toEqual(parameters);
+  });
+});
+
+describe(addReservedParameters.name, () => {
+  it('adds reserved parameters from initial to modified parameters', () => {
+    const initialParameters = {
+      _type: 'int256',
+      _path: 'price',
+    };
+    const modifiedParameters = {
+      normalParam1: 'value1',
+      normalParam2: 'value2',
+    };
+
+    const result = addReservedParameters(initialParameters, modifiedParameters);
+
+    expect(result).toEqual({
+      normalParam1: 'value1',
+      normalParam2: 'value2',
+      _type: 'int256',
+      _path: 'price',
+    });
+  });
+
+  it('does not modify modifiedParameters if no reserved parameters in initialParameters', () => {
+    const initialParameters = {
+      normalParam3: 'value3',
+    };
+    const modifiedParameters = {
+      normalParam1: 'value1',
+      normalParam2: 'value2',
+    };
+
+    const result = addReservedParameters(initialParameters, modifiedParameters);
+
+    expect(result).toEqual(modifiedParameters);
+  });
+});

package/src/processing/processing.ts

@@ -0,0 +1,160 @@
+import { type Endpoint, RESERVED_PARAMETERS } from '@api3/ois';
+import { type GoAsyncOptions, go } from '@api3/promise-utils';
+
+import { type ApiCallParameters, validateApiCallParameters } from './schema';
+import { unsafeEvaluate, unsafeEvaluateAsync } from './unsafe-evaluate';
+
+export const DEFAULT_PROCESSING_TIMEOUT_MS = 10_000;
+
+const reservedParameters = RESERVED_PARAMETERS as string[]; // To avoid strict TS checks.
+
+/**
+ * Removes reserved parameters from the parameters object.
+ * @param parameters The API call parameters from which reserved parameters will be removed.
+ * @returns The parameters object without reserved parameters.
+ */
+export const removeReservedParameters = (parameters: ApiCallParameters): ApiCallParameters => {
+  const result: ApiCallParameters = {};
+
+  for (const key in parameters) {
+    if (!reservedParameters.includes(key)) {
+      result[key] = parameters[key];
+    }
+  }
+
+  return result;
+};
+
+/**
+ * Re-inserts reserved parameters from the initial parameters object into the modified parameters object.
+ * @param initialParameters The initial API call parameters that might contain reserved parameters.
+ * @param modifiedParameters The modified API call parameters to which reserved parameters will be added.
+ * @returns The modified parameters object with re-inserted reserved parameters.
+ */
+export const addReservedParameters = (
+  initialParameters: ApiCallParameters,
+  modifiedParameters: ApiCallParameters
+): ApiCallParameters => {
+  for (const key in initialParameters) {
+    if (reservedParameters.includes(key)) {
+      modifiedParameters[key] = initialParameters[key];
+    }
+  }
+
+  return modifiedParameters;
+};
+
+/**
+ * Pre-processes API call parameters based on the provided endpoint's processing specifications.
+ *
+ * @param endpoint The endpoint containing processing specifications.
+ * @param apiCallParameters The parameters to be pre-processed.
+ * @param processingOptions Options to control the async processing behavior like retries and timeouts.
+ *
+ * @returns A promise that resolves to the pre-processed parameters.
+ */
+export const preProcessApiCallParameters = async (
+  endpoint: Endpoint,
+  apiCallParameters: ApiCallParameters,
+  processingOptions: GoAsyncOptions = { retries: 0, totalTimeoutMs: DEFAULT_PROCESSING_TIMEOUT_MS }
+): Promise<ApiCallParameters> => {
+  const { preProcessingSpecifications } = endpoint;
+  if (!preProcessingSpecifications || preProcessingSpecifications.length === 0) {
+    return apiCallParameters;
+  }
+
+  // We only wrap the code through "go" utils because of the timeout and retry logic.
+  const goProcessedParameters = await go(async () => {
+    let currentValue: unknown = removeReservedParameters(apiCallParameters);
+
+    for (const processing of preProcessingSpecifications) {
+      // Provide endpoint parameters without reserved parameters immutably between steps. Recompute them for each
+      // snippet independently because processing snippets can modify the parameters.
+      const endpointParameters = removeReservedParameters(apiCallParameters);
+
+      switch (processing.environment) {
+        case 'Node': {
+          currentValue = await unsafeEvaluate(
+            processing.value,
+            { input: currentValue, endpointParameters },
+            processing.timeoutMs
+          );
+          break;
+        }
+        case 'Node async': {
+          currentValue = await unsafeEvaluateAsync(
+            processing.value,
+            { input: currentValue, endpointParameters },
+            processing.timeoutMs
+          );
+          break;
+        }
+      }
+    }
+
+    return currentValue;
+  }, processingOptions);
+  if (!goProcessedParameters.success) throw goProcessedParameters.error;
+
+  // Let this throw if the processed parameters are invalid.
+  const parsedParameters = validateApiCallParameters(goProcessedParameters.data);
+
+  // Having removed reserved parameters for pre-processing, we need to re-insert them for the API call.
+  return addReservedParameters(apiCallParameters, parsedParameters);
+};
+
+/**
+ * Post-processes the API call response based on the provided endpoint's processing specifications.
+ *
+ * @param apiCallResponse The raw response obtained from the API call.
+ * @param endpoint The endpoint containing processing specifications.
+ * @param apiCallParameters The parameters used in the API call.
+ * @param processingOptions Options to control the async processing behavior like retries and timeouts.
+ *
+ * @returns A promise that resolves to the post-processed API call response.
+ */
+export const postProcessApiCallResponse = async (
+  apiCallResponse: unknown,
+  endpoint: Endpoint,
+  apiCallParameters: ApiCallParameters,
+  processingOptions: GoAsyncOptions = { retries: 0, totalTimeoutMs: DEFAULT_PROCESSING_TIMEOUT_MS }
+) => {
+  const { postProcessingSpecifications } = endpoint;
+  if (!postProcessingSpecifications || postProcessingSpecifications?.length === 0) {
+    return apiCallResponse;
+  }
+
+  // We only wrap the code through "go" utils because of the timeout and retry logic.
+  const goResult = await go(async () => {
+    let currentValue: unknown = apiCallResponse;
+
+    for (const processing of postProcessingSpecifications) {
+      // Provide endpoint parameters without reserved parameters immutably between steps. Recompute them for each
+      // snippet independently because processing snippets can modify the parameters.
+      const endpointParameters = removeReservedParameters(apiCallParameters);
+      switch (processing.environment) {
+        case 'Node': {
+          currentValue = await unsafeEvaluate(
+            processing.value,
+            { input: currentValue, endpointParameters },
+            processing.timeoutMs
+          );
+          break;
+        }
+        case 'Node async': {
+          currentValue = await unsafeEvaluateAsync(
+            processing.value,
+            { input: currentValue, endpointParameters },
+            processing.timeoutMs
+          );
+          break;
+        }
+      }
+    }
+
+    return currentValue;
+  }, processingOptions);
+  if (!goResult.success) throw goResult.error;
+
+  return goResult.data;
+};

package/src/processing/schema.ts

@@ -0,0 +1,10 @@
+export const validateApiCallParameters = (apiCallParameters: unknown): ApiCallParameters => {
+  // eslint-disable-next-line lodash/prefer-lodash-typecheck
+  if (typeof apiCallParameters !== 'object' || apiCallParameters === null) {
+    throw new TypeError('Invalid API call parameters');
+  }
+
+  return apiCallParameters as ApiCallParameters;
+};
+
+export type ApiCallParameters = Record<string, any>;

package/src/processing/unsafe-evaluate.test.ts

@@ -0,0 +1,103 @@
+/* eslint-disable jest/prefer-strict-equal */ // Because the errors are thrown from the "vm" module (different context), they are not strictly equal.
+import { unsafeEvaluate, unsafeEvaluateAsync } from './unsafe-evaluate';
+
+describe('unsafe evaluate - sync', () => {
+  it('executes harmless code', () => {
+    const result = unsafeEvaluate("const output = {...input, c: 'some-value'}", { input: { a: true, b: 123 } }, 5000);
+
+    expect(result).toEqual({ a: true, b: 123, c: 'some-value' });
+  });
+
+  it('throws on exception', () => {
+    expect(() => unsafeEvaluate("throw new Error('unexpected')", {}, 5000)).toThrow('unexpected');
+  });
+});
+
+describe('unsafe evaluate - async', () => {
+  it('executes harmless code', async () => {
+    const result = unsafeEvaluateAsync(
+      "const output = {...input, c: 'some-value'}; resolve(output);",
+      { input: { a: true, b: 123 } },
+      5000
+    );
+
+    await expect(result).resolves.toEqual({ a: true, b: 123, c: 'some-value' });
+  });
+
+  it('can use setTimeout and setInterval', async () => {
+    const result = unsafeEvaluateAsync(
+      `
+      const fn = async () => {
+        const output = input;
+        output.push('start')
+
+        const tickMs = 35
+        const bufferMs = 25
+        setInterval(() => output.push('ping interval'), tickMs)
+        await new Promise((res) => setTimeout(res, tickMs * 4 + bufferMs));
+
+        output.push('end')
+        resolve(output);
+      };
+
+      fn()
+      `,
+      { input: [] },
+      200
+    );
+
+    await expect(result).resolves.toEqual([
+      'start',
+      'ping interval',
+      'ping interval',
+      'ping interval',
+      'ping interval',
+      'end',
+    ]);
+  });
+
+  it('applies timeout when using setTimeout', async () => {
+    await expect(async () =>
+      unsafeEvaluateAsync(
+        `
+        const fn = () => {
+          setTimeout(() => console.log('ping timeout'), 100)
+        };
+
+        fn()
+        `,
+        {},
+        50
+      )
+    ).rejects.toEqual(new Error('Timeout exceeded'));
+  });
+
+  it('applies timeout when using setInterval', async () => {
+    await expect(async () =>
+      unsafeEvaluateAsync(
+        `
+        const fn = () => {
+          const someFn = () => {}
+          setInterval(someFn, 10)
+        };
+
+        fn()
+        `,
+        {},
+        50
+      )
+    ).rejects.toEqual(new Error('Timeout exceeded'));
+  });
+
+  it('processing can call reject', async () => {
+    await expect(async () =>
+      unsafeEvaluateAsync(`reject(new Error('Rejected by processing snippet.'))`, {}, 50)
+    ).rejects.toEqual(new Error('Rejected by processing snippet.'));
+  });
+
+  it('throws on exception', async () => {
+    await expect(async () => unsafeEvaluateAsync("throw new Error('unexpected')", {}, 5000)).rejects.toEqual(
+      new Error('unexpected')
+    );
+  });
+});