@api3/commons 0.1.0 → 0.2.0

package/src/processing/processing.ts

@@ -0,0 +1,160 @@
+import { type Endpoint, RESERVED_PARAMETERS } from '@api3/ois';
+import { type GoAsyncOptions, go } from '@api3/promise-utils';
+
+import { type ApiCallParameters, apiCallParametersSchema } from './schema';
+import { unsafeEvaluate, unsafeEvaluateAsync } from './unsafe-evaluate';
+
+export const DEFAULT_PROCESSING_TIMEOUT_MS = 10_000;
+
+const reservedParameters = RESERVED_PARAMETERS as string[]; // To avoid strict TS checks.
+
+/**
+ * Removes reserved parameters from the parameters object.
+ * @param parameters The API call parameters from which reserved parameters will be removed.
+ * @returns The parameters object without reserved parameters.
+ */
+export const removeReservedParameters = (parameters: ApiCallParameters): ApiCallParameters => {
+  const result: ApiCallParameters = {};
+
+  for (const key in parameters) {
+    if (!reservedParameters.includes(key)) {
+      result[key] = parameters[key];
+    }
+  }
+
+  return result;
+};
+
+/**
+ * Re-inserts reserved parameters from the initial parameters object into the modified parameters object.
+ * @param initialParameters The initial API call parameters that might contain reserved parameters.
+ * @param modifiedParameters The modified API call parameters to which reserved parameters will be added.
+ * @returns The modified parameters object with re-inserted reserved parameters.
+ */
+export const addReservedParameters = (
+  initialParameters: ApiCallParameters,
+  modifiedParameters: ApiCallParameters
+): ApiCallParameters => {
+  for (const key in initialParameters) {
+    if (reservedParameters.includes(key)) {
+      modifiedParameters[key] = initialParameters[key];
+    }
+  }
+
+  return modifiedParameters;
+};
+
+/**
+ * Pre-processes API call parameters based on the provided endpoint's processing specifications.
+ *
+ * @param endpoint The endpoint containing processing specifications.
+ * @param apiCallParameters The parameters to be pre-processed.
+ * @param processingOptions Options to control the async processing behavior like retries and timeouts.
+ *
+ * @returns A promise that resolves to the pre-processed parameters.
+ */
+export const preProcessApiCallParameters = async (
+  endpoint: Endpoint,
+  apiCallParameters: ApiCallParameters,
+  processingOptions: GoAsyncOptions = { retries: 0, totalTimeoutMs: DEFAULT_PROCESSING_TIMEOUT_MS }
+): Promise<ApiCallParameters> => {
+  const { preProcessingSpecifications } = endpoint;
+  if (!preProcessingSpecifications || preProcessingSpecifications.length === 0) {
+    return apiCallParameters;
+  }
+
+  // We only wrap the code through "go" utils because of the timeout and retry logic.
+  const goProcessedParameters = await go(async () => {
+    let currentValue: unknown = removeReservedParameters(apiCallParameters);
+
+    for (const processing of preProcessingSpecifications) {
+      // Provide endpoint parameters without reserved parameters immutably between steps. Recompute them for each
+      // snippet independently because processing snippets can modify the parameters.
+      const endpointParameters = removeReservedParameters(apiCallParameters);
+
+      switch (processing.environment) {
+        case 'Node': {
+          currentValue = await unsafeEvaluate(
+            processing.value,
+            { input: currentValue, endpointParameters },
+            processing.timeoutMs
+          );
+          break;
+        }
+        case 'Node async': {
+          currentValue = await unsafeEvaluateAsync(
+            processing.value,
+            { input: currentValue, endpointParameters },
+            processing.timeoutMs
+          );
+          break;
+        }
+      }
+    }
+
+    return currentValue;
+  }, processingOptions);
+  if (!goProcessedParameters.success) throw goProcessedParameters.error;
+
+  // Let this throw if the processed parameters are invalid.
+  const parsedParameters = apiCallParametersSchema.parse(goProcessedParameters.data);
+
+  // Having removed reserved parameters for pre-processing, we need to re-insert them for the API call.
+  return addReservedParameters(apiCallParameters, parsedParameters);
+};
+
+/**
+ * Post-processes the API call response based on the provided endpoint's processing specifications.
+ *
+ * @param apiCallResponse The raw response obtained from the API call.
+ * @param endpoint The endpoint containing processing specifications.
+ * @param apiCallParameters The parameters used in the API call.
+ * @param processingOptions Options to control the async processing behavior like retries and timeouts.
+ *
+ * @returns A promise that resolves to the post-processed API call response.
+ */
+export const postProcessApiCallResponse = async (
+  apiCallResponse: unknown,
+  endpoint: Endpoint,
+  apiCallParameters: ApiCallParameters,
+  processingOptions: GoAsyncOptions = { retries: 0, totalTimeoutMs: DEFAULT_PROCESSING_TIMEOUT_MS }
+) => {
+  const { postProcessingSpecifications } = endpoint;
+  if (!postProcessingSpecifications || postProcessingSpecifications?.length === 0) {
+    return apiCallResponse;
+  }
+
+  // We only wrap the code through "go" utils because of the timeout and retry logic.
+  const goResult = await go(async () => {
+    let currentValue: unknown = apiCallResponse;
+
+    for (const processing of postProcessingSpecifications) {
+      // Provide endpoint parameters without reserved parameters immutably between steps. Recompute them for each
+      // snippet independently because processing snippets can modify the parameters.
+      const endpointParameters = removeReservedParameters(apiCallParameters);
+      switch (processing.environment) {
+        case 'Node': {
+          currentValue = await unsafeEvaluate(
+            processing.value,
+            { input: currentValue, endpointParameters },
+            processing.timeoutMs
+          );
+          break;
+        }
+        case 'Node async': {
+          currentValue = await unsafeEvaluateAsync(
+            processing.value,
+            { input: currentValue, endpointParameters },
+            processing.timeoutMs
+          );
+          break;
+        }
+      }
+    }
+
+    return currentValue;
+  }, processingOptions);
+  if (!goResult.success) throw goResult.error;
+
+  return goResult.data;
+};

package/src/processing/schema.ts

@@ -0,0 +1,5 @@
+import { z } from 'zod';
+
+export const apiCallParametersSchema = z.record(z.string(), z.any());
+
+export type ApiCallParameters = z.infer<typeof apiCallParametersSchema>;

package/src/processing/unsafe-evaluate.test.ts

@@ -0,0 +1,103 @@
+/* eslint-disable jest/prefer-strict-equal */ // Because the errors are thrown from the "vm" module (different context), they are not strictly equal.
+import { unsafeEvaluate, unsafeEvaluateAsync } from './unsafe-evaluate';
+
+describe('unsafe evaluate - sync', () => {
+  it('executes harmless code', () => {
+    const result = unsafeEvaluate("const output = {...input, c: 'some-value'}", { input: { a: true, b: 123 } }, 5000);
+
+    expect(result).toEqual({ a: true, b: 123, c: 'some-value' });
+  });
+
+  it('throws on exception', () => {
+    expect(() => unsafeEvaluate("throw new Error('unexpected')", {}, 5000)).toThrow('unexpected');
+  });
+});
+
+describe('unsafe evaluate - async', () => {
+  it('executes harmless code', async () => {
+    const result = unsafeEvaluateAsync(
+      "const output = {...input, c: 'some-value'}; resolve(output);",
+      { input: { a: true, b: 123 } },
+      5000
+    );
+
+    await expect(result).resolves.toEqual({ a: true, b: 123, c: 'some-value' });
+  });
+
+  it('can use setTimeout and setInterval', async () => {
+    const result = unsafeEvaluateAsync(
+      `
+      const fn = async () => {
+        const output = input;
+        output.push('start')
+
+        const tickMs = 35
+        const bufferMs = 25
+        setInterval(() => output.push('ping interval'), tickMs)
+        await new Promise((res) => setTimeout(res, tickMs * 4 + bufferMs));
+
+        output.push('end')
+        resolve(output);
+      };
+
+      fn()
+      `,
+      { input: [] },
+      200
+    );
+
+    await expect(result).resolves.toEqual([
+      'start',
+      'ping interval',
+      'ping interval',
+      'ping interval',
+      'ping interval',
+      'end',
+    ]);
+  });
+
+  it('applies timeout when using setTimeout', async () => {
+    await expect(async () =>
+      unsafeEvaluateAsync(
+        `
+        const fn = () => {
+          setTimeout(() => console.log('ping timeout'), 100)
+        };
+
+        fn()
+        `,
+        {},
+        50
+      )
+    ).rejects.toEqual(new Error('Timeout exceeded'));
+  });
+
+  it('applies timeout when using setInterval', async () => {
+    await expect(async () =>
+      unsafeEvaluateAsync(
+        `
+        const fn = () => {
+          const someFn = () => {}
+          setInterval(someFn, 10)
+        };
+
+        fn()
+        `,
+        {},
+        50
+      )
+    ).rejects.toEqual(new Error('Timeout exceeded'));
+  });
+
+  it('processing can call reject', async () => {
+    await expect(async () =>
+      unsafeEvaluateAsync(`reject(new Error('Rejected by processing snippet.'))`, {}, 50)
+    ).rejects.toEqual(new Error('Rejected by processing snippet.'));
+  });
+
+  it('throws on exception', async () => {
+    await expect(async () => unsafeEvaluateAsync("throw new Error('unexpected')", {}, 5000)).rejects.toEqual(
+      new Error('unexpected')
+    );
+  });
+});

package/src/processing/unsafe-evaluate.ts

@@ -0,0 +1,178 @@
+/* eslint-disable camelcase */
+import assert from 'node:assert';
+import async_hooks from 'node:async_hooks';
+import buffer from 'node:buffer';
+import child_process from 'node:child_process';
+import cluster from 'node:cluster';
+import console from 'node:console';
+import constants from 'node:constants';
+import crypto from 'node:crypto';
+import dgram from 'node:dgram';
+import dns from 'node:dns';
+import events from 'node:events';
+import fs from 'node:fs';
+import http from 'node:http';
+import http2 from 'node:http2';
+import https from 'node:https';
+import inspector from 'node:inspector';
+import module from 'node:module';
+import net from 'node:net';
+import os from 'node:os';
+import path from 'node:path';
+import perf_hooks from 'node:perf_hooks';
+import process from 'node:process';
+import readline from 'node:readline';
+import repl from 'node:repl';
+import stream from 'node:stream';
+import string_decoder from 'node:string_decoder';
+import timers from 'node:timers';
+import tls from 'node:tls';
+import trace_events from 'node:trace_events';
+import tty from 'node:tty';
+import url from 'node:url';
+import util from 'node:util';
+import v8 from 'node:v8';
+import vm from 'node:vm';
+import worker_threads from 'node:worker_threads';
+import zlib from 'node:zlib';
+
+import { createTimers } from './vm-timers';
+
+const builtInNodeModules = {
+  assert,
+  async_hooks,
+  buffer,
+  child_process,
+  cluster,
+  console,
+  constants,
+  crypto,
+  dgram,
+  dns,
+  events,
+  fs,
+  http,
+  http2,
+  https,
+  inspector,
+  module,
+  net,
+  os,
+  path,
+  perf_hooks,
+  process,
+  readline,
+  repl,
+  stream,
+  string_decoder,
+  timers,
+  tls,
+  trace_events,
+  tty,
+  url,
+  util,
+  v8,
+  vm,
+  worker_threads,
+  zlib,
+};
+
+/**
+ * Evaluates the provided code in a new VM context with the specified global variables.
+ *
+ * **Security Warning:** This function executes the provided code and can have unintended side effects or
+ * vulnerabilities if used with untrusted or malicious input. It's imperative to use this function only with code you
+ * trust completely. Avoid using this function with user-generated code or third-party code that hasn't been thoroughly
+ * reviewed.
+ *
+ * @param code The JavaScript code to evaluate.
+ * @param globalVariables A key-value pair of variables to be made available in the context of the executed code.
+ * @param timeout Duration in milliseconds to wait before terminating the execution.
+ *
+ * @returns The result of the evaluated code.
+ *
+ * @throws Throws an error if the execution exceeds the provided timeout or if there's a problem with the code.
+ *
+ * @example
+ *
+ *const result = unsafeEvaluate('const output = input + 1;', { input: 1 }, 1000);
+ *console.log(result); // Outputs: 2
+ */
+export const unsafeEvaluate = (code: string, globalVariables: Record<string, unknown>, timeout: number) => {
+  const vmContext = {
+    ...globalVariables,
+    ...builtInNodeModules,
+    deferredOutput: undefined as unknown,
+  };
+
+  vm.runInNewContext(`${code}; deferredOutput = output;`, vmContext, {
+    displayErrors: true,
+    timeout,
+  });
+
+  return vmContext.deferredOutput;
+};
+
+/**
+ * Asynchronously evaluates the provided code in a new VM context with the specified global variables.
+ *
+ * **Security Warning:** This function executes the provided code and can have unintended side effects or
+ * vulnerabilities if used with untrusted or malicious input. It's imperative to use this function only with code you
+ * trust completely. Avoid using this function with user-generated code or third-party code that hasn't been thoroughly
+ * reviewed.
+ *
+ * @param code The JavaScript code to evaluate. The code should call the `resolve` method to return the result of the
+ * evaluation. You may use async/await syntax in the code.
+ * @param globalVariables A key-value pair of variables to be made available in the context of the executed code.
+ * @param timeout Duration in milliseconds to wait before terminating the execution.
+ *
+ * @returns The result of the evaluated code wrapped in a Promise.
+ *
+ * @throws Throws an error if the execution exceeds the provided timeout or if there's a problem with the code.
+ *
+ * @example
+ *
+ *const result = await unsafeEvaluateAsync(
+ *  "const output = {...input, c: 'some-value'}; resolve(output);",
+ *  { input: { a: true, b: 123 } },
+ *  5000
+ *);
+ *console.log(result); // Outputs: { a: true, b: 123, c: 'some-value' }
+ */
+export const unsafeEvaluateAsync = async (code: string, globalVariables: Record<string, unknown>, timeout: number) => {
+  let vmReject: (reason: unknown) => void;
+
+  // Make sure the timeout is applied. When the processing snippet uses setTimeout or setInterval, the timeout option
+  // from VM is broken. See: https://github.com/nodejs/node/issues/3020.
+  //
+  // We need to manually clear all timers and reject the processing manually.
+  const timeoutTimer = setTimeout(() => {
+    vmReject(new Error('Timeout exceeded'));
+  }, timeout);
+
+  return new Promise((resolve, reject) => {
+    const timers = createTimers();
+    const vmResolve = (value: unknown) => {
+      timers.clearAll();
+      clearTimeout(timeoutTimer);
+      resolve(value);
+    };
+    vmReject = (reason: unknown) => {
+      timers.clearAll();
+      clearTimeout(timeoutTimer);
+      reject(reason);
+    };
+
+    const vmContext = {
+      ...globalVariables,
+      ...builtInNodeModules,
+      resolve: vmResolve,
+      reject: vmReject,
+      setTimeout: timers.customSetTimeout,
+      setInterval: timers.customSetInterval,
+      clearTimeout: timers.customClearTimeout,
+      clearInterval: timers.customClearInterval,
+    };
+    vm.runInNewContext(code, vmContext, { displayErrors: true, timeout });
+  });
+};

package/src/processing/vm-timers.ts

@@ -0,0 +1,58 @@
+/**
+ * Timers (setTimeout, setInterval) do not work in Node.js vm, see: https://github.com/nodejs/help/issues/1875
+ *
+ * The API is wrapped in a "create" function so that every processing snippet keeps track of its timers and properly
+ * cleans them up after use.
+ */
+export const createTimers = () => {
+  let timeouts: NodeJS.Timeout[] = [];
+
+  const customSetTimeout = (fn: () => void, ms: number) => {
+    timeouts.push(setTimeout(fn, ms));
+  };
+
+  const customClearTimeout = (id: NodeJS.Timeout) => {
+    timeouts = timeouts.filter((timeoutId) => timeoutId !== id);
+    clearTimeout(id);
+  };
+
+  const clearAllTimeouts = () => {
+    for (const element of timeouts) {
+      clearTimeout(element);
+    }
+    timeouts = [];
+  };
+
+  let intervals: NodeJS.Timeout[] = [];
+
+  const customSetInterval = (fn: () => void, ms: number) => {
+    intervals.push(setInterval(fn, ms));
+  };
+
+  const customClearInterval = (id: NodeJS.Timeout) => {
+    intervals = intervals.filter((intervalId) => intervalId !== id);
+    clearInterval(id);
+  };
+
+  const clearAllIntervals = () => {
+    for (const element of intervals) {
+      clearInterval(element);
+    }
+    intervals = [];
+  };
+
+  const clearAll = () => {
+    clearAllTimeouts();
+    clearAllIntervals();
+  };
+
+  return {
+    customSetTimeout,
+    customClearTimeout,
+    clearAllTimeouts,
+    customSetInterval,
+    customClearInterval,
+    clearAllIntervals,
+    clearAll,
+  };
+};

package/dist/index.d.ts

@@ -1 +0,0 @@
-//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":""}

package/dist/index.js

@@ -1,4 +0,0 @@
-"use strict";
-// eslint-disable-next-line no-console
-console.log('works');
-//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAAA,sCAAsC;AACtC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC"}

package/src/index.ts

@@ -1,2 +0,0 @@
-// eslint-disable-next-line no-console
-console.log('works');