@reclaimprotocol/attestor-core 5.0.1-beta.2 → 5.0.1-beta.22

package/lib/server/socket.js

@@ -1,112 +1,109 @@
-import { promisify } from "util";
+import { promisify } from 'util';
 import { handleMessage } from "../client/utils/message-handler.js";
 import { DEFAULT_RPC_TIMEOUT_MS } from "../config/index.js";
-import { HANDLERS } from "../server/handlers/index.js";
-import { getApm } from "../server/utils/apm.js";
-import { getInitialMessagesFromQuery } from "../server/utils/generics.js";
+import { HANDLERS } from "./handlers/index.js";
+import { getApm } from "./utils/apm.js";
+import { getInitialMessagesFromQuery } from "./utils/generics.js";
 import { AttestorError, generateSessionId } from "../utils/index.js";
 import { AttestorSocket } from "../utils/socket-base.js";
-class AttestorServerSocket extends AttestorSocket {
-  tunnels = {};
-  sessionId;
-  bgpListener;
-  constructor(socket, sessionId, bgpListener, logger) {
-    super(socket, {}, logger);
-    this.sessionId = sessionId;
-    this.bgpListener = bgpListener;
-    this.addEventListener("rpc-request", handleRpcRequest.bind(this));
-    this.addEventListener("tunnel-message", handleTunnelMessage.bind(this));
-    this.addEventListener("connection-terminated", () => {
-      for (const tunnelId in this.tunnels) {
+export class AttestorServerSocket extends AttestorSocket {
+    tunnels = {};
+    sessionId;
+    bgpListener;
+    constructor(socket, sessionId, bgpListener, logger) {
+        // @ts-ignore
+        super(socket, {}, logger);
+        this.sessionId = sessionId;
+        this.bgpListener = bgpListener;
+        // handle RPC requests
+        this.addEventListener('rpc-request', handleRpcRequest.bind(this));
+        // forward packets to the appropriate tunnel
+        this.addEventListener('tunnel-message', handleTunnelMessage.bind(this));
+        // close all tunnels when the connection is terminated
+        // since this tunnel can no longer be written to
+        this.addEventListener('connection-terminated', () => {
+            for (const tunnelId in this.tunnels) {
+                const tunnel = this.tunnels[tunnelId];
+                void tunnel.close(new Error('WS session terminated'));
+            }
+        });
+    }
+    getTunnel(tunnelId) {
         const tunnel = this.tunnels[tunnelId];
-        void tunnel.close(new Error("WS session terminated"));
-      }
-    });
-  }
-  getTunnel(tunnelId) {
-    const tunnel = this.tunnels[tunnelId];
-    if (!tunnel) {
-      throw new AttestorError(
-        "ERROR_NOT_FOUND",
-        `Tunnel "${tunnelId}" not found`
-      );
+        if (!tunnel) {
+            throw new AttestorError('ERROR_NOT_FOUND', `Tunnel "${tunnelId}" not found`);
+        }
+        return tunnel;
     }
-    return tunnel;
-  }
-  removeTunnel(tunnelId) {
-    delete this.tunnels[tunnelId];
-  }
-  static async acceptConnection(socket, { req, logger, bgpListener }) {
-    const bindSend = socket.send.bind(socket);
-    socket.send = promisify(bindSend);
-    const sessionId = generateSessionId();
-    logger = logger.child({ sessionId });
-    const client = new AttestorServerSocket(
-      socket,
-      sessionId,
-      bgpListener,
-      logger
-    );
-    try {
-      const initMsgs = getInitialMessagesFromQuery(req);
-      logger.trace(
-        { initMsgs: initMsgs.length },
-        "new connection, validating..."
-      );
-      for (const msg of initMsgs) {
-        await handleMessage.call(client, msg);
-      }
-      logger.debug("connection accepted");
-    } catch (err) {
-      logger.error({ err }, "error in new connection");
-      if (client.isOpen) {
-        await client.terminateConnection(
-          err instanceof AttestorError ? err : AttestorError.badRequest(err.message)
-        );
-      }
-      return;
+    removeTunnel(tunnelId) {
+        delete this.tunnels[tunnelId];
+    }
+    static async acceptConnection(socket, { req, logger, bgpListener }) {
+        // promisify ws.send -- so the sendMessage method correctly
+        // awaits the send operation
+        const bindSend = socket.send.bind(socket);
+        socket.send = promisify(bindSend);
+        const sessionId = generateSessionId();
+        logger = logger.child({ sessionId });
+        const client = new AttestorServerSocket(socket, sessionId, bgpListener, logger);
+        try {
+            const initMsgs = getInitialMessagesFromQuery(req);
+            logger.trace({ initMsgs: initMsgs.length }, 'new connection, validating...');
+            for (const msg of initMsgs) {
+                await handleMessage.call(client, msg);
+            }
+            logger.debug('connection accepted');
+        }
+        catch (err) {
+            logger.error({ err }, 'error in new connection');
+            if (client.isOpen) {
+                await client.terminateConnection(err instanceof AttestorError
+                    ? err
+                    : AttestorError.badRequest(err.message));
+            }
+            return;
+        }
+        return client;
     }
-    return client;
-  }
 }
 async function handleTunnelMessage({ data: { tunnelId, message } }) {
-  try {
-    const tunnel = this.getTunnel(tunnelId);
-    await tunnel.write(message);
-  } catch (err) {
-    this.logger?.error({ err, tunnelId }, "error writing to tunnel");
-  }
+    try {
+        const tunnel = this.getTunnel(tunnelId);
+        await tunnel.write(message);
+    }
+    catch (err) {
+        this.logger?.error({ err, tunnelId }, 'error writing to tunnel');
+    }
 }
 async function handleRpcRequest({ data: { data, requestId, respond, type } }) {
-  const logger = this.logger.child({ rpc: type, requestId });
-  const apm = getApm();
-  const tx = apm?.startTransaction(type);
-  tx?.setLabel("requestId", requestId);
-  tx?.setLabel("sessionId", this.sessionId.toString());
-  const userId = this.metadata.auth?.data?.id;
-  if (userId) {
-    tx?.setLabel("authUserId", userId);
-  }
-  const timeout = setTimeout(() => {
-    logger.warn({ type, requestId }, "RPC took too long to respond");
-  }, DEFAULT_RPC_TIMEOUT_MS);
-  try {
-    logger.debug({ data }, "handling RPC request");
-    const handler = HANDLERS[type];
-    const res = await handler(data, { client: this, logger, tx });
-    respond(res);
-    logger.debug({ res }, "handled RPC request");
-    tx?.setOutcome("success");
-  } catch (err) {
-    logger.error({ err }, "error in RPC request");
-    respond(AttestorError.fromError(err));
-    tx?.setOutcome("failure");
-    apm?.captureError(err, { parent: tx });
-  } finally {
-    clearTimeout(timeout);
-    tx?.end();
-  }
+    const logger = this.logger.child({ rpc: type, requestId });
+    const apm = getApm();
+    const tx = apm?.startTransaction(type);
+    tx?.setLabel('requestId', requestId);
+    tx?.setLabel('sessionId', this.sessionId.toString());
+    const userId = this.metadata.auth?.data?.id;
+    if (userId) {
+        tx?.setLabel('authUserId', userId);
+    }
+    const timeout = setTimeout(() => {
+        logger.warn({ type, requestId }, 'RPC took too long to respond');
+    }, DEFAULT_RPC_TIMEOUT_MS);
+    try {
+        logger.debug({ data }, 'handling RPC request');
+        const handler = HANDLERS[type];
+        const res = await handler(data, { client: this, logger, tx });
+        respond(res);
+        logger.debug({ res }, 'handled RPC request');
+        tx?.setOutcome('success');
+    }
+    catch (err) {
+        logger.error({ err }, 'error in RPC request');
+        respond(AttestorError.fromError(err));
+        tx?.setOutcome('failure');
+        apm?.captureError(err, { parent: tx });
+    }
+    finally {
+        clearTimeout(timeout);
+        tx?.end();
+    }
 }
-export {
-  AttestorServerSocket
-};

package/lib/server/tunnels/make-tcp-tunnel.js

@@ -1,202 +1,177 @@
-import { HttpsProxyAgent } from "https-proxy-agent";
-import { Socket } from "net";
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import { Socket } from 'net';
 import { CONNECTION_TIMEOUT_MS } from "../../config/index.js";
-import { resolveHostnames } from "../../server/utils/dns.js";
-import { isValidCountryCode } from "../../server/utils/iso.js";
-import { isValidProxySessionId } from "../../server/utils/proxy-session.js";
+import { resolveHostnames } from "../utils/dns.js";
+import { isValidCountryCode } from "../utils/iso.js";
+import { isValidProxySessionId } from "../utils/proxy-session.js";
 import { getEnvVariable } from "../../utils/env.js";
 import { AttestorError } from "../../utils/index.js";
-const HTTPS_PROXY_URL = getEnvVariable("HTTPS_PROXY_URL");
-const makeTcpTunnel = async ({
-  onClose,
-  onMessage,
-  logger,
-  ...opts
-}) => {
-  const transcript = [];
-  const socket = await connectTcp({ ...opts, logger });
-  let closed = false;
-  socket.on("data", (message) => {
-    if (closed) {
-      logger.warn("socket is closed, dropping message");
-      return;
-    }
-    onMessage?.(message);
-    transcript.push({ sender: "server", message });
-  });
-  socket.once("close", () => onSocketClose(void 0));
-  return {
-    socket,
-    transcript,
-    createRequest: opts,
-    async write(data) {
-      transcript.push({ sender: "client", message: data });
-      await new Promise((resolve, reject) => {
-        socket.write(data, (err) => {
-          if (err) {
-            reject(err);
-          } else {
-            resolve();
-          }
-        });
-      });
-    },
-    close(err) {
-      if (closed) {
-        return;
-      }
-      socket.destroy(err);
-    }
-  };
-  function onSocketClose(err) {
-    if (closed) {
-      return;
+const HTTPS_PROXY_URL = getEnvVariable('HTTPS_PROXY_URL');
+/**
+ * Builds a TCP tunnel to the given host and port.
+ * If a geolocation is provided -- an HTTPS proxy is used
+ * to connect to the host.
+ * If a proxySessionId is provided -- a static ip is used with HTTPS proxy
+ * across multiple requests with this same proxySessionId.
+ *
+ * HTTPS proxy essentially creates an opaque tunnel to the
+ * host using the CONNECT method. Any data can be sent through
+ * this tunnel to the end host.
+ * https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/CONNECT
+ *
+ * The tunnel also retains a transcript of all messages sent and received.
+ */
+export const makeTcpTunnel = async ({ onClose, onMessage, logger, ...opts }) => {
+    const transcript = [];
+    const socket = await connectTcp({ ...opts, logger });
+    let closed = false;
+    socket.on('data', message => {
+        if (closed) {
+            logger.warn('socket is closed, dropping message');
+            return;
+        }
+        onMessage?.(message);
+        transcript.push({ sender: 'server', message });
+    });
+    // socket.once('error', onSocketClose)
+    socket.once('close', () => onSocketClose(undefined));
+    return {
+        socket,
+        transcript,
+        createRequest: opts,
+        async write(data) {
+            transcript.push({ sender: 'client', message: data });
+            await new Promise((resolve, reject) => {
+                socket.write(data, err => {
+                    if (err) {
+                        reject(err);
+                    }
+                    else {
+                        resolve();
+                    }
+                });
+            });
+        },
+        close(err) {
+            if (closed) {
+                return;
+            }
+            socket.destroy(err);
+        }
+    };
+    function onSocketClose(err) {
+        if (closed) {
+            return;
+        }
+        logger.debug({ err }, 'closing socket');
+        closed = true;
+        onClose?.(err);
+        onClose = undefined;
     }
-    logger.debug({ err }, "closing socket");
-    closed = true;
-    onClose?.(err);
-    onClose = void 0;
-  }
 };
 async function connectTcp({ host, port, geoLocation, proxySessionId, logger }) {
-  let connectTimeout;
-  let socket;
-  try {
-    await new Promise(async (resolve, reject) => {
-      try {
-        connectTimeout = setTimeout(
-          () => reject(
-            new AttestorError(
-              "ERROR_NETWORK_ERROR",
-              "Server connection timed out"
-            )
-          ),
-          CONNECTION_TIMEOUT_MS
-        );
-        socket = await getSocket({
-          host,
-          port,
-          geoLocation,
-          proxySessionId,
-          logger
+    let connectTimeout;
+    let socket;
+    try {
+        await new Promise(async (resolve, reject) => {
+            try {
+                // add a timeout to ensure the connection doesn't hang
+                // and cause our gateway to send out a 504
+                connectTimeout = setTimeout(() => reject(new AttestorError('ERROR_NETWORK_ERROR', 'Server connection timed out')), CONNECTION_TIMEOUT_MS);
+                socket = await getSocket({
+                    host,
+                    port,
+                    geoLocation,
+                    proxySessionId,
+                    logger
+                });
+                socket.once('connect', resolve);
+                socket.once('error', reject);
+                socket.once('end', () => (reject(new AttestorError('ERROR_NETWORK_ERROR', 'connection closed'))));
+            }
+            catch (err) {
+                reject(err);
+            }
         });
-        socket.once("connect", resolve);
-        socket.once("error", reject);
-        socket.once("end", () => reject(
-          new AttestorError(
-            "ERROR_NETWORK_ERROR",
-            "connection closed"
-          )
-        ));
-      } catch (err) {
-        reject(err);
-      }
-    });
-    logger.debug({ addr: `${host}:${port}` }, "connected");
-    return socket;
-  } catch (err) {
-    socket?.end();
-    throw err;
-  } finally {
-    clearTimeout(connectTimeout);
-  }
+        logger.debug({ addr: `${host}:${port}` }, 'connected');
+        return socket;
+    }
+    catch (err) {
+        socket?.end();
+        throw err;
+    }
+    finally {
+        clearTimeout(connectTimeout);
+    }
 }
 async function getSocket(opts) {
-  const { logger } = opts;
-  try {
-    return await _getSocket(opts);
-  } catch (err) {
-    if (!(err instanceof AttestorError) || err.data?.code !== 403) {
-      throw err;
+    const { logger } = opts;
+    try {
+        return await _getSocket(opts);
     }
-    const addrs = await resolveHostnames(opts.host);
-    logger.info(
-      { addrs, host: opts.host },
-      "failed to connect due to restricted IP, trying via raw addr"
-    );
-    for (const addr of addrs) {
-      try {
-        return await _getSocket({ ...opts, host: addr });
-      } catch (err2) {
-        logger.error(
-          { addr, err: err2 },
-          "failed to connect to host"
-        );
-      }
+    catch (err) {
+        // see if the proxy is blocking the connection
+        // due to their own arbitrary rules,
+        // if so -- we resolve hostname first &
+        // connect directly via address to
+        // avoid proxy knowing which host we're connecting to
+        if (!(err instanceof AttestorError)
+            || err.data?.code !== 403) {
+            throw err;
+        }
+        const addrs = await resolveHostnames(opts.host);
+        logger.info({ addrs, host: opts.host }, 'failed to connect due to restricted IP, trying via raw addr');
+        for (const addr of addrs) {
+            try {
+                return await _getSocket({ ...opts, host: addr });
+            }
+            catch (err) {
+                logger.error({ addr, err }, 'failed to connect to host');
+            }
+        }
+        throw err;
     }
-    throw err;
-  }
 }
-async function _getSocket({
-  host,
-  port,
-  geoLocation,
-  proxySessionId,
-  logger
-}) {
-  const socket = new Socket();
-  if ((proxySessionId || geoLocation) && !HTTPS_PROXY_URL) {
-    logger.warn(
-      { geoLocation, proxySessionId },
-      "geoLocation or proxySessionId provided but no proxy URL found"
-    );
-    geoLocation = "";
-    proxySessionId = "";
-  }
-  if (!geoLocation && !proxySessionId) {
-    socket.connect({ host, port });
-    return socket;
-  }
-  if (!isValidCountryCode(geoLocation)) {
-    throw AttestorError.badRequest(
-      `Geolocation "${geoLocation}" is invalid. Must be 2 letter ISO country code`,
-      { geoLocation }
-    );
-  }
-  if (proxySessionId && !isValidProxySessionId(proxySessionId)) {
-    throw AttestorError.badRequest(
-      `proxySessionId "${proxySessionId}" is invalid. Must be a lowercase alphanumeric string of length 8-14 characters. eg. "mystring12345", "something1234".`,
-      { proxySessionId }
-    );
-  }
-  const agentUrl = HTTPS_PROXY_URL.replace(
-    "{{geoLocation}}",
-    geoLocation?.toLowerCase() || ""
-  ).replace(
-    "{{proxySessionId}}",
-    proxySessionId ? `-session-${proxySessionId}` : ""
-  );
-  const agent = new HttpsProxyAgent(agentUrl);
-  const waitForProxyRes = new Promise((resolve) => {
-    socket.once("proxyConnect", resolve);
-  });
-  const proxySocket = await agent.connect(
+async function _getSocket({ host, port, geoLocation, proxySessionId, logger }) {
+    const socket = new Socket();
+    if ((proxySessionId || geoLocation) && !HTTPS_PROXY_URL) {
+        logger.warn({ geoLocation, proxySessionId }, 'geoLocation or proxySessionId provided but no proxy URL found');
+        geoLocation = '';
+        proxySessionId = '';
+    }
+    if (!geoLocation && !proxySessionId) {
+        socket.connect({ host, port, });
+        return socket;
+    }
+    if (!isValidCountryCode(geoLocation)) {
+        throw AttestorError.badRequest(`Geolocation "${geoLocation}" is invalid. Must be 2 letter ISO country code`, { geoLocation });
+    }
+    if (proxySessionId && !isValidProxySessionId(proxySessionId)) {
+        throw AttestorError.badRequest(`proxySessionId "${proxySessionId}" is invalid. Must be a lowercase alphanumeric string of length 8-14 characters. eg. "mystring12345", "something1234".`, { proxySessionId });
+    }
+    const agentUrl = HTTPS_PROXY_URL.replace('{{geoLocation}}', geoLocation?.toLowerCase() || '').replace('{{proxySessionId}}', proxySessionId ? `-session-${proxySessionId}` : '');
+    const agent = new HttpsProxyAgent(agentUrl);
+    const waitForProxyRes = new Promise(resolve => {
+        // @ts-ignore
+        socket.once('proxyConnect', resolve);
+    });
+    const proxySocket = await agent.connect(
     // ignore, because https-proxy-agent
     // expects an http request object
     // @ts-ignore
-    socket,
-    { host, port, timeout: CONNECTION_TIMEOUT_MS }
-  );
-  const res = await waitForProxyRes;
-  if (res.statusCode !== 200) {
-    logger.error(
-      { geoLocation, proxySessionId, res },
-      "Proxy geo location or session id failed"
-    );
-    throw new AttestorError(
-      "ERROR_PROXY_ERROR",
-      `Proxy via ${geoLocation ? `geo location "${geoLocation}"` : ""}${geoLocation && proxySessionId ? ", or " : ""}${proxySessionId ? `session id "${proxySessionId}"` : ""} failed with status code: ${res.statusCode}, message: ${res.statusText}`,
-      {
-        code: res.statusCode,
-        message: res.statusText
-      }
-    );
-  }
-  process.nextTick(() => {
-    proxySocket.emit("connect");
-  });
-  return proxySocket;
+    socket, { host, port, timeout: CONNECTION_TIMEOUT_MS });
+    const res = await waitForProxyRes;
+    if (res.statusCode !== 200) {
+        logger.error({ geoLocation, proxySessionId, res }, 'Proxy geo location or session id failed');
+        throw new AttestorError('ERROR_PROXY_ERROR', `Proxy via ${geoLocation ? `geo location "${geoLocation}"` : ''}${geoLocation && proxySessionId ? ', or ' : ''}${proxySessionId ? `session id "${proxySessionId}"` : ''} failed with status code: ${res.statusCode}, message: ${res.statusText}`, {
+            code: res.statusCode,
+            message: res.statusText,
+        });
+    }
+    process.nextTick(() => {
+        // ensure connect event is emitted
+        // so it can be captured by the caller
+        proxySocket.emit('connect');
+    });
+    return proxySocket;
 }
-export {
-  makeTcpTunnel
-};

package/lib/server/utils/apm.js

@@ -1,29 +1,36 @@
-import ElasticAPM from "elastic-apm-node";
+import ElasticAPM from 'elastic-apm-node';
 import { getEnvVariable } from "../../utils/env.js";
 import { logger } from "../../utils/logger.js";
 let apm;
-function getApm() {
-  if (!getEnvVariable("ELASTIC_APM_SERVER_URL") || !getEnvVariable("ELASTIC_APM_SECRET_TOKEN")) {
-    logger.info(
-      "ELASTIC_APM_SERVER_URL or ELASTIC_APM_SECRET_TOKEN not found in env, APM agent not initialised"
-    );
-    return void 0;
-  }
-  if (!apm) {
-    const sampleRate = +(getEnvVariable("ELASTIC_APM_SAMPLE_RATE") || "0.1");
-    apm = ElasticAPM.start({
-      serviceName: "reclaim_attestor",
-      serviceVersion: "4.0.0",
-      transactionSampleRate: sampleRate,
-      instrumentIncomingHTTPRequests: true,
-      usePathAsTransactionName: true,
-      instrument: true,
-      captureHeaders: true
-    });
-    logger.info("initialised APM agent");
-  }
-  return apm;
+/**
+ * Initialises the APM agent if required,
+ * and returns it.
+ * If ELASTIC_APM_SERVER_URL & ELASTIC_APM_SECRET_TOKEN
+ * are not set will return undefined
+ *
+ * Utilises the standard env variables mentioned
+ * here: https://www.elastic.co/guide/en/apm/agent/nodejs/current/custom-stack.html#custom-stack-advanced-configuration
+ */
+export function getApm() {
+    if (!getEnvVariable('ELASTIC_APM_SERVER_URL')
+        || !getEnvVariable('ELASTIC_APM_SECRET_TOKEN')) {
+        logger.info('ELASTIC_APM_SERVER_URL or ELASTIC_APM_SECRET_TOKEN not found'
+            + ' in env, APM agent not initialised');
+        return undefined;
+    }
+    if (!apm) {
+        const sampleRate = +(getEnvVariable('ELASTIC_APM_SAMPLE_RATE')
+            || '0.1');
+        apm = ElasticAPM.start({
+            serviceName: 'reclaim_attestor',
+            serviceVersion: '4.0.0',
+            transactionSampleRate: sampleRate,
+            instrumentIncomingHTTPRequests: true,
+            usePathAsTransactionName: true,
+            instrument: true,
+            captureHeaders: true,
+        });
+        logger.info('initialised APM agent');
+    }
+    return apm;
 }
-export {
-  getApm
-};