@reclaimprotocol/attestor-core 5.0.1-beta.13 → 5.0.1-beta.16

package/lib/server/tunnels/make-tcp-tunnel.js

@@ -1,202 +0,0 @@
-import { HttpsProxyAgent } from "https-proxy-agent";
-import { Socket } from "net";
-import { CONNECTION_TIMEOUT_MS } from "#src/config/index.js";
-import { resolveHostnames } from "#src/server/utils/dns.js";
-import { isValidCountryCode } from "#src/server/utils/iso.js";
-import { isValidProxySessionId } from "#src/server/utils/proxy-session.js";
-import { getEnvVariable } from "#src/utils/env.js";
-import { AttestorError } from "#src/utils/index.js";
-const HTTPS_PROXY_URL = getEnvVariable("HTTPS_PROXY_URL");
-const makeTcpTunnel = async ({
-  onClose,
-  onMessage,
-  logger,
-  ...opts
-}) => {
-  const transcript = [];
-  const socket = await connectTcp({ ...opts, logger });
-  let closed = false;
-  socket.on("data", (message) => {
-    if (closed) {
-      logger.warn("socket is closed, dropping message");
-      return;
-    }
-    onMessage?.(message);
-    transcript.push({ sender: "server", message });
-  });
-  socket.once("close", () => onSocketClose(void 0));
-  return {
-    socket,
-    transcript,
-    createRequest: opts,
-    async write(data) {
-      transcript.push({ sender: "client", message: data });
-      await new Promise((resolve, reject) => {
-        socket.write(data, (err) => {
-          if (err) {
-            reject(err);
-          } else {
-            resolve();
-          }
-        });
-      });
-    },
-    close(err) {
-      if (closed) {
-        return;
-      }
-      socket.destroy(err);
-    }
-  };
-  function onSocketClose(err) {
-    if (closed) {
-      return;
-    }
-    logger.debug({ err }, "closing socket");
-    closed = true;
-    onClose?.(err);
-    onClose = void 0;
-  }
-};
-async function connectTcp({ host, port, geoLocation, proxySessionId, logger }) {
-  let connectTimeout;
-  let socket;
-  try {
-    await new Promise(async (resolve, reject) => {
-      try {
-        connectTimeout = setTimeout(
-          () => reject(
-            new AttestorError(
-              "ERROR_NETWORK_ERROR",
-              "Server connection timed out"
-            )
-          ),
-          CONNECTION_TIMEOUT_MS
-        );
-        socket = await getSocket({
-          host,
-          port,
-          geoLocation,
-          proxySessionId,
-          logger
-        });
-        socket.once("connect", resolve);
-        socket.once("error", reject);
-        socket.once("end", () => reject(
-          new AttestorError(
-            "ERROR_NETWORK_ERROR",
-            "connection closed"
-          )
-        ));
-      } catch (err) {
-        reject(err);
-      }
-    });
-    logger.debug({ addr: `${host}:${port}` }, "connected");
-    return socket;
-  } catch (err) {
-    socket?.end();
-    throw err;
-  } finally {
-    clearTimeout(connectTimeout);
-  }
-}
-async function getSocket(opts) {
-  const { logger } = opts;
-  try {
-    return await _getSocket(opts);
-  } catch (err) {
-    if (!(err instanceof AttestorError) || err.data?.code !== 403) {
-      throw err;
-    }
-    const addrs = await resolveHostnames(opts.host);
-    logger.info(
-      { addrs, host: opts.host },
-      "failed to connect due to restricted IP, trying via raw addr"
-    );
-    for (const addr of addrs) {
-      try {
-        return await _getSocket({ ...opts, host: addr });
-      } catch (err2) {
-        logger.error(
-          { addr, err: err2 },
-          "failed to connect to host"
-        );
-      }
-    }
-    throw err;
-  }
-}
-async function _getSocket({
-  host,
-  port,
-  geoLocation,
-  proxySessionId,
-  logger
-}) {
-  const socket = new Socket();
-  if ((proxySessionId || geoLocation) && !HTTPS_PROXY_URL) {
-    logger.warn(
-      { geoLocation, proxySessionId },
-      "geoLocation or proxySessionId provided but no proxy URL found"
-    );
-    geoLocation = "";
-    proxySessionId = "";
-  }
-  if (!geoLocation && !proxySessionId) {
-    socket.connect({ host, port });
-    return socket;
-  }
-  if (!isValidCountryCode(geoLocation)) {
-    throw AttestorError.badRequest(
-      `Geolocation "${geoLocation}" is invalid. Must be 2 letter ISO country code`,
-      { geoLocation }
-    );
-  }
-  if (proxySessionId && !isValidProxySessionId(proxySessionId)) {
-    throw AttestorError.badRequest(
-      `proxySessionId "${proxySessionId}" is invalid. Must be a lowercase alphanumeric string of length 8-14 characters. eg. "mystring12345", "something1234".`,
-      { proxySessionId }
-    );
-  }
-  const agentUrl = HTTPS_PROXY_URL.replace(
-    "{{geoLocation}}",
-    geoLocation?.toLowerCase() || ""
-  ).replace(
-    "{{proxySessionId}}",
-    proxySessionId ? `-session-${proxySessionId}` : ""
-  );
-  const agent = new HttpsProxyAgent(agentUrl);
-  const waitForProxyRes = new Promise((resolve) => {
-    socket.once("proxyConnect", resolve);
-  });
-  const proxySocket = await agent.connect(
-    // ignore, because https-proxy-agent
-    // expects an http request object
-    // @ts-ignore
-    socket,
-    { host, port, timeout: CONNECTION_TIMEOUT_MS }
-  );
-  const res = await waitForProxyRes;
-  if (res.statusCode !== 200) {
-    logger.error(
-      { geoLocation, proxySessionId, res },
-      "Proxy geo location or session id failed"
-    );
-    throw new AttestorError(
-      "ERROR_PROXY_ERROR",
-      `Proxy via ${geoLocation ? `geo location "${geoLocation}"` : ""}${geoLocation && proxySessionId ? ", or " : ""}${proxySessionId ? `session id "${proxySessionId}"` : ""} failed with status code: ${res.statusCode}, message: ${res.statusText}`,
-      {
-        code: res.statusCode,
-        message: res.statusText
-      }
-    );
-  }
-  process.nextTick(() => {
-    proxySocket.emit("connect");
-  });
-  return proxySocket;
-}
-export {
-  makeTcpTunnel
-};

package/lib/server/utils/apm.js

@@ -1,29 +0,0 @@
-import ElasticAPM from "elastic-apm-node";
-import { getEnvVariable } from "#src/utils/env.js";
-import { logger } from "#src/utils/logger.js";
-let apm;
-function getApm() {
-  if (!getEnvVariable("ELASTIC_APM_SERVER_URL") || !getEnvVariable("ELASTIC_APM_SECRET_TOKEN")) {
-    logger.info(
-      "ELASTIC_APM_SERVER_URL or ELASTIC_APM_SECRET_TOKEN not found in env, APM agent not initialised"
-    );
-    return void 0;
-  }
-  if (!apm) {
-    const sampleRate = +(getEnvVariable("ELASTIC_APM_SAMPLE_RATE") || "0.1");
-    apm = ElasticAPM.start({
-      serviceName: "reclaim_attestor",
-      serviceVersion: "4.0.0",
-      transactionSampleRate: sampleRate,
-      instrumentIncomingHTTPRequests: true,
-      usePathAsTransactionName: true,
-      instrument: true,
-      captureHeaders: true
-    });
-    logger.info("initialised APM agent");
-  }
-  return apm;
-}
-export {
-  getApm
-};

package/lib/server/utils/assert-valid-claim-request.js

@@ -1,354 +0,0 @@
-import { areUint8ArraysEqual, concatenateUint8Arrays } from "@reclaimprotocol/tls";
-import { ClaimTunnelRequest, TranscriptMessageSenderType } from "#src/proto/api.js";
-import { providers } from "#src/providers/index.js";
-import { niceParseJsonObject } from "#src/server/utils/generics.js";
-import { computeOPRFRaw } from "#src/server/utils/oprf-raw.js";
-import { processHandshake } from "#src/server/utils/process-handshake.js";
-import { assertValidateProviderParams } from "#src/server/utils/validation.js";
-import {
-  AttestorError,
-  binaryHashToStr,
-  canonicalStringify,
-  decryptDirect,
-  extractApplicationDataFromTranscript,
-  hashProviderParams,
-  SIGNATURES,
-  verifyZkPacket
-} from "#src/utils/index.js";
-import { getEngineString } from "#src/utils/zk.js";
-async function assertValidClaimRequest(request, metadata, logger) {
-  const {
-    data,
-    signatures: { requestSignature } = {},
-    zkEngine,
-    fixedServerIV,
-    fixedClientIV
-  } = request;
-  if (!data) {
-    throw new AttestorError(
-      "ERROR_INVALID_CLAIM",
-      "No info provided on claim request"
-    );
-  }
-  if (!requestSignature?.length) {
-    throw new AttestorError(
-      "ERROR_INVALID_CLAIM",
-      "No signature provided on claim request"
-    );
-  }
-  const serialisedReq = ClaimTunnelRequest.encode({ ...request, signatures: void 0 }).finish();
-  const { verify: verifySig } = SIGNATURES[metadata.signatureType];
-  const verified = await verifySig(serialisedReq, requestSignature, data.owner);
-  if (!verified) {
-    throw new AttestorError(
-      "ERROR_INVALID_CLAIM",
-      "Invalid signature on claim request"
-    );
-  }
-  const receipt = await decryptTranscript(
-    request.transcript,
-    logger,
-    getEngineString(zkEngine),
-    fixedServerIV,
-    fixedClientIV
-  );
-  const reqHost = request.request?.host;
-  if (receipt.hostname !== reqHost) {
-    throw new Error(
-      `Expected server name ${reqHost}, got ${receipt.hostname}`
-    );
-  }
-  const applData = extractApplicationDataFromTranscript(receipt);
-  const newData = await assertValidProviderTranscript(
-    applData,
-    data,
-    logger,
-    { version: metadata.clientVersion },
-    receipt.oprfRawReplacements
-  );
-  if (newData !== data) {
-    logger.info({ newData }, "updated claim info");
-  }
-  return newData;
-}
-async function assertValidProviderTranscript(applData, info, logger, providerCtx, oprfRawReplacements) {
-  const providerName = info.provider;
-  const provider = providers[providerName];
-  if (!provider) {
-    throw new AttestorError(
-      "ERROR_INVALID_CLAIM",
-      `Unsupported provider: ${providerName}`
-    );
-  }
-  let params = niceParseJsonObject(info.parameters, "params");
-  const ctx = niceParseJsonObject(info.context, "context");
-  if (oprfRawReplacements?.length) {
-    let strParams = canonicalStringify(params) ?? "{}";
-    for (const { originalText, nullifierText } of oprfRawReplacements) {
-      strParams = strParams.replaceAll(originalText, nullifierText);
-    }
-    params = JSON.parse(strParams);
-    info.parameters = strParams;
-    logger.debug(
-      { replacements: oprfRawReplacements.length },
-      "applied oprf-raw parameter replacements"
-    );
-  }
-  assertValidateProviderParams(providerName, params);
-  const rslt = await provider.assertValidProviderReceipt({
-    receipt: applData,
-    params,
-    logger,
-    ctx: providerCtx
-  });
-  ctx.providerHash = hashProviderParams(params);
-  const extractedParameters = rslt?.extractedParameters || {};
-  if (Object.keys(extractedParameters).length) {
-    ctx.extractedParameters = extractedParameters;
-  }
-  info.context = canonicalStringify(ctx) ?? "";
-  return info;
-}
-function assertTranscriptsMatch(clientTranscript, tunnelTranscript) {
-  const clientSends = concatenateUint8Arrays(
-    clientTranscript.filter((m) => m.sender === TranscriptMessageSenderType.TRANSCRIPT_MESSAGE_SENDER_TYPE_CLIENT).map((m) => m.message)
-  );
-  const tunnelSends = concatenateUint8Arrays(
-    tunnelTranscript.filter((m) => m.sender === "client").map((m) => m.message)
-  );
-  if (!areUint8ArraysEqual(clientSends, tunnelSends)) {
-    throw AttestorError.badRequest(
-      "Outgoing messages from client do not match the tunnel transcript"
-    );
-  }
-  const clientRecvs = concatenateUint8Arrays(
-    clientTranscript.filter((m) => m.sender === TranscriptMessageSenderType.TRANSCRIPT_MESSAGE_SENDER_TYPE_SERVER).map((m) => m.message)
-  );
-  const tunnelRecvs = concatenateUint8Arrays(
-    tunnelTranscript.filter((m) => m.sender === "server").map((m) => m.message)
-  ).slice(0, clientRecvs.length);
-  if (!areUint8ArraysEqual(clientRecvs, tunnelRecvs)) {
-    throw AttestorError.badRequest(
-      "Incoming messages from server do not match the tunnel transcript"
-    );
-  }
-}
-async function decryptTranscript(transcript, logger, zkEngine, serverIV, clientIV) {
-  const {
-    tlsVersion,
-    cipherSuite,
-    hostname,
-    nextMsgIndex
-  } = await processHandshake(transcript, logger);
-  let clientRecordNumber = tlsVersion === "TLS1_3" ? -1 : 0;
-  let serverRecordNumber = clientRecordNumber;
-  transcript = transcript.slice(nextMsgIndex);
-  const overshotMap = {};
-  const decryptedTranscript = [];
-  const oprfRawReplacements = [];
-  const pendingOprfRaw = {};
-  for (const [i, {
-    sender,
-    message,
-    reveal: { zkReveal, directReveal } = {}
-  }] of transcript.entries()) {
-    try {
-      await decryptMessage(sender, message, directReveal, zkReveal, i);
-    } catch (error) {
-      const err = new AttestorError(
-        "ERROR_INVALID_CLAIM",
-        `error in handling packet at idx ${i}: ${error}`,
-        { packetIdx: i, error }
-      );
-      if (error.stack) {
-        err.stack = error.stack;
-      }
-      throw err;
-    }
-  }
-  const remainingPending = Object.keys(pendingOprfRaw);
-  if (remainingPending.length) {
-    throw new AttestorError(
-      "ERROR_INVALID_CLAIM",
-      `oprf-raw cross-block markers incomplete: pending for packets ${remainingPending.join(", ")}`
-    );
-  }
-  return {
-    transcript: decryptedTranscript,
-    hostname,
-    tlsVersion,
-    oprfRawReplacements: oprfRawReplacements.length ? oprfRawReplacements : void 0
-  };
-  async function decryptMessage(sender, message, directReveal, zkReveal, i) {
-    const isServer = sender === TranscriptMessageSenderType.TRANSCRIPT_MESSAGE_SENDER_TYPE_SERVER;
-    const recordHeader = message.slice(0, 5);
-    const content = getWithoutHeader(message);
-    if (isServer) {
-      serverRecordNumber++;
-    } else {
-      clientRecordNumber++;
-    }
-    let redacted = true;
-    let plaintext = void 0;
-    let plaintextLength;
-    if (directReveal?.key?.length) {
-      const result = await decryptDirect(
-        directReveal,
-        cipherSuite,
-        recordHeader,
-        tlsVersion,
-        content
-      );
-      plaintext = result.plaintext;
-      redacted = false;
-      plaintextLength = plaintext.length;
-    } else if (zkReveal?.proofs?.length) {
-      const iv = sender === TranscriptMessageSenderType.TRANSCRIPT_MESSAGE_SENDER_TYPE_SERVER ? serverIV : clientIV;
-      const recordNumber = isServer ? serverRecordNumber : clientRecordNumber;
-      const result = await verifyZkPacket(
-        {
-          ciphertext: content,
-          zkReveal,
-          iv,
-          recordNumber,
-          toprfOvershotNullifier: overshotMap[i]?.data,
-          getNextPacket(overshot) {
-            const nextIdx = transcript.findIndex((t, j) => t.sender === sender && j > i);
-            if (nextIdx < 0) {
-              return;
-            }
-            overshotMap[nextIdx] = { data: overshot };
-            return getWithoutHeader(transcript[nextIdx].message);
-          },
-          logger,
-          cipherSuite,
-          zkEngine
-        }
-      );
-      plaintext = result.redactedPlaintext;
-      const pendingForThis = pendingOprfRaw[i];
-      if (pendingForThis && zkReveal?.overshotOprfRawLength) {
-        const overshootLen = zkReveal.overshotOprfRawLength;
-        const overshootData = plaintext.slice(0, overshootLen);
-        const fullData = concatenateUint8Arrays([
-          pendingForThis.partialData,
-          overshootData
-        ]);
-        const expectedLen = pendingForThis.dataLocation.length;
-        if (fullData.length !== expectedLen) {
-          throw new AttestorError(
-            "ERROR_INVALID_CLAIM",
-            `oprf-raw cross-block length mismatch: got ${fullData.length}, expected ${expectedLen}`
-          );
-        }
-        const oprfResults = await computeOPRFRaw(
-          fullData,
-          [{ dataLocation: { fromIndex: 0, length: fullData.length } }],
-          logger
-        );
-        if (oprfResults.length) {
-          const { nullifier } = oprfResults[0];
-          const originalText = new TextDecoder().decode(fullData);
-          const nullifierStr = binaryHashToStr(nullifier, fullData.length);
-          oprfRawReplacements.push({ originalText, nullifierText: nullifierStr });
-          const nullifierBytes = new TextEncoder().encode(nullifierStr);
-          const overshootNullifier = nullifierBytes.slice(pendingForThis.partialData.length);
-          plaintext.set(overshootNullifier, 0);
-          const prevPkt = decryptedTranscript[pendingForThis.originPktIdx];
-          if (prevPkt) {
-            const firstPartNullifier = nullifierBytes.slice(0, pendingForThis.partialData.length);
-            prevPkt.message.set(firstPartNullifier, pendingForThis.dataLocation.fromIndex);
-          }
-        }
-        delete pendingOprfRaw[i];
-      }
-      if (result.oprfRawMarkers?.length) {
-        const { markersThisPacket, pendingMarker } = separateOprfRawMarkers(
-          result.oprfRawMarkers,
-          plaintext.length,
-          () => transcript.findIndex((t, j) => t.sender === sender && j > i),
-          decryptedTranscript.length,
-          logger
-        );
-        if (pendingMarker) {
-          pendingMarker.pending.partialData.set(
-            plaintext.slice(pendingMarker.pending.dataLocation.fromIndex)
-          );
-          pendingOprfRaw[pendingMarker.nextIdx] = pendingMarker.pending;
-        }
-        if (markersThisPacket.length) {
-          const pt = plaintext;
-          const oprfResults = await computeOPRFRaw(pt, markersThisPacket, logger);
-          const originalTexts = oprfResults.map(({ dataLocation }) => new TextDecoder().decode(
-            pt.slice(dataLocation.fromIndex, dataLocation.fromIndex + dataLocation.length)
-          ));
-          for (const [idx, { dataLocation, nullifier }] of oprfResults.entries()) {
-            const originalText = originalTexts[idx];
-            const nullifierStr = binaryHashToStr(nullifier, dataLocation.length);
-            oprfRawReplacements.push({ originalText, nullifierText: nullifierStr });
-            const nullifierBytes = new TextEncoder().encode(nullifierStr);
-            pt.set(nullifierBytes, dataLocation.fromIndex);
-          }
-        }
-      }
-      redacted = false;
-      plaintextLength = plaintext.length;
-    } else {
-      plaintext = content;
-      plaintextLength = plaintext.length;
-    }
-    decryptedTranscript.push({
-      sender: sender === TranscriptMessageSenderType.TRANSCRIPT_MESSAGE_SENDER_TYPE_CLIENT ? "client" : "server",
-      redacted,
-      message: plaintext,
-      recordHeader,
-      plaintextLength
-    });
-  }
-}
-function getWithoutHeader(message) {
-  return message.slice(5);
-}
-function separateOprfRawMarkers(markers, plaintextLength, findNextPacketIdx, currentTranscriptLength, logger) {
-  const markersThisPacket = [];
-  let pendingMarker;
-  for (const marker of markers) {
-    const dataLocation = marker.dataLocation;
-    if (!dataLocation) {
-      continue;
-    }
-    const { fromIndex, length } = dataLocation;
-    const endInPacket = fromIndex + length;
-    if (endInPacket <= plaintextLength) {
-      markersThisPacket.push({ dataLocation });
-      continue;
-    }
-    const nextIdx = findNextPacketIdx();
-    if (nextIdx < 0) {
-      throw new AttestorError(
-        "ERROR_INVALID_CLAIM",
-        "oprf-raw marker spans packets but no next packet found"
-      );
-    }
-    pendingMarker = {
-      nextIdx,
-      pending: {
-        partialData: new Uint8Array(plaintextLength - fromIndex),
-        dataLocation: { fromIndex, length },
-        originPktIdx: currentTranscriptLength
-      }
-    };
-    logger.debug(
-      { fromIndex, length, partialLen: plaintextLength - fromIndex, nextIdx },
-      "oprf-raw marker spans packets, storing partial data"
-    );
-  }
-  return { markersThisPacket, pendingMarker };
-}
-export {
-  assertTranscriptsMatch,
-  assertValidClaimRequest,
-  assertValidProviderTranscript,
-  decryptTranscript,
-  getWithoutHeader
-};

package/lib/server/utils/config-env.js

@@ -1,4 +0,0 @@
-import { config } from "dotenv";
-import { getEnvVariable } from "#src/utils/env.js";
-const nodeEnv = getEnvVariable("NODE_ENV") || "development";
-config({ path: `.env.${nodeEnv}` });

package/lib/server/utils/dns.js

@@ -1,24 +0,0 @@
-import { resolve, setServers } from "dns";
-import { DNS_SERVERS } from "#src/config/index.js";
-setDnsServers();
-async function resolveHostnames(hostname) {
-  return new Promise((_resolve, reject) => {
-    resolve(hostname, (err, addresses) => {
-      if (err) {
-        reject(
-          new Error(
-            `Could not resolve hostname: ${hostname}, ${err.message}`
-          )
-        );
-      } else {
-        _resolve(addresses);
-      }
-    });
-  });
-}
-function setDnsServers() {
-  setServers(DNS_SERVERS);
-}
-export {
-  resolveHostnames
-};