@reclaimprotocol/attestor-core 5.0.7 → 5.0.8

package/README.md

@@ -36,7 +36,7 @@ Read a longer brief [here](/docs/problem-statement.md).
 
 This repository contains **Typescript** implementation of the attestor server & the SDK to interact with it. The SDK is compatible with the browser & NodeJS. However, to run the SDK on React Native/Mobile, refer to our [React Native guide](/docs/browser-rpc.md).
 
-Note: if you're looking to build apps on Reclaim, we'd recommend you to use the [Reclaim SDK](https://docs.reclaimprotocol.org/) instead. This repository is intended for developers looking to contribute to the core protocol.
+Note: if you're looking to build apps on Reclaim, we'd recommend you to use the [Reclaim SDK](https://docs.reclaimprotocol.org/) instead. This repository is intended for developers looking to contribute to the core protocol
 
 ## Install
 

package/lib/config/index.d.ts

@@ -10,10 +10,10 @@ export declare const API_SERVER_PORT = 8001;
 export declare const CONNECTION_TIMEOUT_MS = 10000;
 export declare const DNS_SERVERS: string[];
 export declare const MAX_CLAIM_TIMESTAMP_DIFF_S: number;
-export declare const CURRENT_ATTESTOR_VERSION: 6;
+export declare const CURRENT_ATTESTOR_VERSION: 7;
 export declare const DEFAULT_METADATA: InitRequest;
 export declare const PROVIDER_CTX: {
-    version: 6;
+    version: 7;
 };
 export declare const PING_INTERVAL_MS = 10000;
 /**

package/lib/external-rpc/index.js

@@ -2489,6 +2489,7 @@ var AttestorVersion = {
   ATTESTOR_VERSION_2_0_1: 4,
   ATTESTOR_VERSION_3_0_0: 5,
   ATTESTOR_VERSION_3_1_0: 6,
+  ATTESTOR_VERSION_3_2_0: 7,
   UNRECOGNIZED: -1
 };
 function attestorVersionFromJSON(object) {
@@ -2514,6 +2515,9 @@ function attestorVersionFromJSON(object) {
     case 6:
     case "ATTESTOR_VERSION_3_1_0":
       return AttestorVersion.ATTESTOR_VERSION_3_1_0;
+    case 7:
+    case "ATTESTOR_VERSION_3_2_0":
+      return AttestorVersion.ATTESTOR_VERSION_3_2_0;
     case -1:
     case "UNRECOGNIZED":
     default:
@@ -2536,6 +2540,8 @@ function attestorVersionToJSON(object) {
       return "ATTESTOR_VERSION_3_0_0";
     case AttestorVersion.ATTESTOR_VERSION_3_1_0:
       return "ATTESTOR_VERSION_3_1_0";
+    case AttestorVersion.ATTESTOR_VERSION_3_2_0:
+      return "ATTESTOR_VERSION_3_2_0";
     case AttestorVersion.UNRECOGNIZED:
     default:
       return "UNRECOGNIZED";
@@ -6901,7 +6907,7 @@ var WS_PATHNAME = "/ws";
 var BROWSER_RPC_PATHNAME = "/browser-rpc";
 var DEFAULT_REMOTE_FILE_FETCH_BASE_URL = `${BROWSER_RPC_PATHNAME}/resources`;
 var MAX_CLAIM_TIMESTAMP_DIFF_S = 10 * 60;
-var CURRENT_ATTESTOR_VERSION = AttestorVersion.ATTESTOR_VERSION_3_1_0;
+var CURRENT_ATTESTOR_VERSION = AttestorVersion.ATTESTOR_VERSION_3_2_0;
 var DEFAULT_METADATA = {
   signatureType: ServiceSignatureType.SERVICE_SIGNATURE_TYPE_ETH,
   clientVersion: CURRENT_ATTESTOR_VERSION,
@@ -8422,7 +8428,6 @@ import {
 import { JSONPath } from "jsonpath-plus";
 import { parse } from "parse5";
 import { adapter as htmlAdapter } from "parse5-htmlparser2-tree-adapter";
-import RE2 from "re2";
 import xpath from "xpath";
 function extractHTMLElement(html, xpathExpression, contentsOnly) {
   const { start, end } = extractHTMLElementIndex(html, xpathExpression, contentsOnly);
@@ -8589,8 +8594,16 @@ function parseHttpResponse(buff) {
   parser.streamEnded();
   return parser.res;
 }
+var RE2;
+import("re2").then((m) => {
+  RE2 = m.default;
+}).catch(() => {
+});
 function makeRegex(str) {
-  return RE2(str, "sgiu");
+  if (RE2) {
+    return RE2(str, "sgiu");
+  }
+  return new RegExp(str, "sgiu");
 }
 var TEMPLATE_START_CHARCODE = "{".charCodeAt(0);
 var TEMPLATE_END_CHARCODE = "}".charCodeAt(0);
@@ -8798,6 +8811,7 @@ var HTTP_PROVIDER = {
       logger2.error({ response: encodeBase643(response) });
       throw new Error("Failed to find response body");
     }
+    const revealFraming = shouldRevealChunkFraming(ctx.version);
     const reveals = [
       { fromIndex: 0, toIndex: headerEndIndex }
     ];
@@ -8814,6 +8828,9 @@ var HTTP_PROVIDER = {
     if (res.headerIndices["date"]) {
       reveals.push(res.headerIndices["date"]);
     }
+    if (revealFraming && res.headerIndices["transfer-encoding"]) {
+      reveals.push(res.headerIndices["transfer-encoding"]);
+    }
     const body = uint8ArrayToBinaryStr2(res.body);
     const redactions = [];
     for (const rs of params.responseRedactions || []) {
@@ -8821,23 +8838,34 @@ var HTTP_PROVIDER = {
         body,
         rs,
         bodyStartIdx,
-        res.chunks
+        res.chunks,
+        revealFraming
       );
       for (const { reveal, redactions: reds } of processor) {
         reveals.push(reveal);
         redactions.push(...reds);
       }
     }
-    reveals.sort((a, b) => a.toIndex - b.toIndex);
+    if (revealFraming && res.chunks?.length) {
+      let prev = res.headerEndIdx + 4;
+      for (const chunk of res.chunks) {
+        reveals.push({ fromIndex: prev, toIndex: chunk.fromIndex });
+        prev = chunk.toIndex;
+      }
+      reveals.push({ fromIndex: prev, toIndex: response.length });
+    }
+    reveals.sort((a, b) => a.fromIndex - b.fromIndex);
     if (reveals.length > 1) {
       let currentIndex = 0;
       for (const r of reveals) {
         if (currentIndex < r.fromIndex) {
           redactions.push({ fromIndex: currentIndex, toIndex: r.fromIndex });
         }
-        currentIndex = r.toIndex;
+        currentIndex = Math.max(currentIndex, r.toIndex);
+      }
+      if (currentIndex < response.length) {
+        redactions.push({ fromIndex: currentIndex, toIndex: response.length });
       }
-      redactions.push({ fromIndex: currentIndex, toIndex: response.length });
     }
     for (const r of reveals) {
       if (!r.hash) {
@@ -8887,7 +8915,9 @@ var HTTP_PROVIDER = {
     if (connectionHeader !== "close") {
       throw new Error(`Connection header must be "close", got "${connectionHeader}"`);
     }
-    const serverBlocks = receipt.filter((s) => s.sender === "server").map((r) => r.message).filter((b) => !b.every((b2) => b2 === REDACTION_CHAR_CODE2));
+    const allServerBlocks = receipt.filter((s) => s.sender === "server").map((r) => r.message);
+    const firstRealIdx = allServerBlocks.findIndex((b) => !b.every((x) => x === REDACTION_CHAR_CODE2));
+    const serverBlocks = firstRealIdx === -1 ? [] : allServerBlocks.slice(firstRealIdx);
     const response = concatArrays(...serverBlocks);
     let res;
     res = uint8ArrayToStr(response);
@@ -8930,8 +8960,11 @@ var HTTP_PROVIDER = {
     if (paramBody.length > 0 && !matchRedactedStrings(paramBody, req.body)) {
       throw new Error("request body mismatch");
     }
-    if (!secretParams) {
-      res = res.slice(bodyStart).replace(/(\*){3,}/g, "");
+    if (shouldRevealChunkFraming(clientVersion)) {
+      const headersStr = res.slice(0, bodyStart);
+      res = /transfer-encoding:\s*chunked/i.test(headersStr) ? headersStr + dechunkResponseBody(res.slice(bodyStart)) : res;
+    } else if (!secretParams) {
+      res = res.slice(bodyStart).replace(/\*{3,}/g, "");
     }
     for (const { type, value, invert } of params.responseMatches || []) {
       const inv = Boolean(invert);
@@ -9018,6 +9051,16 @@ function assertNoSmuggle(reqBuffer, params) {
 function shouldRevealCrlf({ version }) {
   return version >= AttestorVersion.ATTESTOR_VERSION_2_0_1;
 }
+function shouldRevealChunkFraming(version) {
+  return version >= AttestorVersion.ATTESTOR_VERSION_3_2_0;
+}
+function dechunkResponseBody(body) {
+  const parser = makeHttpResponseParser();
+  parser.onChunk(
+    strToUint8Array("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n" + body)
+  );
+  return uint8ArrayToStr(parser.res.body ?? new Uint8Array());
+}
 function getHostPort(params, secretParams) {
   const { host } = new URL(getURL(params, secretParams));
   if (!host) {
@@ -9031,7 +9074,7 @@ function getHostHeaderString(url) {
   return port && +port !== DEFAULT_HTTPS_PORT ? `${host}:${port}` : host;
 }
 var paramsRegex = /{{([^{}]+)}}/sgi;
-function* processRedactionRequest(body, rs, bodyStartIdx, resChunks) {
+function* processRedactionRequest(body, rs, bodyStartIdx, resChunks, revealFraming) {
   let element = body;
   let elementIdx = 0;
   let elementLength = -1;
@@ -9121,19 +9164,13 @@ function* processRedactionRequest(body, rs, bodyStartIdx, resChunks) {
     elementLength = element.length;
     yield* addRedaction(null);
   }
-  function* addRedaction(hash = rs.hash, _resChunks = resChunks) {
+  function* addRedaction(hash = rs.hash) {
     if (elementIdx < 0 || !elementLength) {
       return;
     }
     const reveal = getReveal(elementIdx, elementLength, hash || void 0);
-    yield {
-      reveal,
-      redactions: getRedactionsForChunkHeaders(
-        reveal.fromIndex,
-        reveal.toIndex,
-        _resChunks
-      )
-    };
+    const redactions = revealFraming ? [] : getRedactionsForChunkHeaders(reveal.fromIndex, reveal.toIndex, resChunks);
+    yield { reveal, redactions };
   }
   function getReveal(startIdx, len, hash) {
     const from = convertResponsePosToAbsolutePos(
@@ -9692,13 +9729,25 @@ async function _createClaimOnAttestor({
       }
       revealedPackets.push(...packets.filter((p) => p.sender === "server"));
     } else {
-      for (const {
-        block,
-        redactedPlaintext,
-        overshotToprfFromPrevBlock,
-        toprfs,
-        oprfRawMarkers
-      } of serverPacketsToReveal) {
+      const revealByBlock = new Map(
+        serverPacketsToReveal.map((r) => [r.block, r])
+      );
+      for (const sb of serverBlocks) {
+        const reveal = revealByBlock.get(sb);
+        if (!reveal) {
+          revealedPackets.push({
+            sender: "server",
+            message: new Uint8Array(sb.plaintext.length).fill(REDACTION_CHAR_CODE2)
+          });
+          continue;
+        }
+        const {
+          block,
+          redactedPlaintext,
+          overshotToprfFromPrevBlock,
+          toprfs,
+          oprfRawMarkers
+        } = reveal;
         setRevealOfMessage(block.message, {
           type: "zk",
           redactedPlaintext,

package/lib/index.js

@@ -155,6 +155,7 @@ var AttestorVersion = {
   ATTESTOR_VERSION_2_0_1: 4,
   ATTESTOR_VERSION_3_0_0: 5,
   ATTESTOR_VERSION_3_1_0: 6,
+  ATTESTOR_VERSION_3_2_0: 7,
   UNRECOGNIZED: -1
 };
 function attestorVersionFromJSON(object) {
@@ -180,6 +181,9 @@ function attestorVersionFromJSON(object) {
     case 6:
     case "ATTESTOR_VERSION_3_1_0":
       return AttestorVersion.ATTESTOR_VERSION_3_1_0;
+    case 7:
+    case "ATTESTOR_VERSION_3_2_0":
+      return AttestorVersion.ATTESTOR_VERSION_3_2_0;
     case -1:
     case "UNRECOGNIZED":
     default:
@@ -202,6 +206,8 @@ function attestorVersionToJSON(object) {
       return "ATTESTOR_VERSION_3_0_0";
     case AttestorVersion.ATTESTOR_VERSION_3_1_0:
       return "ATTESTOR_VERSION_3_1_0";
+    case AttestorVersion.ATTESTOR_VERSION_3_2_0:
+      return "ATTESTOR_VERSION_3_2_0";
     case AttestorVersion.UNRECOGNIZED:
     default:
       return "UNRECOGNIZED";
@@ -4284,7 +4290,7 @@ var DNS_SERVERS = [
   "8.8.4.4"
 ];
 var MAX_CLAIM_TIMESTAMP_DIFF_S = 10 * 60;
-var CURRENT_ATTESTOR_VERSION = AttestorVersion.ATTESTOR_VERSION_3_1_0;
+var CURRENT_ATTESTOR_VERSION = AttestorVersion.ATTESTOR_VERSION_3_2_0;
 var DEFAULT_METADATA = {
   signatureType: ServiceSignatureType.SERVICE_SIGNATURE_TYPE_ETH,
   clientVersion: CURRENT_ATTESTOR_VERSION,
@@ -6731,7 +6737,6 @@ import {
 import { JSONPath } from "jsonpath-plus";
 import { parse } from "parse5";
 import { adapter as htmlAdapter } from "parse5-htmlparser2-tree-adapter";
-import RE2 from "re2";
 import xpath from "xpath";
 function extractHTMLElementsIndexes(html, xpathExpression, contentsOnly) {
   return extractHTMLElementIndexesParse5(html, xpathExpression, contentsOnly);
@@ -6888,8 +6893,16 @@ function parseHttpResponse(buff) {
   parser.streamEnded();
   return parser.res;
 }
+var RE2;
+import("re2").then((m) => {
+  RE2 = m.default;
+}).catch(() => {
+});
 function makeRegex(str) {
-  return RE2(str, "sgiu");
+  if (RE2) {
+    return RE2(str, "sgiu");
+  }
+  return new RegExp(str, "sgiu");
 }
 var TEMPLATE_START_CHARCODE = "{".charCodeAt(0);
 var TEMPLATE_END_CHARCODE = "}".charCodeAt(0);
@@ -7078,6 +7091,7 @@ var HTTP_PROVIDER = {
       logger2.error({ response: encodeBase643(response) });
       throw new Error("Failed to find response body");
     }
+    const revealFraming = shouldRevealChunkFraming(ctx.version);
     const reveals = [
       { fromIndex: 0, toIndex: headerEndIndex }
     ];
@@ -7094,6 +7108,9 @@ var HTTP_PROVIDER = {
     if (res.headerIndices["date"]) {
       reveals.push(res.headerIndices["date"]);
     }
+    if (revealFraming && res.headerIndices["transfer-encoding"]) {
+      reveals.push(res.headerIndices["transfer-encoding"]);
+    }
     const body = uint8ArrayToBinaryStr2(res.body);
     const redactions = [];
     for (const rs of params.responseRedactions || []) {
@@ -7101,23 +7118,34 @@ var HTTP_PROVIDER = {
         body,
         rs,
         bodyStartIdx,
-        res.chunks
+        res.chunks,
+        revealFraming
       );
       for (const { reveal, redactions: reds } of processor) {
         reveals.push(reveal);
         redactions.push(...reds);
       }
     }
-    reveals.sort((a, b) => a.toIndex - b.toIndex);
+    if (revealFraming && res.chunks?.length) {
+      let prev = res.headerEndIdx + 4;
+      for (const chunk of res.chunks) {
+        reveals.push({ fromIndex: prev, toIndex: chunk.fromIndex });
+        prev = chunk.toIndex;
+      }
+      reveals.push({ fromIndex: prev, toIndex: response.length });
+    }
+    reveals.sort((a, b) => a.fromIndex - b.fromIndex);
     if (reveals.length > 1) {
       let currentIndex = 0;
       for (const r of reveals) {
         if (currentIndex < r.fromIndex) {
           redactions.push({ fromIndex: currentIndex, toIndex: r.fromIndex });
         }
-        currentIndex = r.toIndex;
+        currentIndex = Math.max(currentIndex, r.toIndex);
+      }
+      if (currentIndex < response.length) {
+        redactions.push({ fromIndex: currentIndex, toIndex: response.length });
       }
-      redactions.push({ fromIndex: currentIndex, toIndex: response.length });
     }
     for (const r of reveals) {
       if (!r.hash) {
@@ -7167,7 +7195,9 @@ var HTTP_PROVIDER = {
     if (connectionHeader !== "close") {
       throw new Error(`Connection header must be "close", got "${connectionHeader}"`);
     }
-    const serverBlocks = receipt.filter((s) => s.sender === "server").map((r) => r.message).filter((b) => !b.every((b2) => b2 === REDACTION_CHAR_CODE2));
+    const allServerBlocks = receipt.filter((s) => s.sender === "server").map((r) => r.message);
+    const firstRealIdx = allServerBlocks.findIndex((b) => !b.every((x) => x === REDACTION_CHAR_CODE2));
+    const serverBlocks = firstRealIdx === -1 ? [] : allServerBlocks.slice(firstRealIdx);
     const response = concatArrays(...serverBlocks);
     let res;
     res = uint8ArrayToStr(response);
@@ -7210,8 +7240,11 @@ var HTTP_PROVIDER = {
     if (paramBody.length > 0 && !matchRedactedStrings(paramBody, req.body)) {
       throw new Error("request body mismatch");
     }
-    if (!secretParams) {
-      res = res.slice(bodyStart).replace(/(\*){3,}/g, "");
+    if (shouldRevealChunkFraming(clientVersion)) {
+      const headersStr = res.slice(0, bodyStart);
+      res = /transfer-encoding:\s*chunked/i.test(headersStr) ? headersStr + dechunkResponseBody(res.slice(bodyStart)) : res;
+    } else if (!secretParams) {
+      res = res.slice(bodyStart).replace(/\*{3,}/g, "");
     }
     for (const { type, value, invert } of params.responseMatches || []) {
       const inv = Boolean(invert);
@@ -7298,6 +7331,16 @@ function assertNoSmuggle(reqBuffer, params) {
 function shouldRevealCrlf({ version }) {
   return version >= AttestorVersion.ATTESTOR_VERSION_2_0_1;
 }
+function shouldRevealChunkFraming(version) {
+  return version >= AttestorVersion.ATTESTOR_VERSION_3_2_0;
+}
+function dechunkResponseBody(body) {
+  const parser = makeHttpResponseParser();
+  parser.onChunk(
+    strToUint8Array("HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n" + body)
+  );
+  return uint8ArrayToStr(parser.res.body ?? new Uint8Array());
+}
 function getHostPort(params, secretParams) {
   const { host } = new URL(getURL(params, secretParams));
   if (!host) {
@@ -7311,7 +7354,7 @@ function getHostHeaderString(url) {
   return port && +port !== DEFAULT_HTTPS_PORT ? `${host}:${port}` : host;
 }
 var paramsRegex = /{{([^{}]+)}}/sgi;
-function* processRedactionRequest(body, rs, bodyStartIdx, resChunks) {
+function* processRedactionRequest(body, rs, bodyStartIdx, resChunks, revealFraming) {
   let element = body;
   let elementIdx = 0;
   let elementLength = -1;
@@ -7401,19 +7444,13 @@ function* processRedactionRequest(body, rs, bodyStartIdx, resChunks) {
     elementLength = element.length;
     yield* addRedaction(null);
   }
-  function* addRedaction(hash = rs.hash, _resChunks = resChunks) {
+  function* addRedaction(hash = rs.hash) {
     if (elementIdx < 0 || !elementLength) {
       return;
     }
     const reveal = getReveal(elementIdx, elementLength, hash || void 0);
-    yield {
-      reveal,
-      redactions: getRedactionsForChunkHeaders(
-        reveal.fromIndex,
-        reveal.toIndex,
-        _resChunks
-      )
-    };
+    const redactions = revealFraming ? [] : getRedactionsForChunkHeaders(reveal.fromIndex, reveal.toIndex, resChunks);
+    yield { reveal, redactions };
   }
   function getReveal(startIdx, len, hash) {
     const from = convertResponsePosToAbsolutePos(
@@ -8173,13 +8210,25 @@ async function _createClaimOnAttestor({
       }
       revealedPackets.push(...packets.filter((p) => p.sender === "server"));
     } else {
-      for (const {
-        block,
-        redactedPlaintext,
-        overshotToprfFromPrevBlock,
-        toprfs,
-        oprfRawMarkers
-      } of serverPacketsToReveal) {
+      const revealByBlock = new Map(
+        serverPacketsToReveal.map((r) => [r.block, r])
+      );
+      for (const sb of serverBlocks) {
+        const reveal = revealByBlock.get(sb);
+        if (!reveal) {
+          revealedPackets.push({
+            sender: "server",
+            message: new Uint8Array(sb.plaintext.length).fill(REDACTION_CHAR_CODE2)
+          });
+          continue;
+        }
+        const {
+          block,
+          redactedPlaintext,
+          overshotToprfFromPrevBlock,
+          toprfs,
+          oprfRawMarkers
+        } = reveal;
         setRevealOfMessage(block.message, {
           type: "zk",
           redactedPlaintext,

package/lib/proto/api.d.ts

@@ -41,6 +41,7 @@ export declare const AttestorVersion: {
     readonly ATTESTOR_VERSION_2_0_1: 4;
     readonly ATTESTOR_VERSION_3_0_0: 5;
     readonly ATTESTOR_VERSION_3_1_0: 6;
+    readonly ATTESTOR_VERSION_3_2_0: 7;
     readonly UNRECOGNIZED: -1;
 };
 export type AttestorVersion = typeof AttestorVersion[keyof typeof AttestorVersion];
@@ -52,6 +53,7 @@ export declare namespace AttestorVersion {
     type ATTESTOR_VERSION_2_0_1 = typeof AttestorVersion.ATTESTOR_VERSION_2_0_1;
     type ATTESTOR_VERSION_3_0_0 = typeof AttestorVersion.ATTESTOR_VERSION_3_0_0;
     type ATTESTOR_VERSION_3_1_0 = typeof AttestorVersion.ATTESTOR_VERSION_3_1_0;
+    type ATTESTOR_VERSION_3_2_0 = typeof AttestorVersion.ATTESTOR_VERSION_3_2_0;
     type UNRECOGNIZED = typeof AttestorVersion.UNRECOGNIZED;
 }
 export declare function attestorVersionFromJSON(object: any): AttestorVersion;