@reclaimprotocol/attestor-core 5.0.1-beta.21 → 5.0.1-beta.22

package/lib/server/tunnels/make-tcp-tunnel.js

@@ -0,0 +1,177 @@
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import { Socket } from 'net';
+import { CONNECTION_TIMEOUT_MS } from "../../config/index.js";
+import { resolveHostnames } from "../utils/dns.js";
+import { isValidCountryCode } from "../utils/iso.js";
+import { isValidProxySessionId } from "../utils/proxy-session.js";
+import { getEnvVariable } from "../../utils/env.js";
+import { AttestorError } from "../../utils/index.js";
+const HTTPS_PROXY_URL = getEnvVariable('HTTPS_PROXY_URL');
+/**
+ * Builds a TCP tunnel to the given host and port.
+ * If a geolocation is provided -- an HTTPS proxy is used
+ * to connect to the host.
+ * If a proxySessionId is provided -- a static ip is used with HTTPS proxy
+ * across multiple requests with this same proxySessionId.
+ *
+ * HTTPS proxy essentially creates an opaque tunnel to the
+ * host using the CONNECT method. Any data can be sent through
+ * this tunnel to the end host.
+ * https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/CONNECT
+ *
+ * The tunnel also retains a transcript of all messages sent and received.
+ */
+export const makeTcpTunnel = async ({ onClose, onMessage, logger, ...opts }) => {
+    const transcript = [];
+    const socket = await connectTcp({ ...opts, logger });
+    let closed = false;
+    socket.on('data', message => {
+        if (closed) {
+            logger.warn('socket is closed, dropping message');
+            return;
+        }
+        onMessage?.(message);
+        transcript.push({ sender: 'server', message });
+    });
+    // socket.once('error', onSocketClose)
+    socket.once('close', () => onSocketClose(undefined));
+    return {
+        socket,
+        transcript,
+        createRequest: opts,
+        async write(data) {
+            transcript.push({ sender: 'client', message: data });
+            await new Promise((resolve, reject) => {
+                socket.write(data, err => {
+                    if (err) {
+                        reject(err);
+                    }
+                    else {
+                        resolve();
+                    }
+                });
+            });
+        },
+        close(err) {
+            if (closed) {
+                return;
+            }
+            socket.destroy(err);
+        }
+    };
+    function onSocketClose(err) {
+        if (closed) {
+            return;
+        }
+        logger.debug({ err }, 'closing socket');
+        closed = true;
+        onClose?.(err);
+        onClose = undefined;
+    }
+};
+async function connectTcp({ host, port, geoLocation, proxySessionId, logger }) {
+    let connectTimeout;
+    let socket;
+    try {
+        await new Promise(async (resolve, reject) => {
+            try {
+                // add a timeout to ensure the connection doesn't hang
+                // and cause our gateway to send out a 504
+                connectTimeout = setTimeout(() => reject(new AttestorError('ERROR_NETWORK_ERROR', 'Server connection timed out')), CONNECTION_TIMEOUT_MS);
+                socket = await getSocket({
+                    host,
+                    port,
+                    geoLocation,
+                    proxySessionId,
+                    logger
+                });
+                socket.once('connect', resolve);
+                socket.once('error', reject);
+                socket.once('end', () => (reject(new AttestorError('ERROR_NETWORK_ERROR', 'connection closed'))));
+            }
+            catch (err) {
+                reject(err);
+            }
+        });
+        logger.debug({ addr: `${host}:${port}` }, 'connected');
+        return socket;
+    }
+    catch (err) {
+        socket?.end();
+        throw err;
+    }
+    finally {
+        clearTimeout(connectTimeout);
+    }
+}
+async function getSocket(opts) {
+    const { logger } = opts;
+    try {
+        return await _getSocket(opts);
+    }
+    catch (err) {
+        // see if the proxy is blocking the connection
+        // due to their own arbitrary rules,
+        // if so -- we resolve hostname first &
+        // connect directly via address to
+        // avoid proxy knowing which host we're connecting to
+        if (!(err instanceof AttestorError)
+            || err.data?.code !== 403) {
+            throw err;
+        }
+        const addrs = await resolveHostnames(opts.host);
+        logger.info({ addrs, host: opts.host }, 'failed to connect due to restricted IP, trying via raw addr');
+        for (const addr of addrs) {
+            try {
+                return await _getSocket({ ...opts, host: addr });
+            }
+            catch (err) {
+                logger.error({ addr, err }, 'failed to connect to host');
+            }
+        }
+        throw err;
+    }
+}
+async function _getSocket({ host, port, geoLocation, proxySessionId, logger }) {
+    const socket = new Socket();
+    if ((proxySessionId || geoLocation) && !HTTPS_PROXY_URL) {
+        logger.warn({ geoLocation, proxySessionId }, 'geoLocation or proxySessionId provided but no proxy URL found');
+        geoLocation = '';
+        proxySessionId = '';
+    }
+    if (!geoLocation && !proxySessionId) {
+        socket.connect({ host, port, });
+        return socket;
+    }
+    if (!isValidCountryCode(geoLocation)) {
+        throw AttestorError.badRequest(`Geolocation "${geoLocation}" is invalid. Must be 2 letter ISO country code`, { geoLocation });
+    }
+    if (proxySessionId && !isValidProxySessionId(proxySessionId)) {
+        throw AttestorError.badRequest(`proxySessionId "${proxySessionId}" is invalid. Must be a lowercase alphanumeric string of length 8-14 characters. eg. "mystring12345", "something1234".`, { proxySessionId });
+    }
+    const agentUrl = HTTPS_PROXY_URL.replace('{{geoLocation}}', geoLocation?.toLowerCase() || '').replace('{{proxySessionId}}', proxySessionId ? `-session-${proxySessionId}` : '');
+    const agent = new HttpsProxyAgent(agentUrl);
+    const waitForProxyRes = new Promise(resolve => {
+        // @ts-ignore
+        socket.once('proxyConnect', resolve);
+    });
+    const proxySocket = await agent.connect(
+    // ignore, because https-proxy-agent
+    // expects an http request object
+    // @ts-ignore
+    socket, { host, port, timeout: CONNECTION_TIMEOUT_MS });
+    const res = await waitForProxyRes;
+    if (res.statusCode !== 200) {
+        logger.error({ geoLocation, proxySessionId, res }, 'Proxy geo location or session id failed');
+        throw new AttestorError('ERROR_PROXY_ERROR', `Proxy via ${geoLocation ? `geo location "${geoLocation}"` : ''}${geoLocation && proxySessionId ? ', or ' : ''}${proxySessionId ? `session id "${proxySessionId}"` : ''} failed with status code: ${res.statusCode}, message: ${res.statusText}`, {
+            code: res.statusCode,
+            message: res.statusText,
+        });
+    }
+    process.nextTick(() => {
+        // ensure connect event is emitted
+        // so it can be captured by the caller
+        proxySocket.emit('connect');
+    });
+    return proxySocket;
+}

package/lib/server/utils/apm.js

@@ -0,0 +1,36 @@
+import ElasticAPM from 'elastic-apm-node';
+import { getEnvVariable } from "../../utils/env.js";
+import { logger } from "../../utils/logger.js";
+let apm;
+/**
+ * Initialises the APM agent if required,
+ * and returns it.
+ * If ELASTIC_APM_SERVER_URL & ELASTIC_APM_SECRET_TOKEN
+ * are not set will return undefined
+ *
+ * Utilises the standard env variables mentioned
+ * here: https://www.elastic.co/guide/en/apm/agent/nodejs/current/custom-stack.html#custom-stack-advanced-configuration
+ */
+export function getApm() {
+    if (!getEnvVariable('ELASTIC_APM_SERVER_URL')
+        || !getEnvVariable('ELASTIC_APM_SECRET_TOKEN')) {
+        logger.info('ELASTIC_APM_SERVER_URL or ELASTIC_APM_SECRET_TOKEN not found'
+            + ' in env, APM agent not initialised');
+        return undefined;
+    }
+    if (!apm) {
+        const sampleRate = +(getEnvVariable('ELASTIC_APM_SAMPLE_RATE')
+            || '0.1');
+        apm = ElasticAPM.start({
+            serviceName: 'reclaim_attestor',
+            serviceVersion: '4.0.0',
+            transactionSampleRate: sampleRate,
+            instrumentIncomingHTTPRequests: true,
+            usePathAsTransactionName: true,
+            instrument: true,
+            captureHeaders: true,
+        });
+        logger.info('initialised APM agent');
+    }
+    return apm;
+}

package/lib/server/utils/assert-valid-claim-request.js

@@ -0,0 +1,325 @@
+import { areUint8ArraysEqual, concatenateUint8Arrays } from '@reclaimprotocol/tls';
+import { ClaimTunnelRequest, TranscriptMessageSenderType } from "../../proto/api.js";
+import { providers } from "../../providers/index.js";
+import { niceParseJsonObject } from "./generics.js";
+import { computeOPRFRaw } from "./oprf-raw.js";
+import { processHandshake } from "./process-handshake.js";
+import { assertValidateProviderParams } from "./validation.js";
+import { AttestorError, binaryHashToStr, canonicalStringify, decryptDirect, extractApplicationDataFromTranscript, hashProviderParams, SIGNATURES, verifyZkPacket } from "../../utils/index.js";
+import { getEngineString } from "../../utils/zk.js";
+/**
+ * Asserts that the claim request is valid.
+ *
+ * 1. We begin by verifying the signature of the claim request.
+ * 2. Next, we produce the transcript of the TLS exchange
+ * from the proofs provided by the client.
+ * 3. We then pull the provider the client is trying to claim
+ * from
+ * 4. We then use the provider's verification function to verify
+ *  whether the claim is valid.
+ *
+ * If any of these steps fail, we throw an error.
+ */
+export async function assertValidClaimRequest(request, metadata, logger) {
+    const { data, signatures: { requestSignature } = {}, zkEngine, fixedServerIV, fixedClientIV } = request;
+    if (!data) {
+        throw new AttestorError('ERROR_INVALID_CLAIM', 'No info provided on claim request');
+    }
+    if (!requestSignature?.length) {
+        throw new AttestorError('ERROR_INVALID_CLAIM', 'No signature provided on claim request');
+    }
+    // verify request signature
+    const serialisedReq = ClaimTunnelRequest
+        .encode({ ...request, signatures: undefined })
+        .finish();
+    const { verify: verifySig } = SIGNATURES[metadata.signatureType];
+    const verified = await verifySig(serialisedReq, requestSignature, data.owner);
+    if (!verified) {
+        throw new AttestorError('ERROR_INVALID_CLAIM', 'Invalid signature on claim request');
+    }
+    const receipt = await decryptTranscript(request.transcript, logger, getEngineString(zkEngine), fixedServerIV, fixedClientIV);
+    const reqHost = request.request?.host;
+    if (receipt.hostname !== reqHost) {
+        throw new Error(`Expected server name ${reqHost}, got ${receipt.hostname}`);
+    }
+    // get all application data messages
+    const applData = extractApplicationDataFromTranscript(receipt);
+    const newData = await assertValidProviderTranscript(applData, data, logger, { version: metadata.clientVersion }, receipt.oprfRawReplacements);
+    if (newData !== data) {
+        logger.info({ newData }, 'updated claim info');
+    }
+    return newData;
+}
+/**
+ * Verify that the transcript contains a valid claim
+ * for the provider.
+ */
+export async function assertValidProviderTranscript(applData, info, logger, providerCtx, oprfRawReplacements) {
+    const providerName = info.provider;
+    const provider = providers[providerName];
+    if (!provider) {
+        throw new AttestorError('ERROR_INVALID_CLAIM', `Unsupported provider: ${providerName}`);
+    }
+    let params = niceParseJsonObject(info.parameters, 'params');
+    const ctx = niceParseJsonObject(info.context, 'context');
+    // Apply oprf-raw replacements to parameters (server-side OPRF)
+    if (oprfRawReplacements?.length) {
+        let strParams = canonicalStringify(params) ?? '{}';
+        for (const { originalText, nullifierText } of oprfRawReplacements) {
+            strParams = strParams.replaceAll(originalText, nullifierText);
+        }
+        params = JSON.parse(strParams);
+        // Update info.parameters with replaced values
+        info.parameters = strParams;
+        logger.debug({ replacements: oprfRawReplacements.length }, 'applied oprf-raw parameter replacements');
+    }
+    assertValidateProviderParams(providerName, params);
+    const rslt = await provider.assertValidProviderReceipt({
+        receipt: applData,
+        params,
+        logger,
+        ctx: providerCtx
+    });
+    ctx.providerHash = hashProviderParams(params);
+    const extractedParameters = rslt?.extractedParameters || {};
+    if (Object.keys(extractedParameters).length) {
+        ctx.extractedParameters = extractedParameters;
+    }
+    info.context = canonicalStringify(ctx) ?? '';
+    return info;
+}
+/**
+ * Verify that the transcript provided by the client
+ * matches the transcript of the tunnel, the server
+ * has created.
+ */
+export function assertTranscriptsMatch(clientTranscript, tunnelTranscript) {
+    const clientSends = concatenateUint8Arrays(clientTranscript
+        .filter(m => m.sender === TranscriptMessageSenderType.TRANSCRIPT_MESSAGE_SENDER_TYPE_CLIENT)
+        .map(m => m.message));
+    const tunnelSends = concatenateUint8Arrays(tunnelTranscript
+        .filter(m => m.sender === 'client')
+        .map(m => m.message));
+    if (!areUint8ArraysEqual(clientSends, tunnelSends)) {
+        throw AttestorError.badRequest('Outgoing messages from client do not match the tunnel transcript');
+    }
+    const clientRecvs = concatenateUint8Arrays(clientTranscript
+        .filter(m => m.sender === TranscriptMessageSenderType.TRANSCRIPT_MESSAGE_SENDER_TYPE_SERVER)
+        .map(m => m.message));
+    const tunnelRecvs = concatenateUint8Arrays(tunnelTranscript
+        .filter(m => m.sender === 'server')
+        .map(m => m.message))
+        // We only need to compare the first N messages
+        // that the client claims to have received
+        // the rest are not relevant -- so even if they're
+        // not present in the tunnel transcript, it's fine
+        .slice(0, clientRecvs.length);
+    if (!areUint8ArraysEqual(clientRecvs, tunnelRecvs)) {
+        throw AttestorError.badRequest('Incoming messages from server do not match the tunnel transcript');
+    }
+}
+export async function decryptTranscript(transcript, logger, zkEngine, serverIV, clientIV) {
+    const { tlsVersion, cipherSuite, hostname, nextMsgIndex } = await processHandshake(transcript, logger);
+    // TLS 1.3 has already one record encrypted at this point
+    let clientRecordNumber = tlsVersion === 'TLS1_3' ? -1 : 0;
+    let serverRecordNumber = clientRecordNumber;
+    transcript = transcript.slice(nextMsgIndex);
+    const overshotMap = {};
+    const decryptedTranscript = [];
+    const oprfRawReplacements = [];
+    // Track pending oprf-raw markers that span multiple packets
+    // keyed by packet index that will receive the overshot data
+    const pendingOprfRaw = {};
+    for (const [i, { sender, message, reveal: { zkReveal, directReveal } = {} }] of transcript.entries()) {
+        try {
+            //start with first message after last handshake message
+            await decryptMessage(sender, message, directReveal, zkReveal, i);
+        }
+        catch (error) {
+            const err = new AttestorError('ERROR_INVALID_CLAIM', `error in handling packet at idx ${i}: ${error}`, { packetIdx: i, error });
+            if (error.stack) {
+                err.stack = error.stack;
+            }
+            throw err;
+        }
+    }
+    // Fail if any oprf-raw markers remain incomplete
+    const remainingPending = Object.keys(pendingOprfRaw);
+    if (remainingPending.length) {
+        throw new AttestorError('ERROR_INVALID_CLAIM', `oprf-raw cross-block markers incomplete: pending for packets ${remainingPending.join(', ')}`);
+    }
+    return {
+        transcript: decryptedTranscript,
+        hostname: hostname,
+        tlsVersion: tlsVersion,
+        oprfRawReplacements: oprfRawReplacements.length ? oprfRawReplacements : undefined
+    };
+    async function decryptMessage(sender, message, directReveal, zkReveal, i) {
+        const isServer = sender === TranscriptMessageSenderType
+            .TRANSCRIPT_MESSAGE_SENDER_TYPE_SERVER;
+        const recordHeader = message.slice(0, 5);
+        const content = getWithoutHeader(message);
+        if (isServer) {
+            serverRecordNumber++;
+        }
+        else {
+            clientRecordNumber++;
+        }
+        let redacted = true;
+        let plaintext = undefined;
+        let plaintextLength;
+        if (directReveal?.key?.length) {
+            const result = await decryptDirect(directReveal, cipherSuite, recordHeader, tlsVersion, content);
+            plaintext = result.plaintext;
+            redacted = false;
+            plaintextLength = plaintext.length;
+        }
+        else if (zkReveal?.proofs?.length) {
+            const iv = sender === TranscriptMessageSenderType
+                .TRANSCRIPT_MESSAGE_SENDER_TYPE_SERVER
+                ? serverIV
+                : clientIV;
+            const recordNumber = isServer
+                ? serverRecordNumber
+                : clientRecordNumber;
+            const result = await verifyZkPacket({
+                ciphertext: content,
+                zkReveal,
+                iv,
+                recordNumber,
+                toprfOvershotNullifier: overshotMap[i]?.data,
+                getNextPacket(overshot) {
+                    const nextIdx = transcript
+                        .findIndex((t, j) => t.sender === sender && j > i);
+                    if (nextIdx < 0) {
+                        return;
+                    }
+                    overshotMap[nextIdx] = { data: overshot };
+                    return getWithoutHeader(transcript[nextIdx].message);
+                },
+                logger,
+                cipherSuite,
+                zkEngine: zkEngine,
+            });
+            plaintext = result.redactedPlaintext;
+            // Handle pending oprf-raw data from previous packet (cross-block)
+            const pendingForThis = pendingOprfRaw[i];
+            if (pendingForThis && zkReveal?.overshotOprfRawLength) {
+                const overshootLen = zkReveal.overshotOprfRawLength;
+                // Collect the overshot plaintext from this packet
+                const overshootData = plaintext.slice(0, overshootLen);
+                const fullData = concatenateUint8Arrays([
+                    pendingForThis.partialData,
+                    overshootData
+                ]);
+                // Verify accumulated length matches declared length
+                const expectedLen = pendingForThis.dataLocation.length;
+                if (fullData.length !== expectedLen) {
+                    throw new AttestorError('ERROR_INVALID_CLAIM', `oprf-raw cross-block length mismatch: got ${fullData.length}, expected ${expectedLen}`);
+                }
+                // Compute OPRF for the complete data
+                const oprfResults = await computeOPRFRaw(fullData, [{ dataLocation: { fromIndex: 0, length: fullData.length } }], logger);
+                if (oprfResults.length) {
+                    const { nullifier } = oprfResults[0];
+                    const originalText = new TextDecoder().decode(fullData);
+                    const nullifierStr = binaryHashToStr(nullifier, fullData.length);
+                    oprfRawReplacements.push({ originalText, nullifierText: nullifierStr });
+                    // Replace in original packet (handled when that packet was processed)
+                    // Replace in current packet
+                    const nullifierBytes = new TextEncoder().encode(nullifierStr);
+                    const overshootNullifier = nullifierBytes.slice(pendingForThis.partialData.length);
+                    plaintext.set(overshootNullifier, 0);
+                    // Also need to update the previous packet's plaintext
+                    // The previous packet has the first part of the nullifier
+                    const prevPkt = decryptedTranscript[pendingForThis.originPktIdx];
+                    if (prevPkt) {
+                        const firstPartNullifier = nullifierBytes.slice(0, pendingForThis.partialData.length);
+                        prevPkt.message.set(firstPartNullifier, pendingForThis.dataLocation.fromIndex);
+                    }
+                }
+                delete pendingOprfRaw[i];
+            }
+            // Process oprf-raw markers: compute OPRF server-side and replace with nullifier
+            if (result.oprfRawMarkers?.length) {
+                const { markersThisPacket, pendingMarker } = separateOprfRawMarkers(result.oprfRawMarkers, plaintext.length, () => transcript.findIndex((t, j) => t.sender === sender && j > i), decryptedTranscript.length, logger);
+                // Store pending marker for cross-block processing
+                if (pendingMarker) {
+                    // Copy partial data from plaintext
+                    pendingMarker.pending.partialData.set(plaintext.slice(pendingMarker.pending.dataLocation.fromIndex));
+                    pendingOprfRaw[pendingMarker.nextIdx] = pendingMarker.pending;
+                }
+                // Process markers that fit in this packet
+                if (markersThisPacket.length) {
+                    const pt = plaintext;
+                    const oprfResults = await computeOPRFRaw(pt, markersThisPacket, logger);
+                    // Capture all original texts BEFORE any replacements
+                    // to avoid reading corrupted data when markers are adjacent
+                    const originalTexts = oprfResults.map(({ dataLocation }) => new TextDecoder().decode(pt.slice(dataLocation.fromIndex, dataLocation.fromIndex + dataLocation.length)));
+                    // Now replace plaintext at marker positions with nullifier string
+                    for (const [idx, { dataLocation, nullifier }] of oprfResults.entries()) {
+                        const originalText = originalTexts[idx];
+                        const nullifierStr = binaryHashToStr(nullifier, dataLocation.length);
+                        oprfRawReplacements.push({ originalText, nullifierText: nullifierStr });
+                        const nullifierBytes = new TextEncoder().encode(nullifierStr);
+                        pt.set(nullifierBytes, dataLocation.fromIndex);
+                    }
+                }
+            }
+            redacted = false;
+            plaintextLength = plaintext.length;
+        }
+        else {
+            plaintext = content;
+            plaintextLength = plaintext.length;
+        }
+        decryptedTranscript.push({
+            sender: sender === TranscriptMessageSenderType
+                .TRANSCRIPT_MESSAGE_SENDER_TYPE_CLIENT
+                ? 'client'
+                : 'server',
+            redacted,
+            message: plaintext,
+            recordHeader,
+            plaintextLength,
+        });
+    }
+}
+export function getWithoutHeader(message) {
+    // strip the record header (xx 03 03 xx xx)
+    return message.slice(5);
+}
+/**
+ * Separate oprf-raw markers into those that fit in current packet
+ * vs those that span to the next packet
+ */
+function separateOprfRawMarkers(markers, plaintextLength, findNextPacketIdx, currentTranscriptLength, logger) {
+    const markersThisPacket = [];
+    let pendingMarker;
+    for (const marker of markers) {
+        const dataLocation = marker.dataLocation;
+        if (!dataLocation) {
+            continue;
+        }
+        const { fromIndex, length } = dataLocation;
+        const endInPacket = fromIndex + length;
+        if (endInPacket <= plaintextLength) {
+            markersThisPacket.push({ dataLocation });
+            continue;
+        }
+        // Spans to next packet
+        const nextIdx = findNextPacketIdx();
+        if (nextIdx < 0) {
+            throw new AttestorError('ERROR_INVALID_CLAIM', 'oprf-raw marker spans packets but no next packet found');
+        }
+        pendingMarker = {
+            nextIdx,
+            pending: {
+                partialData: new Uint8Array(plaintextLength - fromIndex),
+                dataLocation: { fromIndex, length },
+                originPktIdx: currentTranscriptLength
+            }
+        };
+        logger.debug({ fromIndex, length, partialLen: plaintextLength - fromIndex, nextIdx }, 'oprf-raw marker spans packets, storing partial data');
+    }
+    return { markersThisPacket, pendingMarker };
+}

package/lib/server/utils/config-env.js

@@ -0,0 +1,4 @@
+import { config } from 'dotenv';
+import { getEnvVariable } from "../../utils/env.js";
+const nodeEnv = getEnvVariable('NODE_ENV') || 'development';
+config({ path: `.env.${nodeEnv}` });

package/lib/server/utils/dns.js

@@ -0,0 +1,18 @@
+import { resolve, setServers } from 'dns';
+import { DNS_SERVERS } from "../../config/index.js";
+setDnsServers();
+export async function resolveHostnames(hostname) {
+    return new Promise((_resolve, reject) => {
+        resolve(hostname, (err, addresses) => {
+            if (err) {
+                reject(new Error(`Could not resolve hostname: ${hostname}, ${err.message}`));
+            }
+            else {
+                _resolve(addresses);
+            }
+        });
+    });
+}
+function setDnsServers() {
+    setServers(DNS_SERVERS);
+}