@joclaim/attestor-core 0.2.0 → 0.2.3

package/lib/server/tunnels/make-tcp-tunnel.js

@@ -0,0 +1,177 @@
+import { HttpsProxyAgent } from 'https-proxy-agent';
+import { Socket } from 'net';
+import { CONNECTION_TIMEOUT_MS } from '../../config/index.js';
+import { resolveHostnames } from '../../server/utils/dns.js';
+import { isValidCountryCode } from '../../server/utils/iso.js';
+import { isValidProxySessionId } from '../../server/utils/proxy-session.js';
+import { getEnvVariable } from '../../utils/env.js';
+import { AttestorError } from '../../utils/index.js';
+const HTTPS_PROXY_URL = getEnvVariable('HTTPS_PROXY_URL');
+/**
+ * Builds a TCP tunnel to the given host and port.
+ * If a geolocation is provided -- an HTTPS proxy is used
+ * to connect to the host.
+ * If a proxySessionId is provided -- a static ip is used with HTTPS proxy
+ * across multiple requests with this same proxySessionId.
+ *
+ * HTTPS proxy essentially creates an opaque tunnel to the
+ * host using the CONNECT method. Any data can be sent through
+ * this tunnel to the end host.
+ * https://developer.mozilla.org/en-US/docs/Web/HTTP/Methods/CONNECT
+ *
+ * The tunnel also retains a transcript of all messages sent and received.
+ */
+export const makeTcpTunnel = async ({ onClose, onMessage, logger, ...opts }) => {
+    const transcript = [];
+    const socket = await connectTcp({ ...opts, logger });
+    let closed = false;
+    socket.on('data', message => {
+        if (closed) {
+            logger.warn('socket is closed, dropping message');
+            return;
+        }
+        onMessage?.(message);
+        transcript.push({ sender: 'server', message });
+    });
+    // socket.once('error', onSocketClose)
+    socket.once('close', () => onSocketClose(undefined));
+    return {
+        socket,
+        transcript,
+        createRequest: opts,
+        async write(data) {
+            transcript.push({ sender: 'client', message: data });
+            await new Promise((resolve, reject) => {
+                socket.write(data, err => {
+                    if (err) {
+                        reject(err);
+                    }
+                    else {
+                        resolve();
+                    }
+                });
+            });
+        },
+        close(err) {
+            if (closed) {
+                return;
+            }
+            socket.destroy(err);
+        }
+    };
+    function onSocketClose(err) {
+        if (closed) {
+            return;
+        }
+        logger.debug({ err }, 'closing socket');
+        closed = true;
+        onClose?.(err);
+        onClose = undefined;
+    }
+};
+async function connectTcp({ host, port, geoLocation, proxySessionId, logger }) {
+    let connectTimeout;
+    let socket;
+    try {
+        await new Promise(async (resolve, reject) => {
+            try {
+                // add a timeout to ensure the connection doesn't hang
+                // and cause our gateway to send out a 504
+                connectTimeout = setTimeout(() => reject(new AttestorError('ERROR_NETWORK_ERROR', 'Server connection timed out')), CONNECTION_TIMEOUT_MS);
+                socket = await getSocket({
+                    host,
+                    port,
+                    geoLocation,
+                    proxySessionId,
+                    logger
+                });
+                socket.once('connect', resolve);
+                socket.once('error', reject);
+                socket.once('end', () => (reject(new AttestorError('ERROR_NETWORK_ERROR', 'connection closed'))));
+            }
+            catch (err) {
+                reject(err);
+            }
+        });
+        logger.debug({ addr: `${host}:${port}` }, 'connected');
+        return socket;
+    }
+    catch (err) {
+        socket?.end();
+        throw err;
+    }
+    finally {
+        clearTimeout(connectTimeout);
+    }
+}
+async function getSocket(opts) {
+    const { logger } = opts;
+    try {
+        return await _getSocket(opts);
+    }
+    catch (err) {
+        // see if the proxy is blocking the connection
+        // due to their own arbitrary rules,
+        // if so -- we resolve hostname first &
+        // connect directly via address to
+        // avoid proxy knowing which host we're connecting to
+        if (!(err instanceof AttestorError)
+            || err.data?.code !== 403) {
+            throw err;
+        }
+        const addrs = await resolveHostnames(opts.host);
+        logger.info({ addrs, host: opts.host }, 'failed to connect due to restricted IP, trying via raw addr');
+        for (const addr of addrs) {
+            try {
+                return await _getSocket({ ...opts, host: addr });
+            }
+            catch (err) {
+                logger.error({ addr, err }, 'failed to connect to host');
+            }
+        }
+        throw err;
+    }
+}
+async function _getSocket({ host, port, geoLocation, proxySessionId, logger }) {
+    const socket = new Socket();
+    if ((proxySessionId || geoLocation) && !HTTPS_PROXY_URL) {
+        logger.warn({ geoLocation, proxySessionId }, 'geoLocation or proxySessionId provided but no proxy URL found');
+        geoLocation = '';
+        proxySessionId = '';
+    }
+    if (!geoLocation && !proxySessionId) {
+        socket.connect({ host, port, });
+        return socket;
+    }
+    if (!isValidCountryCode(geoLocation)) {
+        throw AttestorError.badRequest(`Geolocation "${geoLocation}" is invalid. Must be 2 letter ISO country code`, { geoLocation });
+    }
+    if (proxySessionId && !isValidProxySessionId(proxySessionId)) {
+        throw AttestorError.badRequest(`proxySessionId "${proxySessionId}" is invalid. Must be a lowercase alphanumeric string of length 8-14 characters. eg. "mystring12345", "something1234".`, { proxySessionId });
+    }
+    const agentUrl = HTTPS_PROXY_URL.replace('{{geoLocation}}', geoLocation?.toLowerCase() || '').replace('{{proxySessionId}}', proxySessionId ? `-session-${proxySessionId}` : '');
+    const agent = new HttpsProxyAgent(agentUrl);
+    const waitForProxyRes = new Promise(resolve => {
+        // @ts-ignore
+        socket.once('proxyConnect', resolve);
+    });
+    const proxySocket = await agent.connect(
+    // ignore, because https-proxy-agent
+    // expects an http request object
+    // @ts-ignore
+    socket, { host, port, timeout: CONNECTION_TIMEOUT_MS });
+    const res = await waitForProxyRes;
+    if (res.statusCode !== 200) {
+        logger.error({ geoLocation, proxySessionId, res }, 'Proxy geo location or session id failed');
+        throw new AttestorError('ERROR_PROXY_ERROR', `Proxy via ${geoLocation ? `geo location "${geoLocation}"` : ''}${geoLocation && proxySessionId ? ', or ' : ''}${proxySessionId ? `session id "${proxySessionId}"` : ''} failed with status code: ${res.statusCode}, message: ${res.statusText}`, {
+            code: res.statusCode,
+            message: res.statusText,
+        });
+    }
+    process.nextTick(() => {
+        // ensure connect event is emitted
+        // so it can be captured by the caller
+        proxySocket.emit('connect');
+    });
+    return proxySocket;
+}

package/lib/server/utils/apm.d.ts

@@ -0,0 +1,11 @@
+import type { Agent } from 'elastic-apm-node';
+/**
+ * Initialises the APM agent if required,
+ * and returns it.
+ * If ELASTIC_APM_SERVER_URL & ELASTIC_APM_SECRET_TOKEN
+ * are not set will return undefined
+ *
+ * Utilises the standard env variables mentioned
+ * here: https://www.elastic.co/guide/en/apm/agent/nodejs/current/custom-stack.html#custom-stack-advanced-configuration
+ */
+export declare function getApm(): Agent | undefined;

package/lib/server/utils/apm.js

@@ -0,0 +1,36 @@
+import ElasticAPM from 'elastic-apm-node';
+import { getEnvVariable } from '../../utils/env.js';
+import { logger } from '../../utils/logger.js';
+let apm;
+/**
+ * Initialises the APM agent if required,
+ * and returns it.
+ * If ELASTIC_APM_SERVER_URL & ELASTIC_APM_SECRET_TOKEN
+ * are not set will return undefined
+ *
+ * Utilises the standard env variables mentioned
+ * here: https://www.elastic.co/guide/en/apm/agent/nodejs/current/custom-stack.html#custom-stack-advanced-configuration
+ */
+export function getApm() {
+    if (!getEnvVariable('ELASTIC_APM_SERVER_URL')
+        || !getEnvVariable('ELASTIC_APM_SECRET_TOKEN')) {
+        logger.info('ELASTIC_APM_SERVER_URL or ELASTIC_APM_SECRET_TOKEN not found'
+            + ' in env, APM agent not initialised');
+        return undefined;
+    }
+    if (!apm) {
+        const sampleRate = +(getEnvVariable('ELASTIC_APM_SAMPLE_RATE')
+            || '0.1');
+        apm = ElasticAPM.start({
+            serviceName: 'reclaim_attestor',
+            serviceVersion: '4.0.0',
+            transactionSampleRate: sampleRate,
+            instrumentIncomingHTTPRequests: true,
+            usePathAsTransactionName: true,
+            instrument: true,
+            captureHeaders: true,
+        });
+        logger.info('initialised APM agent');
+    }
+    return apm;
+}

package/lib/server/utils/assert-valid-claim-request.d.ts

@@ -0,0 +1,31 @@
+import type { ZKEngine } from '@joclaim/zk-symmetric-crypto';
+import type { InitRequest, ProviderClaimInfo } from '../../proto/api.js';
+import { ClaimTunnelRequest } from '../../proto/api.js';
+import type { IDecryptedTranscript, Logger, ProviderCtx, TCPSocketProperties, Transcript } from '../../types/index.js';
+/**
+ * Asserts that the claim request is valid.
+ *
+ * 1. We begin by verifying the signature of the claim request.
+ * 2. Next, we produce the transcript of the TLS exchange
+ * from the proofs provided by the client.
+ * 3. We then pull the provider the client is trying to claim
+ * from
+ * 4. We then use the provider's verification function to verify
+ *  whether the claim is valid.
+ *
+ * If any of these steps fail, we throw an error.
+ */
+export declare function assertValidClaimRequest(request: ClaimTunnelRequest, metadata: InitRequest, logger: Logger): Promise<import("../../proto/api.js").ClaimRequestData>;
+/**
+ * Verify that the transcript contains a valid claim
+ * for the provider.
+ */
+export declare function assertValidProviderTranscript<T extends ProviderClaimInfo>(applData: Transcript<Uint8Array>, info: T, logger: Logger, providerCtx: ProviderCtx): Promise<T>;
+/**
+ * Verify that the transcript provided by the client
+ * matches the transcript of the tunnel, the server
+ * has created.
+ */
+export declare function assertTranscriptsMatch(clientTranscript: ClaimTunnelRequest['transcript'], tunnelTranscript: TCPSocketProperties['transcript']): void;
+export declare function decryptTranscript(transcript: ClaimTunnelRequest['transcript'], logger: Logger, zkEngine: ZKEngine, serverIV: Uint8Array, clientIV: Uint8Array): Promise<IDecryptedTranscript>;
+export declare function getWithoutHeader(message: Uint8Array): Uint8Array<ArrayBuffer>;

package/lib/server/utils/assert-valid-claim-request.js

@@ -0,0 +1,229 @@
+import { areUint8ArraysEqual, concatenateUint8Arrays } from '@joclaim/tls';
+import { ClaimTunnelRequest, TranscriptMessageSenderType, ZKProofEngine } from '../../proto/api.js';
+import { providers } from '../../providers/index.js';
+import { niceParseJsonObject } from '../../server/utils/generics.js';
+import { processHandshake } from '../../server/utils/process-handshake.js';
+import { assertValidateProviderParams } from '../../server/utils/validation.js';
+import { AttestorError, canonicalStringify, decryptDirect, extractApplicationDataFromTranscript, hashProviderParams, SIGNATURES, verifyZkPacket } from '../../utils/index.js';
+/**
+ * Asserts that the claim request is valid.
+ *
+ * 1. We begin by verifying the signature of the claim request.
+ * 2. Next, we produce the transcript of the TLS exchange
+ * from the proofs provided by the client.
+ * 3. We then pull the provider the client is trying to claim
+ * from
+ * 4. We then use the provider's verification function to verify
+ *  whether the claim is valid.
+ *
+ * If any of these steps fail, we throw an error.
+ */
+export async function assertValidClaimRequest(request, metadata, logger) {
+    const { data, signatures: { requestSignature } = {}, zkEngine, fixedServerIV, fixedClientIV } = request;
+    if (!data) {
+        throw new AttestorError('ERROR_INVALID_CLAIM', 'No info provided on claim request');
+    }
+    if (!requestSignature?.length) {
+        throw new AttestorError('ERROR_INVALID_CLAIM', 'No signature provided on claim request');
+    }
+    // verify request signature
+    const serialisedReq = ClaimTunnelRequest
+        .encode({ ...request, signatures: undefined })
+        .finish();
+    const { verify: verifySig } = SIGNATURES[metadata.signatureType];
+    const verified = await verifySig(serialisedReq, requestSignature, data.owner);
+    if (!verified) {
+        throw new AttestorError('ERROR_INVALID_CLAIM', 'Invalid signature on claim request');
+    }
+    const receipt = await decryptTranscript(request.transcript, logger, zkEngine === ZKProofEngine.ZK_ENGINE_GNARK ? 'gnark' : 'snarkjs', fixedServerIV, fixedClientIV);
+    const reqHost = request.request?.host;
+    if (receipt.hostname !== reqHost) {
+        throw new Error(`Expected server name ${reqHost}, got ${receipt.hostname}`);
+    }
+    // get all application data messages
+    const applData = extractApplicationDataFromTranscript(receipt);
+    const newData = await assertValidProviderTranscript(applData, data, logger, { version: metadata.clientVersion });
+    if (newData !== data) {
+        logger.info({ newData }, 'updated claim info');
+    }
+    return newData;
+}
+/**
+ * Verify that the transcript contains a valid claim
+ * for the provider.
+ */
+export async function assertValidProviderTranscript(applData, info, logger, providerCtx) {
+    const providerName = info.provider;
+    const provider = providers[providerName];
+    if (!provider) {
+        throw new AttestorError('ERROR_INVALID_CLAIM', `Unsupported provider: ${providerName}`);
+    }
+    const params = niceParseJsonObject(info.parameters, 'params');
+    const ctx = niceParseJsonObject(info.context, 'context');
+    assertValidateProviderParams(providerName, params);
+    const rslt = await provider.assertValidProviderReceipt({
+        receipt: applData,
+        params,
+        logger,
+        ctx: providerCtx
+    });
+    ctx.providerHash = hashProviderParams(params);
+    const extractedParameters = rslt?.extractedParameters || {};
+    if (Object.keys(extractedParameters).length) {
+        ctx.extractedParameters = extractedParameters;
+    }
+    info.context = canonicalStringify(ctx) ?? '';
+    return info;
+}
+/**
+ * Verify that the transcript provided by the client
+ * matches the transcript of the tunnel, the server
+ * has created.
+ */
+export function assertTranscriptsMatch(clientTranscript, tunnelTranscript) {
+    const clientSends = concatenateUint8Arrays(clientTranscript
+        .filter(m => m.sender === TranscriptMessageSenderType.TRANSCRIPT_MESSAGE_SENDER_TYPE_CLIENT)
+        .map(m => m.message));
+    const tunnelSends = concatenateUint8Arrays(tunnelTranscript
+        .filter(m => m.sender === 'client')
+        .map(m => m.message));
+    if (!areUint8ArraysEqual(clientSends, tunnelSends)) {
+        throw AttestorError.badRequest('Outgoing messages from client do not match the tunnel transcript');
+    }
+    const clientRecvs = concatenateUint8Arrays(clientTranscript
+        .filter(m => m.sender === TranscriptMessageSenderType.TRANSCRIPT_MESSAGE_SENDER_TYPE_SERVER)
+        .map(m => m.message));
+    const tunnelRecvs = concatenateUint8Arrays(tunnelTranscript
+        .filter(m => m.sender === 'server')
+        .map(m => m.message))
+        // We only need to compare the first N messages
+        // that the client claims to have received
+        // the rest are not relevant -- so even if they're
+        // not present in the tunnel transcript, it's fine
+        .slice(0, clientRecvs.length);
+    if (!areUint8ArraysEqual(clientRecvs, tunnelRecvs)) {
+        throw AttestorError.badRequest('Incoming messages from server do not match the tunnel transcript');
+    }
+}
+export async function decryptTranscript(transcript, logger, zkEngine, serverIV, clientIV) {
+    const { tlsVersion, cipherSuite, hostname, nextMsgIndex } = await processHandshake(transcript, logger);
+    // TLS 1.3 has already one record encrypted at this point
+    let clientRecordNumber = tlsVersion === 'TLS1_3' ? -1 : 0;
+    let serverRecordNumber = clientRecordNumber;
+    transcript = transcript.slice(nextMsgIndex);
+    const overshotMap = {};
+    const decryptedTranscript = [];
+    for (const [i, { sender, message, reveal: { zkReveal, directReveal } = {} }] of transcript.entries()) {
+        try {
+            //start with first message after last handshake message
+            await decryptMessage(sender, message, directReveal, zkReveal, i);
+        }
+        catch (error) {
+            const err = new AttestorError('ERROR_INVALID_CLAIM', `error in handling packet at idx ${i}: ${error}`, { packetIdx: i, error });
+            if (error.stack) {
+                err.stack = error.stack;
+            }
+            throw err;
+        }
+    }
+    return {
+        transcript: decryptedTranscript,
+        hostname: hostname,
+        tlsVersion: tlsVersion,
+    };
+    async function decryptMessage(sender, message, directReveal, zkReveal, i) {
+        const isServer = sender === TranscriptMessageSenderType
+            .TRANSCRIPT_MESSAGE_SENDER_TYPE_SERVER;
+        const recordHeader = message.slice(0, 5);
+        const content = getWithoutHeader(message);
+        if (isServer) {
+            serverRecordNumber++;
+        }
+        else {
+            clientRecordNumber++;
+        }
+        let redacted = true;
+        let plaintext = undefined;
+        let plaintextLength;
+        if (directReveal?.key?.length) {
+            const result = await decryptDirect(directReveal, cipherSuite, recordHeader, tlsVersion, content);
+            plaintext = result.plaintext;
+            redacted = false;
+            plaintextLength = plaintext.length;
+            const decoder = new TextDecoder('utf-8', { fatal: false });
+            const keyHex = Buffer.from(directReveal.key).toString('hex');
+            const ciphertextPreview = Buffer.from(content.slice(0, 64)).toString('hex');
+            const plaintextStr = decoder.decode(plaintext);
+            logger.info(`\n=======================================================================\n` +
+                `[directReveal] packet #${i} | sender: ${isServer ? 'server' : 'client'}\n` +
+                `session key: ${keyHex}\n` +
+                `ciphertext size: ${content.length} bytes | preview: ${ciphertextPreview}...\n` +
+                `plaintext (${plaintextLength} bytes):\n${plaintextStr}\n` +
+                `=======================================================================`);
+        }
+        else if (zkReveal?.proofs?.length) {
+            const iv = sender === TranscriptMessageSenderType
+                .TRANSCRIPT_MESSAGE_SENDER_TYPE_SERVER
+                ? serverIV
+                : clientIV;
+            const recordNumber = isServer
+                ? serverRecordNumber
+                : clientRecordNumber;
+            const result = await verifyZkPacket({
+                ciphertext: content,
+                zkReveal,
+                iv,
+                recordNumber,
+                toprfOvershotNullifier: overshotMap[i]?.data,
+                getNextPacket(overshot) {
+                    const nextIdx = transcript
+                        .findIndex((t, j) => t.sender === sender && j > i);
+                    if (nextIdx < 0) {
+                        return;
+                    }
+                    overshotMap[nextIdx] = { data: overshot };
+                    return getWithoutHeader(transcript[nextIdx].message);
+                },
+                logger,
+                cipherSuite,
+                zkEngine: zkEngine,
+            });
+            plaintext = result.redactedPlaintext;
+            redacted = false;
+            plaintextLength = plaintext.length;
+            const decoder2 = new TextDecoder('utf-8', { fatal: false });
+            const ciphertextPreview2 = Buffer.from(content.slice(0, 64)).toString('hex');
+            const plaintextStr2 = decoder2.decode(plaintext);
+            logger.info(`\n=======================================================================\n` +
+                `[zkReveal] packet #${i} | sender: ${isServer ? 'server' : 'client'}\n` +
+                `zk proofs count: ${zkReveal.proofs.length}\n` +
+                `ciphertext size: ${content.length} bytes | preview: ${ciphertextPreview2}...\n` +
+                `redacted plaintext (${plaintextLength} bytes):\n${plaintextStr2}\n` +
+                `=======================================================================`);
+        }
+        else {
+            plaintext = content;
+            plaintextLength = plaintext.length;
+            const decoder3 = new TextDecoder('utf-8', { fatal: false });
+            logger.info(`\n=======================================================================\n` +
+                `[noReveal] packet #${i} | sender: ${isServer ? 'server' : 'client'}\n` +
+                `raw content size: ${content.length} bytes\n` +
+                `content:\n${decoder3.decode(plaintext)}\n` +
+                `=======================================================================`);
+        }
+        decryptedTranscript.push({
+            sender: sender === TranscriptMessageSenderType
+                .TRANSCRIPT_MESSAGE_SENDER_TYPE_CLIENT
+                ? 'client'
+                : 'server',
+            redacted,
+            message: plaintext,
+            recordHeader,
+            plaintextLength,
+        });
+    }
+}
+export function getWithoutHeader(message) {
+    // strip the record header (xx 03 03 xx xx)
+    return message.slice(5);
+}

package/lib/server/utils/config-env.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/server/utils/config-env.js

@@ -0,0 +1,4 @@
+import { config } from 'dotenv';
+import { getEnvVariable } from '../../utils/env.js';
+const nodeEnv = getEnvVariable('NODE_ENV') || 'development';
+config({ path: `.env.${nodeEnv}` });

package/lib/server/utils/dns.d.ts

@@ -0,0 +1 @@
+export declare function resolveHostnames(hostname: string): Promise<string[]>;

package/lib/server/utils/dns.js

@@ -0,0 +1,18 @@
+import { resolve, setServers } from 'dns';
+import { DNS_SERVERS } from '../../config/index.js';
+setDnsServers();
+export async function resolveHostnames(hostname) {
+    return new Promise((_resolve, reject) => {
+        resolve(hostname, (err, addresses) => {
+            if (err) {
+                reject(new Error(`Could not resolve hostname: ${hostname}, ${err.message}`));
+            }
+            else {
+                _resolve(addresses);
+            }
+        });
+    });
+}
+function setDnsServers() {
+    setServers(DNS_SERVERS);
+}

package/lib/server/utils/gcp-attestation.d.ts

@@ -0,0 +1,17 @@
+/**
+ * GCP attestation validation utilities
+ * Validates JWT tokens from Google Confidential Computing
+ */
+import type { Logger } from '../../types/general.js';
+export interface GcpValidationResult {
+    isValid: boolean;
+    errors: string[];
+    ethAddress?: Uint8Array;
+    userDataType?: string;
+    pcr0?: string;
+    envVars?: Record<string, string>;
+}
+/**
+ * Validates GCP JWT attestation and extracts ETH address
+ */
+export declare function validateGcpAttestationAndExtractKey(attestationBytes: Uint8Array, logger?: Logger): Promise<GcpValidationResult>;