@learncard/lca-api-service 1.5.1 → 1.5.3

package/src/services/escrow-enclave/remoteEnclave.tls.test.ts

@@ -1,9 +1,10 @@
 import { readFileSync } from 'node:fs';
 import { createServer, type Server } from 'node:https';
 import type { AddressInfo } from 'node:net';
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { createHttpTransport } from './remoteEnclave';
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import { createHttpTransport, createRemoteEnclave } from './remoteEnclave';
 import { pinnedEnclaveCa } from './enclaveTrustAnchors';
+import { logEscrowFailure } from './logFailure';
 
 const fixture = (name: string): string =>
     readFileSync(new URL(`./__fixtures__/tls/${name}`, import.meta.url), 'utf8');
@@ -60,3 +61,182 @@ describe('remote enclave TLS trust', () => {
         expect(pinnedEnclaveCa('https://enclave-host.internal')).toBeUndefined();
     });
 });
+
+describe('remote enclave connection recovery', () => {
+    const servers: Server[] = [];
+    const config = { headers: { Authorization: 'Bearer test-secret' }, timeout: 1_000 };
+    const serve = async (handler: Parameters<typeof createServer>[1]): Promise<string> => {
+        const server = createServer(
+            { cert: fixture('pinned-leaf.pem'), key: fixture('pinned-leaf.key') },
+            handler
+        );
+        servers.push(server);
+        await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));
+        return url(server).replace('/v1/attest', '');
+    };
+
+    afterEach(async () => {
+        vi.restoreAllMocks();
+        await Promise.all(
+            servers.splice(0).map(
+                server =>
+                    new Promise<void>(resolve => {
+                        server.closeAllConnections();
+                        server.close(() => resolve());
+                    })
+            )
+        );
+    });
+
+    it.each(['/v1/attest', '/v1/enrollment-state', '/v1/hold-state'])(
+        'retries %s once on a pre-response reset, on a fresh TLS connection',
+        async path => {
+            const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+            const sockets = new Set();
+            let requests = 0;
+            const origin = await serve((request, response) => {
+                sockets.add(request.socket);
+                requests += 1;
+                if (requests === 1) request.socket.destroy();
+                else response.end('{"ok":true}');
+            });
+            const transport = createHttpTransport(fixture('pinned-ca.pem'));
+            await expect(
+                transport.post(`${origin}${path}`, { secret: 'private-body' }, config)
+            ).resolves.toEqual({ status: 200, data: { ok: true } });
+            expect(requests).toBe(2);
+            expect(sockets.size).toBe(2);
+            expect(warn.mock.calls).toEqual([
+                ['[LCA escrow] retrying enclave read', { code: 'ECONNRESET' }],
+            ]);
+        }
+    );
+
+    it('does not pool successful requests across invocations', async () => {
+        const sockets = new Set();
+        const connections: (string | undefined)[] = [];
+        const origin = await serve((request, response) => {
+            sockets.add(request.socket);
+            connections.push(request.headers.connection);
+            response.end('{"ok":true}');
+        });
+        const transport = createHttpTransport(fixture('pinned-ca.pem'));
+        await transport.post(`${origin}/v1/attest`, {}, config);
+        await transport.post(`${origin}/v1/attest`, {}, config);
+        expect(sockets.size).toBe(2);
+        expect(connections).toEqual(['close', 'close']);
+    });
+
+    it.each([
+        '/v1/create-hold',
+        '/v1/confirm-hold',
+        '/v1/release',
+        '/v1/cancel-hold',
+        '/v1/apply-enrollment',
+        '/v1/authority-rewrap',
+        '/v1/carry-pin-verifier',
+        '/v1/resend-code',
+        '/v1/send-release-warning',
+        '/v1/unknown',
+    ])('never retries %s even when no response arrived', async path => {
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let requests = 0;
+        const origin = await serve(request => {
+            requests += 1;
+            request.socket.destroy();
+        });
+        await expect(
+            createHttpTransport(fixture('pinned-ca.pem')).post(`${origin}${path}`, {}, config)
+        ).rejects.toMatchObject({ code: 'ECONNRESET' });
+        expect(requests).toBe(1);
+        expect(warn).not.toHaveBeenCalled();
+    });
+
+    it.each([429, 503, 302])('does not retry or follow HTTP %s', async status => {
+        let requests = 0;
+        const origin = await serve((_request, response) => {
+            requests += 1;
+            response.writeHead(status, { Location: '/v1/attest' });
+            response.end('{}');
+        });
+        await expect(
+            createHttpTransport(fixture('pinned-ca.pem')).post(`${origin}/v1/attest`, {}, config)
+        ).resolves.toMatchObject({ status });
+        expect(requests).toBe(1);
+    });
+
+    it('does not retry a timeout', async () => {
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let requests = 0;
+        const origin = await serve(() => {
+            requests += 1;
+        });
+        await expect(
+            createHttpTransport(fixture('pinned-ca.pem')).post(
+                `${origin}/v1/attest`,
+                {},
+                { ...config, timeout: 100 }
+            )
+        ).rejects.toThrow();
+        expect(requests).toBe(1);
+        expect(warn).not.toHaveBeenCalled();
+    });
+
+    it('does not retry after even partial response headers', async () => {
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let requests = 0;
+        const origin = await serve(request => {
+            requests += 1;
+            request.socket.write('HTTP/1.1 200 OK\r\n');
+            setTimeout(() => request.socket.destroy(), 20);
+        });
+        await expect(
+            createHttpTransport(fixture('pinned-ca.pem')).post(`${origin}/v1/attest`, {}, config)
+        ).rejects.toThrow();
+        expect(requests).toBe(1);
+        expect(warn).not.toHaveBeenCalled();
+    });
+
+    it('keeps the second attempt within the original wall-clock budget', async () => {
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let requests = 0;
+        const origin = await serve(request => {
+            requests += 1;
+            if (requests === 1) setTimeout(() => request.socket.destroy(), 150);
+        });
+        const start = performance.now();
+        await expect(
+            createHttpTransport(fixture('pinned-ca.pem')).post(
+                `${origin}/v1/attest`,
+                {},
+                { ...config, timeout: 300 }
+            )
+        ).rejects.toThrow();
+        expect(performance.now() - start).toBeLessThan(430);
+        expect(requests).toBe(2);
+        expect(warn).toHaveBeenCalledOnce();
+    });
+
+    it('stops after one retry and preserves sanitized transport failure logging', async () => {
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let requests = 0;
+        const origin = await serve(request => {
+            requests += 1;
+            request.socket.destroy();
+        });
+        const enclave = createRemoteEnclave({
+            baseUrl: origin,
+            token: 'private-token',
+            timeoutMs: 1_000,
+            caPem: fixture('pinned-ca.pem'),
+        });
+        const error = await enclave.getAttestation().catch(error => error);
+        expect(error).toMatchObject({ reason: 'transport' });
+        logEscrowFailure(error);
+        expect(requests).toBe(2);
+        expect(warn.mock.calls).toEqual([
+            ['[LCA escrow] retrying enclave read', { code: 'ECONNRESET' }],
+            ['[LCA escrow] enclave operation failed', { reason: 'transport' }],
+        ]);
+    });
+});

package/src/services/escrow-enclave/remoteEnclave.ts

@@ -1,6 +1,14 @@
 import { Agent } from 'node:https';
+import { Agent as HttpAgent, ClientRequest } from 'node:http';
 import axios from 'axios';
 import { z } from 'zod';
+import {
+    AuthorityCommitValidator,
+    AuthorityHoldReceiptValidator,
+    AuthorityMutationValidator,
+    AuthorityStateResponseValidator,
+    safeInteger,
+} from './authorityContract';
 import {
     ESCROW_ALGORITHM,
     ESCROW_ENVELOPE_VERSION,
@@ -9,14 +17,18 @@ import {
 import {
     EscrowBlobError,
     EscrowPinMismatchError,
+    EscrowCodeError,
     EscrowPolicyError,
     EscrowUnavailableError,
     type CarryPinVerifierInput,
+    type ConfirmHoldRequest,
     type EnclaveAttestation,
     type EscrowEnclave,
     type ReleaseRequest,
     type ReleaseResult,
+    type ResendCodeRequest,
     type RewrapEscrowBlobInput,
+    type SendReleaseWarningRequest,
     type VerifyEscrowBlobInput,
     type VerifyEscrowBlobResult,
 } from './types';
@@ -58,7 +70,6 @@ const verifyBlobResponseValidator = z.union([
 ]);
 
 const carryPinVerifierResponseValidator = z.object({ envelope: envelopeValidator });
-const rewrapEscrowBlobResponseValidator = z.object({ envelope: envelopeValidator });
 
 // Opaque SignedHoldRecord JSON: validate shape while preserving signed extensions.
 const holdRecordValidator = z
@@ -74,23 +85,41 @@ const holdRecordValidator = z
                 clientEphemeralPublicKey: z.string().min(1),
                 createdLo: z.number().int().nonnegative(),
                 createdHi: z.number().int().nonnegative(),
-                policyVersion: z.number().int().positive(),
-                signature: z.string().min(1),
+                policyVersion: z.union([z.literal(1), z.literal(2)]),
+                signature: z.string().min(1).max(8192),
             })
             .passthrough(),
         holdDurationMs: z.number().int().nonnegative(),
         ledgerSeq: z.number().int().nonnegative(),
     })
     .passthrough();
-const validatedHold = (hold: ReleaseRequest['hold']): ReleaseRequest['hold'] => {
-    if (!holdRecordValidator.safeParse(hold).success) throw new EscrowUnavailableError();
+const validatedHold = (
+    hold: ReleaseRequest['hold'],
+    reason: 'invalid_hold' | 'invalid_response' = 'invalid_hold'
+): ReleaseRequest['hold'] => {
+    if (!holdRecordValidator.safeParse(hold).success) throw new EscrowUnavailableError(reason);
+    if (hold.hold.policyVersion === 2) {
+        try {
+            AuthorityCommitValidator.parse(
+                JSON.parse(Buffer.from(hold.hold.signature, 'base64').toString('utf8'))
+            );
+        } catch {
+            throw new EscrowUnavailableError(reason);
+        }
+    }
     return hold;
 };
-const releaseResponseValidator = z.object({ sealed: envelopeValidator });
+const releaseResponseValidator = z.object({
+    sealed: envelopeValidator,
+    receipt: AuthorityCommitValidator.optional(),
+});
 
 const errorCodeValidator = z.enum([
     'policy',
     'pinMismatch',
+    'codeMismatch',
+    'codeExpired',
+    'codeLocked',
     'blob',
     'unavailable',
     'ledger',
@@ -101,8 +130,10 @@ const errorResponseValidator = z.object({ code: errorCodeValidator, message: z.s
 const errorForCode = (code: z.infer<typeof errorCodeValidator>): Error => {
     if (code === 'policy') return new EscrowPolicyError();
     if (code === 'pinMismatch') return new EscrowPinMismatchError();
+    if (code === 'codeMismatch' || code === 'codeExpired' || code === 'codeLocked')
+        return new EscrowCodeError(code);
     if (code === 'blob') return new EscrowBlobError();
-    return new EscrowUnavailableError(); // 'unavailable' | 'ledger' | 'time'
+    return new EscrowUnavailableError(code); // 'unavailable' | 'ledger' | 'time'
 };
 
 export interface RemoteEnclaveTransportResponse {
@@ -120,15 +151,57 @@ export interface RemoteEnclaveTransport {
 
 /** With `caPem`, the connection trusts only that CA (system roots are not consulted). */
 export const createHttpTransport = (caPem?: string): RemoteEnclaveTransport => {
-    const httpsAgent = caPem ? new Agent({ ca: caPem, keepAlive: true }) : undefined;
+    // The host enforces a 60s absolute connection lifetime, even for busy sockets.
+    // Do not retain sockets across Lambda freezes (or use Node's global pool).
+    const httpsAgent = new Agent({ ...(caPem ? { ca: caPem } : {}), keepAlive: false });
+    const httpAgent = new HttpAgent({ keepAlive: false });
     return {
         post: async (url, body, config) => {
-            const response = await axios.post(url, body, {
-                ...config,
-                httpsAgent,
-                validateStatus: () => true,
-            });
-            return { status: response.status, data: response.data };
+            const path = new URL(url).pathname;
+            const safeRead = ['/v1/attest', '/v1/enrollment-state', '/v1/hold-state'].includes(
+                path
+            );
+            const deadline = performance.now() + config.timeout;
+            const controller = new AbortController();
+            // One wall-clock budget covers connection setup and BOTH attempts.
+            const timer = setTimeout(() => controller.abort(), config.timeout);
+            try {
+                for (let attempt = 0; ; attempt += 1) {
+                    try {
+                        const response = await axios.post(url, body, {
+                            ...config,
+                            timeout: Math.max(1, Math.ceil(deadline - performance.now())),
+                            signal: controller.signal,
+                            httpsAgent,
+                            httpAgent,
+                            // Keep the native ClientRequest observable; never redirect a vault POST.
+                            maxRedirects: 0,
+                            validateStatus: () => true,
+                        });
+                        return { status: response.status, data: response.data };
+                    } catch (error) {
+                        const code = axios.isAxiosError(error) ? error.code : undefined;
+                        if (
+                            attempt !== 0 ||
+                            !safeRead ||
+                            (code !== 'ECONNRESET' && code !== 'EPIPE') ||
+                            !axios.isAxiosError(error) ||
+                            error.response ||
+                            !(error.request instanceof ClientRequest) ||
+                            error.request.socket?.bytesRead !== 0 ||
+                            controller.signal.aborted ||
+                            performance.now() >= deadline
+                        )
+                            throw error;
+                        // No response does NOT prove the server did not execute the request.
+                        // Never replay mutations: holds, PIN attempts, delivery and enrollment
+                        // may already be committed. Only the three read paths above are safe.
+                        console.warn('[LCA escrow] retrying enclave read', { code });
+                    }
+                }
+            } finally {
+                clearTimeout(timer);
+            }
         },
     };
 };
@@ -162,25 +235,46 @@ export const createRemoteEnclave = (config: RemoteEnclaveConfig): EscrowEnclave
         let data: unknown;
         try {
             const response = await transport.post(`${config.baseUrl}${path}`, body, {
-                headers: { Authorization: `Bearer ${config.token}` },
+                headers: {
+                    Authorization: `Bearer ${config.token}`,
+                    ...(path === '/v1/create-hold' ? { 'X-Escrow-Delivery-Evidence': '1' } : {}),
+                },
                 timeout: config.timeoutMs,
             });
             status = response.status;
             data = response.data;
         } catch {
-            throw new EscrowUnavailableError();
+            throw new EscrowUnavailableError('transport');
         }
         if (status < 200 || status >= 300) {
             const parsedError = errorResponseValidator.safeParse(data);
             if (parsedError.success) throw errorForCode(parsedError.data.code);
-            throw new EscrowUnavailableError();
+            throw new EscrowUnavailableError(status === 429 ? 'rate_limit' : 'http_refusal');
         }
         const parsed = validator.safeParse(data);
-        if (!parsed.success) throw new EscrowUnavailableError();
+        if (!parsed.success) throw new EscrowUnavailableError('invalid_response');
         return parsed.data;
     };
 
     return {
+        getEnrollmentState: input =>
+            call('/v1/enrollment-state', input, AuthorityStateResponseValidator),
+        applyEnrollmentIntent: async input => {
+            const { result } = await call(
+                '/v1/apply-enrollment',
+                input,
+                z.object({ result: AuthorityMutationValidator }).strict()
+            );
+            if (
+                result.authorityGeneration !== input.signedIntent.intent.authorityGeneration ||
+                result.enrollmentVersion !==
+                    input.signedIntent.intent.expectedEnrollmentVersion + 1 ||
+                result.enrollmentEpoch !== input.signedIntent.intent.enrollmentEpoch ||
+                (result.envelope === null) !== (input.signedIntent.intent.action === 'disable')
+            )
+                throw new EscrowUnavailableError('invalid_response');
+            return result;
+        },
         getAttestation: (nonce?: Uint8Array): Promise<EnclaveAttestation> =>
             call('/v1/attest', { nonce: Array.from(nonce ?? []) }, attestResponseValidator),
         verifyEscrowBlob: (input: VerifyEscrowBlobInput): Promise<VerifyEscrowBlobResult> =>
@@ -207,19 +301,23 @@ export const createRemoteEnclave = (config: RemoteEnclaveConfig): EscrowEnclave
                 },
                 carryPinVerifierResponseValidator
             ),
-        rewrapEscrowBlob: (input: RewrapEscrowBlobInput): Promise<{ envelope: EscrowEnvelope }> =>
-            call(
-                '/v1/rewrap-escrow-blob',
+        rewrapEscrowBlob: async (input: RewrapEscrowBlobInput) => {
+            const { result } = await call(
+                '/v1/authority-rewrap',
                 {
                     envelope: input.envelope,
                     expectedDid: input.expectedDid,
                     expectedShareVersion: input.expectedShareVersion,
                     sourceEnrollmentEpoch: input.sourceEnrollmentEpoch,
                 },
-                rewrapEscrowBlobResponseValidator
-            ),
-        createHold: async input => ({
-            holdRecord: await call(
+                z.object({ result: AuthorityMutationValidator }).strict()
+            );
+            if (!result.envelope || result.enrollmentEpoch !== input.sourceEnrollmentEpoch)
+                throw new EscrowUnavailableError('invalid_response');
+            return { envelope: result.envelope, authority: result };
+        },
+        createHold: async input => {
+            const result = await call(
                 '/v1/create-hold',
                 {
                     envelope: input.envelope,
@@ -230,24 +328,93 @@ export const createRemoteEnclave = (config: RemoteEnclaveConfig): EscrowEnclave
                     releasePolicy: input.releasePolicy,
                     clientEphemeralPublicKey: input.clientEphemeralPublicKey,
                 },
-                holdRecordValidator
-            ),
-        }),
+                holdRecordValidator.extend({
+                    delivery: z
+                        .object({
+                            codeSent: z.boolean(),
+                            proof: AuthorityHoldReceiptValidator,
+                        })
+                        .optional(),
+                })
+            );
+            // Response-only evidence must not be echoed into strict old-image hold DTOs.
+            const { delivery, ...holdRecord } = result;
+            return { holdRecord: validatedHold(holdRecord, 'invalid_response'), delivery };
+        },
+        getHoldState: async input =>
+            call('/v1/hold-state', input, z.object({ proof: AuthorityHoldReceiptValidator })),
         cancelHold: async input => {
-            await call(
+            const result = await call(
                 '/v1/cancel-hold',
                 {
                     envelope: input.envelope,
                     hold: validatedHold(input.hold),
                     clientEphemeralPublicKey: input.clientEphemeralPublicKey,
                     expectedDid: input.expectedDid,
+                    ...(input.requestId ? { requestId: input.requestId } : {}),
+                },
+                z.object({ ok: z.literal(true), receipt: AuthorityCommitValidator.optional() })
+            );
+            if (input.hold.hold.policyVersion === 2 && !result.receipt)
+                throw new EscrowUnavailableError('invalid_response');
+            if (result.receipt) return { receipt: result.receipt };
+            return undefined;
+        },
+        confirmHold: async (input: ConfirmHoldRequest) => {
+            const result = await call(
+                '/v1/confirm-hold',
+                {
+                    envelope: input.envelope,
+                    hold: validatedHold(input.hold),
+                    clientEphemeralPublicKey: input.clientEphemeralPublicKey,
+                    expectedDid: input.expectedDid,
+                    code: input.code,
+                    ...(input.requestId ? { requestId: input.requestId } : {}),
+                },
+                z.object({
+                    confirmed: z.literal(true),
+                    receipt: AuthorityHoldReceiptValidator,
+                    confirmedHiMs: safeInteger,
+                })
+            );
+            return { receipt: result.receipt, confirmedAtMs: result.confirmedHiMs };
+        },
+        resendCode: async (input: ResendCodeRequest) => {
+            const result = await call(
+                '/v1/resend-code',
+                {
+                    envelope: input.envelope,
+                    hold: validatedHold(input.hold),
+                    clientEphemeralPublicKey: input.clientEphemeralPublicKey,
+                    expectedDid: input.expectedDid,
+                    ...(input.requestId ? { requestId: input.requestId } : {}),
+                },
+                z.object({ sent: z.literal(true), receipt: AuthorityCommitValidator })
+            );
+            return { receipt: result.receipt };
+        },
+        sendReleaseWarning: async (input: SendReleaseWarningRequest) => {
+            const result = await call(
+                '/v1/send-release-warning',
+                {
+                    holdId: input.holdId,
+                    expectedDid: input.expectedDid,
+                    ...(input.requestId ? { requestId: input.requestId } : {}),
                 },
-                z.object({ ok: z.literal(true) })
+                z.object({
+                    sent: z.literal(true),
+                    receipt: AuthorityHoldReceiptValidator,
+                    releaseWarningSentHiMs: safeInteger,
+                })
             );
+            return {
+                receipt: result.receipt,
+                releaseWarningSentAtMs: result.releaseWarningSentHiMs,
+            };
         },
         releaseEscrow: async (input: ReleaseRequest): Promise<ReleaseResult> => {
             validatedHold(input.hold);
-            return call(
+            const result = await call(
                 '/v1/release',
                 {
                     envelope: input.envelope,
@@ -255,9 +422,13 @@ export const createRemoteEnclave = (config: RemoteEnclaveConfig): EscrowEnclave
                     clientEphemeralPublicKey: input.clientEphemeralPublicKey,
                     expectedDid: input.expectedDid,
                     ...(input.pinProof !== undefined ? { pinProof: input.pinProof } : {}),
+                    ...(input.requestId ? { requestId: input.requestId } : {}),
                 },
                 releaseResponseValidator
             );
+            if (input.hold.hold.policyVersion === 2 && !result.receipt)
+                throw new EscrowUnavailableError('invalid_response');
+            return result;
         },
     };
 };