@learncard/lca-api-service 1.5.2 → 1.5.3

package/src/services/escrow-enclave/remoteEnclave.tls.test.ts

@@ -1,9 +1,10 @@
 import { readFileSync } from 'node:fs';
 import { createServer, type Server } from 'node:https';
 import type { AddressInfo } from 'node:net';
-import { afterAll, beforeAll, describe, expect, it } from 'vitest';
-import { createHttpTransport } from './remoteEnclave';
+import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from 'vitest';
+import { createHttpTransport, createRemoteEnclave } from './remoteEnclave';
 import { pinnedEnclaveCa } from './enclaveTrustAnchors';
+import { logEscrowFailure } from './logFailure';
 
 const fixture = (name: string): string =>
     readFileSync(new URL(`./__fixtures__/tls/${name}`, import.meta.url), 'utf8');
@@ -60,3 +61,182 @@ describe('remote enclave TLS trust', () => {
         expect(pinnedEnclaveCa('https://enclave-host.internal')).toBeUndefined();
     });
 });
+
+describe('remote enclave connection recovery', () => {
+    const servers: Server[] = [];
+    const config = { headers: { Authorization: 'Bearer test-secret' }, timeout: 1_000 };
+    const serve = async (handler: Parameters<typeof createServer>[1]): Promise<string> => {
+        const server = createServer(
+            { cert: fixture('pinned-leaf.pem'), key: fixture('pinned-leaf.key') },
+            handler
+        );
+        servers.push(server);
+        await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve));
+        return url(server).replace('/v1/attest', '');
+    };
+
+    afterEach(async () => {
+        vi.restoreAllMocks();
+        await Promise.all(
+            servers.splice(0).map(
+                server =>
+                    new Promise<void>(resolve => {
+                        server.closeAllConnections();
+                        server.close(() => resolve());
+                    })
+            )
+        );
+    });
+
+    it.each(['/v1/attest', '/v1/enrollment-state', '/v1/hold-state'])(
+        'retries %s once on a pre-response reset, on a fresh TLS connection',
+        async path => {
+            const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+            const sockets = new Set();
+            let requests = 0;
+            const origin = await serve((request, response) => {
+                sockets.add(request.socket);
+                requests += 1;
+                if (requests === 1) request.socket.destroy();
+                else response.end('{"ok":true}');
+            });
+            const transport = createHttpTransport(fixture('pinned-ca.pem'));
+            await expect(
+                transport.post(`${origin}${path}`, { secret: 'private-body' }, config)
+            ).resolves.toEqual({ status: 200, data: { ok: true } });
+            expect(requests).toBe(2);
+            expect(sockets.size).toBe(2);
+            expect(warn.mock.calls).toEqual([
+                ['[LCA escrow] retrying enclave read', { code: 'ECONNRESET' }],
+            ]);
+        }
+    );
+
+    it('does not pool successful requests across invocations', async () => {
+        const sockets = new Set();
+        const connections: (string | undefined)[] = [];
+        const origin = await serve((request, response) => {
+            sockets.add(request.socket);
+            connections.push(request.headers.connection);
+            response.end('{"ok":true}');
+        });
+        const transport = createHttpTransport(fixture('pinned-ca.pem'));
+        await transport.post(`${origin}/v1/attest`, {}, config);
+        await transport.post(`${origin}/v1/attest`, {}, config);
+        expect(sockets.size).toBe(2);
+        expect(connections).toEqual(['close', 'close']);
+    });
+
+    it.each([
+        '/v1/create-hold',
+        '/v1/confirm-hold',
+        '/v1/release',
+        '/v1/cancel-hold',
+        '/v1/apply-enrollment',
+        '/v1/authority-rewrap',
+        '/v1/carry-pin-verifier',
+        '/v1/resend-code',
+        '/v1/send-release-warning',
+        '/v1/unknown',
+    ])('never retries %s even when no response arrived', async path => {
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let requests = 0;
+        const origin = await serve(request => {
+            requests += 1;
+            request.socket.destroy();
+        });
+        await expect(
+            createHttpTransport(fixture('pinned-ca.pem')).post(`${origin}${path}`, {}, config)
+        ).rejects.toMatchObject({ code: 'ECONNRESET' });
+        expect(requests).toBe(1);
+        expect(warn).not.toHaveBeenCalled();
+    });
+
+    it.each([429, 503, 302])('does not retry or follow HTTP %s', async status => {
+        let requests = 0;
+        const origin = await serve((_request, response) => {
+            requests += 1;
+            response.writeHead(status, { Location: '/v1/attest' });
+            response.end('{}');
+        });
+        await expect(
+            createHttpTransport(fixture('pinned-ca.pem')).post(`${origin}/v1/attest`, {}, config)
+        ).resolves.toMatchObject({ status });
+        expect(requests).toBe(1);
+    });
+
+    it('does not retry a timeout', async () => {
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let requests = 0;
+        const origin = await serve(() => {
+            requests += 1;
+        });
+        await expect(
+            createHttpTransport(fixture('pinned-ca.pem')).post(
+                `${origin}/v1/attest`,
+                {},
+                { ...config, timeout: 100 }
+            )
+        ).rejects.toThrow();
+        expect(requests).toBe(1);
+        expect(warn).not.toHaveBeenCalled();
+    });
+
+    it('does not retry after even partial response headers', async () => {
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let requests = 0;
+        const origin = await serve(request => {
+            requests += 1;
+            request.socket.write('HTTP/1.1 200 OK\r\n');
+            setTimeout(() => request.socket.destroy(), 20);
+        });
+        await expect(
+            createHttpTransport(fixture('pinned-ca.pem')).post(`${origin}/v1/attest`, {}, config)
+        ).rejects.toThrow();
+        expect(requests).toBe(1);
+        expect(warn).not.toHaveBeenCalled();
+    });
+
+    it('keeps the second attempt within the original wall-clock budget', async () => {
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let requests = 0;
+        const origin = await serve(request => {
+            requests += 1;
+            if (requests === 1) setTimeout(() => request.socket.destroy(), 150);
+        });
+        const start = performance.now();
+        await expect(
+            createHttpTransport(fixture('pinned-ca.pem')).post(
+                `${origin}/v1/attest`,
+                {},
+                { ...config, timeout: 300 }
+            )
+        ).rejects.toThrow();
+        expect(performance.now() - start).toBeLessThan(430);
+        expect(requests).toBe(2);
+        expect(warn).toHaveBeenCalledOnce();
+    });
+
+    it('stops after one retry and preserves sanitized transport failure logging', async () => {
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        let requests = 0;
+        const origin = await serve(request => {
+            requests += 1;
+            request.socket.destroy();
+        });
+        const enclave = createRemoteEnclave({
+            baseUrl: origin,
+            token: 'private-token',
+            timeoutMs: 1_000,
+            caPem: fixture('pinned-ca.pem'),
+        });
+        const error = await enclave.getAttestation().catch(error => error);
+        expect(error).toMatchObject({ reason: 'transport' });
+        logEscrowFailure(error);
+        expect(requests).toBe(2);
+        expect(warn.mock.calls).toEqual([
+            ['[LCA escrow] retrying enclave read', { code: 'ECONNRESET' }],
+            ['[LCA escrow] enclave operation failed', { reason: 'transport' }],
+        ]);
+    });
+});

package/src/services/escrow-enclave/remoteEnclave.ts

@@ -1,4 +1,5 @@
 import { Agent } from 'node:https';
+import { Agent as HttpAgent, ClientRequest } from 'node:http';
 import axios from 'axios';
 import { z } from 'zod';
 import {
@@ -16,6 +17,7 @@ import {
 import {
     EscrowBlobError,
     EscrowPinMismatchError,
+    EscrowCodeError,
     EscrowPolicyError,
     EscrowUnavailableError,
     type CarryPinVerifierInput,
@@ -91,15 +93,18 @@ const holdRecordValidator = z
         ledgerSeq: z.number().int().nonnegative(),
     })
     .passthrough();
-const validatedHold = (hold: ReleaseRequest['hold']): ReleaseRequest['hold'] => {
-    if (!holdRecordValidator.safeParse(hold).success) throw new EscrowUnavailableError();
+const validatedHold = (
+    hold: ReleaseRequest['hold'],
+    reason: 'invalid_hold' | 'invalid_response' = 'invalid_hold'
+): ReleaseRequest['hold'] => {
+    if (!holdRecordValidator.safeParse(hold).success) throw new EscrowUnavailableError(reason);
     if (hold.hold.policyVersion === 2) {
         try {
             AuthorityCommitValidator.parse(
                 JSON.parse(Buffer.from(hold.hold.signature, 'base64').toString('utf8'))
             );
         } catch {
-            throw new EscrowUnavailableError();
+            throw new EscrowUnavailableError(reason);
         }
     }
     return hold;
@@ -112,6 +117,9 @@ const releaseResponseValidator = z.object({
 const errorCodeValidator = z.enum([
     'policy',
     'pinMismatch',
+    'codeMismatch',
+    'codeExpired',
+    'codeLocked',
     'blob',
     'unavailable',
     'ledger',
@@ -122,8 +130,10 @@ const errorResponseValidator = z.object({ code: errorCodeValidator, message: z.s
 const errorForCode = (code: z.infer<typeof errorCodeValidator>): Error => {
     if (code === 'policy') return new EscrowPolicyError();
     if (code === 'pinMismatch') return new EscrowPinMismatchError();
+    if (code === 'codeMismatch' || code === 'codeExpired' || code === 'codeLocked')
+        return new EscrowCodeError(code);
     if (code === 'blob') return new EscrowBlobError();
-    return new EscrowUnavailableError(); // 'unavailable' | 'ledger' | 'time'
+    return new EscrowUnavailableError(code); // 'unavailable' | 'ledger' | 'time'
 };
 
 export interface RemoteEnclaveTransportResponse {
@@ -141,15 +151,57 @@ export interface RemoteEnclaveTransport {
 
 /** With `caPem`, the connection trusts only that CA (system roots are not consulted). */
 export const createHttpTransport = (caPem?: string): RemoteEnclaveTransport => {
-    const httpsAgent = caPem ? new Agent({ ca: caPem, keepAlive: true }) : undefined;
+    // The host enforces a 60s absolute connection lifetime, even for busy sockets.
+    // Do not retain sockets across Lambda freezes (or use Node's global pool).
+    const httpsAgent = new Agent({ ...(caPem ? { ca: caPem } : {}), keepAlive: false });
+    const httpAgent = new HttpAgent({ keepAlive: false });
     return {
         post: async (url, body, config) => {
-            const response = await axios.post(url, body, {
-                ...config,
-                httpsAgent,
-                validateStatus: () => true,
-            });
-            return { status: response.status, data: response.data };
+            const path = new URL(url).pathname;
+            const safeRead = ['/v1/attest', '/v1/enrollment-state', '/v1/hold-state'].includes(
+                path
+            );
+            const deadline = performance.now() + config.timeout;
+            const controller = new AbortController();
+            // One wall-clock budget covers connection setup and BOTH attempts.
+            const timer = setTimeout(() => controller.abort(), config.timeout);
+            try {
+                for (let attempt = 0; ; attempt += 1) {
+                    try {
+                        const response = await axios.post(url, body, {
+                            ...config,
+                            timeout: Math.max(1, Math.ceil(deadline - performance.now())),
+                            signal: controller.signal,
+                            httpsAgent,
+                            httpAgent,
+                            // Keep the native ClientRequest observable; never redirect a vault POST.
+                            maxRedirects: 0,
+                            validateStatus: () => true,
+                        });
+                        return { status: response.status, data: response.data };
+                    } catch (error) {
+                        const code = axios.isAxiosError(error) ? error.code : undefined;
+                        if (
+                            attempt !== 0 ||
+                            !safeRead ||
+                            (code !== 'ECONNRESET' && code !== 'EPIPE') ||
+                            !axios.isAxiosError(error) ||
+                            error.response ||
+                            !(error.request instanceof ClientRequest) ||
+                            error.request.socket?.bytesRead !== 0 ||
+                            controller.signal.aborted ||
+                            performance.now() >= deadline
+                        )
+                            throw error;
+                        // No response does NOT prove the server did not execute the request.
+                        // Never replay mutations: holds, PIN attempts, delivery and enrollment
+                        // may already be committed. Only the three read paths above are safe.
+                        console.warn('[LCA escrow] retrying enclave read', { code });
+                    }
+                }
+            } finally {
+                clearTimeout(timer);
+            }
         },
     };
 };
@@ -183,21 +235,24 @@ export const createRemoteEnclave = (config: RemoteEnclaveConfig): EscrowEnclave
         let data: unknown;
         try {
             const response = await transport.post(`${config.baseUrl}${path}`, body, {
-                headers: { Authorization: `Bearer ${config.token}` },
+                headers: {
+                    Authorization: `Bearer ${config.token}`,
+                    ...(path === '/v1/create-hold' ? { 'X-Escrow-Delivery-Evidence': '1' } : {}),
+                },
                 timeout: config.timeoutMs,
             });
             status = response.status;
             data = response.data;
         } catch {
-            throw new EscrowUnavailableError();
+            throw new EscrowUnavailableError('transport');
         }
         if (status < 200 || status >= 300) {
             const parsedError = errorResponseValidator.safeParse(data);
             if (parsedError.success) throw errorForCode(parsedError.data.code);
-            throw new EscrowUnavailableError();
+            throw new EscrowUnavailableError(status === 429 ? 'rate_limit' : 'http_refusal');
         }
         const parsed = validator.safeParse(data);
-        if (!parsed.success) throw new EscrowUnavailableError();
+        if (!parsed.success) throw new EscrowUnavailableError('invalid_response');
         return parsed.data;
     };
 
@@ -217,7 +272,7 @@ export const createRemoteEnclave = (config: RemoteEnclaveConfig): EscrowEnclave
                 result.enrollmentEpoch !== input.signedIntent.intent.enrollmentEpoch ||
                 (result.envelope === null) !== (input.signedIntent.intent.action === 'disable')
             )
-                throw new EscrowUnavailableError();
+                throw new EscrowUnavailableError('invalid_response');
             return result;
         },
         getAttestation: (nonce?: Uint8Array): Promise<EnclaveAttestation> =>
@@ -258,26 +313,36 @@ export const createRemoteEnclave = (config: RemoteEnclaveConfig): EscrowEnclave
                 z.object({ result: AuthorityMutationValidator }).strict()
             );
             if (!result.envelope || result.enrollmentEpoch !== input.sourceEnrollmentEpoch)
-                throw new EscrowUnavailableError();
+                throw new EscrowUnavailableError('invalid_response');
             return { envelope: result.envelope, authority: result };
         },
-        createHold: async input => ({
-            holdRecord: validatedHold(
-                await call(
-                    '/v1/create-hold',
-                    {
-                        envelope: input.envelope,
-                        holdId: input.holdId,
-                        expectedDid: input.expectedDid,
-                        expectedShareVersion: input.expectedShareVersion,
-                        enrollmentEpoch: input.enrollmentEpoch,
-                        releasePolicy: input.releasePolicy,
-                        clientEphemeralPublicKey: input.clientEphemeralPublicKey,
-                    },
-                    holdRecordValidator
-                )
-            ),
-        }),
+        createHold: async input => {
+            const result = await call(
+                '/v1/create-hold',
+                {
+                    envelope: input.envelope,
+                    holdId: input.holdId,
+                    expectedDid: input.expectedDid,
+                    expectedShareVersion: input.expectedShareVersion,
+                    enrollmentEpoch: input.enrollmentEpoch,
+                    releasePolicy: input.releasePolicy,
+                    clientEphemeralPublicKey: input.clientEphemeralPublicKey,
+                },
+                holdRecordValidator.extend({
+                    delivery: z
+                        .object({
+                            codeSent: z.boolean(),
+                            proof: AuthorityHoldReceiptValidator,
+                        })
+                        .optional(),
+                })
+            );
+            // Response-only evidence must not be echoed into strict old-image hold DTOs.
+            const { delivery, ...holdRecord } = result;
+            return { holdRecord: validatedHold(holdRecord, 'invalid_response'), delivery };
+        },
+        getHoldState: async input =>
+            call('/v1/hold-state', input, z.object({ proof: AuthorityHoldReceiptValidator })),
         cancelHold: async input => {
             const result = await call(
                 '/v1/cancel-hold',
@@ -291,7 +356,7 @@ export const createRemoteEnclave = (config: RemoteEnclaveConfig): EscrowEnclave
                 z.object({ ok: z.literal(true), receipt: AuthorityCommitValidator.optional() })
             );
             if (input.hold.hold.policyVersion === 2 && !result.receipt)
-                throw new EscrowUnavailableError();
+                throw new EscrowUnavailableError('invalid_response');
             if (result.receipt) return { receipt: result.receipt };
             return undefined;
         },
@@ -362,7 +427,7 @@ export const createRemoteEnclave = (config: RemoteEnclaveConfig): EscrowEnclave
                 releaseResponseValidator
             );
             if (input.hold.hold.policyVersion === 2 && !result.receipt)
-                throw new EscrowUnavailableError();
+                throw new EscrowUnavailableError('invalid_response');
             return result;
         },
     };

package/src/services/escrow-enclave/types.ts

@@ -127,7 +127,15 @@ export interface EscrowEnclave {
         input: RewrapEscrowBlobInput
     ): Promise<{ envelope: EscrowEnvelope; authority?: AuthorityMutationResult }>;
     releaseEscrow(input: ReleaseRequest): Promise<ReleaseResult>;
-    createHold(input: EnclaveCreateHoldInput): Promise<{ holdRecord: EscrowHoldRecord }>;
+    createHold(input: EnclaveCreateHoldInput): Promise<{
+        holdRecord: EscrowHoldRecord;
+        delivery?: { codeSent: boolean; proof: AuthorityHoldReceipt };
+    }>;
+    getHoldState?(input: {
+        expectedDid: string;
+        hold: EscrowHoldRecord;
+        nonce: string;
+    }): Promise<{ proof: AuthorityRecord }>;
     cancelHold(input: CancelHoldRequest): Promise<void | { receipt: AuthorityRecord }>;
     /** P9.1 (attack C): authority (nitro) mode only — software mode never
      * produces an awaiting-confirmation hold, so it has nothing to confirm.
@@ -150,6 +158,12 @@ export class EscrowPolicyError extends Error {
         this.name = 'EscrowPolicyError';
     }
 }
+export class EscrowCodeError extends Error {
+    constructor(public readonly reason: 'codeMismatch' | 'codeExpired' | 'codeLocked') {
+        super(reason);
+        this.name = 'EscrowCodeError';
+    }
+}
 export class EscrowPinMismatchError extends Error {
     constructor() {
         super('Incorrect PIN.');
@@ -163,7 +177,17 @@ export class EscrowBlobError extends Error {
     }
 }
 export class EscrowUnavailableError extends Error {
-    constructor() {
+    constructor(
+        readonly reason:
+            | 'unavailable'
+            | 'ledger'
+            | 'time'
+            | 'transport'
+            | 'invalid_response'
+            | 'invalid_hold'
+            | 'http_refusal'
+            | 'rate_limit' = 'unavailable'
+    ) {
         super('Escrow recovery is not available.');
         this.name = 'EscrowUnavailableError';
     }

package/test/keys-escrow.spec.ts

@@ -48,6 +48,7 @@ import {
     getEscrowEnclave,
     getEscrowBlobStaleReason,
     EscrowPolicyError,
+    EscrowCodeError,
     EscrowBlobError,
     EscrowPinMismatchError,
     EscrowUnavailableError,
@@ -419,6 +420,26 @@ describe('authority enrollment persistence', () => {
         expect(stored?.requiresConfirmation).toBe(true);
         expect(stored?.startedNotificationPending).toBe(false);
     });
+    it('persists an awaiting-confirmation hold even when code delivery failed', async () => {
+        await enroll();
+        const enclave = getEscrowEnclave();
+        const create = enclave.createHold.bind(enclave);
+        const delivery = { codeSent: false, proof: { ...receipt, kind: 'snapshot' as const } };
+        vi.spyOn(enclave, 'createHold').mockImplementation(async input => ({
+            ...(await create(input)),
+            delivery,
+        }));
+        __setEscrowEnclaveForTests(
+            Object.assign(enclave, { confirmHold: async () => ({ success: true as const }) })
+        );
+        const started = await start();
+        expect(started.codeSent).toBe(false);
+        expect(started.delivery).toEqual(delivery);
+        const stored = await findEscrowHoldById(started.holdId);
+        expect(stored?.holdRecord).toEqual(started.holdRecord);
+        expect(stored?.requiresConfirmation).toBe(true);
+        expect(stored?.confirmedAt).toBeUndefined();
+    });
     it('persists a cancellation commit receipt before clearing the durable pending marker', async () => {
         await enroll();
         const started = await start();
@@ -1464,6 +1485,27 @@ describe('P2-c confirmRecovery reconciliation', () => {
     }));
     beforeEach(() => idempotentConfirmHold.mockClear());
 
+    it.each(['codeMismatch', 'codeExpired', 'codeLocked'] as const)(
+        'returns sanitized confirmation reason %s without confirming',
+        async reason => {
+            const { hold, resumeToken } = await createConfirmableHold();
+            const enclave = getEscrowEnclave();
+            enclave.confirmHold = vi.fn().mockRejectedValue(new EscrowCodeError(reason));
+            try {
+                await expect(
+                    getClient().escrow.confirmRecovery({
+                        holdId: hold._id,
+                        resumeToken,
+                        code: '12345678',
+                    })
+                ).rejects.toMatchObject({ code: 'FORBIDDEN', message: reason });
+                expect((await findEscrowHoldById(hold._id))?.confirmedAt).toBeUndefined();
+            } finally {
+                delete enclave.confirmHold;
+            }
+        }
+    );
+
     it('allows the SDK commit-to-v2-snapshot readback without a second persistence or deadline shift', async () => {
         const { hold, resumeToken } = await createConfirmableHold();
         const snapshotReceipt = {
@@ -1908,10 +1950,14 @@ describe('escrow PIN release', () => {
     it('burns a mismatched hold; a new hold with the correct PIN succeeds', async () => {
         await enrollPin();
         const first = await startPin();
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
         await expect(completePin(first, wrongProof)).rejects.toMatchObject({
             code: 'FORBIDDEN',
             message: 'Incorrect PIN. 9 attempts left.',
         });
+        expect(warn).toHaveBeenCalledExactlyOnceWith('[LCA escrow] enclave operation failed', {
+            reason: 'pin_mismatch',
+        });
         expect(await findEscrowHoldById(first.holdId)).toMatchObject({
             status: 'cancelled',
             cancelledBy: 'system',
@@ -2259,6 +2305,25 @@ describe('escrow PIN release', () => {
         expect((await getClient().escrow.getStatus(auth)).hold?.holdId).toBe(hold.holdId);
     });
 
+    it('logs an unexpected non-PIN mismatch once without refunding a PIN attempt', async () => {
+        await enroll();
+        const hold = await start();
+        advanceToRelease(hold);
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
+        const refund = vi.spyOn(models, 'refundEscrowPinAttempt');
+        vi.spyOn(getEscrowEnclave(), 'releaseEscrow').mockRejectedValueOnce(
+            new EscrowPinMismatchError()
+        );
+        await expect(getClient().escrow.completeRecovery(resume(hold))).rejects.toMatchObject({
+            code: 'INTERNAL_SERVER_ERROR',
+        });
+        expect(warn).toHaveBeenCalledExactlyOnceWith('[LCA escrow] enclave operation failed', {
+            reason: 'pin_mismatch',
+        });
+        expect(refund).not.toHaveBeenCalled();
+        expect(await findEscrowHoldById(hold.holdId)).toMatchObject({ status: 'pending' });
+    });
+
     it.each([new EscrowUnavailableError(), new EscrowBlobError()])(
         'refunds non-mismatch enclave failures: %s',
         async error => {