@learncard/lca-api-service 1.5.3 → 1.6.1

package/src/routes/ai.test.ts

@@ -1,103 +1,80 @@
-import { beforeEach, describe, expect, it, vi } from 'vitest';
-import {
-    SKILL_ICON_MAX_CHARACTERS_PER_REQUEST,
-    SKILL_ICON_MAX_NAMES_PER_REQUEST,
-} from '@learncard/types';
-
-const mocks = vi.hoisted(() => ({
-    create: vi.fn(),
-}));
-
-vi.mock('@environment', () => ({ environment: { OPENAI_API_KEY: 'test-key' } }));
-vi.mock('openai', () => ({
-    default: class {
-        chat = { completions: { create: mocks.create } };
-    },
+import { describe, expect, it, vi } from 'vitest';
+import { z } from 'zod';
+
+const { generate } = vi.hoisted(() => ({ generate: vi.fn() }));
+vi.mock('@environment', () => ({ environment: {} }));
+vi.mock('filestack-js', () => ({ init: vi.fn() }));
+vi.mock('@helpers/bedrockGeneration', () => ({
+    createBedrockClient: vi.fn(() => ({})),
+    generateStructuredOutput: generate,
 }));
-vi.mock('filestack-js', () => ({ init: () => ({}) }));
-vi.mock('@routes', async () => {
-    const { initTRPC } = await import('@trpc/server');
-    const t = initTRPC.create();
-    return { t, didAndChallengeRoute: t.procedure };
+// Capture route resolvers without booting Mongo, Redis, Firebase or the HTTP server.
+vi.mock('@routes', () => {
+    const procedure = {
+        meta: () => procedure,
+        input: () => procedure,
+        output: () => procedure,
+        query: (resolver: unknown) => resolver,
+    };
+    return { t: { router: (routes: unknown) => routes }, didAndChallengeRoute: procedure };
 });
-
 import { aiRouter } from './ai';
 
-const caller = aiRouter.createCaller({ user: { did: 'did:example:123' } } as never);
-
-const respondWithIcons = (): void => {
-    mocks.create.mockImplementation(async ({ messages }: { messages: { content: string }[] }) => {
-        const names = [...messages[1]!.content.matchAll(/^- (.*)$/gm)].map(match => match[1]);
-
-        return {
-            choices: [
-                {
-                    message: {
-                        content: JSON.stringify({
-                            items: names.map(name => ({ name, icon: '🎯' })),
-                        }),
-                    },
-                },
-            ],
+const routes = aiRouter as unknown as Record<
+    string,
+    (params: { input: unknown; ctx?: unknown }) => Promise<unknown>
+>;
+describe('LCA generation route contracts', () => {
+    it('uses Sol for boost generation and preserves locale and schema', async () => {
+        const output = {
+            title: 'Título',
+            description: 'Descripción',
+            category: 'Achievement',
+            type: 'Badge',
+            narrative: 'Narración',
         };
+        generate.mockResolvedValueOnce(output);
+        expect(
+            await routes.generateBoostInfo!({ input: { description: 'Synthetic', locale: 'es' } })
+        ).toEqual(output);
+        const params = generate.mock.calls.at(-1)![1];
+        expect(params.task).toBe('reasoning');
+        expect(params.instructions).toContain('Spanish');
+        expect(params.schema.parse(output)).toEqual(output);
+        expect(() => params.schema.parse({ ...output, category: 'Invented' })).toThrow();
     });
-};
-
-describe('generateSkillIcons input validation', () => {
-    beforeEach(() => {
-        vi.clearAllMocks();
-        respondWithIcons();
-    });
-
-    it('accepts a batch at both limits', async () => {
-        const names = Array.from({ length: SKILL_ICON_MAX_NAMES_PER_REQUEST }, (_, i) =>
-            `${i}`.padStart(
-                SKILL_ICON_MAX_CHARACTERS_PER_REQUEST / SKILL_ICON_MAX_NAMES_PER_REQUEST,
-                'x'
-            )
-        );
-
-        const result = await caller.generateSkillIcons({ names });
-
-        expect(Object.keys(result)).toHaveLength(SKILL_ICON_MAX_NAMES_PER_REQUEST);
-        expect(mocks.create).toHaveBeenCalledTimes(1);
-    });
-
-    it('rejects too many names with BAD_REQUEST before calling the model', async () => {
-        const names = Array.from(
-            { length: SKILL_ICON_MAX_NAMES_PER_REQUEST + 1 },
-            (_, i) => `s${i}`
-        );
-
-        await expect(caller.generateSkillIcons({ names })).rejects.toMatchObject({
-            code: 'BAD_REQUEST',
-        });
-        expect(mocks.create).not.toHaveBeenCalled();
-    });
-
-    it('rejects names totalling more than the character limit with BAD_REQUEST', async () => {
-        const names = ['a'.repeat(SKILL_ICON_MAX_CHARACTERS_PER_REQUEST), 'b'];
-
-        await expect(caller.generateSkillIcons({ names })).rejects.toMatchObject({
-            code: 'BAD_REQUEST',
+    it('uses Sol for hierarchy reasoning and keeps the flattened API shape', async () => {
+        generate.mockResolvedValueOnce({
+            digital: [{ skill: 'cybersecurity', subskills: ['dataPrivacy'] }],
         });
-        expect(mocks.create).not.toHaveBeenCalled();
+        expect(await routes.generateBoostSkills!({ input: { description: 'Synthetic' } })).toEqual([
+            { category: 'digital', skill: 'cybersecurity', subskills: ['dataPrivacy'] },
+        ]);
+        expect(generate.mock.calls.at(-1)![1].task).toBe('reasoning');
     });
-
-    it('rejects a single name over the character limit', async () => {
-        const names = ['a'.repeat(SKILL_ICON_MAX_CHARACTERS_PER_REQUEST + 1)];
-
-        await expect(caller.generateSkillIcons({ names })).rejects.toMatchObject({
-            code: 'BAD_REQUEST',
+    it('uses Luna for icon mapping and keeps missing-name fallback and requested-name filtering', async () => {
+        generate.mockResolvedValueOnce({
+            items: [
+                { name: 'Typing', icon: '⌨️' },
+                { name: 'Extra', icon: '🎯' },
+            ],
         });
+        expect(
+            await routes.generateSkillIcons!({ input: { names: ['Typing', 'Planning'] } })
+        ).toEqual({ Typing: '⌨️', Planning: '⭐' });
+        const params = generate.mock.calls.at(-1)![1];
+        expect(params.task).toBe('formatting');
+        expect(() =>
+            (params.schema as z.ZodType).parse({
+                items: [{ name: 'Typing', icon: 'too long for icon' }],
+            })
+        ).toThrow();
     });
-
-    it('still rejects empty lists and empty names', async () => {
-        await expect(caller.generateSkillIcons({ names: [] })).rejects.toMatchObject({
-            code: 'BAD_REQUEST',
-        });
-        await expect(caller.generateSkillIcons({ names: [''] })).rejects.toMatchObject({
-            code: 'BAD_REQUEST',
-        });
+    it('propagates provider failures rather than manufacturing an icon success', async () => {
+        const error = new Error('AI generation unavailable');
+        generate.mockRejectedValueOnce(error);
+        await expect(routes.generateSkillIcons!({ input: { names: ['Typing'] } })).rejects.toBe(
+            error
+        );
     });
 });

package/src/routes/ai.ts

@@ -1,10 +1,10 @@
 import { environment } from '@environment';
 import OpenAI from 'openai';
-import { zodResponseFormat } from 'openai/helpers/zod';
+import { createBedrockClient, generateStructuredOutput } from '@helpers/bedrockGeneration';
+import { generateStoredImage } from '@helpers/imageGeneration';
 import { z } from 'zod';
 
 import { t, didAndChallengeRoute } from '@routes';
-import { TRPCError } from '@trpc/server';
 import { GenerateSkillIconsInputValidator } from '@learncard/types';
 
 import * as filestack from 'filestack-js';
@@ -80,7 +80,7 @@ const IconMapValidator = z.record(
     z.string().min(1).max(8)
 );
 
-// LLM output schema: object-wrapped list to satisfy response_format type 'object'
+// Required-function output schema: object-wrapped list.
 // We'll convert this into the IconMapValidator shape before returning
 const IconListContainerValidator = z.object({
     items: z.array(
@@ -95,6 +95,10 @@ const openai = environment.OPENAI_API_KEY
     ? new OpenAI({ apiKey: environment.OPENAI_API_KEY })
     : undefined;
 
+let generationClient: OpenAI | undefined;
+const getGenerationClient = (): OpenAI =>
+    (generationClient ??= createBedrockClient(environment.BEDROCK_BASE_URL));
+
 export const aiRouter = t.router({
     generateBoostInfo: didAndChallengeRoute
         .meta({
@@ -110,41 +114,19 @@ export const aiRouter = t.router({
         })
         .input(z.object({ description: z.string().nonempty(), locale: z.string().optional() }))
         .output(AIResponseValidator)
-        .query(async ({ input, ctx }) => {
-            const {
-                user: { did },
-            } = ctx;
-            if (!openai) {
-                throw new TRPCError({
-                    code: 'INTERNAL_SERVER_ERROR',
-                    message: 'No OpenAI Key Set',
-                });
-            }
-
+        .query(async ({ input }) => {
             const { description } = input;
 
-            const completion = await openai.chat.completions.create({
-                model: 'gpt-4o-2024-08-06',
-                messages: [
-                    {
-                        role: 'system',
-                        content:
-                            `Generate a title ${aiTitleCharsetRule(
-                                input.locale
-                            )} with a maximum length of 24 characters and separate words with spaces if there is more than one, description, category, type, and narrative for a boost based on an arbitrary description. The narrative is no longer than 300 characters and answers the question "How do you earn this boost?"\n\nStrict constraints:\n- category MUST be EXACTLY one of: "Social Badge", "Achievement", "Course", "ID", "Work History", "Learning History", "Accomplishment", "Accommodation". Use exact casing and spelling; do NOT invent other categories.\n- type MUST be EXACTLY one of: "Certificate", "Badge", "ID". Use exact casing and spelling; do NOT invent other types.\n\nOutput requirements:\n- Return ONLY a JSON object with the keys "title", "description", "category", "type", and optional "narrative".\n- Do not include explanations, markdown, or extra fields.\n- Ensure the JSON parses without code fences.` +
-                            aiLocaleInstruction(input.locale),
-                    },
-                    { role: 'user', content: description },
-                ],
-                // response_format: zodResponseFormat(AIResponseValidator, 'info'),
-                response_format: { type: 'json_object' },
-                user: did,
+            return generateStructuredOutput(getGenerationClient(), {
+                task: 'reasoning',
+                instructions:
+                    `Generate a title ${aiTitleCharsetRule(
+                        input.locale
+                    )} with a maximum length of 24 characters and separate words with spaces if there is more than one, description, category, type, and narrative for a boost based on an arbitrary description. The narrative is no longer than 300 characters and answers the question "How do you earn this boost?"\n\nStrict constraints:\n- category MUST be EXACTLY one of: "Social Badge", "Achievement", "Course", "ID", "Work History", "Learning History", "Accomplishment", "Accommodation". Use exact casing and spelling; do NOT invent other categories.\n- type MUST be EXACTLY one of: "Certificate", "Badge", "ID". Use exact casing and spelling; do NOT invent other types.\n\nOutput requirements:\n- Return ONLY a JSON object with the keys "title", "description", "category", "type", and "narrative".\n- Do not include explanations, markdown, or extra fields.\n- Ensure the JSON parses without code fences.` +
+                    aiLocaleInstruction(input.locale),
+                input: description,
+                schema: AIResponseValidator,
             });
-
-            const content = completion.choices[0]?.message.content ?? '';
-            const response = JSON.parse(content);
-
-            return AIResponseValidator.parseAsync(response);
         }),
 
     generateBoostSkills: didAndChallengeRoute
@@ -160,36 +142,15 @@ export const aiRouter = t.router({
         })
         .input(z.object({ description: z.string().nonempty() }))
         .output(z.array(BoostSkillsValidator))
-        .query(async ({ input, ctx }) => {
-            const {
-                user: { did },
-            } = ctx;
-            if (!openai) {
-                throw new TRPCError({
-                    code: 'INTERNAL_SERVER_ERROR',
-                    message: 'No OpenAI Key Set',
-                });
-            }
-
+        .query(async ({ input }) => {
             const { description } = input;
 
-            const completion = await openai.chat.completions.create({
-                model: 'gpt-4o-2024-08-06',
-                messages: [
-                    {
-                        role: 'system',
-                        content: `Generate *NO MORE THAN 3* skills for the provided description about a boost that matches the given skills hierarchy. ${aiSkillsHierarchyRule}`,
-                    },
-                    { role: 'user', content: description },
-                ],
-                response_format: zodResponseFormat(BoostSkillHierarchyValidator, 'skills'),
-                user: did,
+            const parsedResponse = await generateStructuredOutput(getGenerationClient(), {
+                task: 'reasoning',
+                instructions: `Generate *NO MORE THAN 3* skills for the provided description about a boost that matches the given skills hierarchy. ${aiSkillsHierarchyRule}`,
+                input: description,
+                schema: BoostSkillHierarchyValidator,
             });
-
-            const response = JSON.parse(completion.choices[0]?.message.content ?? '');
-
-            const parsedResponse = await BoostSkillHierarchyValidator.parseAsync(response);
-
             return transformLLMResultToOutputShape(parsedResponse);
         }),
 
@@ -207,18 +168,7 @@ export const aiRouter = t.router({
         })
         .input(GenerateSkillIconsInputValidator)
         .output(IconMapValidator)
-        .query(async ({ input, ctx }) => {
-            const {
-                user: { did },
-            } = ctx;
-
-            if (!openai) {
-                throw new TRPCError({
-                    code: 'INTERNAL_SERVER_ERROR',
-                    message: 'No OpenAI Key Set',
-                });
-            }
-
+        .query(async ({ input }) => {
             const { names } = input;
 
             const systemPrompt = `You are assigning a single, standard Unicode emoji to represent each skill name.
@@ -234,42 +184,12 @@ Rules:
                     '\n'
                 )}\n\nReturn a JSON object of the form { items: [{ name, icon }] } for these names.`;
 
-            // TODO switch to gpt 5 and use responses API
-            // will have to bump the openai package
-            let completion: any;
-            try {
-                completion = await openai.chat.completions.create({
-                    model: 'gpt-4o-2024-08-06',
-                    messages: [
-                        {
-                            role: 'system',
-                            content: systemPrompt,
-                        },
-                        { role: 'user', content: userContent },
-                    ],
-                    // Use the object-wrapped list schema for structured outputs
-                    //   currently breaks because response is wrapped in a code fence (```json ... ```)
-                    //   likely would need to to bump openai package version
-                    // response_format: zodResponseFormat(IconListContainerValidator, 'icons'),
-                    response_format: { type: 'json_object' },
-                    user: did,
-                });
-            } catch (error) {
-                console.error('🔥🔥🔥🔥🔥🔥🔥🔥🔥');
-                console.error('error:', error);
-            }
-
-            const content = completion.choices[0]?.message.content ?? '';
-
-            const response = JSON.parse(content);
-            // First, validate the model output with the object-wrapped list schema
-            const parsed = await IconListContainerValidator.parseAsync(response);
-            if (!parsed) {
-                throw new TRPCError({
-                    code: 'INTERNAL_SERVER_ERROR',
-                    message: 'No parsed structured output returned.',
-                });
-            }
+            const parsed = await generateStructuredOutput(getGenerationClient(), {
+                task: 'formatting',
+                instructions: systemPrompt,
+                input: userContent,
+                schema: IconListContainerValidator,
+            });
 
             // Transform into a record mapping input names to icons
             const record: Record<string, string> = {};
@@ -307,39 +227,7 @@ Rules:
             const {
                 user: { did },
             } = ctx;
-            if (!openai) {
-                throw new TRPCError({
-                    code: 'INTERNAL_SERVER_ERROR',
-                    message: 'No OpenAI Key Set',
-                });
-            }
-
-            const { prompt } = input;
-            const res = await openai.images.generate({
-                prompt,
-                model: 'dall-e-3',
-                size: '1024x1024',
-                user: did,
-            });
-
-            if (!res.data || !res.data[0]?.url) {
-                throw new TRPCError({
-                    code: 'INTERNAL_SERVER_ERROR',
-                    message: 'No image URL returned from OpenAI',
-                });
-            }
-
-            const filestackRes = (await client.storeURL(res?.data[0]?.url)) as any;
-            if (!filestackRes || !filestackRes.url) {
-                throw new TRPCError({
-                    code: 'INTERNAL_SERVER_ERROR',
-                    message: 'Failed to store image URL',
-                });
-            }
-
-            return {
-                url: filestackRes.url,
-            };
+            return generateStoredImage(input.prompt, did, environment, openai, client);
         }),
 });
 

package/src/routes/keys.ts

@@ -643,6 +643,13 @@ export const keysRouter = t.router({
                     escrowOptedOut: z.boolean(),
                     escrowPin: EscrowPinStatusValidator,
                     escrowStale: z.enum(['mode-mismatch', 'key-rotated']).optional(),
+                    /**
+                     * The account has an escrow recovery method (any confirmation state)
+                     * but none confirmed at the current share version, e.g. `pending`
+                     * after an identity-recovery rebind. A hint only; the client still
+                     * proves prior enrollment against the signed escrow authority.
+                     */
+                    priorEscrowEnrollment: z.boolean(),
                     sssActivationState: z.enum(['provisional', 'active']),
                 })
                 .nullable()
@@ -702,6 +709,10 @@ export const keysRouter = t.router({
                           }
                         : {}),
                 }));
+            const currentVersion = userKey.shareVersion ?? 1;
+            const escrowMethods = (userKey.recoveryMethods ?? []).filter(
+                rm => rm?.type === 'escrow'
+            );
 
             return {
                 authShare,
@@ -720,6 +731,13 @@ export const keysRouter = t.router({
                 escrowStale: userKey.escrowBlob
                     ? await getEscrowBlobStaleReason(userKey.escrowBlob)
                     : undefined,
+                priorEscrowEnrollment:
+                    escrowMethods.length > 0 &&
+                    !escrowMethods.some(
+                        rm =>
+                            rm.shareVersion === currentVersion &&
+                            isRecoveryMethodConfirmed(userKey, rm)
+                    ),
             };
         }),
 

package/src/services/escrow-enclave/enclaveTrustAnchors.ts

@@ -27,6 +27,25 @@ MAoGCCqGSM49BAMDA2cAMGQCMGp/wTQLmtB6/GK6hMFaFTA27gXZcxrgJ0Uy+8C0
 Ztl5uoRwN99inHNWv1cnKGdNQgIwSoQgKs8uMnBUcfiB9ZR6xb98z/9athp5pDIE
 /KUv6cNmJtuCgPOJxC+4fVKl8xC3
 -----END CERTIFICATE-----
+`,
+    // Production CA, valid until 2029-10-26. SHA-256 C017B1482D96679D6F725F8A46A52DAC
+    // CE9921AB47B4FEDFDB4C814B38584859; source: SSM
+    // /learncard/escrow-enclave/production/ca-cert in account 934654685669.
+    'escrow-enclave.production.internal': `-----BEGIN CERTIFICATE-----
+MIICaDCCAe6gAwIBAgIUCMYQNgdrxBILfOeoVNOWuZBiG1cwCgYIKoZIzj0EAwMw
+VzEkMCIGA1UECgwbTGVhcm5pbmcgRWNvbm9teSBGb3VuZGF0aW9uMS8wLQYDVQQD
+DCZMZWFybkNhcmQgRXNjcm93IEVuY2xhdmUgUHJvZHVjdGlvbiBDQTAeFw0yNjEw
+MjcxOTE5MzBaFw0yOTEwMjYxOTE5MzBaMFcxJDAiBgNVBAoMG0xlYXJuaW5nIEVj
+b25vbXkgRm91bmRhdGlvbjEvMC0GA1UEAwwmTGVhcm5DYXJkIEVzY3JvdyBFbmNs
+YXZlIFByb2R1Y3Rpb24gQ0EwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAARE/z2wlD3g
+oj5XyHZEJOjdffpvXHav9802AHod0h7QQP4NCLpw4Aofhx6+Qrx/le8KUGdu63V7
+948yzbrvW08XL20lmd2kFkb/23589kOzaTPeE1Ngs2y08n5cAkCdxWOjezB5MBIG
+A1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDQGA1UdHgEB/wQqMCig
+JjAkgiJlc2Nyb3ctZW5jbGF2ZS5wcm9kdWN0aW9uLmludGVybmFsMB0GA1UdDgQW
+BBTHUF7Yd+mLuA2XOthizgvTc8WW3zAKBggqhkjOPQQDAwNoADBlAjEAiFXqrpcc
+nR6ttLI4EEZ5qLMMGTksNypiTa+WyiUmJCtW3o6Luj9o/wlFXsFParINAjA8zeDx
+b1+05o+sCl0kQzEq1b2pN9/Xco5NQpMDfG2/BvKuSOVLhOQ1tdkZLo+uTgk=
+-----END CERTIFICATE-----
 `,
 };
 

package/src/services/escrow-enclave/logFailure.test.ts

@@ -7,6 +7,7 @@ afterEach(() => vi.restoreAllMocks());
 
 describe('escrow failure reasons', () => {
     it('logs invalid outbound holds without error details', () => {
+        const log = vi.spyOn(console, 'log').mockImplementation(() => undefined);
         const warn = vi.spyOn(console, 'warn').mockImplementation(() => undefined);
         const error = new EscrowUnavailableError('invalid_hold');
         error.message = 'private hold contents';
@@ -14,6 +15,13 @@ describe('escrow failure reasons', () => {
         expect(warn).toHaveBeenCalledExactlyOnceWith('[LCA escrow] enclave operation failed', {
             reason: 'invalid_hold',
         });
+        const metric = JSON.parse(String(log.mock.calls[0]?.[0]));
+        expect(metric).toMatchObject({ Reason: 'invalid_hold', EnclaveFailure: 1 });
+        expect(metric._aws.CloudWatchMetrics[0]).toMatchObject({
+            Namespace: 'LearnCard/Escrow',
+            Dimensions: [['Reason']],
+        });
+        expect(JSON.stringify(log.mock.calls)).not.toContain('private hold contents');
     });
     it.each([
         [403, 'policy', 'policy'],

package/src/services/escrow-enclave/logFailure.ts

@@ -26,4 +26,21 @@ export const logEscrowFailure = (error: unknown): void => {
                     : 'unexpected';
     // lca-api uses structured console warnings rather than the browser logger.
     console.warn('[LCA escrow] enclave operation failed', { reason });
+    // One EMF event alongside the existing warning; no error text, request or identity fields.
+    console.log(
+        JSON.stringify({
+            _aws: {
+                Timestamp: Date.now(),
+                CloudWatchMetrics: [
+                    {
+                        Namespace: 'LearnCard/Escrow',
+                        Dimensions: [['Reason']],
+                        Metrics: [{ Name: 'EnclaveFailure', Unit: 'Count' }],
+                    },
+                ],
+            },
+            Reason: reason,
+            EnclaveFailure: 1,
+        })
+    );
 };

package/src/services/escrow-enclave/remoteEnclave.tls.test.ts

@@ -60,6 +60,16 @@ describe('remote enclave TLS trust', () => {
         );
         expect(pinnedEnclaveCa('https://enclave-host.internal')).toBeUndefined();
     });
+
+    it('pins a distinct production CA by hostname only', () => {
+        const production = pinnedEnclaveCa('https://escrow-enclave.production.internal:8443');
+
+        expect(production).toContain('BEGIN CERTIFICATE');
+        expect(production).not.toEqual(
+            pinnedEnclaveCa('https://escrow-enclave.staging.internal:8443')
+        );
+        expect(pinnedEnclaveCa('https://production.internal:8443')).toBeUndefined();
+    });
 });
 
 describe('remote enclave connection recovery', () => {

package/test/jobs/escrowHoldReminders.spec.ts

@@ -15,6 +15,13 @@ import {
     hashEscrowResumeToken,
     recordEscrowHoldNotification,
     computeEffectiveReleaseAfter,
+    findEscrowHoldsDueForReminder,
+    findEscrowHoldsPendingStartNotification,
+    findEscrowHoldsPendingEnclaveCancellation,
+    claimEscrowHoldForReminder,
+    releaseEscrowReminderClaim,
+    recordEscrowStartNotificationAttempt,
+    recordEscrowEnclaveCancelAttempt,
     ESCROW_RELEASE_TIME_MARGIN_MS,
     ESCROW_RELEASE_WARNING_GAP_MS,
     type AuthProviderMapping,
@@ -41,7 +48,11 @@ const makeProvider = (): AuthProviderMapping => {
 
 /** Every seed() call gets its own authProvider (see test/models/escrow.spec.ts
  * for why: only one pending hold-policy hold is allowed per authProvider). */
-const seed = async (releaseAfter: Date, releasePolicy: 'hold' | 'pin' = 'hold') => {
+const seed = async (
+    releaseAfter: Date,
+    releasePolicy: 'hold' | 'pin' = 'hold',
+    holdId = randomUUID()
+) => {
     const provider = makeProvider();
     await upsertUserKeyByAuthProvider(
         { type: 'email', value: `${provider.id}@example.com` },
@@ -54,7 +65,7 @@ const seed = async (releaseAfter: Date, releasePolicy: 'hold' | 'pin' = 'hold')
     return createEscrowHold({
         holdRecord: {
             hold: {
-                holdId: randomUUID(),
+                holdId,
                 did: 'did:key:test',
                 shareVersion: 1,
                 blobHash: 'ab'.repeat(32),
@@ -117,6 +128,75 @@ afterAll(async () => {
 });
 
 describe('runEscrowHoldReminders', () => {
+    describe.each(['reminder', 'started', 'cancelled'] as const)('%s queue ordering', queue => {
+        it.each([false, true])(
+            'keeps tied batches stable and rotates retries (previously attempted: %s)',
+            async previouslyAttempted => {
+                const now = new Date();
+                const previousAttempt = new Date(now.getTime() - HOUR);
+                // More than a default delivery batch. Insert backwards so physical
+                // insertion order cannot accidentally satisfy the expected ID order.
+                const ids = Array.from(
+                    { length: 13 },
+                    (_, index) => `00000000-0000-4000-8000-${String(index + 1).padStart(12, '0')}`
+                );
+                for (const id of [...ids].reverse()) {
+                    await seed(new Date(now.getTime() + HOUR), 'hold', id);
+                }
+                await getEscrowHoldsCollection().updateMany(
+                    {},
+                    {
+                        $set: {
+                            requestedAt: previousAttempt,
+                            ...(queue === 'started' ? { startedNotificationPending: true } : {}),
+                            ...(queue === 'cancelled'
+                                ? {
+                                      status: 'cancelled' as const,
+                                      enclaveCancelPendingAt: previousAttempt,
+                                      enclaveCancelAttempts: previouslyAttempted ? 2 : 0,
+                                  }
+                                : previouslyAttempted
+                                  ? queue === 'reminder'
+                                      ? { reminderAttemptedAt: previousAttempt }
+                                      : { startedNotificationAttemptedAt: previousAttempt }
+                                  : {}),
+                        },
+                    }
+                );
+                const readBatch = () =>
+                    queue === 'reminder'
+                        ? findEscrowHoldsDueForReminder(now, 5)
+                        : queue === 'started'
+                          ? findEscrowHoldsPendingStartNotification(5)
+                          : findEscrowHoldsPendingEnclaveCancellation(5);
+
+                for (let repeat = 0; repeat < 3; repeat += 1) {
+                    expect((await readBatch()).map(hold => hold._id)).toEqual(ids.slice(0, 5));
+                }
+                // Failed deliveries remain eligible. Updating their real attempt
+                // markers must put untouched rows first, even with tied timestamps.
+                for (let offset = 0; offset < ids.length; offset += 5) {
+                    const batch = await readBatch();
+                    const remaining = Math.min(5, ids.length - offset);
+                    expect(batch.slice(0, remaining).map(hold => hold._id)).toEqual(
+                        ids.slice(offset, offset + remaining)
+                    );
+                    for (const hold of batch) {
+                        if (queue === 'reminder') {
+                            const claimed = await claimEscrowHoldForReminder(hold._id, now);
+                            expect(claimed).not.toBeNull();
+                            if (claimed) await releaseEscrowReminderClaim(claimed);
+                        } else if (queue === 'started') {
+                            await recordEscrowStartNotificationAttempt(hold._id, now);
+                        } else {
+                            expect(await recordEscrowEnclaveCancelAttempt(hold._id)).toBe(true);
+                        }
+                    }
+                }
+            }
+        );
+    });
+
     it('bounds cancellation retries and gives unattempted cancellations priority', async () => {
         const now = new Date();
         const holds = await Promise.all(
@@ -129,9 +209,13 @@ describe('runEscrowHoldReminders', () => {
         await runEscrowHoldReminders({ now, limit: 200 });
         const firstRun = await Promise.all(holds.map(hold => findEscrowHoldById(hold._id)));
         expect(firstRun.map(hold => hold?.enclaveCancelAttempts).sort()).toEqual([0, 0, 1, 1]);
-        const unattempted = firstRun.find(hold => hold?.enclaveCancelAttempts === 0)!;
+        // UUIDs are ASCII: default string ordering matches Mongo's byte ordering.
+        const unattemptedId = firstRun
+            .filter(hold => hold?.enclaveCancelAttempts === 0)
+            .map(hold => hold!._id)
+            .sort()[0]!;
         await runEscrowHoldReminders({ now, limit: 1 });
-        expect((await findEscrowHoldById(unattempted._id))?.enclaveCancelAttempts).toBe(1);
+        expect((await findEscrowHoldById(unattemptedId))?.enclaveCancelAttempts).toBe(1);
         for (const hold of holds) {
             const stored = await findEscrowHoldById(hold._id);
             expect(stored?.enclaveCancelPendingAt).toBeInstanceOf(Date);