@open-mercato/core 0.6.8-develop.6983.1.e4f794099e → 0.6.8-develop.6985.1.fb93574faa

package/src/modules/push_notifications/lib/push-fanout.ts

@@ -0,0 +1,237 @@
+import type { EntityManager } from '@mikro-orm/postgresql'
+import type { EntityName } from '@mikro-orm/core'
+import { sql } from 'kysely'
+import { findWithDecryption } from '@open-mercato/shared/lib/encryption/find'
+import type { PushOptions } from '@open-mercato/core/modules/communication_channels/lib/push-envelope'
+import type { UserDevice } from '@open-mercato/core/modules/devices/data/entities'
+import type { CommunicationChannel } from '@open-mercato/core/modules/communication_channels/data/entities'
+import { defaultLocale } from '@open-mercato/shared/lib/i18n/config'
+import { resolveSupportedLocale } from '@open-mercato/shared/lib/i18n/locale'
+import { createLogger } from '@open-mercato/shared/lib/logger'
+import {
+  resolveNotificationCopy,
+  type NotificationCopySource,
+} from '@open-mercato/core/modules/notifications/lib/notificationCopy'
+import { enqueuePushDelivery } from './queue'
+
+const logger = createLogger('push_notifications')
+
+export const PUSH_CHANNEL = 'push'
+
+type Resolve = <T = unknown>(name: string) => T
+
+/** The push payload persisted on each delivery row and unpacked by the worker into the send envelope. */
+export interface PushFanoutPayload {
+  title?: string
+  body?: string | null
+  data: Record<string, string>
+  options?: PushOptions
+  silent?: boolean
+}
+
+export interface FanOutPushDeliveriesArgs {
+  em: EntityManager
+  resolve: Resolve
+  scope: { tenantId: string; organizationId: string | null }
+  userId: string
+  /**
+   * Restrict the fan-out to a single one of the user's devices (`UserDevice.id`). Still scoped to the
+   * same (tenant, org, user), so an id that isn't the user's own matches nothing. Omit to fan out to
+   * all of the user's push-capable devices (the default).
+   */
+  userDeviceId?: string
+  /** Source in-app notification id, or `null` for a silent push (no Notification row). */
+  notificationId: string | null
+  notificationTypeId: string
+  payload: PushFanoutPayload
+  /**
+   * Per-device localizable copy for visible notifications. When present, `title`/`body` are
+   * resolved per device using its `locale`. Omit for silent pushes (no user-facing copy).
+   */
+  copy?: NotificationCopySource
+}
+
+function tokenSnapshot(token: string): string {
+  // Persist at most the last 8 chars of the (long) provider token — never the full secret.
+  return token.slice(-8)
+}
+
+/**
+ * Resolve a recipient's push-capable devices, route each to its provider's tenant push
+ * `CommunicationChannel`, persist one `pending` push delivery row per device, and enqueue a send
+ * job per row. Shared by the `push` delivery strategy (visible + silent notifications) and
+ * `sendCustomPush` (admin one-off pushes); it is preference-agnostic — the caller decides whether
+ * to consult per-channel preferences before fanning out.
+ *
+ * Cross-module entities are resolved via DI tokens (registered `asValue` by their owning modules)
+ * so this stays decoupled from those modules' internals. Returns the number of jobs enqueued.
+ */
+export async function fanOutPushDeliveries(args: FanOutPushDeliveriesArgs): Promise<{ enqueued: number }> {
+  const { em, resolve, scope, userId, userDeviceId, notificationId, notificationTypeId, payload, copy } = args
+  const { tenantId, organizationId } = scope
+  const silent = payload.silent === true
+
+  // Require at least one active push CommunicationChannel for the tenant (push not configured ⇒ skip).
+  // This is the cheapest, most selective short-circuit (most tenants have no push channel), so it runs
+  // before the per-recipient device lookup. Channels are indexed by providerKey so each device can be
+  // routed to its matching provider below (ios→apns, android→fcm, expo→expo). Oldest-first so that when
+  // a tenant has more than one active channel for the same provider, the pick is deterministic.
+  const ChannelRef = resolve('CommunicationChannel') as EntityName<CommunicationChannel>
+  const channels = await em.find(
+    ChannelRef,
+    {
+      tenantId,
+      channelType: PUSH_CHANNEL,
+      isActive: true,
+      deletedAt: null,
+    },
+    { orderBy: { createdAt: 'asc' } },
+  )
+  if (channels.length === 0) return { enqueued: 0 }
+  const channelsByProvider = new Map<string, CommunicationChannel>()
+  for (const channel of channels) {
+    if (!channelsByProvider.has(channel.providerKey)) channelsByProvider.set(channel.providerKey, channel)
+  }
+
+  // Load the recipient's devices that can receive push (active + has a token). Scoped to the
+  // notification's organization so an org-scoped notification never fans out to a device the user
+  // registered under a different org — device identity is per (tenant, org, user, device) (see devices
+  // module). A tenant-level (null-org) notification targets the user's tenant-level (null-org) devices.
+  // `push_token` is encrypted at rest; decrypt on read (no-op when encryption is disabled).
+  const DeviceRef = resolve('UserDevice') as EntityName<UserDevice>
+  const devices = await findWithDecryption(
+    em,
+    DeviceRef,
+    {
+      tenantId,
+      organizationId,
+      userId,
+      // Optionally restrict to a single device; still scoped to this user so a foreign id matches none.
+      ...(userDeviceId ? { id: userDeviceId } : {}),
+      deletedAt: null,
+      pushToken: { $ne: null },
+    },
+    undefined,
+    { tenantId, organizationId },
+  )
+  if (devices.length === 0) return { enqueued: 0 }
+
+  // Resolve the per-device localized copy. The default-locale case (the common one: device has no
+  // locale, an unsupported one, or already speaks the default) reuses the copy already resolved
+  // upstream — no dictionary load. Other locales translate via the shared notification-copy helper
+  // (dictionaries are memoized by loadDictionary). Silent pushes carry no copy and skip this.
+  // Memoized by resolved locale so a user with many same-locale devices derives copy once
+  // (O(distinct locales) instead of O(devices)).
+  const localizedByLocale = new Map<string, Promise<PushFanoutPayload>>()
+  function resolveLocalizedPayload(deviceLocale?: string | null): Promise<PushFanoutPayload> {
+    if (!copy) return Promise.resolve(payload)
+    const locale = resolveSupportedLocale(deviceLocale) ?? defaultLocale
+    let cached = localizedByLocale.get(locale)
+    if (!cached) {
+      cached = (async () => {
+        if (locale === defaultLocale) {
+          return { ...payload, title: copy.title, body: copy.body ?? null }
+        }
+        const { title, body } = await resolveNotificationCopy(copy, locale)
+        return { ...payload, title, body }
+      })()
+      localizedByLocale.set(locale, cached)
+    }
+    return cached
+  }
+
+  // Insert one pending delivery row per device, routing each device to the push channel whose
+  // providerKey matches the device's pushProvider. Devices with no provider, or no matching configured
+  // channel, are skipped. Insert via INSERT ... ON CONFLICT DO NOTHING on the (notification_id,
+  // user_device_id) partial unique index: the `push` strategy runs inside the at-least-once persistent
+  // `notifications:deliver` subscriber, so a redelivered event re-runs it — the conflict clause makes
+  // the re-fan-out a no-op instead of inserting a duplicate set of rows (and duplicate pushes). Silent
+  // pushes carry a null notification_id and are excluded from the partial index (they are triggered
+  // explicitly, not via at-least-once redelivery). Only the rows actually inserted are enqueued.
+  const builtRows = await Promise.all(
+    devices.map(async (device) => {
+      const providerKey = device.pushProvider
+      if (!providerKey) return null
+      const channel = channelsByProvider.get(providerKey)
+      if (!channel) return null
+      const rowPayload = await resolveLocalizedPayload(device.locale)
+      return {
+        tenant_id: tenantId,
+        organization_id: organizationId,
+        notification_id: notificationId,
+        notification_type_id: notificationTypeId,
+        user_device_id: device.id,
+        user_id: userId,
+        provider: channel.providerKey,
+        token_snapshot: tokenSnapshot(device.pushToken as string),
+        silent,
+        status: 'pending',
+        attempts: 0,
+        payload: sql`${JSON.stringify(rowPayload)}::jsonb`,
+        created_at: sql`now()`,
+        updated_at: sql`now()`,
+      }
+    }),
+  )
+  const rows = builtRows.filter((row): row is NonNullable<typeof row> => row !== null)
+  // A registered device whose pushProvider has no matching active channel (e.g. an Expo device on an
+  // FCM-only tenant) is correctly skipped above — but it produces no delivery row and would otherwise
+  // be an invisible no-op. Surface a count so a provider-config gap is diagnosable.
+  const skippedNoChannel = devices.length - rows.length
+  if (skippedNoChannel > 0) {
+    logger.warn('Skipped devices with no matching push channel', {
+      tenantId,
+      userId,
+      notificationId,
+      skipped: skippedNoChannel,
+    })
+  }
+  if (rows.length === 0) return { enqueued: 0 }
+
+  const db = em.getKysely<any>()
+  const inserted = (await db
+    .insertInto('push_notification_deliveries')
+    .values(rows)
+    .onConflict((oc: any) =>
+      oc.columns(['notification_id', 'user_device_id']).where('notification_id', 'is not', null).doNothing(),
+    )
+    .returning(['id'])
+    .execute()) as Array<{ id: string }>
+
+  // Enqueue one send job per inserted row (per-device retry isolation, idempotent on delivery id).
+  // Enqueue all rows concurrently — the jobs are independent, so there is no reason to serialize the
+  // queue round-trips. If an enqueue fails, mark that row failed instead of leaving it orphaned in
+  // `pending` forever (the worker only ever processes rows it receives a job for). Failed rows are
+  // grouped by reason and marked in one UPDATE per reason to keep DB round-trips minimal.
+  const outcomes = await Promise.allSettled(
+    inserted.map((row) => enqueuePushDelivery({ deliveryId: row.id, tenantId, organizationId })),
+  )
+  const failedIdsByReason = new Map<string, string[]>()
+  let enqueued = 0
+  outcomes.forEach((outcome, index) => {
+    if (outcome.status === 'fulfilled') {
+      enqueued += 1
+      return
+    }
+    const error = outcome.reason
+    const reason = error instanceof Error ? `enqueue_failed: ${error.message}` : 'enqueue_failed'
+    const ids = failedIdsByReason.get(reason) ?? []
+    ids.push(inserted[index].id)
+    failedIdsByReason.set(reason, ids)
+  })
+  if (failedIdsByReason.size > 0) {
+    await Promise.allSettled(
+      Array.from(failedIdsByReason.entries()).map(([reason, ids]) =>
+        db
+          .updateTable('push_notification_deliveries')
+          .set({ status: 'failed', last_error: reason, updated_at: sql`now()` })
+          .where('id', 'in', ids)
+          // Guard on `pending`: if the enqueue reported failure yet the job actually landed and a
+          // worker already claimed/finished the row, don't clobber its later status back to `failed`.
+          .where('status', '=', 'pending')
+          .execute(),
+      ),
+    )
+  }
+  return { enqueued }
+}

package/src/modules/push_notifications/lib/push-health.ts

@@ -0,0 +1,47 @@
+import type { ZodType } from 'zod'
+import type { EmailHealthCheck } from '@open-mercato/core/modules/communication_channels/lib/provider-health'
+import { isPushFakeProvidersEnabled, PUSH_FAKE_PROVIDERS_ENV } from './fake-provider-recorder'
+
+/**
+ * Liveness probe for a push provider (FCM/APNs/Expo).
+ *
+ * A push provider has NO OAuth client — its credentials are a service-account JSON, a `.p8` signing
+ * key, or an Expo access token. So this probe validates the tenant-scoped credentials against the
+ * provider schema and reports push-appropriate copy, rather than the OAuth-client wording the shared
+ * `makeClientConfigHealthCheck` emits (which is correct for the Gmail-style channels but misleading
+ * here). It additionally surfaces fake-provider mode: when `OM_PUSH_FAKE_PROVIDERS` is set every send is
+ * short-circuited to a recorded no-op, so a plain "credentials valid → healthy" probe would report green
+ * while nothing is delivered — this reports `degraded` instead. Pairs with `warnPushFakeProvidersActive`
+ * (a one-time log at registration).
+ */
+export function makePushClientConfigHealthCheck<T>(options: {
+  schema: ZodType<T>
+  providerLabel: string
+  healthyDetails?: (parsed: T) => Record<string, unknown>
+}): EmailHealthCheck {
+  return {
+    async check(credentials) {
+      if (isPushFakeProvidersEnabled()) {
+        return {
+          status: 'degraded',
+          message: `${options.providerLabel} is running in FAKE mode (${PUSH_FAKE_PROVIDERS_ENV} is set): pushes are recorded, not delivered. Unset this flag in production.`,
+          details: { fakeProviders: true, env: PUSH_FAKE_PROVIDERS_ENV },
+        }
+      }
+      const parsed = options.schema.safeParse(credentials ?? {})
+      if (!parsed.success) {
+        const first = parsed.error.issues[0]
+        return {
+          status: 'unhealthy',
+          message: `${options.providerLabel} credentials invalid: ${first?.message ?? 'unknown validation error'}`,
+          details: { reason: 'invalid_credentials' },
+        }
+      }
+      return {
+        status: 'healthy',
+        message: `${options.providerLabel} credentials configured`,
+        details: { credentialsConfigured: true, ...(options.healthyDetails?.(parsed.data) ?? {}) },
+      }
+    },
+  }
+}

package/src/modules/push_notifications/lib/push-reaper.ts

@@ -0,0 +1,140 @@
+import type { EntityManager } from '@mikro-orm/postgresql'
+import { PushNotificationDelivery } from '../data/entities'
+import { emitPushNotificationsEvent } from '../events'
+import { MAX_ATTEMPTS } from './push-delivery'
+import { resolveStuckThresholdMs } from './reclaim-window'
+import { enqueuePushDelivery } from './queue'
+
+// The stuck-reclaim window (minutes a row may sit in `sending`/`pending` before a dead-worker reclaim)
+// lives in ./reclaim-window so the send-path timeout shares the same source of truth.
+//
+// INVARIANT: this window MUST exceed the worst-case single provider send time. `updated_at` is stamped
+// once when the row is claimed (`pending` → `sending`) and is NOT refreshed mid-send (no heartbeat), so
+// a legitimate send that runs longer than the window would be reclaimed and re-enqueued → a duplicate
+// push. The send path caps each provider send below this window (see push-delivery.ts
+// `resolvePushSendTimeoutMs`), and duplicates are otherwise bounded by MAX_ATTEMPTS (incremented at
+// claim time) and inherent to at-least-once delivery.
+
+// Bound the per-tick scan so a stranded backlog (provider/queue outage) cannot load an unbounded row
+// set into memory and re-enqueue it serially in one tick. Oldest-stuck rows are drained first; the
+// remainder is picked up by subsequent ticks. Tunable via OM_PUSH_STUCK_RECLAIM_BATCH_LIMIT (values
+// below MIN_RECLAIM_BATCH_LIMIT, negative, or non-numeric → default). Mirrors the receipt reaper's
+// OM_PUSH_RECEIPT_BATCH_LIMIT.
+const DEFAULT_RECLAIM_BATCH_LIMIT = 500
+const MIN_RECLAIM_BATCH_LIMIT = 1
+
+export type ReclaimStuckResult = { reEnqueued: number; expired: number }
+
+function resolveReclaimBatchLimit(): number {
+  const raw = Number.parseInt(process.env.OM_PUSH_STUCK_RECLAIM_BATCH_LIMIT ?? '', 10)
+  return Number.isFinite(raw) && raw >= MIN_RECLAIM_BATCH_LIMIT ? raw : DEFAULT_RECLAIM_BATCH_LIMIT
+}
+
+async function emitFailed(delivery: PushNotificationDelivery, willRetry: boolean): Promise<void> {
+  await emitPushNotificationsEvent(
+    'push_notifications.delivery.failed',
+    {
+      deliveryId: delivery.id,
+      tenantId: delivery.tenantId,
+      organizationId: delivery.organizationId ?? null,
+      userId: delivery.userId,
+      provider: delivery.provider,
+      status: delivery.status,
+      ...(willRetry ? { willRetry: true } : {}),
+    },
+    { persistent: true },
+  )
+}
+
+/**
+ * Recover push delivery rows stranded by a worker (or an enqueue) that never finished:
+ *
+ *  - `sending` rows — a worker crashed between claiming the row (`pending` → `sending`) and finalizing
+ *    it. Such a row has no outstanding job the claim can re-match.
+ *  - `pending` rows — the fan-out committed the delivery row (a plain INSERT, auto-committed) but the
+ *    subsequent enqueue never landed (process died, or the queue dropped the job). The row then sits
+ *    `pending` forever: the send-path claim only runs when a job fires, so no job ⇒ no claim ⇒ the row
+ *    is never retried, expired, nor surfaced as failed. Only a *synchronous* enqueue throw is handled
+ *    inline by the fan-out; a lost job after the INSERT committed is invisible without this sweep.
+ *
+ * Both are recovered identically: re-enqueue if retry budget remains, else finalize `expired`. Driven
+ * by the `push_notifications:reclaim-stuck` scheduler tick (one per tenant), so the query is scoped to
+ * the tenant only (covers org-bound and tenant-level rows alike); there is no cross-tenant read.
+ *
+ * The stale-`updated_at < cutoff` guard is what makes sweeping `pending` safe: a freshly fan-outed row
+ * (with a live job in flight) is younger than the cutoff and is left alone; only a row that has sat
+ * past the reclaim window — long after any real job would have claimed it — is re-enqueued. A rare
+ * duplicate job is a no-op because the send-path claim is atomic (`pending` → `sending`, exactly one
+ * winner), and provider-send duplicates are bounded by MAX_ATTEMPTS.
+ *
+ * Each transition is an atomic `nativeUpdate` guarded on the row's observed `status` AND still-stale
+ * `updated_at < cutoff`, so overlapping ticks (or a worker that re-claimed the row in the meantime)
+ * can never re-open an actively-processing delivery — exactly one actor wins each row.
+ */
+export async function reclaimStuckPushDeliveries(
+  em: EntityManager,
+  scope: { tenantId: string },
+  now: Date = new Date(),
+): Promise<ReclaimStuckResult> {
+  const cutoff = new Date(now.getTime() - resolveStuckThresholdMs())
+  const stuck = await em.find(
+    PushNotificationDelivery,
+    {
+      tenantId: scope.tenantId,
+      status: { $in: ['sending', 'pending'] },
+      updatedAt: { $lt: cutoff },
+    },
+    { limit: resolveReclaimBatchLimit(), orderBy: { updatedAt: 'asc' } },
+  )
+
+  let reEnqueued = 0
+  let expired = 0
+  for (const delivery of stuck) {
+    const claimGuard = { id: delivery.id, tenantId: scope.tenantId, status: delivery.status, updatedAt: { $lt: cutoff } }
+
+    if (delivery.attempts >= MAX_ATTEMPTS) {
+      const claimed = await em.nativeUpdate(PushNotificationDelivery, claimGuard, {
+        status: 'expired',
+        lastError: 'stuck_reclaimed',
+        nextRetryAt: null,
+        updatedAt: new Date(),
+      })
+      if (claimed === 0) continue
+      delivery.status = 'expired'
+      delivery.lastError = 'stuck_reclaimed'
+      await emitFailed(delivery, false)
+      expired += 1
+      continue
+    }
+
+    const claimed = await em.nativeUpdate(PushNotificationDelivery, claimGuard, {
+      status: 'pending',
+      nextRetryAt: null,
+      updatedAt: new Date(),
+    })
+    if (claimed === 0) continue
+    delivery.status = 'pending'
+
+    try {
+      await enqueuePushDelivery({
+        deliveryId: delivery.id,
+        tenantId: scope.tenantId,
+        organizationId: delivery.organizationId ?? null,
+      })
+      reEnqueued += 1
+    } catch (error) {
+      // Re-enqueue failed: don't leave the row pending with no job. Fail it terminally instead.
+      const reason = error instanceof Error ? `reclaim_enqueue_failed: ${error.message}` : 'reclaim_enqueue_failed'
+      await em.nativeUpdate(
+        PushNotificationDelivery,
+        { id: delivery.id, tenantId: scope.tenantId, status: 'pending' },
+        { status: 'failed', lastError: reason, nextRetryAt: null, updatedAt: new Date() },
+      )
+      delivery.status = 'failed'
+      delivery.lastError = reason
+      await emitFailed(delivery, false)
+    }
+  }
+
+  return { reEnqueued, expired }
+}

package/src/modules/push_notifications/lib/push-receipt-reaper.ts

@@ -0,0 +1,210 @@
+import type { EntityManager } from '@mikro-orm/postgresql'
+import { raw, type EntityName } from '@mikro-orm/core'
+import type { ChannelAdapter } from '@open-mercato/core/modules/communication_channels/lib/adapter'
+import {
+  DEVICE_UNREGISTERED,
+  supportsReceiptChecking,
+} from '@open-mercato/core/modules/communication_channels/lib/push-adapter'
+import type { CommunicationChannel } from '@open-mercato/core/modules/communication_channels/data/entities'
+import { createLogger } from '@open-mercato/shared/lib/logger'
+import { PushNotificationDelivery } from '../data/entities'
+import { softDeleteUnregisteredDevice } from './push-delivery'
+
+const logger = createLogger('push_notifications')
+
+type Resolve = <T = unknown>(name: string) => T
+
+interface ChannelAdapterRegistryLike {
+  get(providerKey: string): ChannelAdapter | undefined
+}
+
+interface CredentialsServiceLike {
+  resolve(
+    integrationId: string,
+    scope: { tenantId: string; organizationId: string; userId?: string | null },
+  ): Promise<Record<string, unknown> | null>
+}
+
+export type ReceiptReaperResult = { checked: number; unregistered: number }
+
+// A receipt only exists after the provider has attempted handoff to FCM/APNs, so a just-`sent` row has
+// nothing to poll yet. Wait MIN_AGE before the first check (Expo recommends ~15m) and stop looking after
+// MAX_AGE — a row not resolved within the window is left as `sent` (best-effort hygiene, not a
+// correctness guarantee). Both tunable; MAX is clamped above MIN so the window is always non-empty.
+// Parse an int env var, falling back to `fallback` only when it is unset/non-numeric — a legitimately
+// configured `0` is preserved (`|| fallback` would coerce it, since `0` is falsy).
+function envInt(name: string, fallback: number): number {
+  const parsed = Number.parseInt(process.env[name] ?? '', 10)
+  return Number.isFinite(parsed) ? parsed : fallback
+}
+// A negative value must NOT collapse to 0 (that would poll before any receipt exists and mark the
+// delivery `receiptChecked` forever). Preserve a legitimately-configured 0 but floor a negative back to
+// the default — mirroring the reaper's `raw >= MIN ? raw : DEFAULT` guard (resolveReclaimBatchLimit).
+const RECEIPT_MIN_AGE_MINUTES_RAW = envInt('OM_PUSH_RECEIPT_MIN_AGE_MINUTES', 15)
+const RECEIPT_MIN_AGE_MS = (RECEIPT_MIN_AGE_MINUTES_RAW >= 0 ? RECEIPT_MIN_AGE_MINUTES_RAW : 15) * 60 * 1000
+const RECEIPT_MAX_AGE_MS = Math.max(
+  RECEIPT_MIN_AGE_MS + 60 * 1000,
+  envInt('OM_PUSH_RECEIPT_MAX_AGE_MINUTES', 60) * 60 * 1000,
+)
+// Bound the per-tick scan so a busy tenant's `sent` backlog cannot fetch an unbounded row set.
+const RECEIPT_BATCH_LIMIT = Math.max(1, envInt('OM_PUSH_RECEIPT_BATCH_LIMIT', 500))
+
+/** The provider ticket id persisted by the send path, or null when there is nothing to poll. */
+function ticketIdOf(delivery: PushNotificationDelivery): string | null {
+  const id = (delivery.providerResponse as { externalMessageId?: unknown } | null | undefined)?.externalMessageId
+  return typeof id === 'string' && id.length > 0 ? id : null
+}
+
+/** A row whose receipt was already resolved on a prior sweep — skip it (marker is JSON, no migration). */
+function isReceiptChecked(delivery: PushNotificationDelivery): boolean {
+  return (delivery.providerResponse as { receiptChecked?: unknown } | null | undefined)?.receiptChecked === true
+}
+
+function markReceiptChecked(delivery: PushNotificationDelivery, unregistered: boolean): void {
+  delivery.providerResponse = {
+    ...(delivery.providerResponse ?? {}),
+    receiptChecked: true,
+    ...(unregistered ? { unregistered: true } : {}),
+  }
+}
+
+/**
+ * Second-pass hygiene for push providers whose "device unregistered" signal arrives in an ASYNCHRONOUS
+ * receipt phase rather than the synchronous send ticket (Expo). Runs on the per-tenant reclaim tick.
+ *
+ * For each `sent` delivery old enough to have a receipt, group ticket ids by provider, and for every
+ * provider whose adapter implements the optional `PushReceiptChecker` capability, poll receipts using
+ * the same channel credentials as the send path. A `DeviceNotRegistered` receipt is routed through the
+ * SAME `softDeleteUnregisteredDevice` contract the synchronous path uses — no separate deletion path.
+ * `MessageRateExceeded` and other transient receipt errors resolve the receipt but never kill the token.
+ *
+ * Idempotent: each resolved row is marked (`providerResponse.receiptChecked`) so it is never re-polled,
+ * and `devices.deactivate` is itself idempotent, so an overlapping tick cannot double-delete.
+ */
+export async function checkPushReceipts(
+  em: EntityManager,
+  scope: { tenantId: string },
+  resolve: Resolve,
+  now: Date = new Date(),
+): Promise<ReceiptReaperResult> {
+  const registry = resolve('channelAdapterRegistry') as ChannelAdapterRegistryLike | undefined
+  if (!registry) return { checked: 0, unregistered: 0 }
+
+  // Exclude non-workable rows in SQL, not in memory. A resolved row keeps `status: 'sent'` (the
+  // `receiptChecked` marker lives in the `provider_response` JSON, not the status column), and rows
+  // with no pollable ticket id (fcm/apns report unregistered synchronously and persist an empty
+  // `externalMessageId`) can never be receipt-checked. Filtering those in memory after a
+  // `LIMIT … ORDER BY sent_at ASC` fetch lets an oldest-first backlog of them fill the bounded batch
+  // and starve every still-unchecked Expo row in the tenant — the batch does zero work forever until
+  // the backlog ages past MAX_AGE. Push both predicates into the query so the budget only ever loads
+  // rows that can actually make progress. (`provider_response` is `json`; `->>` reads it as text.)
+  const workableRow = raw(
+    `(nullif(provider_response->>'externalMessageId', '') is not null and coalesce((provider_response->>'receiptChecked')::boolean, false) = false)`,
+  )
+  const candidates = await em.find(
+    PushNotificationDelivery,
+    {
+      tenantId: scope.tenantId,
+      status: 'sent',
+      sentAt: { $gte: new Date(now.getTime() - RECEIPT_MAX_AGE_MS), $lte: new Date(now.getTime() - RECEIPT_MIN_AGE_MS) },
+      [workableRow as unknown as string]: true,
+    },
+    { limit: RECEIPT_BATCH_LIMIT, orderBy: { sentAt: 'asc' } },
+  )
+
+  // Group unchecked rows that have a real ticket id by provider. Providers whose adapter doesn't
+  // implement receipt checking (fcm/apns) never issue a network call — they are dropped below.
+  const byProvider = new Map<string, { ticketId: string; delivery: PushNotificationDelivery }[]>()
+  for (const delivery of candidates) {
+    if (isReceiptChecked(delivery)) continue
+    const ticketId = ticketIdOf(delivery)
+    if (!ticketId) continue
+    const list = byProvider.get(delivery.provider) ?? []
+    list.push({ ticketId, delivery })
+    byProvider.set(delivery.provider, list)
+  }
+  if (byProvider.size === 0) return { checked: 0, unregistered: 0 }
+
+  let credentialsService: CredentialsServiceLike | undefined
+  try {
+    credentialsService = resolve('integrationCredentialsService') as CredentialsServiceLike | undefined
+  } catch {
+    credentialsService = undefined
+  }
+  const ChannelRef = resolve('CommunicationChannel') as EntityName<CommunicationChannel>
+
+  let checked = 0
+  let unregistered = 0
+
+  for (const [provider, entries] of byProvider) {
+    const adapter = registry.get(provider)
+    if (!adapter || !supportsReceiptChecking(adapter)) continue
+
+    const channel = await em.findOne(ChannelRef, {
+      tenantId: scope.tenantId,
+      providerKey: provider,
+      channelType: 'push',
+      isActive: true,
+      deletedAt: null,
+    })
+    if (!channel) continue
+
+    // Credentials are keyed by the CHANNEL's org context, mirroring the send path (see push-delivery).
+    const credentialScope = {
+      tenantId: scope.tenantId,
+      organizationId: channel.organizationId ?? scope.tenantId,
+      userId: channel.userId ?? null,
+    }
+    let credentials: Record<string, unknown> = {}
+    if (channel.credentialsRef && credentialsService) {
+      credentials = (await credentialsService.resolve(`channel_${provider}`, credentialScope).catch(() => null)) ?? {}
+    }
+
+    const byTicket = new Map<string, PushNotificationDelivery[]>()
+    for (const entry of entries) {
+      const list = byTicket.get(entry.ticketId) ?? []
+      list.push(entry.delivery)
+      byTicket.set(entry.ticketId, list)
+    }
+
+    let outcomes
+    try {
+      outcomes = await adapter.checkReceipts([...byTicket.keys()], credentials)
+    } catch (error) {
+      logger.error('Push receipt check failed', {
+        tenantId: scope.tenantId,
+        provider,
+        error: error instanceof Error ? error.message : String(error),
+      })
+      continue
+    }
+
+    // Persist the receipt-checked marks first, then run the device deactivations AFTER the flush so a
+    // command that reads on the same EntityManager can never discard the pending scalar marks.
+    const toDeactivate: Array<{ id: string; tenantId: string; userId: string; organizationId: string | null }> = []
+    for (const outcome of outcomes) {
+      const deliveries = byTicket.get(outcome.ticketId)
+      if (!deliveries) continue
+      for (const delivery of deliveries) {
+        markReceiptChecked(delivery, outcome.unregistered)
+        checked += 1
+        if (outcome.unregistered) {
+          delivery.lastError = DEVICE_UNREGISTERED
+          toDeactivate.push({
+            id: delivery.userDeviceId,
+            tenantId: delivery.tenantId,
+            userId: delivery.userId,
+            organizationId: delivery.organizationId ?? null,
+          })
+          unregistered += 1
+        }
+      }
+    }
+    await em.flush()
+    for (const input of toDeactivate) {
+      await softDeleteUnregisteredDevice(resolve, input)
+    }
+  }
+
+  return { checked, unregistered }
+}