nolo-cli 0.1.26 → 0.1.29

package/auth/server/emailDelivery.ts

@@ -0,0 +1,960 @@
+import { ulid } from "ulid";
+import serverDb from "../../database/server/db";
+import { DB_PREFIX } from "../../database/keys";
+import {
+  emailProviderSupportsMarketingBroadcast,
+  sendProviderBatchEmails,
+  sendProviderEmail,
+  sendProviderMarketingBroadcast,
+} from "./emailProvider";
+import { balanceLockManager } from "./locks";
+import { logger } from "./shared";
+import {
+  getReengagementDeliveryMode,
+  shouldRunSchedulerOnThisInstance,
+} from "./emailAutomationConfig";
+import {
+  confirmIdempotencySent,
+  normalizeEmailTag,
+  releaseIdempotency,
+  reserveIdempotency,
+  shouldSuppressOnNonLeader,
+  type EmailIdempotencyReservation,
+} from "./emailDeliveryGuards";
+
+const EMAIL_RECORD_PREFIX = "email:delivery:record:";
+const EMAIL_STATS_PREFIX = "email:delivery:stats:";
+const EMAIL_RETRY_PREFIX = "email:delivery:retry:";
+const EMAIL_MARKETING_SEGMENT_MEMBERS_PREFIX = "email:delivery:marketing:segment-members:";
+
+const parseNumber = (value: string | undefined, fallback: number, min = 1) => {
+  const parsed = Number(value);
+  if (!Number.isFinite(parsed)) return fallback;
+  return Math.max(min, Math.floor(parsed));
+};
+const parseBoolean = (value: string | undefined, fallback: boolean) => {
+  if (value == null || value === "") return fallback;
+  return !["0", "false", "off", "no"].includes(String(value).toLowerCase());
+};
+const normalizeHost = (value: string | undefined) => {
+  const raw = String(value || "").trim().toLowerCase();
+  if (!raw) return "";
+  if (raw.startsWith("[") && raw.includes("]")) {
+    return raw.slice(1, raw.indexOf("]"));
+  }
+  return raw.split("/")[0].split(":")[0];
+};
+const resolveHostFromUrl = (value: string | undefined) => {
+  const raw = String(value || "").trim();
+  if (!raw) return "";
+  try {
+    return normalizeHost(new URL(raw).hostname);
+  } catch {
+    return "";
+  }
+};
+const getCurrentEmailHost = () =>
+  normalizeHost(process.env.EMAIL_APP_HOST || process.env.APP_HOST || process.env.HOST) ||
+  resolveHostFromUrl(process.env.EMAIL_APP_URL || process.env.APP_URL);
+const getAllowedSignupHost = () =>
+  normalizeHost(process.env.EMAIL_SIGNUP_ALLOWED_HOST || "nolo.chat") || "nolo.chat";
+const isFreezeModeEnabled = () =>
+  // 临时策略：仅放行 nolo.chat 的 signup-welcome；其余邮件类型统一冻结。
+  parseBoolean(process.env.EMAIL_ONLY_SIGNUP_WELCOME_ENABLED, false);
+const getEmailFreezeDecision = (tag: string) => {
+  if (!isFreezeModeEnabled()) {
+    return {
+      allowed: true as const,
+      reason: "",
+      currentHost: "",
+      allowedHost: "",
+    };
+  }
+  const normalizedTag = String(tag || "").trim().toLowerCase();
+  const currentHost = getCurrentEmailHost();
+  const allowedHost = getAllowedSignupHost();
+  const allowed = normalizedTag === "signup-welcome" && currentHost === allowedHost;
+  return {
+    allowed,
+    reason: allowed
+      ? "allowed"
+      : normalizedTag !== "signup-welcome"
+        ? "frozen_non_signup_welcome"
+        : "frozen_non_allowed_host",
+    currentHost,
+    allowedHost,
+  };
+};
+const getRetryConfig = () => ({
+  maxRetryAttempts: parseNumber(process.env.EMAIL_RETRY_MAX_ATTEMPTS, 3, 1),
+  retryDelayMs: parseNumber(process.env.EMAIL_RETRY_DELAY_MS, 5 * 60 * 1000, 1_000),
+  retryBatchSize: parseNumber(process.env.EMAIL_RETRY_BATCH_SIZE, 100, 1),
+  retryIntervalMinutes: parseNumber(process.env.EMAIL_RETRY_INTERVAL_MINUTES, 10, 1),
+  retryEnabled: !["0", "false", "off", "no"].includes(
+    String(process.env.EMAIL_RETRY_ENABLED ?? "1").toLowerCase()
+  ),
+});
+
+const normalizeEmailAddress = (value: string) => String(value || "").trim().toLowerCase();
+
+const getReengagementMarketingSegmentId = () =>
+  String(process.env.EMAIL_REENGAGEMENT_MARKETING_SEGMENT_ID || "").trim();
+
+const getMarketingSegmentMembersKey = (segmentId: string) =>
+  `${EMAIL_MARKETING_SEGMENT_MEMBERS_PREFIX}${segmentId}:reengagement`;
+
+const isMarketingReengagementBatch = (payloads: ManagedEmailPayload[]) =>
+  payloads.length > 0 &&
+  getReengagementDeliveryMode() === "marketing" &&
+  payloads.every((payload) => normalizeEmailTag(payload.tag) === "reengagement");
+
+const hasUniformBatchContent = (payloads: ManagedEmailPayload[]) => {
+  if (payloads.length <= 1) return true;
+  const first = payloads[0];
+  return payloads.every(
+    (payload) =>
+      payload.subject === first.subject &&
+      payload.textBody === first.textBody &&
+      payload.htmlBody === first.htmlBody
+  );
+};
+
+const readMarketingSegmentMembers = async (segmentId: string) => {
+  const raw = (await serverDb.get(getMarketingSegmentMembersKey(segmentId)).catch(() => null)) as
+    | { emails?: unknown }
+    | null;
+  if (!raw || !Array.isArray(raw.emails)) return [];
+  return Array.from(
+    new Set(
+      raw.emails
+        .map((item) => normalizeEmailAddress(String(item || "")))
+        .filter(Boolean)
+    )
+  );
+};
+
+const writeMarketingSegmentMembers = async (segmentId: string, emails: string[]) => {
+  const normalizedEmails = Array.from(
+    new Set(emails.map((email) => normalizeEmailAddress(email)).filter(Boolean))
+  );
+  await serverDb.put(getMarketingSegmentMembersKey(segmentId), {
+    emails: normalizedEmails,
+    updatedAt: Date.now(),
+  });
+};
+
+const getDayKey = (time: number = Date.now()) =>
+  new Date(time).toISOString().slice(0, 10).replace(/-/g, "");
+
+const padTimestamp = (time: number) => String(Math.max(0, Math.floor(time))).padStart(13, "0");
+
+const getStatsKey = (dayKey: string, tag: string) =>
+  `${EMAIL_STATS_PREFIX}${dayKey}:${tag || "untagged"}`;
+
+const getRetryKey = (nextRetryAt: number, retryId: string) =>
+  `${EMAIL_RETRY_PREFIX}${padTimestamp(nextRetryAt)}:${retryId}`;
+
+type DeliveryStatus = "sent" | "failed";
+
+export interface ManagedEmailPayload {
+  userId?: string;
+  to: string;
+  subject: string;
+  textBody?: string;
+  htmlBody?: string;
+  tag: string;
+  metadata?: Record<string, any>;
+}
+
+interface RetryJob {
+  retryId: string;
+  payload: ManagedEmailPayload;
+  attempt: number;
+  maxAttempts: number;
+  nextRetryAt: number;
+  lastError?: string;
+  createdAt: number;
+  updatedAt: number;
+}
+
+interface SendManagedEmailOptions {
+  enqueueOnFail?: boolean;
+  attempt?: number;
+  maxAttempts?: number;
+  retryDelayMs?: number;
+}
+
+export interface SendManagedEmailResult {
+  success: boolean;
+  queued: boolean;
+  error?: string;
+  suppressed?: boolean;
+}
+
+export interface SendManagedEmailBatchItemResult extends SendManagedEmailResult {
+  userId?: string;
+  to: string;
+}
+
+interface RecordStatsDelta {
+  sent?: number;
+  failed?: number;
+  retried?: number;
+  queued?: number;
+}
+
+interface EmailDeliveryRecord {
+  id: string;
+  timestamp: number;
+  status: DeliveryStatus;
+  userId?: string;
+  to: string;
+  subject: string;
+  tag: string;
+  attempt: number;
+  error?: string;
+  metadata?: Record<string, any>;
+}
+
+let retryTimer: ReturnType<typeof setInterval> | null = null;
+let retryRunning = false;
+
+const incrementStats = async (
+  dayKey: string,
+  tag: string,
+  delta: RecordStatsDelta
+) => {
+  const statsKey = getStatsKey(dayKey, tag);
+  const statsLock = balanceLockManager.getLock(`email-stats:${statsKey}`);
+  await statsLock.runExclusive(async () => {
+    const currentRaw = await serverDb.get(statsKey).catch(() => null);
+    const current =
+      currentRaw && typeof currentRaw === "object" && !Array.isArray(currentRaw)
+        ? (currentRaw as Record<string, any>)
+        : {};
+    await serverDb.put(statsKey, {
+      ...current,
+      dayKey,
+      tag,
+      sent: Number(current.sent || 0) + Number(delta.sent || 0),
+      failed: Number(current.failed || 0) + Number(delta.failed || 0),
+      retried: Number(current.retried || 0) + Number(delta.retried || 0),
+      queued: Number(current.queued || 0) + Number(delta.queued || 0),
+      updatedAt: Date.now(),
+    });
+  });
+};
+
+const writeDeliveryRecord = async (record: EmailDeliveryRecord) => {
+  await serverDb.put(`${EMAIL_RECORD_PREFIX}${record.id}`, record);
+};
+
+const enqueueRetryJob = async (job: RetryJob) => {
+  await serverDb.put(getRetryKey(job.nextRetryAt, job.retryId), job);
+  await incrementStats(getDayKey(job.updatedAt), job.payload.tag, { queued: 1 });
+};
+
+const markWelcomeEmailSent = async (userId: string, sentAt: number) => {
+  const lock = balanceLockManager.getLock(userId);
+  await lock.runExclusive(async () => {
+    const key = `${DB_PREFIX.USER}${userId}`;
+    const current = await serverDb.get(key).catch(() => null);
+    if (!current || typeof current !== "object") return;
+    await serverDb.put(key, {
+      ...current,
+      welcomeEmailSentAt: sentAt,
+    });
+  });
+};
+
+interface FinalizeManagedEmailAttemptArgs {
+  payload: ManagedEmailPayload;
+  status: DeliveryStatus;
+  errorMessage?: string;
+  attempt: number;
+  maxAttempts: number;
+  retryDelayMs: number;
+  enqueueOnFail: boolean;
+  suppressed?: boolean;
+  metadata?: Record<string, any>;
+}
+
+const finalizeManagedEmailAttempt = async ({
+  payload,
+  status,
+  errorMessage,
+  attempt,
+  maxAttempts,
+  retryDelayMs,
+  enqueueOnFail,
+  suppressed = false,
+  metadata,
+}: FinalizeManagedEmailAttemptArgs): Promise<SendManagedEmailResult> => {
+  const now = Date.now();
+  const tag = payload.tag || "untagged";
+
+  await writeDeliveryRecord({
+    id: ulid(now),
+    timestamp: now,
+    status,
+    userId: payload.userId,
+    to: payload.to,
+    subject: payload.subject,
+    tag,
+    attempt,
+    error: errorMessage,
+    metadata: metadata ?? payload.metadata,
+  });
+
+  await incrementStats(getDayKey(now), tag, {
+    sent: status === "sent" ? 1 : 0,
+    failed: status === "failed" ? 1 : 0,
+    retried: attempt > 1 && status === "sent" ? 1 : 0,
+  });
+
+  if (status === "sent") {
+    return { success: true as const, queued: false };
+  }
+
+  if (suppressed) {
+    return {
+      success: false as const,
+      queued: false,
+      suppressed: true,
+      error: errorMessage,
+    };
+  }
+
+  if (!enqueueOnFail || attempt >= maxAttempts) {
+    return { success: false as const, queued: false, error: errorMessage };
+  }
+
+  const retryId = ulid(now);
+  const nextRetryAt = now + retryDelayMs * attempt;
+  await enqueueRetryJob({
+    retryId,
+    payload,
+    attempt,
+    maxAttempts,
+    nextRetryAt,
+    lastError: errorMessage,
+    createdAt: now,
+    updatedAt: now,
+  });
+
+  return { success: false as const, queued: true, error: errorMessage };
+};
+
+export async function sendManagedEmail(
+  payload: ManagedEmailPayload,
+  options: SendManagedEmailOptions = {}
+): Promise<SendManagedEmailResult> {
+  const retryConfig = getRetryConfig();
+  const attempt = options.attempt ?? 1;
+  const maxAttempts = options.maxAttempts ?? retryConfig.maxRetryAttempts;
+  const retryDelayMs = options.retryDelayMs ?? retryConfig.retryDelayMs;
+  const enqueueOnFail = options.enqueueOnFail ?? true;
+  const tag = normalizeEmailTag(payload.tag);
+  const normalizedPayload: ManagedEmailPayload = { ...payload, tag };
+
+  if (tag === "reengagement" && getReengagementDeliveryMode() === "marketing") {
+    const [result] = await sendManagedEmailBatch([normalizedPayload], options);
+    return {
+      success: Boolean(result?.success),
+      queued: Boolean(result?.queued),
+      error: result?.error,
+      suppressed: result?.suppressed,
+    };
+  }
+
+  if (shouldSuppressOnNonLeader(tag)) {
+    const errorMessage = "Email suppressed by leader-only policy (non_leader_instance)";
+    logger.warn(
+      {
+        tag,
+        userId: normalizedPayload.userId,
+        to: normalizedPayload.to,
+        reason: "non_leader_instance",
+      },
+      "Email suppressed by leader-only policy"
+    );
+    return finalizeManagedEmailAttempt({
+      payload: normalizedPayload,
+      status: "failed",
+      errorMessage,
+      attempt,
+      maxAttempts,
+      retryDelayMs,
+      enqueueOnFail,
+      suppressed: true,
+      metadata: {
+        ...(normalizedPayload.metadata || {}),
+        suppressedByLeaderOnly: true,
+        suppressionReason: "non_leader_instance",
+      },
+    });
+  }
+
+  const freezeDecision = getEmailFreezeDecision(tag);
+
+  if (!freezeDecision.allowed) {
+    const errorMessage = `Email suppressed by freeze policy (${freezeDecision.reason})`;
+    logger.warn(
+      {
+        tag,
+        userId: normalizedPayload.userId,
+        to: normalizedPayload.to,
+        reason: freezeDecision.reason,
+        currentHost: freezeDecision.currentHost,
+        allowedHost: freezeDecision.allowedHost,
+      },
+      "Email suppressed by freeze policy"
+    );
+    return finalizeManagedEmailAttempt({
+      payload: normalizedPayload,
+      status: "failed",
+      errorMessage,
+      attempt,
+      maxAttempts,
+      retryDelayMs,
+      enqueueOnFail,
+      suppressed: true,
+      metadata: {
+        ...(normalizedPayload.metadata || {}),
+        suppressedByFreeze: true,
+        suppressionReason: freezeDecision.reason,
+        currentHost: freezeDecision.currentHost,
+        allowedHost: freezeDecision.allowedHost,
+      },
+    });
+  }
+
+  const idempotencyDecision = await reserveIdempotency(normalizedPayload, tag);
+  if (!idempotencyDecision.allowed) {
+    const errorMessage = `Email suppressed by idempotency policy (${idempotencyDecision.reason})`;
+    logger.warn(
+      {
+        tag,
+        userId: normalizedPayload.userId,
+        to: normalizedPayload.to,
+        reason: idempotencyDecision.reason,
+      },
+      "Email suppressed by idempotency policy"
+    );
+    return finalizeManagedEmailAttempt({
+      payload: normalizedPayload,
+      status: "failed",
+      errorMessage,
+      attempt,
+      maxAttempts,
+      retryDelayMs,
+      enqueueOnFail,
+      suppressed: true,
+      metadata: {
+        ...(normalizedPayload.metadata || {}),
+        suppressedByIdempotency: true,
+        suppressionReason: idempotencyDecision.reason,
+      },
+    });
+  }
+
+  const reservation = idempotencyDecision.reservation;
+
+  let status: DeliveryStatus = "sent";
+  let errorMessage: string | undefined;
+  try {
+    await sendProviderEmail({
+      to: normalizedPayload.to,
+      subject: normalizedPayload.subject,
+      textBody: normalizedPayload.textBody,
+      htmlBody: normalizedPayload.htmlBody,
+      tag,
+    });
+  } catch (error: any) {
+    status = "failed";
+    errorMessage = error?.message || String(error);
+  }
+
+  if (status === "sent") {
+    await confirmIdempotencySent(reservation);
+  } else {
+    await releaseIdempotency(reservation);
+  }
+
+  return finalizeManagedEmailAttempt({
+    payload: normalizedPayload,
+    status,
+    errorMessage,
+    attempt,
+    maxAttempts,
+    retryDelayMs,
+    enqueueOnFail,
+  });
+}
+
+export async function sendManagedEmailBatch(
+  payloads: ManagedEmailPayload[],
+  options: SendManagedEmailOptions = {}
+): Promise<SendManagedEmailBatchItemResult[]> {
+  if (!Array.isArray(payloads) || payloads.length === 0) return [];
+
+  const retryConfig = getRetryConfig();
+  const attempt = options.attempt ?? 1;
+  const maxAttempts = options.maxAttempts ?? retryConfig.maxRetryAttempts;
+  const retryDelayMs = options.retryDelayMs ?? retryConfig.retryDelayMs;
+  const enqueueOnFail = options.enqueueOnFail ?? true;
+
+  const results: Array<SendManagedEmailBatchItemResult | undefined> = new Array(
+    payloads.length
+  );
+  const batchPayloads: ManagedEmailPayload[] = [];
+  const batchIndexes: number[] = [];
+  const batchReservations: Array<EmailIdempotencyReservation | null> = [];
+
+  for (let index = 0; index < payloads.length; index += 1) {
+    const payload = payloads[index];
+    const tag = normalizeEmailTag(payload.tag);
+    const normalizedPayload: ManagedEmailPayload = { ...payload, tag };
+
+    if (shouldSuppressOnNonLeader(tag)) {
+      const errorMessage = "Email suppressed by leader-only policy (non_leader_instance)";
+      logger.warn(
+        {
+          tag,
+          userId: normalizedPayload.userId,
+          to: normalizedPayload.to,
+          reason: "non_leader_instance",
+        },
+        "Email suppressed by leader-only policy"
+      );
+      const result = await finalizeManagedEmailAttempt({
+        payload: normalizedPayload,
+        status: "failed",
+        errorMessage,
+        attempt,
+        maxAttempts,
+        retryDelayMs,
+        enqueueOnFail,
+        suppressed: true,
+        metadata: {
+          ...(normalizedPayload.metadata || {}),
+          suppressedByLeaderOnly: true,
+          suppressionReason: "non_leader_instance",
+        },
+      });
+      results[index] = {
+        ...result,
+        userId: normalizedPayload.userId,
+        to: normalizedPayload.to,
+      };
+      continue;
+    }
+
+    const freezeDecision = getEmailFreezeDecision(tag);
+
+    if (!freezeDecision.allowed) {
+      const errorMessage = `Email suppressed by freeze policy (${freezeDecision.reason})`;
+      logger.warn(
+        {
+          tag,
+          userId: normalizedPayload.userId,
+          to: normalizedPayload.to,
+          reason: freezeDecision.reason,
+          currentHost: freezeDecision.currentHost,
+          allowedHost: freezeDecision.allowedHost,
+        },
+        "Email suppressed by freeze policy"
+      );
+      const result = await finalizeManagedEmailAttempt({
+        payload: normalizedPayload,
+        status: "failed",
+        errorMessage,
+        attempt,
+        maxAttempts,
+        retryDelayMs,
+        enqueueOnFail,
+        suppressed: true,
+        metadata: {
+          ...(normalizedPayload.metadata || {}),
+          suppressedByFreeze: true,
+          suppressionReason: freezeDecision.reason,
+          currentHost: freezeDecision.currentHost,
+          allowedHost: freezeDecision.allowedHost,
+        },
+      });
+      results[index] = {
+        ...result,
+        userId: payload.userId,
+        to: payload.to,
+      };
+      continue;
+    }
+
+    const idempotencyDecision = await reserveIdempotency(normalizedPayload, tag);
+    if (!idempotencyDecision.allowed) {
+      const errorMessage = `Email suppressed by idempotency policy (${idempotencyDecision.reason})`;
+      logger.warn(
+        {
+          tag,
+          userId: normalizedPayload.userId,
+          to: normalizedPayload.to,
+          reason: idempotencyDecision.reason,
+        },
+        "Email suppressed by idempotency policy"
+      );
+      const result = await finalizeManagedEmailAttempt({
+        payload: normalizedPayload,
+        status: "failed",
+        errorMessage,
+        attempt,
+        maxAttempts,
+        retryDelayMs,
+        enqueueOnFail,
+        suppressed: true,
+        metadata: {
+          ...(normalizedPayload.metadata || {}),
+          suppressedByIdempotency: true,
+          suppressionReason: idempotencyDecision.reason,
+        },
+      });
+      results[index] = {
+        ...result,
+        userId: normalizedPayload.userId,
+        to: normalizedPayload.to,
+      };
+      continue;
+    }
+
+    batchPayloads.push(normalizedPayload);
+    batchIndexes.push(index);
+    batchReservations.push(idempotencyDecision.reservation);
+  }
+
+  if (batchPayloads.length > 0) {
+    const perItemErrors = new Map<number, string>();
+    let batchError: string | undefined;
+    let suppressFailedResults = false;
+    const useMarketingBatch =
+      isMarketingReengagementBatch(batchPayloads) && hasUniformBatchContent(batchPayloads);
+
+    if (useMarketingBatch && emailProviderSupportsMarketingBroadcast()) {
+      const segmentId = getReengagementMarketingSegmentId();
+      if (!segmentId) {
+        batchError = "EMAIL_REENGAGEMENT_MARKETING_SEGMENT_ID is not defined";
+        suppressFailedResults = true;
+      } else {
+        const firstPayload = batchPayloads[0];
+        if (!firstPayload) {
+          batchError = "Batch payload missing";
+          suppressFailedResults = true;
+        } else {
+          try {
+            const previousRecipientEmails = await readMarketingSegmentMembers(segmentId);
+            await sendProviderMarketingBroadcast({
+              segmentId,
+              recipients: batchPayloads.map((payload) => ({ email: payload.to })),
+              previousRecipientEmails,
+              subject: firstPayload.subject,
+              textBody: firstPayload.textBody,
+              htmlBody: firstPayload.htmlBody,
+              tag: firstPayload.tag || "reengagement",
+            });
+            await writeMarketingSegmentMembers(
+              segmentId,
+              batchPayloads.map((payload) => payload.to)
+            );
+          } catch (error: any) {
+            batchError = error?.message || String(error);
+          }
+        }
+      }
+    } else {
+      try {
+        const batchResponse = await sendProviderBatchEmails(
+          batchPayloads.map((payload) => ({
+            to: payload.to,
+            subject: payload.subject,
+            textBody: payload.textBody,
+            htmlBody: payload.htmlBody,
+            tag: payload.tag || "untagged",
+          }))
+        );
+
+        if (Array.isArray((batchResponse as any)?.errors)) {
+          for (const entry of (batchResponse as any).errors) {
+            if (!Number.isInteger(entry?.index)) continue;
+            perItemErrors.set(
+              Number(entry.index),
+              String(entry?.message || "Batch delivery failed")
+            );
+          }
+        }
+      } catch (error: any) {
+        batchError = error?.message || String(error);
+      }
+    }
+
+    for (let localIndex = 0; localIndex < batchPayloads.length; localIndex += 1) {
+      const payload = batchPayloads[localIndex];
+      const targetIndex = batchIndexes[localIndex];
+      const reservation = batchReservations[localIndex] || null;
+      const itemError = perItemErrors.get(localIndex);
+      const status: DeliveryStatus = batchError || itemError ? "failed" : "sent";
+      if (status === "sent") {
+        await confirmIdempotencySent(reservation);
+      } else {
+        await releaseIdempotency(reservation);
+      }
+      const result = await finalizeManagedEmailAttempt({
+        payload,
+        status,
+        errorMessage: batchError || itemError,
+        attempt,
+        maxAttempts,
+        retryDelayMs,
+        enqueueOnFail,
+        suppressed: status === "failed" && suppressFailedResults,
+      });
+      results[targetIndex] = {
+        ...result,
+        userId: payload.userId,
+        to: payload.to,
+      };
+    }
+  }
+
+  return results.map((item, index) => {
+    if (item) return item;
+    return {
+      userId: payloads[index]?.userId,
+      to: payloads[index]?.to || "",
+      success: false,
+      queued: false,
+      error: "Batch delivery result missing",
+    };
+  });
+}
+
+export async function processEmailRetryQueue() {
+  if (retryRunning) {
+    return { processed: 0, retried: 0, queued: 0, failed: 0, skipped: true };
+  }
+
+  retryRunning = true;
+  try {
+    const retryConfig = getRetryConfig();
+    const now = Date.now();
+    const jobs: Array<{ key: string; job: RetryJob }> = [];
+
+    const iter = serverDb.iterator({
+      gt: EMAIL_RETRY_PREFIX,
+      lt: `${EMAIL_RETRY_PREFIX}\xFF`,
+    });
+
+    try {
+      for await (const [key, value] of iter) {
+        const job = value as RetryJob;
+        if (!job?.nextRetryAt) continue;
+        if (job.nextRetryAt > now) break;
+        jobs.push({ key, job });
+        if (jobs.length >= retryConfig.retryBatchSize) break;
+      }
+    } finally {
+      await iter.close();
+    }
+
+    let retried = 0;
+    let queued = 0;
+    let failed = 0;
+
+    for (const item of jobs) {
+      await serverDb.del(item.key).catch(() => undefined);
+      const result = await sendManagedEmail(item.job.payload, {
+        attempt: item.job.attempt + 1,
+        maxAttempts: item.job.maxAttempts,
+        enqueueOnFail: true,
+      });
+      if (result.success) {
+        retried += 1;
+        if (
+          item.job.payload.tag === "signup-welcome" &&
+          typeof item.job.payload.userId === "string" &&
+          item.job.payload.userId
+        ) {
+          try {
+            await markWelcomeEmailSent(item.job.payload.userId, Date.now());
+          } catch (error) {
+            logger.warn(
+              { userId: item.job.payload.userId, error },
+              "Failed to mark welcome email as sent after retry"
+            );
+          }
+        }
+      } else if (result.queued) {
+        queued += 1;
+      } else {
+        failed += 1;
+      }
+    }
+
+    return { processed: jobs.length, retried, queued, failed, skipped: false };
+  } finally {
+    retryRunning = false;
+  }
+}
+
+export function startEmailRetryScheduler() {
+  const retryConfig = getRetryConfig();
+  if (!retryConfig.retryEnabled) {
+    logger.info("Email retry scheduler disabled by env");
+    return;
+  }
+  if (!shouldRunSchedulerOnThisInstance()) {
+    logger.info(
+      {
+        nodeAppInstance: process.env.NODE_APP_INSTANCE ?? null,
+      },
+      "Email retry scheduler skipped on non-leader instance"
+    );
+    return;
+  }
+  if (retryTimer) return;
+
+  const intervalMs = retryConfig.retryIntervalMinutes * 60 * 1000;
+  retryTimer = setInterval(() => {
+    void processEmailRetryQueue();
+  }, intervalMs);
+  void processEmailRetryQueue();
+}
+
+export function stopEmailRetryScheduler() {
+  if (!retryTimer) return;
+  clearInterval(retryTimer);
+  retryTimer = null;
+}
+
+const listPendingRetryCount = async () => {
+  let count = 0;
+  const iter = serverDb.iterator({
+    gt: EMAIL_RETRY_PREFIX,
+    lt: `${EMAIL_RETRY_PREFIX}\xFF`,
+  });
+  try {
+    for await (const _entry of iter) {
+      count += 1;
+    }
+  } finally {
+    await iter.close();
+  }
+  return count;
+};
+
+export async function getEmailDeliveryReport(days: number = 7, recentLimit: number = 50) {
+  const safeDays = Math.max(1, Math.min(90, Math.floor(days)));
+  const normalizedRecentLimit = Number.isFinite(Number(recentLimit))
+    ? Math.floor(Number(recentLimit))
+    : 50;
+  const includeAllRecent = normalizedRecentLimit <= 0;
+  const safeRecentLimit = includeAllRecent
+    ? 0
+    : Math.max(1, Math.min(5_000, normalizedRecentLimit));
+  const now = Date.now();
+  const earliestTimestamp = now - (safeDays - 1) * 24 * 60 * 60 * 1000;
+  const earliestDay = getDayKey(now - (safeDays - 1) * 24 * 60 * 60 * 1000);
+
+  const stats: Array<{
+    dayKey: string;
+    tag: string;
+    sent: number;
+    failed: number;
+    retried: number;
+    queued: number;
+  }> = [];
+
+  const statsIter = serverDb.iterator({
+    gt: EMAIL_STATS_PREFIX,
+    lt: `${EMAIL_STATS_PREFIX}\xFF`,
+  });
+  try {
+    for await (const [key, value] of statsIter) {
+      const suffix = key.slice(EMAIL_STATS_PREFIX.length);
+      const [dayKey, ...tagParts] = suffix.split(":");
+      if (!dayKey || dayKey < earliestDay) continue;
+      const tag = tagParts.join(":") || "untagged";
+      stats.push({
+        dayKey,
+        tag,
+        sent: Number(value?.sent || 0),
+        failed: Number(value?.failed || 0),
+        retried: Number(value?.retried || 0),
+        queued: Number(value?.queued || 0),
+      });
+    }
+  } finally {
+    await statsIter.close();
+  }
+
+  stats.sort((a, b) =>
+    a.dayKey === b.dayKey ? a.tag.localeCompare(b.tag) : a.dayKey.localeCompare(b.dayKey)
+  );
+
+  const totals = stats.reduce(
+    (acc, item) => {
+      acc.sent += item.sent;
+      acc.failed += item.failed;
+      acc.retried += item.retried;
+      acc.queued += item.queued;
+      return acc;
+    },
+    { sent: 0, failed: 0, retried: 0, queued: 0 }
+  );
+
+  const recent: EmailDeliveryRecord[] = [];
+  const recentFailures: EmailDeliveryRecord[] = [];
+  const recordIter = serverDb.iterator({
+    gt: EMAIL_RECORD_PREFIX,
+    lt: `${EMAIL_RECORD_PREFIX}\xFF`,
+    reverse: true,
+  });
+  try {
+    for await (const [_key, value] of recordIter) {
+      const record = value as EmailDeliveryRecord;
+      const recordTimestamp = Number(record?.timestamp || 0);
+      if (recordTimestamp > 0 && recordTimestamp < earliestTimestamp) {
+        break;
+      }
+      if (includeAllRecent || recent.length < safeRecentLimit) {
+        recent.push(record);
+      }
+      if (
+        record.status === "failed" &&
+        (includeAllRecent || recentFailures.length < safeRecentLimit)
+      ) {
+        recentFailures.push(record);
+      }
+      if (
+        !includeAllRecent &&
+        recent.length >= safeRecentLimit &&
+        recentFailures.length >= safeRecentLimit
+      ) {
+        break;
+      }
+    }
+  } finally {
+    await recordIter.close();
+  }
+
+  const pendingRetryCount = await listPendingRetryCount();
+
+  return {
+    generatedAt: now,
+    days: safeDays,
+    totals: {
+      ...totals,
+      pendingRetryCount,
+    },
+    stats,
+    recent,
+    recentFailures,
+  };
+}