@k2b/cloud 0.26.0 → 0.28.0

package/src/services/notifications/runtime.ts

@@ -3,7 +3,7 @@ import { lazySync } from "../../_internal/process-sync";
 import { logger, trace } from "../logging";
 import { createRuntimeLifecycle, createRuntimeTaskTracker, stopRuntimeJobs } from "../runtime-lifecycle";
 
-import { processNotificationDelivery, recoverNotificationDeliveries } from "./dispatcher";
+import { processNotificationDelivery, reconcileNotificationMessages, recoverNotificationDeliveries } from "./dispatcher";
 
 type DeliveryMessage = { deliveryId: string };
 
@@ -58,10 +58,10 @@ const handleDelivery = async (context: JobContext<DeliveryMessage>): Promise<voi
       spanKey: trace.syncSpanKey("job", "cloud-notification-deliveries", context.jobId),
       attributes: { "cloud.notification.delivery_id": context.input.deliveryId },
     },
-    () => processNotificationDelivery(context.input.deliveryId),
+    () => processNotificationDelivery(context.input.deliveryId, context.signal),
   );
   if (result.activatedIds?.length) await enqueueNotificationDeliveries(result.activatedIds);
-  if (result.status === "retry") context.resubmit({ delayMs: result.retryAfterMs });
+  if (result.status === "retry" || result.status === "pending") context.resubmit({ delayMs: result.retryAfterMs });
 };
 
 type RuntimeOptions = { concurrency?: number; recoveryIntervalMs?: number };
@@ -79,6 +79,12 @@ const recover = (): Promise<void> => {
       } catch (error) {
         log.error("Delivery recovery failed", { error: error instanceof Error ? error.message : String(error) });
       }
+      // Message history must never hold back delivery recovery.
+      try {
+        await reconcileNotificationMessages();
+      } catch (error) {
+        log.error("Notification message reconciliation failed", { error: error instanceof Error ? error.message : String(error) });
+      }
     }) ?? Promise.resolve()
   ).finally(() => {
     recovery = undefined;

package/src/services/outgoing-mail/admin.ts

@@ -0,0 +1,84 @@
+import { sql } from "bun";
+import {
+  type AdminMailFilter,
+  type AdminMailRecord,
+  AdminMailRecordSchema,
+  type MailPage,
+  type MailPageParams,
+} from "../../contracts/outgoing-mail";
+import { audit } from "../audit";
+import { cleanupMailAttachments, publishMailSettled } from "./dispatcher";
+import { adminMessageRecord, type MessageRow, outgoingMailMessages } from "./messages";
+import { type MailAuditContext, OutgoingMailError } from "./store";
+
+const get = async (id: string): Promise<AdminMailRecord> => {
+  const record = await outgoingMailMessages.metadata(id);
+  if (!record) throw new OutgoingMailError("message_unknown", "Outgoing mail message does not exist.", 404);
+  return record;
+};
+const cancel = async (id: string, context: MailAuditContext): Promise<AdminMailRecord> => {
+  const row = await sql.begin(async (tx) => {
+    const [row] = await tx<MessageRow[]>`UPDATE outgoing_mail.messages SET status = 'cancelled', error_code = 'cancelled_by_admin',
+      error_message = 'Cancelled by an administrator.', next_attempt_at = NULL, updated_at = now() WHERE id = ${id}::uuid AND status = 'queued'
+      RETURNING *, created_at::text AS cursor_created_at`;
+    if (!row) {
+      if (!(await outgoingMailMessages.read(id, undefined, tx)))
+        throw new OutgoingMailError("message_unknown", "Outgoing mail message does not exist.", 404);
+      throw new OutgoingMailError("message_not_queued", "Only queued outgoing mail can be cancelled.", 409);
+    }
+    await audit.record(
+      {
+        ...context,
+        action: "outgoing_mail.message.cancel",
+        outcome: "allowed",
+        target: { type: "outgoing_mail_message", id },
+        metadata: { appId: row.app_id, profile: row.profile_key },
+      },
+      tx,
+    );
+    return row;
+  });
+  await cleanupMailAttachments(id).catch(() => {});
+  await publishMailSettled(id).catch(() => {});
+  return adminMessageRecord(row);
+};
+export const outgoingMailLog = {
+  get,
+  cancel,
+  async cancelBatch(batchId: string, context: MailAuditContext): Promise<{ batchId: string; cancelled: number }> {
+    const cancelled = await sql.begin(async (tx) => {
+      const apps = await tx<{ app_id: string }[]>`SELECT DISTINCT app_id FROM outgoing_mail.messages WHERE batch_id = ${batchId}::uuid`;
+      if (!apps.length) throw new OutgoingMailError("batch_unknown", "Outgoing mail batch does not exist.", 404);
+      const rows = await tx<{ id: string }[]>`UPDATE outgoing_mail.messages SET status = 'cancelled', error_code = 'cancelled_by_admin',
+        error_message = 'Cancelled by an administrator.', next_attempt_at = NULL, updated_at = now()
+        WHERE batch_id = ${batchId}::uuid AND status = 'queued' RETURNING id`;
+      await audit.record(
+        {
+          ...context,
+          action: "outgoing_mail.batch.cancel",
+          outcome: "allowed",
+          target: { type: "outgoing_mail_batch", id: batchId },
+          metadata: { appIds: apps.map((row) => row.app_id), count: rows.length },
+        },
+        tx,
+      );
+      return rows;
+    });
+    for (const row of cancelled) {
+      await cleanupMailAttachments(row.id).catch(() => {});
+      await publishMailSettled(row.id).catch(() => {});
+    }
+    return { batchId, cancelled: cancelled.length };
+  },
+  content: outgoingMailMessages.content,
+  list: async (filter: AdminMailFilter, page: MailPageParams): Promise<MailPage<AdminMailRecord>> => {
+    const result = await outgoingMailMessages.list(filter, page, true);
+    return {
+      ...result,
+      items: result.items.map((item) => {
+        // The metadata mapper deliberately omits content; validate its public boundary.
+        return AdminMailRecordSchema.parse(item);
+      }),
+    };
+  },
+};

package/src/services/outgoing-mail/attachments.ts

@@ -0,0 +1,135 @@
+import { createHash } from "node:crypto";
+import { type ObjectRef, StoreFullError } from "@k2b/sync";
+import type { MailAttachment, MailRecord } from "../../contracts/outgoing-mail";
+import { OutgoingMailError } from "./store";
+import { mailAttachments } from "./sync";
+
+export type UploadedAttachments = { metadata: MailRecord["attachments"]; refs: ObjectRef[] };
+export const MAIL_ATTACHMENT_UPLOAD_MS = 60_000;
+export const cancelMailStreams = (attachments: unknown = []): void => {
+  if (!Array.isArray(attachments)) return;
+  for (const attachment of attachments) {
+    if (
+      attachment &&
+      typeof attachment === "object" &&
+      "content" in attachment &&
+      attachment.content instanceof ReadableStream &&
+      !attachment.content.locked
+    )
+      void attachment.content.cancel().catch(() => {});
+  }
+};
+export const deleteMailObjects = async (refs: readonly ObjectRef[]): Promise<void> => {
+  // Sequential deletion bounds load even for a large attachment collection.
+  for (const ref of refs) await mailAttachments().delete({ tenantId: ref.tenantId, key: ref.key });
+};
+const contentStream = (content: MailAttachment["content"]): ReadableStream<Uint8Array> => {
+  if (content instanceof ReadableStream) return content;
+  if (content instanceof Blob) return content.stream();
+  return new ReadableStream({
+    start(controller) {
+      controller.enqueue(content);
+      controller.close();
+    },
+  });
+};
+export const uploadMailAttachments = async (
+  id: string,
+  attachments: readonly MailAttachment[],
+  limit: number,
+  uploadDeadline?: number,
+): Promise<UploadedAttachments> => {
+  const result: UploadedAttachments = { metadata: [], refs: [] };
+  let total = 0;
+  const controller = new AbortController();
+  // The caller's signal cancels the wait only. All attachments share one bounded upload.
+  if (uploadDeadline !== undefined && uploadDeadline <= Date.now() && attachments.length) {
+    cancelMailStreams(attachments);
+    throw new OutgoingMailError("mail_unavailable", "Attachment upload timed out.");
+  }
+  const timer = setTimeout(
+    () => controller.abort(new Error("Attachment upload timed out")),
+    uploadDeadline === undefined ? MAIL_ATTACHMENT_UPLOAD_MS : Math.max(0, uploadDeadline - Date.now()),
+  );
+  try {
+    for (const [index, attachment] of attachments.entries()) {
+      const hash = createHash("sha256");
+      let size = 0;
+      let exceeded = false;
+      const key = `${id}-${index}`;
+      try {
+        const stream = contentStream(attachment.content).pipeThrough(
+          new TransformStream<Uint8Array, Uint8Array>({
+            transform(chunk, output) {
+              total += chunk.byteLength;
+              size += chunk.byteLength;
+              if (total > limit) {
+                exceeded = true;
+                throw new OutgoingMailError("attachments_too_large", "Attachments exceed the profile's total byte limit.");
+              }
+              hash.update(chunk);
+              output.enqueue(chunk);
+            },
+          }),
+          { signal: controller.signal },
+        );
+        const ref = await mailAttachments().put({ tenantId: id, key, body: stream, signal: controller.signal });
+        result.refs.push(ref);
+        result.metadata.push({ filename: attachment.filename, contentType: attachment.contentType, size, sha256: hash.digest("hex") });
+      } catch (error) {
+        // A failed put can have written chunks before the final metadata was published.
+        await mailAttachments()
+          .delete({ tenantId: id, key })
+          .catch(() => {});
+        if (exceeded) throw new OutgoingMailError("attachments_too_large", "Attachments exceed the profile's total byte limit.");
+        if (
+          error instanceof StoreFullError ||
+          (error instanceof Error &&
+            /maximum bytes|maximum storage|insufficient (?:storage|resources)|storage resources|store.*full|resource limits exceeded/i.test(
+              error.message,
+            ))
+        )
+          throw new OutgoingMailError("attachment_storage_full", "Outgoing mail attachment storage is full.");
+        throw error;
+      }
+    }
+    return result;
+  } catch (error) {
+    await deleteMailObjects(result.refs).catch(() => {});
+    cancelMailStreams(attachments);
+    throw error;
+  } finally {
+    clearTimeout(timer);
+    controller.abort();
+  }
+};
+
+/** Verify before SMTP DATA; open a second, verified stream for nodemailer. */
+export const verifyMailAttachment = async (
+  ref: ObjectRef,
+  metadata: MailRecord["attachments"][number],
+  signal?: AbortSignal,
+): Promise<void> => {
+  const object = await mailAttachments().get(ref, { signal });
+  if (!object) throw new OutgoingMailError("attachment_lost", "An outgoing mail attachment is missing or changed.");
+  const reader = object.body.getReader();
+  const hash = createHash("sha256");
+  let size = 0;
+  try {
+    while (true) {
+      const chunk = await reader.read();
+      if (chunk.done) break;
+      size += chunk.value.byteLength;
+      if (size > metadata.size) throw new Error("Attachment size mismatch");
+      hash.update(chunk.value);
+    }
+    if (size !== metadata.size || hash.digest("hex") !== metadata.sha256) throw new Error("Attachment digest mismatch");
+  } catch (error) {
+    if (signal?.aborted || (error instanceof Error && /stalled|timeout|timed out|connection/i.test(error.message)))
+      throw new Error("Attachment verification was interrupted.");
+    throw new OutgoingMailError("attachment_lost", "An outgoing mail attachment is missing or changed.");
+  } finally {
+    await reader.cancel().catch(() => {});
+    reader.releaseLock();
+  }
+};

package/src/services/outgoing-mail/bulk.ts

@@ -0,0 +1,11 @@
+export const MAIL_RECOVERY_MS = 30_000;
+export const mailSlotMs = (pacePerMinute: number): number => 60_000 / pacePerMinute;
+export const mailBacklogFull = (queued: number, added: number, pacePerMinute: number): boolean =>
+  added > 0 && queued + added > pacePerMinute * 1440;
+export const mailBatchId = (proposed: string, rows: readonly ({ batch_id: string | null } | undefined)[]): string => {
+  const previous = rows[0]?.batch_id;
+  return previous && rows.every((row) => row?.batch_id === previous) ? previous : proposed;
+};
+// Later submissions join a coalesced continuation, so long backoffs must not hide fresh mail.
+export const mailNextDrainDelay = (now: number, slot: number, due: number): number =>
+  Math.min(MAIL_RECOVERY_MS, Math.max(0, Math.max(slot, due) - now));

package/src/services/outgoing-mail/dispatcher.ts

@@ -0,0 +1,231 @@
+import { createHash } from "node:crypto";
+import { Socket } from "node:net";
+import { Readable } from "node:stream";
+import { sql } from "bun";
+import { z } from "zod";
+import { listApps } from "../../_internal/registry";
+import { parsePgJsonValue } from "../postgres";
+import { get } from "../settings";
+import { deleteMailObjects, verifyMailAttachment } from "./attachments";
+import { mailBackoffMs, mailEnvelope } from "./message";
+import { allowedMailProfile, type MessageRow, messageAttachmentRefs, messageRecord, outgoingMailMessages } from "./messages";
+import { OutgoingMailError, resolveMailCredentials } from "./store";
+import { mailAttachments, mailSettled } from "./sync";
+import { smtpFailureMessage } from "./test-send";
+import { buildMailTransport } from "./transport";
+
+export const cleanupMailAttachments = async (id: string): Promise<void> => {
+  const row = await outgoingMailMessages.read(id);
+  if (!row || ["queued", "sending"].includes(row.status)) return;
+  await deleteMailObjects(messageAttachmentRefs(row));
+  await sql`UPDATE outgoing_mail.messages SET attachment_refs = NULL WHERE id = ${id}::uuid AND status NOT IN ('queued','sending')`;
+};
+export const publishMailSettled = async (id: string): Promise<void> => {
+  await mailSettled().publish({ tenantId: id, data: id });
+};
+const smtpResponseRetryable = (error: unknown): boolean => {
+  const responseCode = error && typeof error === "object" && "responseCode" in error ? Number(error.responseCode) : undefined;
+  return responseCode === undefined || !Number.isFinite(responseCode) || responseCode < 500;
+};
+export const smtpRetryable = (error: unknown): boolean => {
+  if (error && typeof error === "object" && "rejectedErrors" in error && Array.isArray(error.rejectedErrors) && error.rejectedErrors.length)
+    return error.rejectedErrors.some(smtpResponseRetryable);
+  return smtpResponseRetryable(error);
+};
+const rejectReason = (error: unknown, fallback: string): string =>
+  error && typeof error === "object" && "response" in error && typeof error.response === "string" ? error.response : fallback;
+const cancelled = async (row: MessageRow, code: "profile_removed" | "profile_not_allowed", message?: string) => {
+  await sql`UPDATE outgoing_mail.messages SET status = 'cancelled', error_code = ${code}, error_message = ${message ?? (code === "profile_removed" ? "Outgoing mail profile was removed." : "Outgoing mail profile access was revoked.")},
+    next_attempt_at = NULL, updated_at = now() WHERE id = ${row.id}::uuid AND status = 'sending' AND attempt_count = ${row.attempt_count}`;
+};
+export const processOutgoingMail = async (id: string, signal?: AbortSignal): Promise<void> => {
+  const [row] = await sql<
+    MessageRow[]
+  >`UPDATE outgoing_mail.messages SET status = 'sending', attempt_count = attempt_count + 1, updated_at = now()
+    WHERE id = ${id}::uuid AND status = 'queued' AND lane = 'immediate' AND (next_attempt_at IS NULL OR next_attempt_at <= now() OR deadline_at <= now())
+    RETURNING *, created_at::text AS cursor_created_at`;
+  if (!row) return;
+  await attemptOutgoingMail(row, signal);
+};
+
+/** Both lanes share policy, attachments, SMTP outcomes and retry handling. */
+export const attemptOutgoingMail = async (row: MessageRow, signal?: AbortSignal): Promise<void> => {
+  const id = row.id;
+  let credentials: Awaited<ReturnType<typeof resolveMailCredentials>> | undefined;
+  try {
+    if (new Date(row.deadline_at).getTime() <= Date.now()) {
+      await sql`UPDATE outgoing_mail.messages SET status = 'failed', error_code = COALESCE(error_code, 'smtp_failed'), error_message = COALESCE(error_message, 'Outgoing mail delivery deadline expired.'), next_attempt_at = NULL, updated_at = now() WHERE id = ${id}::uuid AND status = 'sending' AND attempt_count = ${row.attempt_count}`;
+      return;
+    }
+    if (!row.profile_id) {
+      await cancelled(row, "profile_removed");
+      return;
+    }
+    const apps = await listApps();
+    const app = apps.find((app) => app.id === row.app_id);
+    if (app && !app.platformPermissions?.includes("mail:send")) {
+      await cancelled(row, "profile_not_allowed", "The application no longer declares mail:send.");
+      return;
+    }
+    // Core mail speaks for the installation, as system email did before it used outgoing mail.
+    const senderName = row.app_id === "core" ? await get<string>("app.name") : app?.name || row.app_id;
+    try {
+      await sql.begin(async (tx) => {
+        await tx`SELECT pg_advisory_xact_lock_shared(hashtextextended('outgoing_mail.policy', 0))`;
+        const profile = await allowedMailProfile(tx, row.app_id, row.profile_key);
+        if (profile.id !== row.profile_id) throw new OutgoingMailError("profile_removed", "Outgoing mail profile was replaced.");
+      });
+    } catch (error) {
+      if (
+        error instanceof OutgoingMailError &&
+        ["profile_unknown", "profile_removed", "mail_unavailable", "profile_not_allowed", "profile_required"].includes(error.code)
+      ) {
+        await cancelled(
+          row,
+          ["profile_unknown", "profile_removed", "mail_unavailable"].includes(error.code) ? "profile_removed" : "profile_not_allowed",
+        );
+        return;
+      }
+      throw error;
+    }
+    const refs = messageAttachmentRefs(row);
+    const metadata = messageRecord(row).attachments;
+    if (refs.length !== metadata.length) throw new OutgoingMailError("attachment_lost", "An outgoing mail attachment is missing.");
+    const attempt = new AbortController();
+    const abort = () => attempt.abort();
+    signal?.addEventListener("abort", abort, { once: true });
+    if (signal?.aborted) abort();
+    const timer = setTimeout(abort, 60_000);
+    const streams: Readable[] = [];
+    try {
+      for (const [index, ref] of refs.entries()) await verifyMailAttachment(ref, metadata[index]!, attempt.signal);
+      const attachments = [];
+      for (const [index, ref] of refs.entries()) {
+        const stored = await mailAttachments().get(ref, { signal: attempt.signal });
+        if (!stored) throw new OutgoingMailError("attachment_lost", "An outgoing mail attachment is missing.");
+        const item = metadata[index]!;
+        const stream = Readable.from(
+          (async function* () {
+            const reader = stored.body.getReader();
+            const hash = createHash("sha256");
+            let bytes = 0;
+            try {
+              while (true) {
+                const chunk = await reader.read();
+                if (chunk.done) break;
+                bytes += chunk.value.byteLength;
+                if (bytes > item.size) throw new OutgoingMailError("attachment_lost", "Attachment size changed.");
+                hash.update(chunk.value);
+                yield chunk.value;
+              }
+              if (bytes !== item.size || hash.digest("hex") !== item.sha256)
+                throw new OutgoingMailError("attachment_lost", "Attachment checksum changed.");
+            } finally {
+              await reader.cancel().catch(() => {});
+              reader.releaseLock();
+            }
+          })(),
+        );
+        streams.push(stream);
+        attachments.push({ filename: item.filename, contentType: item.contentType, content: stream });
+      }
+      // Load the checked profile by ID: a profile deleted (or recreated under its key) since the check is profile_removed.
+      credentials = await resolveMailCredentials({ id: row.profile_id });
+      const socket = new Socket();
+      // DNS can complete after cancellation; tear down a late connection too.
+      socket.once("connect", () => {
+        if (attempt.signal.aborted) socket.destroy();
+      });
+      const transport = buildMailTransport(credentials, socket);
+      const stop = () => {
+        socket.destroy();
+        transport.close();
+        for (const stream of streams) stream.destroy();
+      };
+      attempt.signal.addEventListener("abort", stop, { once: true });
+      try {
+        if (attempt.signal.aborted) throw new Error("SMTP attempt cancelled");
+        const result = await transport.sendMail({
+          ...mailEnvelope(credentials, senderName, row.from_name),
+          envelope: { from: credentials.fromAddress, to: row.to_addresses },
+          to: row.to_addresses,
+          subject: row.subject,
+          text: row.text_body ?? "",
+          html: row.html_body ?? undefined,
+          replyTo: row.reply_to ?? undefined,
+          messageId: row.message_id_header,
+          headers: row.headers === null ? undefined : z.record(z.string(), z.string()).parse(parsePgJsonValue(row.headers)),
+          attachments,
+          disableFileAccess: true,
+          disableUrlAccess: true,
+        });
+        const failures = (result.rejected ?? []).map((recipient: string, index: number) => ({
+          recipient,
+          reason: rejectReason(result.rejectedErrors?.[index], "Recipient rejected."),
+          at: new Date().toISOString(),
+        }));
+        await sql`UPDATE outgoing_mail.messages SET status = 'sent', sent_at = now(), next_attempt_at = NULL, error_code = NULL, error_message = NULL,
+          smtp_response = ${result.response}, failures = ${JSON.stringify(failures)}::text::jsonb, updated_at = now()
+          WHERE id = ${id}::uuid AND status = 'sending' AND attempt_count = ${row.attempt_count}`;
+      } finally {
+        attempt.signal.removeEventListener("abort", stop);
+        socket.destroy();
+        transport.close();
+      }
+    } finally {
+      clearTimeout(timer);
+      signal?.removeEventListener("abort", abort);
+      attempt.abort();
+      for (const stream of streams) stream.destroy();
+    }
+  } catch (error) {
+    if (error instanceof OutgoingMailError && ["profile_unknown", "profile_removed"].includes(error.code)) {
+      await cancelled(row, "profile_removed");
+      return;
+    }
+    const secrets = credentials
+      ? [
+          credentials.smtpUser,
+          credentials.smtpPassword,
+          credentials.smtpUser ? `\0${credentials.smtpUser}\0${credentials.smtpPassword ?? ""}` : null,
+        ]
+      : [];
+    const message =
+      credentials || error instanceof OutgoingMailError
+        ? smtpFailureMessage(error, secrets)
+        : "Outgoing mail delivery is temporarily unavailable.";
+    const lost = error instanceof OutgoingMailError && error.code === "attachment_lost";
+    const retry = !lost && smtpRetryable(error) && new Date(row.deadline_at).getTime() > Date.now();
+    const next = new Date(Date.now() + mailBackoffMs(row.attempt_count));
+    await sql`UPDATE outgoing_mail.messages SET status = ${retry ? "queued" : "failed"}, error_code = ${lost ? "attachment_lost" : "smtp_failed"}, error_message = ${message},
+      next_attempt_at = ${retry ? next : null}, updated_at = now() WHERE id = ${id}::uuid AND status = 'sending' AND attempt_count = ${row.attempt_count}`;
+  } finally {
+    // Cleanup failure must never turn SMTP success into another delivery attempt.
+    await cleanupMailAttachments(id).catch(() => {});
+    await publishMailSettled(id).catch(() => {});
+  }
+};
+export const recoverOutgoingMail = async (): Promise<string[]> => {
+  await sql`WITH stale AS (SELECT id FROM outgoing_mail.messages WHERE status = 'sending' AND updated_at < now() - INTERVAL '5 minutes' LIMIT 1000 FOR UPDATE SKIP LOCKED)
+    UPDATE outgoing_mail.messages SET status = 'queued', next_attempt_at = now(), updated_at = now() WHERE id IN (SELECT id FROM stale)`;
+  const orphaned = await sql<{ id: string }[]>`WITH orphaned AS (
+    SELECT id FROM outgoing_mail.messages WHERE status = 'queued' AND lane = 'bulk' AND profile_id IS NULL
+    LIMIT 1000 FOR UPDATE SKIP LOCKED
+  ) UPDATE outgoing_mail.messages SET status = 'cancelled', error_code = 'profile_removed',
+    error_message = 'Outgoing mail profile was removed.', next_attempt_at = NULL, updated_at = now()
+    WHERE id IN (SELECT id FROM orphaned) RETURNING id`;
+  const expired = await sql<{ id: string }[]>`WITH expired AS (
+    SELECT id FROM outgoing_mail.messages WHERE status = 'queued' AND lane = 'bulk' AND deadline_at <= now()
+    LIMIT 1000 FOR UPDATE SKIP LOCKED
+  ) UPDATE outgoing_mail.messages SET status = 'failed', error_code = COALESCE(error_code, 'smtp_failed'),
+    error_message = COALESCE(error_message, 'Outgoing mail delivery deadline expired.'), next_attempt_at = NULL, updated_at = now()
+    WHERE id IN (SELECT id FROM expired) RETURNING id`;
+  for (const row of [...orphaned, ...expired]) await publishMailSettled(row.id).catch(() => {});
+  const cleanup = await sql<
+    { id: string }[]
+  >`SELECT id FROM outgoing_mail.messages WHERE status NOT IN ('queued','sending') AND attachment_refs IS NOT NULL LIMIT 1000`;
+  for (const row of cleanup) await cleanupMailAttachments(row.id).catch(() => {});
+  const due = await sql<{ id: string }[]>`SELECT id FROM outgoing_mail.messages WHERE lane = 'immediate' AND status = 'queued'
+    AND (next_attempt_at IS NULL OR next_attempt_at <= now() OR deadline_at <= now()) ORDER BY next_attempt_at NULLS FIRST, created_at LIMIT 1000`;
+  return due.map((row) => row.id);
+};

package/src/services/outgoing-mail/drain.ts

@@ -0,0 +1,60 @@
+import { sql } from "bun";
+import { mailNextDrainDelay } from "./bulk";
+import { attemptOutgoingMail } from "./dispatcher";
+import type { MessageRow } from "./messages";
+
+export const MAIL_DRAIN_MS = 60_000;
+export const claimOutgoingBulkMail = async (
+  profileId: string,
+): Promise<{ row: MessageRow; slotAt: string; grantedAt: string } | undefined> =>
+  sql.begin(async (tx) => {
+    // Wait for acceptance's share lock before reading the clock and reserving a slot.
+    await tx`SELECT 1 FROM outgoing_mail.profiles WHERE id = ${profileId}::uuid FOR NO KEY UPDATE`;
+    const [slot] = await tx<{ slot_at: string; granted_at: string }[]>`UPDATE outgoing_mail.profiles p
+      SET next_bulk_slot_at = greatest(p.next_bulk_slot_at, c.at) + (60000.0 / p.pace_per_minute) * INTERVAL '1 millisecond'
+      FROM (SELECT clock_timestamp() AS at) c WHERE p.id = ${profileId}::uuid AND p.next_bulk_slot_at <= c.at
+      RETURNING p.next_bulk_slot_at::text AS slot_at, c.at::text AS granted_at`;
+    if (!slot) return undefined;
+    const [row] = await tx<MessageRow[]>`WITH due AS (
+      SELECT id FROM outgoing_mail.messages WHERE profile_id = ${profileId}::uuid AND status = 'queued' AND lane = 'bulk'
+      AND (next_attempt_at IS NULL OR next_attempt_at <= now()) AND deadline_at > now()
+      ORDER BY COALESCE(next_attempt_at, created_at), id LIMIT 1 FOR UPDATE SKIP LOCKED
+    ) UPDATE outgoing_mail.messages SET status = 'sending', attempt_count = attempt_count + 1, updated_at = now()
+      WHERE id IN (SELECT id FROM due) RETURNING *, created_at::text AS cursor_created_at`;
+    // An idle claim consumes at most one slot. No refund race or extra state is needed.
+    return row ? { row, slotAt: slot.slot_at, grantedAt: slot.granted_at } : undefined;
+  });
+export const nextOutgoingBulkDelay = async (profileId: string): Promise<number | undefined> => {
+  const [next] = await sql<{ slot: Date; due: Date; current: Date }[]>`SELECT p.next_bulk_slot_at AS slot, now() AS current, m.due
+    FROM outgoing_mail.profiles p CROSS JOIN LATERAL (
+      SELECT COALESCE(next_attempt_at, created_at) AS due FROM outgoing_mail.messages
+      WHERE profile_id = p.id AND lane = 'bulk' AND status = 'queued' AND deadline_at > now()
+      ORDER BY COALESCE(next_attempt_at, created_at), id LIMIT 1
+    ) m WHERE p.id = ${profileId}::uuid`;
+  return next
+    ? mailNextDrainDelay(new Date(next.current).getTime(), new Date(next.slot).getTime(), new Date(next.due).getTime())
+    : undefined;
+};
+export const dueOutgoingBulkProfiles = async (): Promise<string[]> => {
+  const rows = await sql<{ id: string }[]>`SELECT p.id FROM outgoing_mail.profiles p WHERE EXISTS (
+    SELECT 1 FROM outgoing_mail.messages m WHERE m.profile_id = p.id AND m.lane = 'bulk' AND m.status = 'queued'
+    AND COALESCE(m.next_attempt_at, m.created_at) <= now() AND m.deadline_at > now()
+  ) LIMIT 1000`;
+  return rows.map((row) => row.id);
+};
+/** Stop at the next closed gate and continue durably instead of occupying a worker while waiting. */
+export const drainOutgoingMail = async (
+  profileId: string,
+  signal?: AbortSignal,
+  heartbeat?: () => Promise<void>,
+): Promise<number | undefined> => {
+  const until = Date.now() + MAIL_DRAIN_MS;
+  // The run window bounds claims; only shutdown may cancel the current attempt.
+  while (!signal?.aborted && Date.now() < until) {
+    const claimed = await claimOutgoingBulkMail(profileId);
+    if (!claimed) break;
+    await attemptOutgoingMail(claimed.row, signal);
+    await heartbeat?.();
+  }
+  return nextOutgoingBulkDelay(profileId);
+};