@antelopejs/dms-mailing 0.3.4 → 0.4.0

package/src/services/events.ts

@@ -1,14 +1,21 @@
 import { timingSafeEqual } from "node:crypto";
+import { Logging } from "@antelopejs/interface-core/logging";
 import { GetModel } from "@antelopejs/interface-database-decorators";
 import {
   emitEmailEvent,
   type EmailEventPayload,
 } from "../automation/email-event-trigger";
-import { type MailingSend, SendEventModel, SendModel } from "../db";
+import {
+  type MailingSend,
+  type MailingSendEvent,
+  SendEventModel,
+  SendModel,
+} from "../db";
 import type { EmailEvent } from "@antelopejs/interface-dms-mailing";
 import { publishSendUpdated } from "../realtime";
 import type { SendEventType, SendStatus } from "../types";
-import { notifyBounce } from "./notify";
+import { notifyProblems } from "./notify";
+import { suppressAddress, suppressionReasonOf } from "./suppressions";
 
 export interface SendCounters {
   status: SendStatus;
@@ -28,6 +35,7 @@ const RANK: Record<SendStatus, number> = {
   spam: 6,
   bounced: 7,
   failed: 7,
+  suppressed: 8,
 };
 
 const COUNTERS: Partial<Record<SendEventType, CounterKey>> = {
@@ -37,6 +45,40 @@ const COUNTERS: Partial<Record<SendEventType, CounterKey>> = {
 
 const MAX_ACTIVITY_ATTEMPTS = 5;
 
+const PROBLEM_EVENTS = new Set<SendStatus>(["bounced", "failed", "spam"]);
+
+/**
+ * Statuses a later sign of delivery overrides: a provider failure reported by
+ * webhook (`421 4.3.0`) is often a deferral the provider retried.
+ */
+const DEFERRABLE_STATUSES = new Set<SendStatus>(["failed"]);
+const CLEARED_ERROR = "";
+
+const SOFT_BOUNCE_TYPE = /soft|transient|temporary/i;
+const HARD_BOUNCE_TYPE = /hard|permanent/i;
+/** A temporary SMTP answer: a `4xx` reply code or a `4.x.x` enhanced status. */
+const SOFT_SMTP_ANSWER = /^\s*4\d\d\b|\b4\.\d{1,3}\.\d{1,3}\b/;
+
+const textOf = (value: unknown): string =>
+  typeof value === "string" ? value : "";
+
+/**
+ * The provider's own words for a problem event (`550 5.1.1 Mailbox does not
+ * exist`), kept on the send so the log shows why it failed. Other events
+ * leave the stored error alone.
+ */
+export function problemReason(event: EmailEvent): string | undefined {
+  if (!PROBLEM_EVENTS.has(event.type)) return undefined;
+  const reason = event.details?.reason;
+  return typeof reason === "string" && reason ? reason : undefined;
+}
+
+function overridesStatus(current: SendStatus, next: SendStatus): boolean {
+  if (DEFERRABLE_STATUSES.has(current))
+    return next !== current && RANK[next] >= RANK.delivered;
+  return RANK[next] > RANK[current];
+}
+
 export function applyEvent(
   current: SendCounters,
   type: SendEventType,
@@ -44,14 +86,101 @@ export function applyEvent(
   const counter = COUNTERS[type];
   const next: SendCounters = { ...current };
   if (counter) next[counter] = current[counter] + 1;
-  if (RANK[type] > RANK[current.status]) next.status = type;
+  if (overridesStatus(current.status, type)) next.status = type;
   return next;
 }
 
-async function publishEmailEvent(payload: EmailEventPayload): Promise<void> {
+/**
+ * Whether a bounce is temporary (a full mailbox, a greylisting): the
+ * provider's `bounceType` when it gives one, else the SMTP answer's class.
+ */
+export function isSoftBounce(event: EmailEvent): boolean {
+  const bounceType = textOf(event.details?.bounceType);
+  if (SOFT_BOUNCE_TYPE.test(bounceType)) return true;
+  if (HARD_BOUNCE_TYPE.test(bounceType)) return false;
+  return SOFT_SMTP_ANSWER.test(textOf(event.details?.reason));
+}
+
+/**
+ * The error the send keeps after an event: the provider's words for a
+ * problem, none once a deferred failure turns out delivered, else unchanged.
+ */
+export function errorAfter(
+  event: EmailEvent,
+  previous: SendStatus,
+  next: SendStatus,
+): string | undefined {
+  const reason = problemReason(event);
+  if (reason) return reason;
+  const recovered = DEFERRABLE_STATUSES.has(previous) && previous !== next;
+  return recovered && !PROBLEM_EVENTS.has(next) ? CLEARED_ERROR : undefined;
+}
+
+function activityOf(send: MailingSend, event: EmailEvent) {
+  const counters = applyEvent(
+    { status: send.status, opens: send.opens, clicks: send.clicks },
+    event.type,
+  );
+  return {
+    ...counters,
+    lastEventAt: event.at ?? new Date(),
+    provider: event.provider,
+    error: errorAfter(event, send.status, counters.status),
+    isSoftBounce: event.type === "bounced" ? isSoftBounce(event) : undefined,
+  };
+}
+
+async function publishEmailEvent(
+  payload: EmailEventPayload,
+  send: MailingSend,
+): Promise<void> {
   emitEmailEvent(payload);
   publishSendUpdated(payload.sendId);
-  await notifyBounce(payload.tenantId, payload);
+  await notifyProblems(payload.tenantId, {
+    sendId: send._id,
+    templateSlug: send.templateSlug,
+    type: payload.type,
+    isTest: send.isTest,
+  });
+}
+
+const eventIdOf = (details: Record<string, unknown> | undefined): string =>
+  textOf(details?.eventId);
+
+/**
+ * Whether a stored event is the one a webhook delivers again: the same
+ * provider event id, or the same type, time and details when it has none.
+ */
+export function isSameEvent(
+  stored: MailingSendEvent,
+  event: EmailEvent,
+): boolean {
+  const eventId = eventIdOf(event.details);
+  if (eventId) {
+    const storedDetails = JSON.parse(stored.json_details || "{}") as Record<
+      string,
+      unknown
+    >;
+    return eventIdOf(storedDetails) === eventId;
+  }
+  if (!event.at || stored.type !== event.type) return false;
+  return (
+    new Date(stored.at).getTime() === event.at.getTime() &&
+    stored.json_details === JSON.stringify(event.details ?? {})
+  );
+}
+
+/** A provider retrying its webhook must not count the event twice. */
+async function isRepeatedEvent(
+  tenantId: string,
+  event: EmailEvent,
+): Promise<boolean> {
+  const send = await GetModel(SendModel, tenantId).getByProviderMessageId(
+    event.messageId,
+  );
+  if (!send) return false;
+  const stored = await GetModel(SendEventModel, tenantId).getBySend(send._id);
+  return stored.some((candidate) => isSameEvent(candidate, event));
 }
 
 async function acceptEmailActivity(
@@ -62,23 +191,44 @@ async function acceptEmailActivity(
   for (let attempt = 0; attempt < MAX_ACTIVITY_ATTEMPTS; attempt++) {
     const send = await sends.getByProviderMessageId(event.messageId);
     if (!send || send.isRetiring) return undefined;
-    const accepted = await sends.recordActivity(send, {
-      ...applyEvent(
-        { status: send.status, opens: send.opens, clicks: send.clicks },
-        event.type,
-      ),
-      lastEventAt: event.at ?? new Date(),
-      provider: event.provider,
-    });
+    const accepted = await sends.recordActivity(send, activityOf(send, event));
     if (accepted) return send;
   }
   throw new Error("Mailing activity conflicted repeatedly; retry the webhook");
 }
 
+/**
+ * Puts the send's address on the suppression list when the event says
+ * nothing more must reach it. The event is recorded whatever happens here.
+ */
+async function suppressAfterEvent(
+  tenantId: string,
+  send: MailingSend,
+  event: EmailEvent,
+): Promise<void> {
+  const reason = suppressionReasonOf({
+    type: event.type,
+    isTest: send.isTest,
+    isSoftBounce: event.type === "bounced" && isSoftBounce(event),
+  });
+  if (!reason) return;
+  try {
+    await suppressAddress(tenantId, {
+      email: send.recipientEmail,
+      reason,
+      sendId: send._id,
+      detail: textOf(event.details?.reason) || undefined,
+    });
+  } catch (error) {
+    Logging.Error("[dms-mailing] Could not suppress an address:", error);
+  }
+}
+
 export async function recordEmailEvent(
   tenantId: string,
   event: EmailEvent,
 ): Promise<boolean> {
+  if (await isRepeatedEvent(tenantId, event)) return false;
   const send = await acceptEmailActivity(tenantId, event);
   if (!send) return false;
   const at = event.at ?? new Date();
@@ -91,13 +241,17 @@ export async function recordEmailEvent(
     },
   );
   if (!recorded) return false;
-  await publishEmailEvent({
-    tenantId,
-    sendId: send._id,
-    templateSlug: send.templateSlug,
-    type: event.type,
-    recipientEmail: send.recipientEmail,
-  });
+  await suppressAfterEvent(tenantId, send, event);
+  await publishEmailEvent(
+    {
+      tenantId,
+      sendId: send._id,
+      templateSlug: send.templateSlug,
+      type: event.type,
+      recipientEmail: send.recipientEmail,
+    },
+    send,
+  );
   return true;
 }
 

package/src/services/metrics.ts

@@ -21,6 +21,13 @@ export function readAudience(raw: string | null | undefined): SendAudience {
   return AUDIENCE_BY_NAME[raw ?? ""] ?? BUSINESS_AUDIENCE;
 }
 
+const STATS_WINDOW_DAYS = 30;
+
+/** Where the per-template figures start: the last 30 days. */
+export function statsWindowStart(now: number = Date.now()): Date {
+  return new Date(now - STATS_WINDOW_DAYS * DAY_MS);
+}
+
 export function forAudience(
   sends: SendSummary[],
   audience: SendAudience,
@@ -28,6 +35,10 @@ export function forAudience(
   return sends.filter(AUDIENCE_KEEPS[audience]);
 }
 
+/**
+ * The counters of a set of sends and the rates drawn from them. A rate is
+ * `null` when its base is empty: "no e-mail delivered" is not "0 % opened".
+ */
 export interface Totals {
   sends: number;
   queued: number;
@@ -39,12 +50,15 @@ export interface Totals {
   failed: number;
   spam: number;
   unsubscribed: number;
-  deliverability: number;
-  openRate: number;
-  clickRate: number;
+  deliverability: number | null;
+  openRate: number | null;
+  clickRate: number | null;
   latencyMedian: number;
 }
 
+/** Reads one figure out of the totals; `null` when it cannot be stated. */
+export type TotalsExtractor = (totals: Totals) => number | null;
+
 export interface SeriesPoint {
   x: string;
   y: number;
@@ -55,6 +69,13 @@ export interface DateRange {
   to: Date;
 }
 
+/** The provider features the Overview's figures depend on. */
+export interface ProviderTracking {
+  name: string;
+  openTracking: boolean;
+  clickTracking: boolean;
+}
+
 const PERCENT = 100;
 const DAY_MS = 86_400_000;
 const DAY_KEY_LENGTH = 10;
@@ -67,9 +88,11 @@ const DELIVERED_LIKE: SendStatus[] = [
   "spam",
 ];
 const OPENED_LIKE: SendStatus[] = ["opened", "clicked"];
+/** Statuses only an event reported by the provider can set. */
+const PROVIDER_EVENT_STATUSES: SendStatus[] = [...DELIVERED_LIKE, "bounced"];
 
-export const ratio = (part: number, whole: number): number =>
-  whole ? (part / whole) * PERCENT : 0;
+export const ratio = (part: number, whole: number): number | null =>
+  whole ? (part / whole) * PERCENT : null;
 
 const countIf = (
   sends: SendSummary[],
@@ -81,18 +104,36 @@ const byStatus =
   (send: SendSummary): boolean =>
     send.status === status;
 
+const isDelivered = (send: SendSummary): boolean =>
+  DELIVERED_LIKE.includes(send.status);
+
+/**
+ * Engagement outlives the final status: a send opened, clicked and then
+ * unsubscribed (or reported as spam) was still opened and clicked, and a
+ * click implies an open the pixel may have missed.
+ */
+const wasClicked = (send: SendSummary): boolean =>
+  isDelivered(send) && (send.status === "clicked" || send.clicks > 0);
+
+const wasOpened = (send: SendSummary): boolean =>
+  isDelivered(send) &&
+  (OPENED_LIKE.includes(send.status) || send.opens > 0 || send.clicks > 0);
+
 function median(values: number[]): number {
   if (!values.length) return 0;
   const sorted = [...values].sort((left, right) => left - right);
   return sorted[Math.floor(sorted.length / 2)] as number;
 }
 
-export function aggregate(sends: SendSummary[]): Totals {
-  const delivered = countIf(sends, (send) =>
-    DELIVERED_LIKE.includes(send.status),
-  );
-  const opened = countIf(sends, (send) => OPENED_LIKE.includes(send.status));
-  const clicked = countIf(sends, byStatus("clicked"));
+/** A send to a suppressed address never left: no figure counts it. */
+export const leftForProvider = (send: SendSummary): boolean =>
+  send.status !== "suppressed";
+
+export function aggregate(logged: SendSummary[]): Totals {
+  const sends = logged.filter(leftForProvider);
+  const delivered = countIf(sends, isDelivered);
+  const opened = countIf(sends, wasOpened);
+  const clicked = countIf(sends, wasClicked);
   return {
     sends: sends.length,
     queued: countIf(sends, byStatus("queued")),
@@ -107,22 +148,55 @@ export function aggregate(sends: SendSummary[]): Totals {
     deliverability: ratio(delivered, sends.length),
     openRate: ratio(opened, delivered),
     clickRate: ratio(clicked, delivered),
-    latencyMedian: median(sends.map((send) => send.latencyMs)),
+    latencyMedian: median(sends.flatMap((send) => send.latencyMs ?? [])),
   };
 }
 
-export function deltaPercent(current: number, previous: number): number {
-  if (!previous) return 0;
+/**
+ * The change from `previous` to `current`, in percent; `null` when no share
+ * can be stated, a preceding zero included: growth away from nothing is
+ * unbounded, and a "0 %" stand-in would read as "no change".
+ */
+export function deltaPercent(
+  current: number | null,
+  previous: number | null,
+): number | null {
+  if (current === null || !previous) return null;
   return Number(
     (((current - previous) / previous) * PERCENT).toFixed(DELTA_PRECISION),
   );
 }
 
+const showsProviderEvent = (send: SendSummary): boolean =>
+  PROVIDER_EVENT_STATUSES.includes(send.status) ||
+  send.opens > 0 ||
+  send.clicks > 0;
+
+/**
+ * Whether the window's delivery figures are unmeasured: the provider tracks
+ * nothing itself (SMTP) and no event came back by webhook either, so every
+ * send stays "sent" and a 0 % deliverability would be a false alarm. Unknown
+ * capabilities (the provider unreachable) claim nothing.
+ */
+export function isDeliveryUntracked(
+  sends: SendSummary[],
+  tracking: ProviderTracking | null,
+): boolean {
+  if (!tracking || tracking.openTracking || tracking.clickTracking)
+    return false;
+  return sends.length > 0 && !sends.some(showsProviderEvent);
+}
+
 const dayKey = (date: Date): string =>
   date.toISOString().slice(0, DAY_KEY_LENGTH);
 
+interface DayBucket {
+  x: string;
+  sends: SendSummary[];
+}
+
 /**
- * One point per day of the window, counted from the window's own boundaries.
+ * One bucket per day of the window, counted from the window's own boundaries.
  *
  * The DMS period selector cuts on *local* midnights, so a UTC+2 tenant asks for
  * 2026-09-03T22:00Z → 2026-09-10T21:59:59.999Z. Buckets are therefore offsets
@@ -137,24 +211,59 @@ const dayKey = (date: Date): string =>
  * to drop it, so a send made today was missing from the chart while the KPI
  * still counted it.
  */
-export function dailySeries(
-  sends: SendSummary[],
-  from: Date,
-  to: Date,
-): SeriesPoint[] {
+function dailyBuckets(sends: SendSummary[], from: Date, to: Date): DayBucket[] {
   const start = from.getTime();
-  const span = to.getTime() - start;
-  const dayCount = Math.max(1, Math.ceil(span / DAY_MS));
-  const counts: number[] = Array.from({ length: dayCount }, () => 0);
+  const dayCount = Math.max(1, Math.ceil((to.getTime() - start) / DAY_MS));
+  const buckets: DayBucket[] = Array.from({ length: dayCount }, (_, index) => ({
+    x: dayKey(new Date(start + index * DAY_MS + DAY_MS / 2)),
+    sends: [],
+  }));
   for (const send of sends) {
     const offset = new Date(send.createdAt).getTime() - start;
     if (offset < 0 || offset >= dayCount * DAY_MS) continue;
-    const index = Math.floor(offset / DAY_MS);
-    counts[index] = (counts[index] ?? 0) + 1;
+    buckets[Math.floor(offset / DAY_MS)]?.sends.push(send);
   }
-  return counts.map((y, index) => ({
-    x: dayKey(new Date(start + index * DAY_MS + DAY_MS / 2)),
-    y,
+  return buckets;
+}
+
+/** The number of sends of each day of the window. */
+export function dailySeries(
+  sends: SendSummary[],
+  from: Date,
+  to: Date,
+): SeriesPoint[] {
+  return dailyBuckets(sends, from, to).map((bucket) => ({
+    x: bucket.x,
+    y: bucket.sends.length,
+  }));
+}
+
+/**
+ * One figure's own value for each day of the window, the trend a KPI's
+ * sparkline draws. A day the figure cannot be stated on (a rate without
+ * deliveries) is left out rather than drawn as a fall to zero.
+ */
+export function dailyValues(
+  sends: SendSummary[],
+  range: DateRange,
+  extract: TotalsExtractor,
+): number[] {
+  return dailyBuckets(sends, range.from, range.to)
+    .map((bucket) => extract(aggregate(bucket.sends)))
+    .filter((value): value is number => value !== null);
+}
+
+/**
+ * Lays a comparison series on the current one's axis, day 1 on day 1, so the
+ * two periods overlap instead of standing side by side.
+ */
+export function alignSeries(
+  points: SeriesPoint[],
+  axis: SeriesPoint[],
+): SeriesPoint[] {
+  return points.slice(0, axis.length).map((point, index) => ({
+    x: (axis[index] as SeriesPoint).x,
+    y: point.y,
   }));
 }