@antelopejs/dms-mailing 0.2.2

package/src/test/integration/retention.test.ts

@@ -0,0 +1,363 @@
+import { randomUUID } from "node:crypto";
+import { mock } from "node:test";
+import { expect } from "chai";
+import { GetModel } from "@antelopejs/interface-database-decorators";
+import { AtomicMutationQuery } from "@antelopejs/interface-database/atomic";
+import {
+  purgeInPages,
+  purgeOrphanEvents,
+  purgeSend,
+} from "../../crons/retention";
+import { type MailingSend, SendEventModel, SendModel } from "../../db";
+import { recordEmailEvent } from "../../services/events";
+
+const OLD = new Date("2020-01-01T00:00:00Z");
+const CUTOFF = new Date("2021-01-01T00:00:00Z");
+const PAGE_SIZE = 2;
+const MAX_ROWS = 10;
+
+interface Fixture {
+  tenantId: string;
+  sends: SendModel;
+  events: SendEventModel;
+}
+
+function fixture(): Fixture {
+  const tenantId = randomUUID();
+  return {
+    tenantId,
+    sends: GetModel(SendModel, tenantId),
+    events: GetModel(SendEventModel, tenantId),
+  };
+}
+
+async function seed(
+  f: Fixture,
+  changes: Partial<MailingSend> = {},
+): Promise<string> {
+  const [id] = await f.sends.insertSend({
+    templateId: "template",
+    templateSlug: "retention",
+    locale: "en",
+    recipientEmail: "retention@example.test",
+    status: "delivered",
+    providerMessageId: randomUUID(),
+    latencyMs: 1,
+    json_variables: "{}",
+    opens: 0,
+    clicks: 0,
+    isTest: true,
+  });
+  await f.sends.table
+    .get(id)
+    .update({ createdAt: OLD, ...changes })
+    .run();
+  await f.events.insert({
+    sendId: id,
+    type: "delivered",
+    at: OLD,
+    json_details: "{}",
+  });
+  return id;
+}
+
+function sweep(f: Fixture): Promise<number> {
+  return purgeInPages(
+    {
+      async loadPage(size) {
+        return (await f.sends.listOlderThan(CUTOFF, size)).map(
+          (send) => send._id,
+        );
+      },
+      purgeSend: (id) => purgeSend(f.tenantId, id, CUTOFF),
+    },
+    { pageSize: PAGE_SIZE, maxRows: MAX_ROWS },
+  );
+}
+
+async function loseAcknowledgement(
+  action: () => Promise<unknown>,
+): Promise<number> {
+  const original: AtomicMutationQuery["run"] = Object.getOwnPropertyDescriptor(
+    AtomicMutationQuery.prototype,
+    "run",
+  )?.value;
+  let calls = 0;
+  const run = mock.method(AtomicMutationQuery.prototype, "run");
+  run.mock.mockImplementation(async function (this: AtomicMutationQuery) {
+    calls++;
+    await original.call(this);
+    return "unknown";
+  });
+  try {
+    await action().then(
+      () => {
+        throw new Error("expected unknown outcome failure");
+      },
+      (error: Error) =>
+        expect(error.message).to.include("outcome is unknown; not retrying"),
+    );
+  } finally {
+    run.mock.restore();
+  }
+  return calls;
+}
+
+describe("[integration] replay-safe Mongo retention", () => {
+  it("initializes a fresh identity and revision on every send", async () => {
+    const f = fixture();
+    const sourceId = await seed(f);
+    const source = await f.sends.getSnapshot(sourceId);
+    expect(source?.revision).to.be.a("string");
+    const [first] = await f.sends.insertSend(source!);
+    const [second] = await f.sends.insertSend(source!);
+    const firstSend = await f.sends.getSnapshot(first);
+    const secondSend = await f.sends.getSnapshot(second);
+    expect(new Set([sourceId, first, second]).size).to.equal(3);
+    expect(firstSend?.revision)
+      .to.be.a("string")
+      .and.not.equal(source?.revision);
+    expect(secondSend?.revision)
+      .to.be.a("string")
+      .and.not.equal(firstSend?.revision);
+    expect(firstSend!.createdAt.getTime()).to.be.greaterThan(CUTOFF.getTime());
+    expect(await f.sends.retire(sourceId, CUTOFF)).to.equal(true);
+    expect((await f.sends.getSnapshot(sourceId))?.revision)
+      .to.be.a("string")
+      .and.not.equal(source?.revision);
+    expect(await sweep(f)).to.equal(1);
+    expect(await f.sends.getAll()).to.have.length(2);
+  });
+
+  it("overlapping paged sweeps count each deleted send once", async () => {
+    const f = fixture();
+    const ids = await Promise.all(Array.from({ length: 5 }, () => seed(f)));
+    const counts = await Promise.all([sweep(f), sweep(f)]);
+    expect(counts[0] + counts[1]).to.equal(ids.length);
+    expect(await f.sends.getAll()).to.deep.equal([]);
+    expect(await f.events.getAll()).to.deep.equal([]);
+    expect(await sweep(f)).to.equal(0);
+  });
+
+  it("preserves new, queued, cutoff-boundary and recently active sends and their events", async () => {
+    const f = fixture();
+    const changes: Partial<MailingSend>[] = [
+      { createdAt: new Date() },
+      { createdAt: CUTOFF },
+      { status: "queued" },
+      { lastEventAt: CUTOFF },
+      { lastActivityAt: CUTOFF },
+    ];
+    const ids = await Promise.all(changes.map((change) => seed(f, change)));
+    expect(await f.sends.listOlderThan(CUTOFF, MAX_ROWS)).to.deep.equal([]);
+    for (const id of ids)
+      expect(await purgeSend(f.tenantId, id, CUTOFF)).to.equal(0);
+    expect(await f.sends.getAll()).to.have.length(ids.length);
+    expect(await f.events.getAll()).to.have.length(ids.length);
+  });
+
+  it("rechecks stale candidates after a backdated webhook activates the send", async () => {
+    const f = fixture();
+    const id = await seed(f);
+    const [candidate] = await f.sends.listOlderThan(CUTOFF, PAGE_SIZE);
+    expect(candidate._id).to.equal(id);
+    expect(
+      await recordEmailEvent(f.tenantId, {
+        messageId: candidate.providerMessageId!,
+        provider: "fixture",
+        type: "opened",
+        at: OLD,
+      }),
+    ).to.equal(true);
+    expect(await purgeSend(f.tenantId, id, CUTOFF)).to.equal(0);
+    expect((await f.sends.get(id))?.opens).to.equal(1);
+    expect(await f.events.getBySend(id)).to.have.length(2);
+  });
+
+  it("serializes retirement against concurrent activity at the send document", async () => {
+    const f = fixture();
+    const id = await seed(f);
+    const candidate = await f.sends.get(id);
+    const [retired, accepted] = await Promise.all([
+      f.sends.retire(id, CUTOFF),
+      f.sends.recordActivity(candidate!, {
+        status: "opened",
+        opens: 1,
+        clicks: 0,
+        lastEventAt: OLD,
+      }),
+    ]);
+    expect(Number(retired) + Number(accepted)).to.equal(1);
+    const send = await f.sends.get(id);
+    expect(send?.isRetiring === true).to.equal(retired);
+    expect(send?.opens).to.equal(Number(accepted));
+  });
+
+  it("retries known conflicts and counts both concurrent opened webhooks", async () => {
+    const f = fixture();
+    const id = await seed(f);
+    const send = await f.sends.get(id);
+    const accepted = await Promise.all(
+      Array.from({ length: 2 }, () =>
+        recordEmailEvent(f.tenantId, {
+          messageId: send!.providerMessageId!,
+          provider: "fixture",
+          type: "opened",
+        }),
+      ),
+    );
+    expect(accepted).to.deep.equal([true, true]);
+    expect((await f.sends.get(id))?.opens).to.equal(2);
+    expect(await f.events.getBySend(id)).to.have.length(3);
+    expect(await purgeSend(f.tenantId, id, CUTOFF)).to.equal(0);
+  });
+
+  it("does not retry an uncertain webhook write or emit an unacknowledged event", async () => {
+    const f = fixture();
+    const id = await seed(f);
+    const send = await f.sends.get(id);
+    const calls = await loseAcknowledgement(() =>
+      recordEmailEvent(f.tenantId, {
+        messageId: send!.providerMessageId!,
+        provider: "fixture",
+        type: "opened",
+      }),
+    );
+    expect(calls).to.equal(1);
+    expect((await f.sends.get(id))?.opens).to.equal(1);
+    expect(await f.events.getBySend(id)).to.have.length(1);
+  });
+
+  it("leaves uncertain retirement replayable and never counts an uncertain deletion", async () => {
+    const f = fixture();
+    const id = await seed(f);
+    expect(
+      await loseAcknowledgement(() => purgeSend(f.tenantId, id, CUTOFF)),
+    ).to.equal(1);
+    expect((await f.sends.get(id))?.isRetiring).to.equal(true);
+    expect(await f.events.getBySend(id)).to.have.length(1);
+    expect(
+      await loseAcknowledgement(() => purgeSend(f.tenantId, id, CUTOFF)),
+    ).to.equal(1);
+    expect(await f.sends.get(id)).to.equal(undefined);
+    expect(await f.events.getBySend(id)).to.deep.equal([]);
+    expect(await sweep(f)).to.equal(0);
+  });
+
+  it("rejects a webhook after retirement and removes an already-paused writer's late event", async () => {
+    const f = fixture();
+    const id = await seed(f);
+    const send = await f.sends.get(id);
+    expect(await f.sends.retire(id, CUTOFF)).to.equal(true);
+    expect(
+      await recordEmailEvent(f.tenantId, {
+        messageId: send!.providerMessageId!,
+        provider: "fixture",
+        type: "opened",
+      }),
+    ).to.equal(false);
+    expect(
+      await f.events.insertForActiveSend({
+        sendId: id,
+        type: "opened",
+        at: new Date(),
+        json_details: "{}",
+      }),
+    ).to.equal(false);
+    expect(await f.events.getBySend(id)).to.have.length(1);
+    expect(await purgeSend(f.tenantId, id, CUTOFF)).to.equal(1);
+    expect(
+      await f.events.insertForActiveSend({
+        sendId: id,
+        type: "clicked",
+        at: new Date(),
+        json_details: "{}",
+      }),
+    ).to.equal(false);
+    expect(await f.events.getBySend(id)).to.deep.equal([]);
+  });
+
+  it("retries event cleanup failures after marking", async () => {
+    const f = fixture();
+    const id = await seed(f);
+    const deletion = mock.method(
+      SendEventModel.prototype,
+      "deleteBySend",
+      async () => {
+        throw new Error("cleanup unavailable");
+      },
+    );
+    try {
+      await purgeSend(f.tenantId, id, CUTOFF).then(
+        () => {
+          throw new Error("expected cleanup failure");
+        },
+        (error: Error) => expect(error.message).to.equal("cleanup unavailable"),
+      );
+    } finally {
+      deletion.mock.restore();
+    }
+    expect((await f.sends.get(id))?.isRetiring).to.equal(true);
+    expect(await f.events.getBySend(id)).to.have.length(1);
+    expect(await sweep(f)).to.equal(1);
+    expect(await sweep(f)).to.equal(0);
+  });
+
+  it("retries parent deletion failures after the events are gone", async () => {
+    const f = fixture();
+    const id = await seed(f);
+    const deletion = mock.method(
+      SendModel.prototype,
+      "deleteRetired",
+      async () => {
+        throw new Error("delete unavailable");
+      },
+    );
+    try {
+      await purgeSend(f.tenantId, id, CUTOFF).then(
+        () => {
+          throw new Error("expected delete failure");
+        },
+        (error: Error) => expect(error.message).to.equal("delete unavailable"),
+      );
+    } finally {
+      deletion.mock.restore();
+    }
+    expect((await f.sends.get(id))?.isRetiring).to.equal(true);
+    expect(await f.events.getBySend(id)).to.deep.equal([]);
+    expect(await sweep(f)).to.equal(1);
+    expect(await sweep(f)).to.equal(0);
+  });
+
+  it("reconciles crash-orphaned events across pages without removing active history", async () => {
+    const f = fixture();
+    const active = await seed(f, { createdAt: new Date() });
+    const retired = await seed(f);
+    expect(await purgeSend(f.tenantId, retired, CUTOFF)).to.equal(1);
+    await f.events.insert(
+      Array.from({ length: 5 }, (_value, index) => ({
+        sendId: index === 0 ? retired : randomUUID(),
+        type: "opened" as const,
+        at: new Date(),
+        json_details: "{}",
+      })),
+    );
+    await Promise.all([
+      purgeOrphanEvents(f.tenantId, PAGE_SIZE),
+      purgeOrphanEvents(f.tenantId, PAGE_SIZE),
+    ]);
+    expect(await f.events.getBySend(active)).to.have.length(1);
+    expect(await f.events.getAll()).to.have.length(1);
+    expect(await f.events.getBySend(retired)).to.deep.equal([]);
+    await f.events.insert({
+      sendId: retired,
+      type: "clicked",
+      at: new Date(),
+      json_details: "{}",
+    });
+    await purgeOrphanEvents(f.tenantId, PAGE_SIZE);
+    await purgeOrphanEvents(f.tenantId, PAGE_SIZE);
+    expect(await f.events.getBySend(active)).to.have.length(1);
+    expect(await f.events.getAll()).to.have.length(1);
+  });
+});

package/src/test/integration/sends.test.ts

@@ -0,0 +1,200 @@
+import { expect } from "chai";
+import { authorizedClient } from "../helpers/http";
+import { ensureOwnerSession } from "../helpers/owner";
+
+const HTTP_OK = 200;
+const HTTP_UNPROCESSABLE = 422;
+const HTTP_NOT_FOUND = 404;
+const DAY_MS = 86_400_000;
+const TABLE = "/api/mailing/tables/templates";
+
+interface TemplateRow {
+  _id: string;
+  slug: string;
+}
+
+interface SendResult {
+  sendId: string;
+}
+
+describe("[integration] sends", () => {
+  it("sends a test e-mail through the SMTP fixture and logs it", async () => {
+    const session = await ensureOwnerSession();
+    const client = authorizedClient(session.accessToken);
+    const created = await client.post(`${TABLE}/new`, {
+      slug: "test-send",
+      name: "Test send",
+      category: "orders",
+    });
+    expect(created.status, JSON.stringify(created.data)).to.equal(HTTP_OK);
+    const id = created.data[0] as string;
+    const sent = await client.post(`/api/mailing/templates/${id}/test-send`, {
+      to: ["sofie@example.test"],
+      locale: "en",
+      data: { customer: { firstName: "Sofie" }, detailsUrl: "https://x" },
+    });
+    expect(sent.status, JSON.stringify(sent.data)).to.equal(HTTP_OK);
+    expect(sent.data.results[0].status).to.be.oneOf(["sent", "queued"]);
+
+    const detail = await client.get(
+      `/api/mailing/sends/${sent.data.results[0].sendId}`,
+    );
+    expect(detail.status).to.equal(HTTP_OK);
+    expect(detail.data.send.isTest).to.equal(true);
+    expect(detail.data.send.templateSlug).to.equal("test-send");
+    expect(detail.data.events[0].type).to.equal(detail.data.send.status);
+  });
+
+  it("refuses a real send on a draft template", async () => {
+    const session = await ensureOwnerSession();
+    const client = authorizedClient(session.accessToken);
+    const created = await client.post(`${TABLE}/new`, {
+      slug: "real-draft",
+      name: "Real draft",
+      category: "orders",
+    });
+    const id = created.data[0] as string;
+    const sent = await client.post(`/api/mailing/templates/${id}/send`, {
+      to: ["sofie@example.test"],
+      locale: "en",
+    });
+    expect(sent.status).to.equal(HTTP_UNPROCESSABLE);
+    expect(JSON.stringify(sent.data)).to.contain("template_not_live");
+  });
+
+  it("sends for real from a live template and counts it in the metrics", async () => {
+    const session = await ensureOwnerSession();
+    const client = authorizedClient(session.accessToken);
+    const created = await client.post(`${TABLE}/new`, {
+      slug: "real-send",
+      name: "Real send",
+      category: "orders",
+    });
+    const id = created.data[0] as string;
+    await client.post(`/api/mailing/templates/${id}/publish`, {});
+    const sent = await client.post(`/api/mailing/templates/${id}/send`, {
+      to: ["sofie@example.test"],
+      locale: "en",
+      data: { customer: { firstName: "Sofie" }, detailsUrl: "https://x" },
+    });
+    expect(sent.status, JSON.stringify(sent.data)).to.equal(HTTP_OK);
+    const detail = await client.get(
+      `/api/mailing/sends/${sent.data.results[0].sendId}`,
+    );
+    expect(detail.data.send.isTest).to.equal(false);
+    expect(detail.data.send.source).to.equal("dms-mailing:manual");
+  });
+
+  it("records one send per recipient of a multi-recipient test send", async () => {
+    const session = await ensureOwnerSession();
+    const client = authorizedClient(session.accessToken);
+    const list = await client.get(`${TABLE}/list`);
+    const id = (
+      list.data.results.find(
+        (item: TemplateRow) => item.slug === "test-send",
+      ) as TemplateRow
+    )._id;
+    const sent = await client.post(`/api/mailing/templates/${id}/test-send`, {
+      to: ["first@example.test", "second@example.test"],
+      locale: "en",
+      data: { customer: { firstName: "Sofie" }, detailsUrl: "https://x" },
+    });
+    expect(sent.status, JSON.stringify(sent.data)).to.equal(HTTP_OK);
+    expect(sent.data.results).to.have.length(2);
+    const sendIds = sent.data.results.map(
+      (result: SendResult) => result.sendId,
+    );
+    expect(new Set(sendIds).size).to.equal(2);
+  });
+
+  it("refuses to send when required variables are missing and blocking is on", async () => {
+    const session = await ensureOwnerSession();
+    const client = authorizedClient(session.accessToken);
+    const settings = await client.get("/api/mailing/settings");
+    await client.post("/api/mailing/settings", {
+      ...settings.data,
+      blockOnMissingVariables: true,
+    });
+    const list = await client.get(`${TABLE}/list`);
+    const id = (
+      list.data.results.find(
+        (item: TemplateRow) => item.slug === "test-send",
+      ) as TemplateRow
+    )._id;
+    // Blank content references nothing, so give it a token to miss.
+    await client.post(`/api/mailing/templates/${id}/content`, {
+      content: {
+        locales: {
+          en: {
+            subject: "Hi {{customer.firstName}}",
+            preheader: "",
+            blocks: [],
+          },
+        },
+      },
+    });
+    await client.post(`/api/mailing/templates/${id}/variables`, {
+      variables: [
+        { path: "customer.firstName", type: "string", required: true },
+      ],
+    });
+    const blocked = await client.post(
+      `/api/mailing/templates/${id}/test-send`,
+      { to: ["sofie@example.test"], locale: "en", data: {} },
+    );
+    expect(blocked.status).to.equal(HTTP_UNPROCESSABLE);
+    await client.post("/api/mailing/settings", {
+      ...settings.data,
+      blockOnMissingVariables: false,
+    });
+  });
+
+  it("replays a send and lists it in the sends table", async () => {
+    const session = await ensureOwnerSession();
+    const client = authorizedClient(session.accessToken);
+    const list = await client.get("/api/mailing/tables/sends/list");
+    expect(list.status, JSON.stringify(list.data)).to.equal(HTTP_OK);
+    const original = list.data.results[0] as TemplateRow;
+    const replay = await client.post(
+      `/api/mailing/sends/${original._id}/replay`,
+      {},
+    );
+    expect(replay.status, JSON.stringify(replay.data)).to.equal(HTTP_OK);
+    expect(replay.data.sendId).to.not.equal(original._id);
+    const html = await client.get(`/api/mailing/sends/${original._id}/html`);
+    expect(html.status).to.equal(HTTP_OK);
+    expect(html.data.html).to.include("<!DOCTYPE html>");
+  });
+
+  it("exposes metrics and the provider status", async () => {
+    const session = await ensureOwnerSession();
+    const client = authorizedClient(session.accessToken);
+    const from = new Date(Date.now() - DAY_MS).toISOString();
+    const to = new Date(Date.now() + DAY_MS).toISOString();
+    const kpiUrl = `/api/mailing/metrics/kpi/sends?from=${from}&to=${to}`;
+    const kpi = await client.get(kpiUrl);
+    expect(kpi.status, JSON.stringify(kpi.data)).to.equal(HTTP_OK);
+    expect(kpi.data.value).to.be.a("number");
+
+    const volume = await client.get(
+      `/api/mailing/metrics/volume?from=${from}&to=${to}`,
+    );
+    expect(volume.status).to.equal(HTTP_OK);
+    expect(volume.data.series[0].data).to.not.have.length(0);
+
+    const unknown = await client.get("/api/mailing/metrics/kpi/nope");
+    expect(unknown.status).to.equal(HTTP_NOT_FOUND);
+
+    const provider = await client.get("/api/mailing/provider");
+    expect(provider.status).to.equal(HTTP_OK);
+    expect(provider.data.connected).to.equal(true);
+
+    const funnel = await client.get("/api/mailing/metrics/funnel");
+    expect(funnel.status).to.equal(HTTP_OK);
+    expect(funnel.data.steps).to.have.length(5);
+
+    const attention = await client.get("/api/mailing/metrics/attention");
+    expect(attention.status).to.equal(HTTP_OK);
+    expect(attention.data.items).to.be.an("array");
+  });
+});

package/src/test/integration/settings.test.ts

@@ -0,0 +1,50 @@
+import { expect } from "chai";
+import { authorizedClient } from "../helpers/http";
+import { ensureOwnerSession } from "../helpers/owner";
+
+const HTTP_OK = 200;
+const HTTP_BAD_REQUEST = 400;
+const DEFAULT_CATEGORY_COUNT = 5;
+const DEFAULT_RETENTION_DAYS = 90;
+const UPDATED_RETENTION_DAYS = 30;
+const INVALID_RETENTION_DAYS = 0;
+
+describe("[integration] settings", () => {
+  it("returns defaults, accepts an update and rejects an invalid retention", async () => {
+    const session = await ensureOwnerSession();
+    const client = authorizedClient(session.accessToken);
+    const initial = await client.get("/api/mailing/settings");
+    expect(initial.status).to.equal(HTTP_OK);
+    expect(initial.data.fallbackLocale).to.equal("en");
+    expect(initial.data.logRetentionDays).to.equal(DEFAULT_RETENTION_DAYS);
+    expect(initial.data.categories).to.have.length(DEFAULT_CATEGORY_COUNT);
+
+    const update = await client.post("/api/mailing/settings", {
+      ...initial.data,
+      fallbackLocale: "fr",
+      logRetentionDays: UPDATED_RETENTION_DAYS,
+      blockOnMissingVariables: true,
+    });
+    expect(update.status).to.equal(HTTP_OK);
+    const after = await client.get("/api/mailing/settings");
+    expect(after.data.fallbackLocale).to.equal("fr");
+    expect(after.data.blockOnMissingVariables).to.equal(true);
+
+    const invalid = await client.post("/api/mailing/settings", {
+      ...after.data,
+      logRetentionDays: INVALID_RETENTION_DAYS,
+    });
+    expect(invalid.status).to.equal(HTTP_BAD_REQUEST);
+
+    // Blanking the secret would leave the unauthenticated event endpoint with
+    // no gate, so the write has to be refused rather than silently accepted.
+    const blankSecret = await client.post("/api/mailing/settings", {
+      ...after.data,
+      webhookSecret: "",
+    });
+    expect(blankSecret.status).to.equal(HTTP_BAD_REQUEST);
+
+    const restored = await client.post("/api/mailing/settings", initial.data);
+    expect(restored.status).to.equal(HTTP_OK);
+  });
+});

package/src/test/integration/setup.test.ts

@@ -0,0 +1,35 @@
+import { createClient } from "../helpers/http";
+
+const READY_TIMEOUT_MS = 30_000;
+const POLL_INTERVAL_MS = 100;
+const HTTP_OK = 200;
+
+function delay(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function waitFor(
+  description: string,
+  check: () => Promise<boolean>,
+): Promise<void> {
+  const deadline = Date.now() + READY_TIMEOUT_MS;
+  while (Date.now() < deadline) {
+    if (await check()) return;
+    await delay(POLL_INTERVAL_MS);
+  }
+  throw new Error(`Timed out waiting for ${description}`);
+}
+
+before(async function () {
+  this.timeout(READY_TIMEOUT_MS + 5_000);
+
+  const client = createClient();
+  await waitFor("the DMS runtime to accept onboarding calls", async () => {
+    try {
+      const response = await client.get("/api/onboarding/informations");
+      return response.status === HTTP_OK;
+    } catch {
+      return false;
+    }
+  });
+});