@hogsend/engine 0.6.0 → 0.7.0

package/src/routes/admin/journeys.ts

@@ -662,7 +662,9 @@ export const journeysRouter = new OpenAPIHono<AppEnv>()
         event: meta.trigger.event,
         userId: body.userId,
         userEmail: body.userEmail,
-        properties: body.properties ?? {},
+        // Public request field stays `properties` (decision #14); it maps to
+        // the event-property bag of the IngestEvent (D2 split).
+        eventProperties: body.properties ?? {},
       },
     });
 

package/src/routes/admin/preferences.ts

@@ -104,7 +104,7 @@ export const preferencesRouter = new OpenAPIHono<AppEnv>()
     const rows = await db
       .select()
       .from(emailPreferences)
-      .where(eq(emailPreferences.userId, contact.externalId))
+      .where(eq(emailPreferences.userId, contact.externalId ?? contact.id))
       .limit(1);
 
     if (rows.length === 0) {
@@ -131,7 +131,7 @@ export const preferencesRouter = new OpenAPIHono<AppEnv>()
     const [upserted] = await db
       .insert(emailPreferences)
       .values({
-        userId: contact.externalId,
+        userId: contact.externalId ?? contact.id,
         email: contact.email,
         unsubscribedAll: body.unsubscribedAll ?? false,
         suppressed: body.suppressed ?? false,

package/src/routes/admin/reporting.ts

@@ -12,7 +12,7 @@ import {
   sql,
 } from "drizzle-orm";
 import type { AppEnv } from "../../app.js";
-import { resolveContact } from "../../lib/contacts.js";
+import { contactKey, resolveContact } from "../../lib/contacts.js";
 import { rate, TRUNC_SQL } from "../../lib/metrics-sql.js";
 import { errorSchema } from "../../lib/schemas.js";
 
@@ -109,7 +109,7 @@ const contactActivityRoute = createRoute({
         "application/json": {
           schema: z.object({
             contact: z.object({
-              externalId: z.string(),
+              externalId: z.string().nullable(),
               email: z.string().nullable(),
             }),
             sends: z.array(
@@ -244,7 +244,7 @@ export const reportingRouter = new OpenAPIHono<AppEnv>()
 
     // Denormalized identity makes this single-table; fall back to the contact's
     // email so journeyless sends still surface.
-    const idConds = [eq(emailSends.userId, contact.externalId)];
+    const idConds = [eq(emailSends.userId, contactKey(contact))];
     if (contact.email) idConds.push(eq(emailSends.userEmail, contact.email));
     const where = or(...idConds);
 

package/src/routes/admin/timeline.ts

@@ -2,7 +2,7 @@ import { emailSends, journeyStates, userEvents } from "@hogsend/db";
 import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
 import { and, count, desc, eq, isNull } from "drizzle-orm";
 import type { AppEnv } from "../../app.js";
-import { resolveContact } from "../../lib/contacts.js";
+import { contactKey, resolveContact } from "../../lib/contacts.js";
 
 const timelineEntrySchema = z.object({
   type: z.enum(["event", "journey", "email"]),
@@ -64,7 +64,10 @@ export const timelineRouter = new OpenAPIHono<AppEnv>().openapi(
       return c.json({ error: "Contact not found" }, 404);
     }
 
-    const externalId = contact.externalId;
+    // Resolved string key the history tables (user_events/journey_states/
+    // email_sends) were written under: external_id, else anonymous_id, else the
+    // contact uuid (matches ingestEvent's resolvedKey).
+    const externalId = contactKey(contact);
     const entries: TimelineEntry[] = [];
 
     const shouldFetch = (t: string) => !type || type === t;

package/src/routes/campaigns/index.ts

@@ -0,0 +1,252 @@
+import { campaigns } from "@hogsend/db";
+import { getTemplateNames, type TemplateName } from "@hogsend/email";
+import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
+import { eq } from "drizzle-orm";
+import type { AppEnv } from "../../app.js";
+import { errorSchema } from "../../lib/schemas.js";
+import { sendCampaignTask } from "../../workflows/send-campaign.js";
+
+const createCampaignSchema = z
+  .object({
+    name: z.string().min(1).optional(),
+    list: z.string().min(1).optional(),
+    bucket: z.string().min(1).optional(),
+    template: z.string().min(1),
+    props: z.record(z.string(), z.unknown()).optional(),
+    from: z.string().optional(),
+    subject: z.string().optional(),
+    /**
+     * Optional client idempotency key. A retried create with the same key
+     * resolves to the EXISTING campaign instead of spawning a second broadcast
+     * (which would double-send to the same recipients). The `Idempotency-Key`
+     * header wins over this body field (mirrors /v1/emails + /v1/events).
+     */
+    idempotencyKey: z.string().min(1).optional(),
+  })
+  // EXACTLY ONE of list|bucket — XOR. Both-or-neither is a 400.
+  .refine((b) => (b.list ? 1 : 0) + (b.bucket ? 1 : 0) === 1, {
+    message: "Exactly one of `list` or `bucket` is required",
+  });
+
+const createResponseSchema = z.object({
+  campaignId: z.string(),
+  status: z.string(),
+});
+
+const campaignSchema = z.object({
+  id: z.string(),
+  name: z.string(),
+  status: z.string(),
+  audienceKind: z.string(),
+  audienceId: z.string(),
+  templateKey: z.string(),
+  totalRecipients: z.number(),
+  sentCount: z.number(),
+  skippedCount: z.number(),
+  failedCount: z.number(),
+  startedAt: z.string().nullable(),
+  completedAt: z.string().nullable(),
+  createdAt: z.string(),
+});
+
+const createRouteDef = createRoute({
+  method: "post",
+  path: "/",
+  tags: ["Campaigns"],
+  summary: "Create + enqueue a campaign",
+  description:
+    "Sends one template to every subscribed member of a list (or every active member of a bucket). Validates the template + audience, inserts a `queued` campaign, and enqueues the durable `send-campaign` task. Exactly one of `list` or `bucket` is required.",
+  request: {
+    body: {
+      content: {
+        "application/json": { schema: createCampaignSchema },
+      },
+    },
+  },
+  responses: {
+    202: {
+      content: {
+        "application/json": { schema: createResponseSchema },
+      },
+      description: "Campaign created and enqueued",
+    },
+    400: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Invalid audience selector or unknown template",
+    },
+    404: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Unknown list or bucket id",
+    },
+  },
+});
+
+const getRouteDef = createRoute({
+  method: "get",
+  path: "/{id}",
+  tags: ["Campaigns"],
+  summary: "Get a campaign",
+  request: {
+    params: z.object({ id: z.string() }),
+  },
+  responses: {
+    200: {
+      content: {
+        "application/json": { schema: campaignSchema },
+      },
+      description: "The campaign",
+    },
+    404: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Unknown campaign id",
+    },
+  },
+});
+
+// The campaigns router does NOT re-apply auth internally — the data-plane prefix
+// guards in `routes/index.ts` (decision #16) apply `requireApiKey` +
+// `requireScope("ingest")` to `/v1/campaigns` (bare + `/*`) before requests
+// reach this router.
+export const campaignsRouter = new OpenAPIHono<AppEnv>()
+  .openapi(createRouteDef, async (c) => {
+    const { db, templates, listRegistry, bucketRegistry, logger } =
+      c.get("container");
+    const body = c.req.valid("json");
+
+    // Enqueue the durable send task, swallowing a transient broker/transport
+    // failure: the campaign row is already committed in `queued`, so a failed
+    // enqueue is recovered by the reaper cron re-enqueueing the orphan rather
+    // than 500-ing the request (which a keyless client would retry into a
+    // duplicate broadcast).
+    const enqueue = async (campaignId: string): Promise<void> => {
+      try {
+        await sendCampaignTask.run({ campaignId });
+      } catch (err) {
+        logger.warn("POST /v1/campaigns: enqueue failed (reaper will retry)", {
+          campaignId,
+          error: err instanceof Error ? err.message : String(err),
+        });
+      }
+    };
+
+    // Validate the template against the wired registry (mirrors /v1/emails).
+    if (!getTemplateNames(templates).includes(body.template as TemplateName)) {
+      return c.json({ error: `Unknown template: ${body.template}` }, 400);
+    }
+
+    // XOR is already enforced by the schema refine, so exactly one is present.
+    const audienceKind = body.list ? "list" : "bucket";
+    const audienceId = (body.list ?? body.bucket) as string;
+
+    if (audienceKind === "list" && !listRegistry.has(audienceId)) {
+      return c.json({ error: `Unknown list: ${audienceId}` }, 404);
+    }
+    if (audienceKind === "bucket" && !bucketRegistry.has(audienceId)) {
+      return c.json({ error: `Unknown bucket: ${audienceId}` }, 404);
+    }
+
+    // Idempotency: the `Idempotency-Key` header wins over the body field
+    // (mirrors /v1/emails + /v1/events). A retried create with the same key must
+    // resolve to the SAME campaign — a fresh row would give the recipients a
+    // different per-send idempotency key and double-send the blast.
+    const headerKey = c.req.header("idempotency-key");
+    const idempotencyKey = headerKey ?? body.idempotencyKey ?? null;
+
+    if (idempotencyKey) {
+      const existing = await db
+        .select({ id: campaigns.id, status: campaigns.status })
+        .from(campaigns)
+        .where(eq(campaigns.idempotencyKey, idempotencyKey))
+        .limit(1);
+      const prior = existing[0];
+      if (prior) {
+        // Re-enqueue is safe (terminal-`sent` short-circuits; the per-send key
+        // dedups), so an idempotent retry both returns the existing campaign AND
+        // ensures it is running — covering a first attempt that committed the
+        // row but failed to enqueue.
+        await enqueue(prior.id);
+        return c.json({ campaignId: prior.id, status: prior.status }, 202);
+      }
+    }
+
+    const baseInsert = db.insert(campaigns).values({
+      name: body.name ?? `Campaign to ${audienceKind} ${audienceId}`,
+      status: "queued",
+      audienceKind,
+      audienceId,
+      templateKey: body.template,
+      props: (body.props ?? {}) as Record<string, unknown>,
+      fromEmail: body.from ?? null,
+      subject: body.subject ?? null,
+      idempotencyKey,
+    });
+
+    // With a key, swallow a concurrent-insert collision on the partial-unique
+    // index (the select-then-insert above is not atomic) and resolve the winner.
+    const insertRows = idempotencyKey
+      ? await baseInsert
+          .onConflictDoNothing({ target: campaigns.idempotencyKey })
+          .returning({ id: campaigns.id })
+      : await baseInsert.returning({ id: campaigns.id });
+
+    const campaignId = insertRows[0]?.id;
+    if (!campaignId && idempotencyKey) {
+      const winner = await db
+        .select({ id: campaigns.id, status: campaigns.status })
+        .from(campaigns)
+        .where(eq(campaigns.idempotencyKey, idempotencyKey))
+        .limit(1);
+      if (winner[0]) {
+        await enqueue(winner[0].id);
+        return c.json(
+          { campaignId: winner[0].id, status: winner[0].status },
+          202,
+        );
+      }
+    }
+    if (!campaignId) throw new Error("Failed to create campaign");
+
+    // Enqueue the durable task. The campaign row is already committed in
+    // `queued`, so a transient enqueue failure (broker down) is NON-fatal: we
+    // still return 202 and the reaper cron re-enqueues the orphaned `queued`
+    // row. This keeps the request from 500-ing AFTER a committed row, which a
+    // client would otherwise retry into a duplicate (sans idempotency key).
+    await enqueue(campaignId);
+
+    return c.json({ campaignId, status: "queued" }, 202);
+  })
+  .openapi(getRouteDef, async (c) => {
+    const { db } = c.get("container");
+    const { id } = c.req.valid("param");
+
+    const rows = await db
+      .select()
+      .from(campaigns)
+      .where(eq(campaigns.id, id))
+      .limit(1);
+    const campaign = rows[0];
+    if (!campaign) {
+      return c.json({ error: `Unknown campaign: ${id}` }, 404);
+    }
+
+    return c.json(
+      {
+        id: campaign.id,
+        name: campaign.name,
+        status: campaign.status,
+        audienceKind: campaign.audienceKind,
+        audienceId: campaign.audienceId,
+        templateKey: campaign.templateKey,
+        totalRecipients: campaign.totalRecipients,
+        sentCount: campaign.sentCount,
+        skippedCount: campaign.skippedCount,
+        failedCount: campaign.failedCount,
+        startedAt: campaign.startedAt ? campaign.startedAt.toISOString() : null,
+        completedAt: campaign.completedAt
+          ? campaign.completedAt.toISOString()
+          : null,
+        createdAt: campaign.createdAt.toISOString(),
+      },
+      200,
+    );
+  });

package/src/routes/contacts/index.ts

@@ -0,0 +1,188 @@
+import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
+import type { AppEnv } from "../../app.js";
+import {
+  findContacts,
+  resolveOrCreateContact,
+  serializeContact,
+  softDeleteContact,
+} from "../../lib/contacts.js";
+import { applyListMembership } from "../../lib/preferences.js";
+import { errorSchema } from "../../lib/schemas.js";
+import { listMembershipError, requireIdentity } from "../_shared.js";
+
+// The public, serialized contact shape (§2.5). `externalId` is nullable (D1 —
+// email-only / anonymous contacts) and timestamps are ISO strings.
+const contactSchema = z.object({
+  id: z.string(),
+  externalId: z.string().nullable(),
+  email: z.string().nullable(),
+  properties: z.record(z.string(), z.unknown()),
+  firstSeenAt: z.string(),
+  lastSeenAt: z.string(),
+  createdAt: z.string(),
+  updatedAt: z.string(),
+});
+
+const upsertRoute = createRoute({
+  method: "put",
+  path: "/",
+  tags: ["Contacts"],
+  summary: "Upsert a contact",
+  description:
+    "Resolves (create / fill-in-link / merge) a contact by email and/or userId, applies contactProperties, and optionally writes list membership.",
+  request: {
+    body: {
+      content: {
+        "application/json": {
+          schema: z.object({
+            email: z.string().email().optional(),
+            userId: z.string().min(1).optional(),
+            properties: z.record(z.string(), z.unknown()).optional(),
+            lists: z.record(z.string(), z.boolean()).optional(),
+          }),
+        },
+      },
+    },
+  },
+  responses: {
+    200: {
+      content: {
+        "application/json": {
+          schema: z.object({
+            id: z.string(),
+            created: z.boolean(),
+            linked: z.boolean(),
+          }),
+        },
+      },
+      description: "Contact resolved",
+    },
+    400: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Missing recipient or unmanageable list membership",
+    },
+  },
+});
+
+const findRoute = createRoute({
+  method: "get",
+  path: "/find",
+  tags: ["Contacts"],
+  summary: "Find contacts by email or userId",
+  request: {
+    query: z.object({
+      email: z.string().optional(),
+      userId: z.string().optional(),
+    }),
+  },
+  responses: {
+    200: {
+      content: {
+        "application/json": {
+          schema: z.object({ contacts: z.array(contactSchema) }),
+        },
+      },
+      description: "Matching contacts (non-deleted)",
+    },
+    400: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Missing query key",
+    },
+  },
+});
+
+const deleteRoute = createRoute({
+  method: "delete",
+  path: "/",
+  tags: ["Contacts"],
+  summary: "Soft-delete a contact",
+  request: {
+    body: {
+      content: {
+        "application/json": {
+          schema: z.object({
+            email: z.string().optional(),
+            userId: z.string().optional(),
+          }),
+        },
+      },
+    },
+  },
+  responses: {
+    200: {
+      content: {
+        "application/json": {
+          schema: z.object({ deleted: z.literal(true) }),
+        },
+      },
+      description: "Contact soft-deleted",
+    },
+    400: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Missing recipient key",
+    },
+    404: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Contact not found",
+    },
+  },
+});
+
+export const contactsRouter = new OpenAPIHono<AppEnv>()
+  .openapi(upsertRoute, async (c) => {
+    const { db } = c.get("container");
+    const body = c.req.valid("json");
+
+    const guard = requireIdentity(c, body);
+    if (guard) return guard;
+
+    const { id, created, linked } = await resolveOrCreateContact({
+      db,
+      userId: body.userId,
+      email: body.email,
+      contactProperties: body.properties,
+    });
+
+    // Lists applied AFTER the resolve so the contact exists (§2.5 lists
+    // ordering). `applyListMembership` requires a resolvable email — surface the
+    // "no email" case as a 400 rather than a 500.
+    if (body.lists && Object.keys(body.lists).length > 0) {
+      try {
+        await applyListMembership({
+          db,
+          userId: body.userId,
+          email: body.email,
+          lists: body.lists,
+        });
+      } catch (err) {
+        return c.json({ error: listMembershipError(err) }, 400);
+      }
+    }
+
+    return c.json({ id, created, linked }, 200);
+  })
+  .openapi(findRoute, async (c) => {
+    const { db } = c.get("container");
+    const { email, userId } = c.req.valid("query");
+
+    const guard = requireIdentity(c, { email, userId });
+    if (guard) return guard;
+
+    const rows = await findContacts({ db, email, userId });
+
+    return c.json({ contacts: rows.map((row) => serializeContact(row)) }, 200);
+  })
+  .openapi(deleteRoute, async (c) => {
+    const { db } = c.get("container");
+    const { email, userId } = c.req.valid("json");
+
+    const guard = requireIdentity(c, { email, userId });
+    if (guard) return guard;
+
+    const deleted = await softDeleteContact({ db, email, userId });
+    if (!deleted) {
+      return c.json({ error: "Contact not found" }, 404);
+    }
+
+    return c.json({ deleted: true as const }, 200);
+  });

package/src/routes/email/preferences.ts

@@ -9,8 +9,12 @@ import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
 import { and, eq } from "drizzle-orm";
 import type { AppEnv } from "../../app.js";
 import { htmlPage } from "../../lib/html.js";
+import { getListRegistry } from "../../lists/registry-singleton.js";
 
-const EMAIL_CATEGORIES = [
+// The built-in journey/lifecycle category is always shown. Defined lists (D3)
+// are appended from the registry so the preference center and the mailer's
+// suppression check share ONE polarity source (`ListRegistry.isSubscribed`).
+const BUILTIN_CATEGORIES = [
   { id: "journey", label: "Journey & lifecycle emails" },
 ] as const;
 
@@ -91,9 +95,29 @@ export const preferencesRouter = new OpenAPIHono<AppEnv>().openapi(
       return `${env.API_PUBLIC_URL}/v1/email/unsubscribe?token=${encodeURIComponent(actionToken)}`;
     }
 
+    // Built-in journey category + every enabled defined list, deduped by id
+    // (a list MAY NOT reuse a reserved category id, but guard anyway).
+    const listRegistry = getListRegistry();
+    const seen = new Set<string>();
+    const renderableCategories: { id: string; label: string }[] = [];
+    for (const cat of BUILTIN_CATEGORIES) {
+      seen.add(cat.id);
+      renderableCategories.push({ id: cat.id, label: cat.label });
+    }
+    for (const list of listRegistry.getEnabled()) {
+      if (seen.has(list.id)) continue;
+      seen.add(list.id);
+      renderableCategories.push({ id: list.id, label: list.name });
+    }
+
     let categoryRows = "";
-    for (const cat of EMAIL_CATEGORIES) {
-      const isSubscribed = categories[cat.id] !== false && !globalUnsub;
+    for (const cat of renderableCategories) {
+      // Registry-driven polarity (§2.6): defined lists honour their
+      // `defaultOptIn`; unknown ids (e.g. the built-in `journey`) fall through
+      // to opt-in default (blocked only on explicit `false`). A global
+      // unsubscribe overrides every per-category state.
+      const isSubscribed =
+        listRegistry.isSubscribed(categories, cat.id) && !globalUnsub;
       const statusClass = isSubscribed ? "subscribed" : "unsubscribed";
       const statusText = isSubscribed ? "Subscribed" : "Unsubscribed";
       const actionLabel = isSubscribed ? "Unsubscribe" : "Resubscribe";

package/src/routes/email/unsubscribe.ts

@@ -1,5 +1,3 @@
-import type { Database } from "@hogsend/db";
-import { emailPreferences } from "@hogsend/db";
 import type { UnsubscribeTokenPayload } from "@hogsend/email";
 import {
   generatePreferenceCenterUrl,
@@ -7,9 +5,9 @@ import {
   validateUnsubscribeToken,
 } from "@hogsend/email";
 import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
-import { sql } from "drizzle-orm";
 import type { AppEnv } from "../../app.js";
 import { htmlPage } from "../../lib/html.js";
+import { upsertEmailPreference } from "../../lib/preferences.js";
 
 const unsubscribeRoute = createRoute({
   method: "get",
@@ -33,46 +31,6 @@ const unsubscribeRoute = createRoute({
   },
 });
 
-async function upsertPreference(
-  db: Database,
-  externalId: string,
-  email: string,
-  update: {
-    unsubscribedAll?: boolean;
-    categoryKey?: string;
-    categoryValue?: boolean;
-  },
-) {
-  const setClause: Record<string, unknown> = { updatedAt: new Date() };
-
-  if (update.unsubscribedAll !== undefined) {
-    setClause.unsubscribedAll = update.unsubscribedAll;
-  }
-  if (update.categoryKey !== undefined) {
-    const jsonValue = update.categoryValue ? "true" : "false";
-    setClause.categories = sql`jsonb_set(COALESCE(${emailPreferences.categories}, '{}'::jsonb), ${`{${update.categoryKey}}`}, ${jsonValue}::jsonb)`;
-  }
-
-  await db
-    .insert(emailPreferences)
-    .values({
-      userId: externalId,
-      email,
-      ...(update.unsubscribedAll !== undefined
-        ? { unsubscribedAll: update.unsubscribedAll }
-        : {}),
-      ...(update.categoryKey !== undefined
-        ? {
-            categories: { [update.categoryKey]: update.categoryValue ?? false },
-          }
-        : {}),
-    })
-    .onConflictDoUpdate({
-      target: [emailPreferences.userId, emailPreferences.email],
-      set: setClause,
-    });
-}
-
 export const unsubscribeRouter = new OpenAPIHono<AppEnv>().openapi(
   unsubscribeRoute,
   async (c) => {
@@ -110,18 +68,18 @@ export const unsubscribeRouter = new OpenAPIHono<AppEnv>().openapi(
     }
 
     if (action === "resubscribe") {
-      await upsertPreference(
+      await upsertEmailPreference({
         db,
         externalId,
         email,
-        category
+        update: category
           ? {
               categoryKey: category,
               categoryValue: true,
               unsubscribedAll: false,
             }
           : { unsubscribedAll: false },
-      );
+      });
 
       return c.html(
         htmlPage({
@@ -132,14 +90,14 @@ export const unsubscribeRouter = new OpenAPIHono<AppEnv>().openapi(
       );
     }
 
-    await upsertPreference(
+    await upsertEmailPreference({
       db,
       externalId,
       email,
-      category
+      update: category
         ? { categoryKey: category, categoryValue: false }
         : { unsubscribedAll: true },
-    );
+    });
 
     const preferenceCenterUrl = generatePreferenceCenterUrl({
       baseUrl: env.API_PUBLIC_URL,