@hogsend/engine 0.5.0 → 0.7.0

package/src/routes/email/unsubscribe.ts

@@ -1,5 +1,3 @@
-import type { Database } from "@hogsend/db";
-import { emailPreferences } from "@hogsend/db";
 import type { UnsubscribeTokenPayload } from "@hogsend/email";
 import {
   generatePreferenceCenterUrl,
@@ -7,9 +5,9 @@ import {
   validateUnsubscribeToken,
 } from "@hogsend/email";
 import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
-import { sql } from "drizzle-orm";
 import type { AppEnv } from "../../app.js";
 import { htmlPage } from "../../lib/html.js";
+import { upsertEmailPreference } from "../../lib/preferences.js";
 
 const unsubscribeRoute = createRoute({
   method: "get",
@@ -33,46 +31,6 @@ const unsubscribeRoute = createRoute({
   },
 });
 
-async function upsertPreference(
-  db: Database,
-  externalId: string,
-  email: string,
-  update: {
-    unsubscribedAll?: boolean;
-    categoryKey?: string;
-    categoryValue?: boolean;
-  },
-) {
-  const setClause: Record<string, unknown> = { updatedAt: new Date() };
-
-  if (update.unsubscribedAll !== undefined) {
-    setClause.unsubscribedAll = update.unsubscribedAll;
-  }
-  if (update.categoryKey !== undefined) {
-    const jsonValue = update.categoryValue ? "true" : "false";
-    setClause.categories = sql`jsonb_set(COALESCE(${emailPreferences.categories}, '{}'::jsonb), ${`{${update.categoryKey}}`}, ${jsonValue}::jsonb)`;
-  }
-
-  await db
-    .insert(emailPreferences)
-    .values({
-      userId: externalId,
-      email,
-      ...(update.unsubscribedAll !== undefined
-        ? { unsubscribedAll: update.unsubscribedAll }
-        : {}),
-      ...(update.categoryKey !== undefined
-        ? {
-            categories: { [update.categoryKey]: update.categoryValue ?? false },
-          }
-        : {}),
-    })
-    .onConflictDoUpdate({
-      target: [emailPreferences.userId, emailPreferences.email],
-      set: setClause,
-    });
-}
-
 export const unsubscribeRouter = new OpenAPIHono<AppEnv>().openapi(
   unsubscribeRoute,
   async (c) => {
@@ -110,18 +68,18 @@ export const unsubscribeRouter = new OpenAPIHono<AppEnv>().openapi(
     }
 
     if (action === "resubscribe") {
-      await upsertPreference(
+      await upsertEmailPreference({
         db,
         externalId,
         email,
-        category
+        update: category
           ? {
               categoryKey: category,
               categoryValue: true,
               unsubscribedAll: false,
             }
           : { unsubscribedAll: false },
-      );
+      });
 
       return c.html(
         htmlPage({
@@ -132,14 +90,14 @@ export const unsubscribeRouter = new OpenAPIHono<AppEnv>().openapi(
       );
     }
 
-    await upsertPreference(
+    await upsertEmailPreference({
       db,
       externalId,
       email,
-      category
+      update: category
         ? { categoryKey: category, categoryValue: false }
         : { unsubscribedAll: true },
-    );
+    });
 
     const preferenceCenterUrl = generatePreferenceCenterUrl({
       baseUrl: env.API_PUBLIC_URL,

package/src/routes/emails/index.ts

@@ -0,0 +1,133 @@
+import { getTemplateNames, type TemplateName } from "@hogsend/email";
+import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
+import type { AppEnv } from "../../app.js";
+import { resolveRecipient } from "../../lib/contacts.js";
+import { errorSchema } from "../../lib/schemas.js";
+import { hasScope } from "../../middleware/api-key.js";
+import { requireIdentity } from "../_shared.js";
+
+const emailRequestSchema = z.object({
+  to: z.string().email().optional(),
+  userId: z.string().min(1).optional(),
+  template: z.string().min(1),
+  props: z.record(z.string(), z.unknown()).optional(),
+  from: z.string().optional(),
+  subject: z.string().optional(),
+  replyTo: z.union([z.string(), z.array(z.string())]).optional(),
+  category: z.string().optional(),
+  skipPreferenceCheck: z.boolean().optional(),
+  idempotencyKey: z.string().optional(),
+});
+
+const emailResponseSchema = z.object({
+  emailSendId: z.string(),
+  status: z.enum(["queued", "sent", "suppressed", "unsubscribed", "skipped"]),
+  reason: z.string().optional(),
+});
+
+const sendRoute = createRoute({
+  method: "post",
+  path: "/",
+  tags: ["Emails"],
+  summary: "Send a transactional email",
+  description:
+    "Resolves a recipient by `to` or `userId`, then sends the named template through the engine-owned tracked mailer (journeyless — link-click + open tracking still applies). `skipPreferenceCheck` requires a full-admin key.",
+  request: {
+    body: {
+      content: {
+        "application/json": { schema: emailRequestSchema },
+      },
+    },
+  },
+  responses: {
+    202: {
+      content: {
+        "application/json": { schema: emailResponseSchema },
+      },
+      description: "Email send queued / dispatched",
+    },
+    400: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Missing recipient or unknown template",
+    },
+    403: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "skipPreferenceCheck requires a full-admin key",
+    },
+    404: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "userId has no resolvable email",
+    },
+  },
+});
+
+export const emailsRouter = new OpenAPIHono<AppEnv>().openapi(
+  sendRoute,
+  async (c) => {
+    const { db, emailService, templates } = c.get("container");
+    const apiKey = c.get("apiKey");
+    const body = c.req.valid("json");
+
+    const guard = requireIdentity(
+      c,
+      { email: body.to, userId: body.userId },
+      { field: "to" },
+    );
+    if (guard) return guard;
+
+    // `skipPreferenceCheck` is a privileged bypass — gate it on full-admin
+    // (§2.5). The data-plane prefix guard already required the `ingest` scope.
+    if (body.skipPreferenceCheck) {
+      if (!apiKey || !hasScope(apiKey.scopes, "full-admin")) {
+        return c.json(
+          { error: "skipPreferenceCheck requires a full-admin key" },
+          403,
+        );
+      }
+    }
+
+    // Validate the template server-side against the wired registry (§2.5).
+    if (!getTemplateNames(templates).includes(body.template as TemplateName)) {
+      return c.json({ error: `Unknown template: ${body.template}` }, 400);
+    }
+
+    const recipient = await resolveRecipient({
+      db,
+      userId: body.userId,
+      email: body.to,
+    });
+    if (!recipient) {
+      return c.json({ error: "No resolvable email for recipient" }, 404);
+    }
+
+    // The `Idempotency-Key` header wins over the body field (mirrors /v1/events).
+    const headerKey = c.req.header("idempotency-key");
+    const idempotencyKey = headerKey ?? body.idempotencyKey;
+
+    // Journeyless send (no journeyStateId) so §5 tracking runs. The
+    // denormalized `userId` on the send row is external_id when present, else
+    // the contact id fallback (§2.5).
+    const result = await emailService.send({
+      template: body.template as TemplateName,
+      props: (body.props ?? {}) as never,
+      to: recipient.email,
+      from: body.from,
+      subject: body.subject,
+      replyTo: body.replyTo,
+      category: body.category,
+      userId: recipient.externalId ?? recipient.contactId,
+      userEmail: recipient.email,
+      skipPreferenceCheck: body.skipPreferenceCheck,
+      idempotencyKey,
+    });
+
+    return c.json(
+      {
+        emailSendId: result.emailSendId,
+        status: result.status,
+        ...(result.reason ? { reason: result.reason } : {}),
+      },
+      202,
+    );
+  },
+);

package/src/routes/events/index.ts

@@ -0,0 +1,119 @@
+import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
+import type { AppEnv } from "../../app.js";
+import { ingestEvent } from "../../lib/ingestion.js";
+import { applyListMembership } from "../../lib/preferences.js";
+import { errorSchema } from "../../lib/schemas.js";
+import { listMembershipError, requireIdentity } from "../_shared.js";
+
+const eventRequestSchema = z.object({
+  name: z.string().min(1),
+  email: z.string().email().optional(),
+  userId: z.string().min(1).optional(),
+  eventProperties: z.record(z.string(), z.unknown()).optional(),
+  contactProperties: z.record(z.string(), z.unknown()).optional(),
+  lists: z.record(z.string(), z.boolean()).optional(),
+  idempotencyKey: z.string().optional(),
+  timestamp: z.string().datetime().optional(),
+});
+
+const eventResponseSchema = z.object({
+  stored: z.boolean(),
+  exits: z.array(
+    z.object({
+      journeyId: z.string(),
+      stateId: z.string(),
+      exited: z.boolean(),
+    }),
+  ),
+  // Present only when the event was durably ingested but the (non-atomic,
+  // post-ingest) list-membership write failed. The ingest itself succeeded —
+  // surfaced as a warning on a 202, not a 400 that conflates "nothing happened"
+  // with "event happened, lists failed" (and would tempt a retry double-ingest).
+  listsError: z.string().optional(),
+});
+
+const eventRoute = createRoute({
+  method: "post",
+  path: "/",
+  tags: ["Events"],
+  summary: "Ingest an event",
+  description:
+    "Stores the event (with eventProperties), merges contactProperties onto the contact, pushes to Hatchet for journey routing, processes exit conditions, and optionally writes list membership. The `Idempotency-Key` header takes precedence over the body field.",
+  request: {
+    body: {
+      content: {
+        "application/json": { schema: eventRequestSchema },
+      },
+    },
+  },
+  responses: {
+    202: {
+      content: {
+        "application/json": { schema: eventResponseSchema },
+      },
+      description: "Event accepted and dispatched",
+    },
+    400: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Missing recipient or unmanageable list membership",
+    },
+  },
+});
+
+export const eventsRouter = new OpenAPIHono<AppEnv>().openapi(
+  eventRoute,
+  async (c) => {
+    const { db, registry, hatchet, logger } = c.get("container");
+    const body = c.req.valid("json");
+
+    const guard = requireIdentity(c, body);
+    if (guard) return guard;
+
+    // The `Idempotency-Key` header wins over the body field (§2.5).
+    const headerKey = c.req.header("idempotency-key");
+    const idempotencyKey = headerKey ?? body.idempotencyKey;
+
+    const result = await ingestEvent({
+      db,
+      registry,
+      hatchet,
+      logger,
+      event: {
+        event: body.name,
+        userId: body.userId,
+        userEmail: body.email,
+        eventProperties: body.eventProperties ?? {},
+        contactProperties: body.contactProperties,
+        idempotencyKey,
+        // §2.5: caller-supplied event time (backfill/replay). The validated ISO
+        // string is coerced to a Date inside ingestEvent.
+        occurredAt: body.timestamp,
+      },
+    });
+
+    // Lists applied AFTER ingest so the contact exists (§2.5 lists ordering).
+    // `applyListMembership` writes `email_preferences` independently of the
+    // contacts row, so it doesn't race the resolve. Requires a resolvable email.
+    //
+    // The ingest above is already durable (event stored, journeys dispatched,
+    // exits processed). A list-write failure here must NOT be reported as a 400
+    // — that would (a) hide a successful ingest behind a "nothing happened"
+    // status and (b) tempt a client to retry, re-ingesting the event. Surface it
+    // as a `listsError` warning on the 202 instead.
+    let listsError: string | undefined;
+    if (body.lists && Object.keys(body.lists).length > 0) {
+      try {
+        await applyListMembership({
+          db,
+          userId: body.userId,
+          email: body.email,
+          lists: body.lists,
+        });
+      } catch (err) {
+        listsError = listMembershipError(err);
+      }
+    }
+
+    return c.json({ ...result, ...(listsError ? { listsError } : {}) }, 202);
+  },
+);

package/src/routes/index.ts

@@ -1,10 +1,16 @@
 import { OpenAPIHono } from "@hono/zod-openapi";
 import type { AppEnv } from "../app.js";
+import { requireApiKey, requireScope } from "../middleware/api-key.js";
+import { createRateLimit } from "../middleware/rate-limit.js";
 import type { DefinedWebhookSource } from "../webhook-sources/define-webhook-source.js";
 import { adminRouter } from "./admin/index.js";
+import { campaignsRouter } from "./campaigns/index.js";
+import { contactsRouter } from "./contacts/index.js";
 import { emailRouter } from "./email/index.js";
+import { emailsRouter } from "./emails/index.js";
+import { eventsRouter } from "./events/index.js";
 import { healthRouter } from "./health.js";
-import { ingestRouter } from "./ingest.js";
+import { listsRouter } from "./lists/index.js";
 import { trackingRouter } from "./tracking/index.js";
 import { registerWebhookRoutes } from "./webhooks/index.js";
 
@@ -12,18 +18,62 @@ export interface RegisterRoutesOptions {
   webhookSources: DefinedWebhookSource[];
 }
 
+// Conservative per-key email budget. `/v1/emails` MUST use a distinct prefix so
+// transactional sends don't share the sliding-window budget with contact
+// upserts / event ingest (open risk #15). 30/min/key is well under the
+// contact-upsert default (100/min) — an integration loop sending more than that
+// is almost certainly a runaway.
+const EMAIL_RATE_LIMIT_MAX = 30;
+
 export function registerRoutes(
   app: OpenAPIHono<AppEnv>,
   opts: RegisterRoutesOptions,
 ) {
   const v1 = new OpenAPIHono<AppEnv>();
 
+  // Open routes: health + tracking pixels/redirects are intentionally
+  // unauthenticated (links land in recipient inboxes), and the admin router
+  // owns its own `requireAdmin` guard.
   v1.route("/health", healthRouter);
-  v1.route("/ingest", ingestRouter);
   v1.route("/email", emailRouter);
   v1.route("/admin", adminRouter);
   v1.route("/t", trackingRouter);
 
+  // The guarded data plane (D5 / decision #16): `requireApiKey` →
+  // `requireScope("ingest")` on `/contacts`, `/events`, `/emails`, `/lists`.
+  // Each prefix is guarded EXPLICITLY rather than via a root-mounted catch-all
+  // sub-app — a sub-app at "/" with `use("*")` also intercepts sibling paths
+  // (e.g. `/v1/webhooks`) and 401s them before they reach their own handlers.
+  // Both the bare path and its `/*` subtree are covered (Hono treats them as
+  // distinct match patterns). `/emails` layers the per-key email rate-limit on
+  // top, in strict order auth → scope → rateLimit.
+  const emailRateLimit = createRateLimit({
+    prefix: "ratelimit:emails",
+    max: EMAIL_RATE_LIMIT_MAX,
+  });
+  for (const base of [
+    "/contacts",
+    "/events",
+    "/emails",
+    "/lists",
+    "/campaigns",
+  ]) {
+    v1.use(base, requireApiKey, requireScope("ingest"));
+    v1.use(`${base}/*`, requireApiKey, requireScope("ingest"));
+  }
+  // Register the email rate-limit ONCE. The wildcard pattern `/emails/*` matches
+  // BOTH the bare `POST /v1/emails` and any subtree, so a single registration
+  // covers the whole emails surface. Registering both bare AND wildcard with the
+  // SAME stateful instance double-counts every send (two sliding-window entries
+  // per request), halving the effective per-key budget (decision #16 / risk 15).
+  v1.use("/emails/*", emailRateLimit);
+
+  v1.route("/contacts", contactsRouter);
+  v1.route("/events", eventsRouter);
+  v1.route("/emails", emailsRouter);
+  v1.route("/lists", listsRouter);
+  v1.route("/campaigns", campaignsRouter);
+
   app.route("/v1", v1);
 
   // Webhooks (built-in Resend + injected content sources) are registered on the

package/src/routes/lists/index.ts

@@ -0,0 +1,222 @@
+import type { Database } from "@hogsend/db";
+import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
+import type { AppEnv } from "../../app.js";
+import { resolveOrCreateContact } from "../../lib/contacts.js";
+import { applyListMembership } from "../../lib/preferences.js";
+import { errorSchema } from "../../lib/schemas.js";
+import { getListRegistry } from "../../lists/registry-singleton.js";
+import { listMembershipError } from "../_shared.js";
+
+const listSummarySchema = z.object({
+  id: z.string(),
+  name: z.string(),
+  description: z.string().optional(),
+  defaultOptIn: z.boolean(),
+});
+
+const bodySchema = z.object({
+  email: z.string().optional(),
+  userId: z.string().optional(),
+});
+
+const listRoute = createRoute({
+  method: "get",
+  path: "/",
+  tags: ["Lists"],
+  summary: "List defined email lists",
+  description:
+    "Returns the enabled, code-defined email lists (D3). Membership lives in `email_preferences.categories`; this only enumerates the catalog.",
+  responses: {
+    200: {
+      content: {
+        "application/json": {
+          schema: z.object({ lists: z.array(listSummarySchema) }),
+        },
+      },
+      description: "Enabled lists",
+    },
+  },
+});
+
+const subscribeRoute = createRoute({
+  method: "post",
+  path: "/{id}/subscribe",
+  tags: ["Lists"],
+  summary: "Subscribe a contact to a list",
+  request: {
+    params: z.object({ id: z.string() }),
+    body: {
+      content: { "application/json": { schema: bodySchema } },
+    },
+  },
+  responses: {
+    200: {
+      content: {
+        "application/json": {
+          schema: z.object({
+            list: z.string(),
+            subscribed: z.literal(true),
+          }),
+        },
+      },
+      description: "Contact subscribed",
+    },
+    400: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Missing recipient or no resolvable email",
+    },
+    404: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Unknown list id",
+    },
+  },
+});
+
+const unsubscribeRoute = createRoute({
+  method: "post",
+  path: "/{id}/unsubscribe",
+  tags: ["Lists"],
+  summary: "Unsubscribe a contact from a list",
+  request: {
+    params: z.object({ id: z.string() }),
+    body: {
+      content: { "application/json": { schema: bodySchema } },
+    },
+  },
+  responses: {
+    200: {
+      content: {
+        "application/json": {
+          schema: z.object({
+            list: z.string(),
+            subscribed: z.literal(false),
+          }),
+        },
+      },
+      description: "Contact unsubscribed",
+    },
+    400: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Missing recipient or no resolvable email",
+    },
+    404: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Unknown list id",
+    },
+  },
+});
+
+/**
+ * The shared side-effect of subscribe + unsubscribe (identical apart from the
+ * boolean polarity): validate the list id, guard identity, then resolve/create
+ * the contact FIRST (mirroring /v1/contacts + /v1/events) so a real row (and
+ * uuid id) exists — without it `resolveRecipient` returns the raw email as the
+ * contactId fallback and `email_preferences.user_id` is written as the raw email
+ * instead of the `external_id ?? contact.id` uuid, breaking risk-10 key
+ * consistency.
+ *
+ * Returns a discriminated result the caller maps to a status: `unknown_list` /
+ * `missing_identity` → 404 / 400; `failed` → 400 with the error message; `ok` →
+ * the caller's literally-typed success body. The `valid()` reads stay in each
+ * typed `.openapi()` handler; only the polarity-invariant work lives here.
+ */
+async function applyListSubscription(opts: {
+  db: Database;
+  id: string;
+  email?: string;
+  userId?: string;
+  subscribed: boolean;
+}): Promise<
+  | { kind: "unknown_list" }
+  | { kind: "missing_identity" }
+  | { kind: "failed"; message: string }
+  | { kind: "ok" }
+> {
+  const { db, id, email, userId, subscribed } = opts;
+
+  if (!getListRegistry().has(id)) {
+    return { kind: "unknown_list" };
+  }
+
+  if (!email && !userId) {
+    return { kind: "missing_identity" };
+  }
+
+  try {
+    await resolveOrCreateContact({ db, userId, email });
+    await applyListMembership({
+      db,
+      userId,
+      email,
+      lists: { [id]: subscribed },
+    });
+  } catch (err) {
+    return { kind: "failed", message: listMembershipError(err) };
+  }
+
+  return { kind: "ok" };
+}
+
+// The lists router does NOT re-apply auth internally — the data-plane prefix
+// guards in `routes/index.ts` (decision #16) apply `requireApiKey` +
+// `requireScope("ingest")` to `/v1/lists` (bare + `/*`) before requests reach
+// this router. Mounting auth here too would double the middleware.
+export const listsRouter = new OpenAPIHono<AppEnv>()
+  .openapi(listRoute, (c) => {
+    const lists = getListRegistry()
+      .getEnabled()
+      .map((l) => ({
+        id: l.id,
+        name: l.name,
+        ...(l.description !== undefined ? { description: l.description } : {}),
+        defaultOptIn: l.defaultOptIn,
+      }));
+
+    return c.json({ lists }, 200);
+  })
+  .openapi(subscribeRoute, async (c) => {
+    const { db } = c.get("container");
+    const { id } = c.req.valid("param");
+    const { email, userId } = c.req.valid("json");
+
+    const result = await applyListSubscription({
+      db,
+      id,
+      email,
+      userId,
+      subscribed: true,
+    });
+    if (result.kind === "unknown_list") {
+      return c.json({ error: `Unknown list: ${id}` }, 404);
+    }
+    if (result.kind === "missing_identity") {
+      return c.json({ error: "email or userId is required" }, 400);
+    }
+    if (result.kind === "failed") {
+      return c.json({ error: result.message }, 400);
+    }
+    return c.json({ list: id, subscribed: true as const }, 200);
+  })
+  .openapi(unsubscribeRoute, async (c) => {
+    const { db } = c.get("container");
+    const { id } = c.req.valid("param");
+    const { email, userId } = c.req.valid("json");
+
+    const result = await applyListSubscription({
+      db,
+      id,
+      email,
+      userId,
+      subscribed: false,
+    });
+    if (result.kind === "unknown_list") {
+      return c.json({ error: `Unknown list: ${id}` }, 404);
+    }
+    if (result.kind === "missing_identity") {
+      return c.json({ error: "email or userId is required" }, 400);
+    }
+    if (result.kind === "failed") {
+      return c.json({ error: result.message }, 400);
+    }
+    return c.json({ list: id, subscribed: false as const }, 200);
+  });