@hogsend/engine 0.6.0 → 0.8.0

package/src/routes/emails/index.ts

@@ -0,0 +1,133 @@
+import { getTemplateNames, type TemplateName } from "@hogsend/email";
+import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
+import type { AppEnv } from "../../app.js";
+import { resolveRecipient } from "../../lib/contacts.js";
+import { errorSchema } from "../../lib/schemas.js";
+import { hasScope } from "../../middleware/api-key.js";
+import { requireIdentity } from "../_shared.js";
+
+const emailRequestSchema = z.object({
+  to: z.string().email().optional(),
+  userId: z.string().min(1).optional(),
+  template: z.string().min(1),
+  props: z.record(z.string(), z.unknown()).optional(),
+  from: z.string().optional(),
+  subject: z.string().optional(),
+  replyTo: z.union([z.string(), z.array(z.string())]).optional(),
+  category: z.string().optional(),
+  skipPreferenceCheck: z.boolean().optional(),
+  idempotencyKey: z.string().optional(),
+});
+
+const emailResponseSchema = z.object({
+  emailSendId: z.string(),
+  status: z.enum(["queued", "sent", "suppressed", "unsubscribed", "skipped"]),
+  reason: z.string().optional(),
+});
+
+const sendRoute = createRoute({
+  method: "post",
+  path: "/",
+  tags: ["Emails"],
+  summary: "Send a transactional email",
+  description:
+    "Resolves a recipient by `to` or `userId`, then sends the named template through the engine-owned tracked mailer (journeyless — link-click + open tracking still applies). `skipPreferenceCheck` requires a full-admin key.",
+  request: {
+    body: {
+      content: {
+        "application/json": { schema: emailRequestSchema },
+      },
+    },
+  },
+  responses: {
+    202: {
+      content: {
+        "application/json": { schema: emailResponseSchema },
+      },
+      description: "Email send queued / dispatched",
+    },
+    400: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Missing recipient or unknown template",
+    },
+    403: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "skipPreferenceCheck requires a full-admin key",
+    },
+    404: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "userId has no resolvable email",
+    },
+  },
+});
+
+export const emailsRouter = new OpenAPIHono<AppEnv>().openapi(
+  sendRoute,
+  async (c) => {
+    const { db, emailService, templates } = c.get("container");
+    const apiKey = c.get("apiKey");
+    const body = c.req.valid("json");
+
+    const guard = requireIdentity(
+      c,
+      { email: body.to, userId: body.userId },
+      { field: "to" },
+    );
+    if (guard) return guard;
+
+    // `skipPreferenceCheck` is a privileged bypass — gate it on full-admin
+    // (§2.5). The data-plane prefix guard already required the `ingest` scope.
+    if (body.skipPreferenceCheck) {
+      if (!apiKey || !hasScope(apiKey.scopes, "full-admin")) {
+        return c.json(
+          { error: "skipPreferenceCheck requires a full-admin key" },
+          403,
+        );
+      }
+    }
+
+    // Validate the template server-side against the wired registry (§2.5).
+    if (!getTemplateNames(templates).includes(body.template as TemplateName)) {
+      return c.json({ error: `Unknown template: ${body.template}` }, 400);
+    }
+
+    const recipient = await resolveRecipient({
+      db,
+      userId: body.userId,
+      email: body.to,
+    });
+    if (!recipient) {
+      return c.json({ error: "No resolvable email for recipient" }, 404);
+    }
+
+    // The `Idempotency-Key` header wins over the body field (mirrors /v1/events).
+    const headerKey = c.req.header("idempotency-key");
+    const idempotencyKey = headerKey ?? body.idempotencyKey;
+
+    // Journeyless send (no journeyStateId) so §5 tracking runs. The
+    // denormalized `userId` on the send row is external_id when present, else
+    // the contact id fallback (§2.5).
+    const result = await emailService.send({
+      template: body.template as TemplateName,
+      props: (body.props ?? {}) as never,
+      to: recipient.email,
+      from: body.from,
+      subject: body.subject,
+      replyTo: body.replyTo,
+      category: body.category,
+      userId: recipient.externalId ?? recipient.contactId,
+      userEmail: recipient.email,
+      skipPreferenceCheck: body.skipPreferenceCheck,
+      idempotencyKey,
+    });
+
+    return c.json(
+      {
+        emailSendId: result.emailSendId,
+        status: result.status,
+        ...(result.reason ? { reason: result.reason } : {}),
+      },
+      202,
+    );
+  },
+);

package/src/routes/events/index.ts

@@ -0,0 +1,119 @@
+import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
+import type { AppEnv } from "../../app.js";
+import { ingestEvent } from "../../lib/ingestion.js";
+import { applyListMembership } from "../../lib/preferences.js";
+import { errorSchema } from "../../lib/schemas.js";
+import { listMembershipError, requireIdentity } from "../_shared.js";
+
+const eventRequestSchema = z.object({
+  name: z.string().min(1),
+  email: z.string().email().optional(),
+  userId: z.string().min(1).optional(),
+  eventProperties: z.record(z.string(), z.unknown()).optional(),
+  contactProperties: z.record(z.string(), z.unknown()).optional(),
+  lists: z.record(z.string(), z.boolean()).optional(),
+  idempotencyKey: z.string().optional(),
+  timestamp: z.string().datetime().optional(),
+});
+
+const eventResponseSchema = z.object({
+  stored: z.boolean(),
+  exits: z.array(
+    z.object({
+      journeyId: z.string(),
+      stateId: z.string(),
+      exited: z.boolean(),
+    }),
+  ),
+  // Present only when the event was durably ingested but the (non-atomic,
+  // post-ingest) list-membership write failed. The ingest itself succeeded —
+  // surfaced as a warning on a 202, not a 400 that conflates "nothing happened"
+  // with "event happened, lists failed" (and would tempt a retry double-ingest).
+  listsError: z.string().optional(),
+});
+
+const eventRoute = createRoute({
+  method: "post",
+  path: "/",
+  tags: ["Events"],
+  summary: "Ingest an event",
+  description:
+    "Stores the event (with eventProperties), merges contactProperties onto the contact, pushes to Hatchet for journey routing, processes exit conditions, and optionally writes list membership. The `Idempotency-Key` header takes precedence over the body field.",
+  request: {
+    body: {
+      content: {
+        "application/json": { schema: eventRequestSchema },
+      },
+    },
+  },
+  responses: {
+    202: {
+      content: {
+        "application/json": { schema: eventResponseSchema },
+      },
+      description: "Event accepted and dispatched",
+    },
+    400: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Missing recipient or unmanageable list membership",
+    },
+  },
+});
+
+export const eventsRouter = new OpenAPIHono<AppEnv>().openapi(
+  eventRoute,
+  async (c) => {
+    const { db, registry, hatchet, logger } = c.get("container");
+    const body = c.req.valid("json");
+
+    const guard = requireIdentity(c, body);
+    if (guard) return guard;
+
+    // The `Idempotency-Key` header wins over the body field (§2.5).
+    const headerKey = c.req.header("idempotency-key");
+    const idempotencyKey = headerKey ?? body.idempotencyKey;
+
+    const result = await ingestEvent({
+      db,
+      registry,
+      hatchet,
+      logger,
+      event: {
+        event: body.name,
+        userId: body.userId,
+        userEmail: body.email,
+        eventProperties: body.eventProperties ?? {},
+        contactProperties: body.contactProperties,
+        idempotencyKey,
+        // §2.5: caller-supplied event time (backfill/replay). The validated ISO
+        // string is coerced to a Date inside ingestEvent.
+        occurredAt: body.timestamp,
+      },
+    });
+
+    // Lists applied AFTER ingest so the contact exists (§2.5 lists ordering).
+    // `applyListMembership` writes `email_preferences` independently of the
+    // contacts row, so it doesn't race the resolve. Requires a resolvable email.
+    //
+    // The ingest above is already durable (event stored, journeys dispatched,
+    // exits processed). A list-write failure here must NOT be reported as a 400
+    // — that would (a) hide a successful ingest behind a "nothing happened"
+    // status and (b) tempt a client to retry, re-ingesting the event. Surface it
+    // as a `listsError` warning on the 202 instead.
+    let listsError: string | undefined;
+    if (body.lists && Object.keys(body.lists).length > 0) {
+      try {
+        await applyListMembership({
+          db,
+          userId: body.userId,
+          email: body.email,
+          lists: body.lists,
+        });
+      } catch (err) {
+        listsError = listMembershipError(err);
+      }
+    }
+
+    return c.json({ ...result, ...(listsError ? { listsError } : {}) }, 202);
+  },
+);

package/src/routes/index.ts

@@ -1,10 +1,16 @@
 import { OpenAPIHono } from "@hono/zod-openapi";
 import type { AppEnv } from "../app.js";
+import { requireApiKey, requireScope } from "../middleware/api-key.js";
+import { createRateLimit } from "../middleware/rate-limit.js";
 import type { DefinedWebhookSource } from "../webhook-sources/define-webhook-source.js";
 import { adminRouter } from "./admin/index.js";
+import { campaignsRouter } from "./campaigns/index.js";
+import { contactsRouter } from "./contacts/index.js";
 import { emailRouter } from "./email/index.js";
+import { emailsRouter } from "./emails/index.js";
+import { eventsRouter } from "./events/index.js";
 import { healthRouter } from "./health.js";
-import { ingestRouter } from "./ingest.js";
+import { listsRouter } from "./lists/index.js";
 import { trackingRouter } from "./tracking/index.js";
 import { registerWebhookRoutes } from "./webhooks/index.js";
 
@@ -12,18 +18,62 @@ export interface RegisterRoutesOptions {
   webhookSources: DefinedWebhookSource[];
 }
 
+// Conservative per-key email budget. `/v1/emails` MUST use a distinct prefix so
+// transactional sends don't share the sliding-window budget with contact
+// upserts / event ingest (open risk #15). 30/min/key is well under the
+// contact-upsert default (100/min) — an integration loop sending more than that
+// is almost certainly a runaway.
+const EMAIL_RATE_LIMIT_MAX = 30;
+
 export function registerRoutes(
   app: OpenAPIHono<AppEnv>,
   opts: RegisterRoutesOptions,
 ) {
   const v1 = new OpenAPIHono<AppEnv>();
 
+  // Open routes: health + tracking pixels/redirects are intentionally
+  // unauthenticated (links land in recipient inboxes), and the admin router
+  // owns its own `requireAdmin` guard.
   v1.route("/health", healthRouter);
-  v1.route("/ingest", ingestRouter);
   v1.route("/email", emailRouter);
   v1.route("/admin", adminRouter);
   v1.route("/t", trackingRouter);
 
+  // The guarded data plane (D5 / decision #16): `requireApiKey` →
+  // `requireScope("ingest")` on `/contacts`, `/events`, `/emails`, `/lists`.
+  // Each prefix is guarded EXPLICITLY rather than via a root-mounted catch-all
+  // sub-app — a sub-app at "/" with `use("*")` also intercepts sibling paths
+  // (e.g. `/v1/webhooks`) and 401s them before they reach their own handlers.
+  // Both the bare path and its `/*` subtree are covered (Hono treats them as
+  // distinct match patterns). `/emails` layers the per-key email rate-limit on
+  // top, in strict order auth → scope → rateLimit.
+  const emailRateLimit = createRateLimit({
+    prefix: "ratelimit:emails",
+    max: EMAIL_RATE_LIMIT_MAX,
+  });
+  for (const base of [
+    "/contacts",
+    "/events",
+    "/emails",
+    "/lists",
+    "/campaigns",
+  ]) {
+    v1.use(base, requireApiKey, requireScope("ingest"));
+    v1.use(`${base}/*`, requireApiKey, requireScope("ingest"));
+  }
+  // Register the email rate-limit ONCE. The wildcard pattern `/emails/*` matches
+  // BOTH the bare `POST /v1/emails` and any subtree, so a single registration
+  // covers the whole emails surface. Registering both bare AND wildcard with the
+  // SAME stateful instance double-counts every send (two sliding-window entries
+  // per request), halving the effective per-key budget (decision #16 / risk 15).
+  v1.use("/emails/*", emailRateLimit);
+
+  v1.route("/contacts", contactsRouter);
+  v1.route("/events", eventsRouter);
+  v1.route("/emails", emailsRouter);
+  v1.route("/lists", listsRouter);
+  v1.route("/campaigns", campaignsRouter);
+
   app.route("/v1", v1);
 
   // Webhooks (built-in Resend + injected content sources) are registered on the

package/src/routes/lists/index.ts

@@ -0,0 +1,258 @@
+import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
+import type { Database } from "@hogsend/db";
+import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
+import type { AppEnv } from "../../app.js";
+import {
+  resolveContact,
+  resolveOrCreateContact,
+  serializeContact,
+} from "../../lib/contacts.js";
+import type { Logger } from "../../lib/logger.js";
+import { emitOutbound } from "../../lib/outbound.js";
+import { applyListMembership } from "../../lib/preferences.js";
+import { errorSchema } from "../../lib/schemas.js";
+import { getListRegistry } from "../../lists/registry-singleton.js";
+import { listMembershipError } from "../_shared.js";
+
+const listSummarySchema = z.object({
+  id: z.string(),
+  name: z.string(),
+  description: z.string().optional(),
+  defaultOptIn: z.boolean(),
+});
+
+const bodySchema = z.object({
+  email: z.string().optional(),
+  userId: z.string().optional(),
+});
+
+const listRoute = createRoute({
+  method: "get",
+  path: "/",
+  tags: ["Lists"],
+  summary: "List defined email lists",
+  description:
+    "Returns the enabled, code-defined email lists (D3). Membership lives in `email_preferences.categories`; this only enumerates the catalog.",
+  responses: {
+    200: {
+      content: {
+        "application/json": {
+          schema: z.object({ lists: z.array(listSummarySchema) }),
+        },
+      },
+      description: "Enabled lists",
+    },
+  },
+});
+
+const subscribeRoute = createRoute({
+  method: "post",
+  path: "/{id}/subscribe",
+  tags: ["Lists"],
+  summary: "Subscribe a contact to a list",
+  request: {
+    params: z.object({ id: z.string() }),
+    body: {
+      content: { "application/json": { schema: bodySchema } },
+    },
+  },
+  responses: {
+    200: {
+      content: {
+        "application/json": {
+          schema: z.object({
+            list: z.string(),
+            subscribed: z.literal(true),
+          }),
+        },
+      },
+      description: "Contact subscribed",
+    },
+    400: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Missing recipient or no resolvable email",
+    },
+    404: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Unknown list id",
+    },
+  },
+});
+
+const unsubscribeRoute = createRoute({
+  method: "post",
+  path: "/{id}/unsubscribe",
+  tags: ["Lists"],
+  summary: "Unsubscribe a contact from a list",
+  request: {
+    params: z.object({ id: z.string() }),
+    body: {
+      content: { "application/json": { schema: bodySchema } },
+    },
+  },
+  responses: {
+    200: {
+      content: {
+        "application/json": {
+          schema: z.object({
+            list: z.string(),
+            subscribed: z.literal(false),
+          }),
+        },
+      },
+      description: "Contact unsubscribed",
+    },
+    400: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Missing recipient or no resolvable email",
+    },
+    404: {
+      content: { "application/json": { schema: errorSchema } },
+      description: "Unknown list id",
+    },
+  },
+});
+
+/**
+ * The shared side-effect of subscribe + unsubscribe (identical apart from the
+ * boolean polarity): validate the list id, guard identity, then resolve/create
+ * the contact FIRST (mirroring /v1/contacts + /v1/events) so a real row (and
+ * uuid id) exists — without it `resolveRecipient` returns the raw email as the
+ * contactId fallback and `email_preferences.user_id` is written as the raw email
+ * instead of the `external_id ?? contact.id` uuid, breaking risk-10 key
+ * consistency.
+ *
+ * Returns a discriminated result the caller maps to a status: `unknown_list` /
+ * `missing_identity` → 404 / 400; `failed` → 400 with the error message; `ok` →
+ * the caller's literally-typed success body. The `valid()` reads stay in each
+ * typed `.openapi()` handler; only the polarity-invariant work lives here.
+ */
+async function applyListSubscription(opts: {
+  db: Database;
+  hatchet: HatchetClient;
+  logger: Logger;
+  id: string;
+  email?: string;
+  userId?: string;
+  subscribed: boolean;
+}): Promise<
+  | { kind: "unknown_list" }
+  | { kind: "missing_identity" }
+  | { kind: "failed"; message: string }
+  | { kind: "ok" }
+> {
+  const { db, hatchet, logger, id, email, userId, subscribed } = opts;
+
+  if (!getListRegistry().has(id)) {
+    return { kind: "unknown_list" };
+  }
+
+  if (!email && !userId) {
+    return { kind: "missing_identity" };
+  }
+
+  try {
+    const { id: contactId, created } = await resolveOrCreateContact({
+      db,
+      userId,
+      email,
+    });
+
+    // INTENT-LAYER outbound emit (decision #3): the lists route emits
+    // `contact.created` ONLY on first creation (a list flip is not a contact
+    // property delta, so no `contact.updated`). Fire-and-forget after a read-back.
+    if (created) {
+      void resolveContact({ db, id: contactId })
+        .then((row) => {
+          if (!row) return;
+          return emitOutbound({
+            db,
+            hatchet,
+            logger,
+            event: "contact.created",
+            payload: serializeContact(row),
+          });
+        })
+        .catch(logger.warn);
+    }
+
+    await applyListMembership({
+      db,
+      userId,
+      email,
+      lists: { [id]: subscribed },
+    });
+  } catch (err) {
+    return { kind: "failed", message: listMembershipError(err) };
+  }
+
+  return { kind: "ok" };
+}
+
+// The lists router does NOT re-apply auth internally — the data-plane prefix
+// guards in `routes/index.ts` (decision #16) apply `requireApiKey` +
+// `requireScope("ingest")` to `/v1/lists` (bare + `/*`) before requests reach
+// this router. Mounting auth here too would double the middleware.
+export const listsRouter = new OpenAPIHono<AppEnv>()
+  .openapi(listRoute, (c) => {
+    const lists = getListRegistry()
+      .getEnabled()
+      .map((l) => ({
+        id: l.id,
+        name: l.name,
+        ...(l.description !== undefined ? { description: l.description } : {}),
+        defaultOptIn: l.defaultOptIn,
+      }));
+
+    return c.json({ lists }, 200);
+  })
+  .openapi(subscribeRoute, async (c) => {
+    const { db, hatchet, logger } = c.get("container");
+    const { id } = c.req.valid("param");
+    const { email, userId } = c.req.valid("json");
+
+    const result = await applyListSubscription({
+      db,
+      hatchet,
+      logger,
+      id,
+      email,
+      userId,
+      subscribed: true,
+    });
+    if (result.kind === "unknown_list") {
+      return c.json({ error: `Unknown list: ${id}` }, 404);
+    }
+    if (result.kind === "missing_identity") {
+      return c.json({ error: "email or userId is required" }, 400);
+    }
+    if (result.kind === "failed") {
+      return c.json({ error: result.message }, 400);
+    }
+    return c.json({ list: id, subscribed: true as const }, 200);
+  })
+  .openapi(unsubscribeRoute, async (c) => {
+    const { db, hatchet, logger } = c.get("container");
+    const { id } = c.req.valid("param");
+    const { email, userId } = c.req.valid("json");
+
+    const result = await applyListSubscription({
+      db,
+      hatchet,
+      logger,
+      id,
+      email,
+      userId,
+      subscribed: false,
+    });
+    if (result.kind === "unknown_list") {
+      return c.json({ error: `Unknown list: ${id}` }, 404);
+    }
+    if (result.kind === "missing_identity") {
+      return c.json({ error: "email or userId is required" }, 400);
+    }
+    if (result.kind === "failed") {
+      return c.json({ error: result.message }, 400);
+    }
+    return c.json({ list: id, subscribed: false as const }, 200);
+  });

package/src/routes/tracking/click.ts

@@ -2,8 +2,12 @@ import { emailSends, linkClicks, trackedLinks } from "@hogsend/db";
 import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
 import { and, eq, isNull, sql } from "drizzle-orm";
 import type { AppEnv } from "../../app.js";
+import { emitOutbound } from "../../lib/outbound.js";
 import { EMAIL_LINK_CLICKED } from "../../lib/tracking-event-names.js";
-import { pushTrackingEvent } from "../../lib/tracking-events.js";
+import {
+  pushTrackingEvent,
+  resolveEmailSendContext,
+} from "../../lib/tracking-events.js";
 
 const clickRoute = createRoute({
   method: "get",
@@ -48,7 +52,10 @@ export const clickRouter = new OpenAPIHono<AppEnv>().openapi(
       null;
     const userAgent = c.req.header("user-agent") ?? null;
 
-    await Promise.all([
+    // The `clickedAt` first-touch UPDATE is split OUT of the Promise.all so it can
+    // `.returning({ id })` — the `WHERE clickedAt IS NULL` makes a row come back
+    // ONLY on the first click, which gates the outbound `email.clicked` emit.
+    const [, , clicked] = await Promise.all([
       db.insert(linkClicks).values({
         trackedLinkId: link.id,
         ipAddress: ip,
@@ -72,7 +79,8 @@ export const clickRouter = new OpenAPIHono<AppEnv>().openapi(
             eq(emailSends.id, link.emailSendId),
             isNull(emailSends.clickedAt),
           ),
-        ),
+        )
+        .returning({ id: emailSends.id }),
     ]);
 
     const {
@@ -82,21 +90,54 @@ export const clickRouter = new OpenAPIHono<AppEnv>().openapi(
       analytics: posthog,
     } = c.get("container");
 
-    pushTrackingEvent({
-      db,
-      hatchet,
-      registry,
-      logger,
-      posthog,
-      event: EMAIL_LINK_CLICKED,
-      emailSendId: link.emailSendId,
-      properties: { linkUrl: link.originalUrl, linkId: link.id },
-    }).catch((err) => {
-      logger.warn("Failed to push click tracking event", {
-        linkId: link.id,
-        error: err instanceof Error ? err.message : String(err),
-      });
-    });
+    // Resolve the send context ONCE (off the response path) and feed both the
+    // re-ingest (every click) and the first-touch outbound emit (first click
+    // only) — avoiding a duplicate `resolveEmailSendContext` read on the click
+    // hot path. `dedupeKey` = `email.clicked:<emailSendId>` is defence-in-depth
+    // alongside the first-touch gate (`clicked.length > 0`); first-party is the
+    // SINGLE emitter for `email.clicked` (the provider-webhook echo is suppressed).
+    const emailSendId = link.emailSendId;
+    const isFirstClick = clicked.length > 0;
+    void resolveEmailSendContext(db, emailSendId)
+      .then(async (ctx) => {
+        await pushTrackingEvent({
+          db,
+          hatchet,
+          registry,
+          logger,
+          posthog,
+          event: EMAIL_LINK_CLICKED,
+          emailSendId,
+          properties: { linkUrl: link.originalUrl, linkId: link.id },
+          resolvedContext: ctx,
+        }).catch((err) => {
+          logger.warn("Failed to push click tracking event", {
+            linkId: link.id,
+            error: err instanceof Error ? err.message : String(err),
+          });
+        });
+
+        if (isFirstClick) {
+          await emitOutbound({
+            db,
+            hatchet,
+            logger,
+            event: "email.clicked",
+            dedupeKey: `email.clicked:${emailSendId}`,
+            payload: {
+              emailSendId,
+              resendId: ctx?.resendId ?? null,
+              templateKey: ctx?.templateKey ?? null,
+              userId: ctx?.userId ?? null,
+              to: ctx?.to ?? ctx?.userEmail ?? "",
+              at: new Date().toISOString(),
+              linkUrl: link.originalUrl,
+              linkId: link.id,
+            },
+          });
+        }
+      })
+      .catch(logger.warn);
 
     return c.redirect(link.originalUrl, 302);
   },