@hogsend/engine 0.7.0 → 0.9.0

package/src/routes/lists/index.ts

@@ -1,7 +1,14 @@
+import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
 import type { Database } from "@hogsend/db";
 import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
 import type { AppEnv } from "../../app.js";
-import { resolveOrCreateContact } from "../../lib/contacts.js";
+import {
+  resolveContact,
+  resolveOrCreateContact,
+  serializeContact,
+} from "../../lib/contacts.js";
+import type { Logger } from "../../lib/logger.js";
+import { emitOutbound } from "../../lib/outbound.js";
 import { applyListMembership } from "../../lib/preferences.js";
 import { errorSchema } from "../../lib/schemas.js";
 import { getListRegistry } from "../../lists/registry-singleton.js";
@@ -122,6 +129,8 @@ const unsubscribeRoute = createRoute({
  */
 async function applyListSubscription(opts: {
   db: Database;
+  hatchet: HatchetClient;
+  logger: Logger;
   id: string;
   email?: string;
   userId?: string;
@@ -132,7 +141,7 @@ async function applyListSubscription(opts: {
   | { kind: "failed"; message: string }
   | { kind: "ok" }
 > {
-  const { db, id, email, userId, subscribed } = opts;
+  const { db, hatchet, logger, id, email, userId, subscribed } = opts;
 
   if (!getListRegistry().has(id)) {
     return { kind: "unknown_list" };
@@ -143,7 +152,30 @@ async function applyListSubscription(opts: {
   }
 
   try {
-    await resolveOrCreateContact({ db, userId, email });
+    const { id: contactId, created } = await resolveOrCreateContact({
+      db,
+      userId,
+      email,
+    });
+
+    // INTENT-LAYER outbound emit (decision #3): the lists route emits
+    // `contact.created` ONLY on first creation (a list flip is not a contact
+    // property delta, so no `contact.updated`). Fire-and-forget after a read-back.
+    if (created) {
+      void resolveContact({ db, id: contactId })
+        .then((row) => {
+          if (!row) return;
+          return emitOutbound({
+            db,
+            hatchet,
+            logger,
+            event: "contact.created",
+            payload: serializeContact(row),
+          });
+        })
+        .catch(logger.warn);
+    }
+
     await applyListMembership({
       db,
       userId,
@@ -175,12 +207,14 @@ export const listsRouter = new OpenAPIHono<AppEnv>()
     return c.json({ lists }, 200);
   })
   .openapi(subscribeRoute, async (c) => {
-    const { db } = c.get("container");
+    const { db, hatchet, logger } = c.get("container");
     const { id } = c.req.valid("param");
     const { email, userId } = c.req.valid("json");
 
     const result = await applyListSubscription({
       db,
+      hatchet,
+      logger,
       id,
       email,
       userId,
@@ -198,12 +232,14 @@ export const listsRouter = new OpenAPIHono<AppEnv>()
     return c.json({ list: id, subscribed: true as const }, 200);
   })
   .openapi(unsubscribeRoute, async (c) => {
-    const { db } = c.get("container");
+    const { db, hatchet, logger } = c.get("container");
     const { id } = c.req.valid("param");
     const { email, userId } = c.req.valid("json");
 
     const result = await applyListSubscription({
       db,
+      hatchet,
+      logger,
       id,
       email,
       userId,

package/src/routes/tracking/click.ts

@@ -2,8 +2,12 @@ import { emailSends, linkClicks, trackedLinks } from "@hogsend/db";
 import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
 import { and, eq, isNull, sql } from "drizzle-orm";
 import type { AppEnv } from "../../app.js";
+import { emitOutbound } from "../../lib/outbound.js";
 import { EMAIL_LINK_CLICKED } from "../../lib/tracking-event-names.js";
-import { pushTrackingEvent } from "../../lib/tracking-events.js";
+import {
+  pushTrackingEvent,
+  resolveEmailSendContext,
+} from "../../lib/tracking-events.js";
 
 const clickRoute = createRoute({
   method: "get",
@@ -48,6 +52,10 @@ export const clickRouter = new OpenAPIHono<AppEnv>().openapi(
       null;
     const userAgent = c.req.header("user-agent") ?? null;
 
+    // First-touch state UPDATE: the `WHERE clickedAt IS NULL` sets `clickedAt`
+    // exactly once (the first click), which is the row-level state we keep. The
+    // outbound emit is NO LONGER gated on this — every destination must receive
+    // EVERY click (owner decision 1), so the emit below fires per-hit.
     await Promise.all([
       db.insert(linkClicks).values({
         trackedLinkId: link.id,
@@ -75,28 +83,56 @@ export const clickRouter = new OpenAPIHono<AppEnv>().openapi(
         ),
     ]);
 
-    const {
-      hatchet,
-      registry,
-      logger,
-      analytics: posthog,
-    } = c.get("container");
+    const { hatchet, registry, logger } = c.get("container");
 
-    pushTrackingEvent({
-      db,
-      hatchet,
-      registry,
-      logger,
-      posthog,
-      event: EMAIL_LINK_CLICKED,
-      emailSendId: link.emailSendId,
-      properties: { linkUrl: link.originalUrl, linkId: link.id },
-    }).catch((err) => {
-      logger.warn("Failed to push click tracking event", {
-        linkId: link.id,
-        error: err instanceof Error ? err.message : String(err),
-      });
-    });
+    // Resolve the send context ONCE (off the response path) and feed both the
+    // re-ingest and the PER-HIT outbound emit — avoiding a duplicate
+    // `resolveEmailSendContext` read on the click hot path. NO `dedupeKey`: a
+    // NULL dedupe key is distinct in Postgres, so every click creates a fresh
+    // delivery to every subscribed destination (per-hit, not first-touch).
+    const emailSendId = link.emailSendId;
+    void resolveEmailSendContext(db, emailSendId)
+      .then(async (ctx) => {
+        await pushTrackingEvent({
+          db,
+          hatchet,
+          registry,
+          logger,
+          event: EMAIL_LINK_CLICKED,
+          emailSendId,
+          properties: { linkUrl: link.originalUrl, linkId: link.id },
+          resolvedContext: ctx,
+        }).catch((err) => {
+          logger.warn("Failed to push click tracking event", {
+            linkId: link.id,
+            error: err instanceof Error ? err.message : String(err),
+          });
+        });
+
+        // Only emit when the send-context resolved. A missing emailSends row
+        // (orphaned tracked link / deleted send) has no userId or recipient to
+        // attribute, and a keyed destination (PostHog) would otherwise receive
+        // an empty distinct_id. A normal click always resolves a non-null userId.
+        if (ctx) {
+          await emitOutbound({
+            db,
+            hatchet,
+            logger,
+            event: "email.clicked",
+            payload: {
+              emailSendId,
+              resendId: ctx.resendId ?? null,
+              templateKey: ctx.templateKey ?? null,
+              userId: ctx.userId ?? null,
+              to: ctx.to ?? ctx.userEmail ?? "",
+              at: new Date().toISOString(),
+              linkUrl: link.originalUrl,
+              linkId: link.id,
+            },
+          });
+        }
+      })
+      .catch(logger.warn);
 
     return c.redirect(link.originalUrl, 302);
   },

package/src/routes/tracking/open.ts

@@ -2,8 +2,12 @@ import { emailSends } from "@hogsend/db";
 import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
 import { and, eq, isNull } from "drizzle-orm";
 import type { AppEnv } from "../../app.js";
+import { emitOutbound } from "../../lib/outbound.js";
 import { EMAIL_OPENED } from "../../lib/tracking-event-names.js";
-import { pushTrackingEvent } from "../../lib/tracking-events.js";
+import {
+  pushTrackingEvent,
+  resolveEmailSendContext,
+} from "../../lib/tracking-events.js";
 
 const TRANSPARENT_GIF = Buffer.from(
   "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7",
@@ -29,8 +33,12 @@ export const openRouter = new OpenAPIHono<AppEnv>().openapi(
   openRoute,
   async (c) => {
     const { id } = c.req.valid("param");
-    const { db } = c.get("container");
+    const { db, hatchet, registry, logger } = c.get("container");
 
+    // First-touch state UPDATE: the `WHERE openedAt IS NULL` sets `openedAt`
+    // exactly once (the first open), which is the row-level state we keep. The
+    // outbound emit is NO LONGER gated on this — every destination must receive
+    // EVERY open (owner decision 1), so the emit below fires per-hit.
     await db
       .update(emailSends)
       .set({
@@ -39,27 +47,50 @@ export const openRouter = new OpenAPIHono<AppEnv>().openapi(
       })
       .where(and(eq(emailSends.id, id), isNull(emailSends.openedAt)));
 
-    const {
-      hatchet,
-      registry,
-      logger,
-      analytics: posthog,
-    } = c.get("container");
+    // Resolve the send context ONCE (off the response path) and feed both the
+    // re-ingest and the PER-HIT outbound emit — avoiding a duplicate
+    // `resolveEmailSendContext` read on the pixel hot path. NO `dedupeKey`: a
+    // NULL dedupe key is distinct in Postgres, so every open creates a fresh
+    // delivery to every subscribed destination (per-hit, not first-touch).
+    void resolveEmailSendContext(db, id)
+      .then(async (ctx) => {
+        await pushTrackingEvent({
+          db,
+          hatchet,
+          registry,
+          logger,
+          event: EMAIL_OPENED,
+          emailSendId: id,
+          resolvedContext: ctx,
+        }).catch((err) => {
+          logger.warn("Failed to push open tracking event", {
+            emailSendId: id,
+            error: err instanceof Error ? err.message : String(err),
+          });
+        });
 
-    pushTrackingEvent({
-      db,
-      hatchet,
-      registry,
-      logger,
-      posthog,
-      event: EMAIL_OPENED,
-      emailSendId: id,
-    }).catch((err) => {
-      logger.warn("Failed to push open tracking event", {
-        emailSendId: id,
-        error: err instanceof Error ? err.message : String(err),
-      });
-    });
+        // Only emit when the send-context resolved. A missing emailSends row
+        // (orphaned tracked pixel / deleted send) has no userId or recipient to
+        // attribute, and a keyed destination (PostHog) would otherwise receive
+        // an empty distinct_id. A normal open always resolves a non-null userId.
+        if (ctx) {
+          await emitOutbound({
+            db,
+            hatchet,
+            logger,
+            event: "email.opened",
+            payload: {
+              emailSendId: id,
+              resendId: ctx.resendId ?? null,
+              templateKey: ctx.templateKey ?? null,
+              userId: ctx.userId ?? null,
+              to: ctx.to ?? ctx.userEmail ?? "",
+              at: new Date().toISOString(),
+            },
+          });
+        }
+      })
+      .catch(logger.warn);
 
     return c.body(TRANSPARENT_GIF, 200, {
       "Content-Type": "image/gif",

package/src/routes/webhooks/sources.ts

@@ -2,6 +2,7 @@ import { createRoute, type OpenAPIHono, z } from "@hono/zod-openapi";
 import type { AppEnv } from "../../app.js";
 import { ingestEvent } from "../../lib/ingestion.js";
 import type { DefinedWebhookSource } from "../../webhook-sources/define-webhook-source.js";
+import { verifySignature } from "../../webhook-sources/verify.js";
 
 export function registerWebhookSourceRoutes(
   app: OpenAPIHono<AppEnv>,
@@ -52,22 +53,75 @@ export function registerWebhookSourceRoutes(
 
     const { db, logger, env, registry, hatchet } = c.get("container");
 
-    // Auth is enforced only when the source's secret is configured. An
-    // unconfigured source is treated as open (parity with the pre-engine route).
+    // Read the body ONCE as the EXACT received bytes — signature schemes verify
+    // over these bytes, so we must not re-stringify. JSON.parse only AFTER auth.
+    const rawBody = await c.req.text();
+    const headers: Record<string, string> = {};
+    for (const [key, value] of c.req.raw.headers.entries()) {
+      headers[key.toLowerCase()] = value;
+    }
+
     const secret = env[source.auth.envKey as keyof typeof env] as
       | string
       | undefined;
-    if (secret) {
-      const provided =
-        c.req.header(source.auth.header) ??
-        c.req.header("authorization")?.replace("Bearer ", "");
 
-      if (provided !== secret) {
-        return c.json({ error: "Invalid webhook secret" }, 401);
+    if (source.auth.type === "signature") {
+      // Signature sources FAIL CLOSED: an unset secret is a 401, never an open
+      // pass-through (deliberate divergence from the "match" variant).
+      if (!secret) {
+        logger.warn("Webhook signature secret not configured", {
+          source: sourceId,
+        });
+        return c.json({ error: "Webhook signature not configured" }, 401);
+      }
+
+      const auth = source.auth;
+      let verified = false;
+
+      if (auth.verify) {
+        verified = await auth.verify({ rawBody, headers, secret });
+      } else {
+        verified = verifySignature(
+          auth.scheme,
+          { rawBody, headers, secret },
+          auth.header,
+        );
+      }
+
+      // Optional plain shared-secret fallback (e.g. Supabase's
+      // `x-supabase-webhook-secret`) when the signature headers are absent.
+      if (!verified && auth.fallbackMatchHeader) {
+        const provided = headers[auth.fallbackMatchHeader.toLowerCase()];
+        verified = provided === secret;
+      }
+
+      if (!verified) {
+        return c.json({ error: "Invalid webhook signature" }, 401);
+      }
+    } else {
+      // "match": shared-secret equality. An unconfigured source stays OPEN
+      // (parity with the pre-engine route).
+      if (secret) {
+        const provided =
+          headers[source.auth.header.toLowerCase()] ??
+          headers.authorization?.replace("Bearer ", "");
+
+        if (provided !== secret) {
+          return c.json({ error: "Invalid webhook secret" }, 401);
+        }
       }
     }
 
-    let payload: unknown = await c.req.json();
+    let payload: unknown;
+    try {
+      payload = JSON.parse(rawBody);
+    } catch {
+      return c.json(
+        { error: "Invalid payload", details: "Malformed JSON" },
+        400,
+      );
+    }
+
     if (source.schema) {
       const parsed = source.schema.safeParse(payload);
       if (!parsed.success) {
@@ -79,7 +133,12 @@ export function registerWebhookSourceRoutes(
       payload = parsed.data;
     }
 
-    const event = await source.transform(payload, { db, logger });
+    const event = await source.transform(payload, {
+      db,
+      logger,
+      rawBody,
+      headers,
+    });
     if (!event) {
       logger.info("Webhook event skipped", { source: sourceId });
       return c.json({ ok: true, skipped: true });

package/src/webhook-sources/define-webhook-source.ts

@@ -2,16 +2,62 @@ import type { Database } from "@hogsend/db";
 import type { z } from "zod";
 import type { IngestEvent } from "../lib/ingestion.js";
 import type { Logger } from "../lib/logger.js";
+import type { SignatureScheme, VerifySignatureArgs } from "./verify.js";
 
-export interface WebhookSourceAuth {
-  header: string;
-  envKey: string;
-  type: "match";
-}
+/**
+ * How a webhook source authenticates inbound requests.
+ *
+ * A discriminated union on `type`:
+ *
+ *  - `"match"` — plain shared-secret equality. The route compares a configured
+ *    secret against the request header (or `Authorization: Bearer`). When the
+ *    secret is UNSET the source stays OPEN (parity with the pre-engine route);
+ *    this variant is unchanged so PostHog + all consumer sources keep compiling.
+ *
+ *  - `"signature"` — provider HMAC signature verification (Svix / Stripe /
+ *    generic hex HMAC). The route resolves the secret from `env[envKey]`, reads
+ *    the EXACT raw request body, and calls `verifySignature` (or the optional
+ *    per-source `verify` override) over those bytes. Signature sources FAIL
+ *    CLOSED (401) when their secret is unset — they are security-sensitive.
+ */
+export type WebhookSourceAuth =
+  | {
+      type: "match";
+      header: string;
+      envKey: string;
+    }
+  | {
+      type: "signature";
+      scheme: SignatureScheme;
+      envKey: string;
+      header: string;
+      /**
+       * For schemes (notably `"svix"`) whose providers may also send a plain
+       * shared-secret header: when the scheme's signature headers are absent but
+       * this header matches the secret verbatim, accept the request. Lets
+       * Supabase's `x-supabase-webhook-secret` plain-secret mode coexist with
+       * its Svix mode.
+       */
+      fallbackMatchHeader?: string;
+      /**
+       * Optional per-source override of the built-in scheme verification. When
+       * provided, the route calls this INSTEAD of `verifySignature(scheme, …)`.
+       * Receives the EXACT received bytes; must return (or resolve to) a boolean.
+       */
+      verify?(args: VerifySignatureArgs): boolean | Promise<boolean>;
+    };
 
 export interface WebhookSourceCtx {
   db: Database;
   logger: Logger;
+  /**
+   * The EXACT raw request body bytes (text), populated by the route. Required by
+   * signature schemes (the signature covers these bytes) and available to any
+   * `transform()` that needs provider-specific raw access.
+   */
+  rawBody?: string;
+  /** The inbound request headers (lowercased keys), populated by the route. */
+  headers?: Record<string, string>;
 }
 
 export interface WebhookSourceMeta {
@@ -32,3 +78,9 @@ export function defineWebhookSource<T>(
 ): DefinedWebhookSource<T> {
   return def;
 }
+
+export {
+  type SignatureScheme,
+  type VerifySignatureArgs,
+  verifySignature,
+} from "./verify.js";

package/src/webhook-sources/presets/clerk.ts

@@ -0,0 +1,185 @@
+import { z } from "zod";
+import type { IngestEvent } from "../../lib/ingestion.js";
+import { defineWebhookSource } from "../define-webhook-source.js";
+
+/**
+ * Clerk webhook preset.
+ *
+ * Auth: Svix-signed (`svix-id`/`svix-timestamp`/`svix-signature`). Clerk's
+ * webhook signing secret is a `whsec_…` value — set `CLERK_WEBHOOK_SECRET` to
+ * auto-enable this source at `POST /v1/webhooks/clerk`. Signature sources FAIL
+ * CLOSED when the secret is unset.
+ *
+ * Event mapping (decision #16, normalized to the outbound vocabulary):
+ *  - `user.created`            → `contact.created`
+ *  - `user.updated`            → `contact.updated`
+ *  - `user.deleted`            → `contact.deleted` (EVENT only — decision #15)
+ *  - `waitlistEntry.created`   → `waitlist.joined`
+ *
+ * D2 split (decision, mirrors `webhook-sources/posthog.ts`): identity/profile
+ * fields → `contactProperties` ONLY; behavioral/source fields → `eventProperties`
+ * ONLY. The two bags are NEVER merged.
+ */
+
+const clerkEmailAddressSchema = z
+  .object({
+    id: z.string().optional(),
+    email_address: z.string().optional(),
+  })
+  .catchall(z.unknown());
+
+const clerkUserDataSchema = z
+  .object({
+    id: z.string().optional(),
+    primary_email_address_id: z.string().nullish(),
+    email_addresses: z.array(clerkEmailAddressSchema).nullish(),
+    first_name: z.string().nullish(),
+    last_name: z.string().nullish(),
+    image_url: z.string().nullish(),
+    profile_image_url: z.string().nullish(),
+    public_metadata: z.record(z.string(), z.unknown()).nullish(),
+  })
+  .catchall(z.unknown());
+
+const clerkWaitlistDataSchema = z
+  .object({
+    id: z.string().optional(),
+    email_address: z.string().nullish(),
+  })
+  .catchall(z.unknown());
+
+const clerkWebhookSchema = z
+  .object({
+    type: z.string(),
+    object: z.string().optional(),
+    data: z.object({}).catchall(z.unknown()).nullish(),
+  })
+  .catchall(z.unknown());
+
+type ClerkPayload = z.infer<typeof clerkWebhookSchema>;
+
+/** Resolve the primary email address from Clerk's `email_addresses` array. */
+function resolveClerkEmail(
+  data: z.infer<typeof clerkUserDataSchema>,
+): string | undefined {
+  const addresses = data.email_addresses ?? [];
+  if (addresses.length === 0) {
+    return undefined;
+  }
+  const primaryId = data.primary_email_address_id;
+  if (primaryId) {
+    const primary = addresses.find((a) => a.id === primaryId);
+    if (primary?.email_address) {
+      return primary.email_address;
+    }
+  }
+  return addresses[0]?.email_address ?? undefined;
+}
+
+export const clerkSource = defineWebhookSource({
+  meta: {
+    id: "clerk",
+    name: "Clerk",
+    description:
+      "Receives Clerk user lifecycle + waitlist webhooks (Svix-signed).",
+  },
+  auth: {
+    type: "signature",
+    scheme: "svix",
+    envKey: "CLERK_WEBHOOK_SECRET",
+    header: "svix-signature",
+  },
+  schema: clerkWebhookSchema,
+  async transform(payload: ClerkPayload): Promise<IngestEvent | null> {
+    const type = payload.type;
+
+    if (type === "waitlistEntry.created") {
+      const data = clerkWaitlistDataSchema.parse(payload.data ?? {});
+      const userEmail =
+        typeof data.email_address === "string" ? data.email_address : "";
+      const userId = data.id ?? userEmail;
+      if (!userId) {
+        return null;
+      }
+      return {
+        event: "waitlist.joined",
+        userId,
+        userEmail,
+        eventProperties: {
+          source: "clerk",
+          _clerkEvent: type,
+        },
+        contactProperties: {},
+      };
+    }
+
+    let event: string;
+    switch (type) {
+      case "user.created":
+        event = "contact.created";
+        break;
+      case "user.updated":
+        event = "contact.updated";
+        break;
+      case "user.deleted":
+        event = "contact.deleted";
+        break;
+      default:
+        return null;
+    }
+
+    const data = clerkUserDataSchema.parse(payload.data ?? {});
+    const userId = data.id;
+    if (!userId) {
+      return null;
+    }
+    const userEmail = resolveClerkEmail(data) ?? "";
+
+    // Deletes carry no profile to merge — emit the event only (decision #15).
+    if (event === "contact.deleted") {
+      return {
+        event,
+        userId,
+        userEmail,
+        eventProperties: {
+          source: "clerk",
+          clerkUserId: userId,
+          _clerkEvent: type,
+        },
+        contactProperties: {},
+      };
+    }
+
+    const avatarUrl =
+      (typeof data.image_url === "string" ? data.image_url : undefined) ??
+      (typeof data.profile_image_url === "string"
+        ? data.profile_image_url
+        : undefined);
+
+    const contactProperties: Record<string, unknown> = {
+      ...(data.public_metadata ?? {}),
+    };
+    if (typeof data.first_name === "string") {
+      contactProperties.firstName = data.first_name;
+    }
+    if (typeof data.last_name === "string") {
+      contactProperties.lastName = data.last_name;
+    }
+    if (avatarUrl) {
+      contactProperties.avatarUrl = avatarUrl;
+    }
+    contactProperties.clerkUserId = userId;
+
+    return {
+      event,
+      userId,
+      userEmail,
+      eventProperties: {
+        source: "clerk",
+        clerkUserId: userId,
+        _clerkEvent: type,
+      },
+      contactProperties,
+    };
+  },
+});