@hogsend/engine 0.6.0 → 0.8.0

package/src/routes/tracking/open.ts

@@ -2,8 +2,12 @@ import { emailSends } from "@hogsend/db";
 import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
 import { and, eq, isNull } from "drizzle-orm";
 import type { AppEnv } from "../../app.js";
+import { emitOutbound } from "../../lib/outbound.js";
 import { EMAIL_OPENED } from "../../lib/tracking-event-names.js";
-import { pushTrackingEvent } from "../../lib/tracking-events.js";
+import {
+  pushTrackingEvent,
+  resolveEmailSendContext,
+} from "../../lib/tracking-events.js";
 
 const TRANSPARENT_GIF = Buffer.from(
   "R0lGODlhAQABAIAAAAAAAP///yH5BAEAAAAALAAAAAABAAEAAAIBRAA7",
@@ -29,37 +33,71 @@ export const openRouter = new OpenAPIHono<AppEnv>().openapi(
   openRoute,
   async (c) => {
     const { id } = c.req.valid("param");
-    const { db } = c.get("container");
+    const {
+      db,
+      hatchet,
+      registry,
+      logger,
+      analytics: posthog,
+    } = c.get("container");
 
-    await db
+    // First-touch gate: the `WHERE openedAt IS NULL` makes this UPDATE return a
+    // row ONLY on the FIRST open. `.returning({ id })` lets the outbound emit fire
+    // exactly once — first-party is the SINGLE emitter for `email.opened` (the
+    // provider-webhook echo in the mailer is suppressed — risk 4).
+    const opened = await db
       .update(emailSends)
       .set({
         openedAt: new Date(),
         updatedAt: new Date(),
       })
-      .where(and(eq(emailSends.id, id), isNull(emailSends.openedAt)));
+      .where(and(eq(emailSends.id, id), isNull(emailSends.openedAt)))
+      .returning({ id: emailSends.id });
 
-    const {
-      hatchet,
-      registry,
-      logger,
-      analytics: posthog,
-    } = c.get("container");
+    // Resolve the send context ONCE (off the response path) and feed both the
+    // re-ingest (every open) and the first-touch outbound emit (first open
+    // only) — avoiding a duplicate `resolveEmailSendContext` read on the pixel
+    // hot path. `dedupeKey` = `email.opened:<id>` is defence-in-depth alongside
+    // the first-touch gate (`opened.length > 0`); first-party is the SINGLE
+    // emitter for `email.opened` (the provider-webhook echo is suppressed).
+    const isFirstOpen = opened.length > 0;
+    void resolveEmailSendContext(db, id)
+      .then(async (ctx) => {
+        await pushTrackingEvent({
+          db,
+          hatchet,
+          registry,
+          logger,
+          posthog,
+          event: EMAIL_OPENED,
+          emailSendId: id,
+          resolvedContext: ctx,
+        }).catch((err) => {
+          logger.warn("Failed to push open tracking event", {
+            emailSendId: id,
+            error: err instanceof Error ? err.message : String(err),
+          });
+        });
 
-    pushTrackingEvent({
-      db,
-      hatchet,
-      registry,
-      logger,
-      posthog,
-      event: EMAIL_OPENED,
-      emailSendId: id,
-    }).catch((err) => {
-      logger.warn("Failed to push open tracking event", {
-        emailSendId: id,
-        error: err instanceof Error ? err.message : String(err),
-      });
-    });
+        if (isFirstOpen) {
+          await emitOutbound({
+            db,
+            hatchet,
+            logger,
+            event: "email.opened",
+            dedupeKey: `email.opened:${id}`,
+            payload: {
+              emailSendId: id,
+              resendId: ctx?.resendId ?? null,
+              templateKey: ctx?.templateKey ?? null,
+              userId: ctx?.userId ?? null,
+              to: ctx?.to ?? ctx?.userEmail ?? "",
+              at: new Date().toISOString(),
+            },
+          });
+        }
+      })
+      .catch(logger.warn);
 
     return c.body(TRANSPARENT_GIF, 200, {
       "Content-Type": "image/gif",

package/src/routes/webhooks/sources.ts

@@ -2,6 +2,7 @@ import { createRoute, type OpenAPIHono, z } from "@hono/zod-openapi";
 import type { AppEnv } from "../../app.js";
 import { ingestEvent } from "../../lib/ingestion.js";
 import type { DefinedWebhookSource } from "../../webhook-sources/define-webhook-source.js";
+import { verifySignature } from "../../webhook-sources/verify.js";
 
 export function registerWebhookSourceRoutes(
   app: OpenAPIHono<AppEnv>,
@@ -52,22 +53,75 @@ export function registerWebhookSourceRoutes(
 
     const { db, logger, env, registry, hatchet } = c.get("container");
 
-    // Auth is enforced only when the source's secret is configured. An
-    // unconfigured source is treated as open (parity with the pre-engine route).
+    // Read the body ONCE as the EXACT received bytes — signature schemes verify
+    // over these bytes, so we must not re-stringify. JSON.parse only AFTER auth.
+    const rawBody = await c.req.text();
+    const headers: Record<string, string> = {};
+    for (const [key, value] of c.req.raw.headers.entries()) {
+      headers[key.toLowerCase()] = value;
+    }
+
     const secret = env[source.auth.envKey as keyof typeof env] as
       | string
       | undefined;
-    if (secret) {
-      const provided =
-        c.req.header(source.auth.header) ??
-        c.req.header("authorization")?.replace("Bearer ", "");
 
-      if (provided !== secret) {
-        return c.json({ error: "Invalid webhook secret" }, 401);
+    if (source.auth.type === "signature") {
+      // Signature sources FAIL CLOSED: an unset secret is a 401, never an open
+      // pass-through (deliberate divergence from the "match" variant).
+      if (!secret) {
+        logger.warn("Webhook signature secret not configured", {
+          source: sourceId,
+        });
+        return c.json({ error: "Webhook signature not configured" }, 401);
+      }
+
+      const auth = source.auth;
+      let verified = false;
+
+      if (auth.verify) {
+        verified = await auth.verify({ rawBody, headers, secret });
+      } else {
+        verified = verifySignature(
+          auth.scheme,
+          { rawBody, headers, secret },
+          auth.header,
+        );
+      }
+
+      // Optional plain shared-secret fallback (e.g. Supabase's
+      // `x-supabase-webhook-secret`) when the signature headers are absent.
+      if (!verified && auth.fallbackMatchHeader) {
+        const provided = headers[auth.fallbackMatchHeader.toLowerCase()];
+        verified = provided === secret;
+      }
+
+      if (!verified) {
+        return c.json({ error: "Invalid webhook signature" }, 401);
+      }
+    } else {
+      // "match": shared-secret equality. An unconfigured source stays OPEN
+      // (parity with the pre-engine route).
+      if (secret) {
+        const provided =
+          headers[source.auth.header.toLowerCase()] ??
+          headers.authorization?.replace("Bearer ", "");
+
+        if (provided !== secret) {
+          return c.json({ error: "Invalid webhook secret" }, 401);
+        }
       }
     }
 
-    let payload: unknown = await c.req.json();
+    let payload: unknown;
+    try {
+      payload = JSON.parse(rawBody);
+    } catch {
+      return c.json(
+        { error: "Invalid payload", details: "Malformed JSON" },
+        400,
+      );
+    }
+
     if (source.schema) {
       const parsed = source.schema.safeParse(payload);
       if (!parsed.success) {
@@ -79,7 +133,12 @@ export function registerWebhookSourceRoutes(
       payload = parsed.data;
     }
 
-    const event = await source.transform(payload, { db, logger });
+    const event = await source.transform(payload, {
+      db,
+      logger,
+      rawBody,
+      headers,
+    });
     if (!event) {
       logger.info("Webhook event skipped", { source: sourceId });
       return c.json({ ok: true, skipped: true });

package/src/webhook-sources/define-webhook-source.ts

@@ -2,16 +2,62 @@ import type { Database } from "@hogsend/db";
 import type { z } from "zod";
 import type { IngestEvent } from "../lib/ingestion.js";
 import type { Logger } from "../lib/logger.js";
+import type { SignatureScheme, VerifySignatureArgs } from "./verify.js";
 
-export interface WebhookSourceAuth {
-  header: string;
-  envKey: string;
-  type: "match";
-}
+/**
+ * How a webhook source authenticates inbound requests.
+ *
+ * A discriminated union on `type`:
+ *
+ *  - `"match"` — plain shared-secret equality. The route compares a configured
+ *    secret against the request header (or `Authorization: Bearer`). When the
+ *    secret is UNSET the source stays OPEN (parity with the pre-engine route);
+ *    this variant is unchanged so PostHog + all consumer sources keep compiling.
+ *
+ *  - `"signature"` — provider HMAC signature verification (Svix / Stripe /
+ *    generic hex HMAC). The route resolves the secret from `env[envKey]`, reads
+ *    the EXACT raw request body, and calls `verifySignature` (or the optional
+ *    per-source `verify` override) over those bytes. Signature sources FAIL
+ *    CLOSED (401) when their secret is unset — they are security-sensitive.
+ */
+export type WebhookSourceAuth =
+  | {
+      type: "match";
+      header: string;
+      envKey: string;
+    }
+  | {
+      type: "signature";
+      scheme: SignatureScheme;
+      envKey: string;
+      header: string;
+      /**
+       * For schemes (notably `"svix"`) whose providers may also send a plain
+       * shared-secret header: when the scheme's signature headers are absent but
+       * this header matches the secret verbatim, accept the request. Lets
+       * Supabase's `x-supabase-webhook-secret` plain-secret mode coexist with
+       * its Svix mode.
+       */
+      fallbackMatchHeader?: string;
+      /**
+       * Optional per-source override of the built-in scheme verification. When
+       * provided, the route calls this INSTEAD of `verifySignature(scheme, …)`.
+       * Receives the EXACT received bytes; must return (or resolve to) a boolean.
+       */
+      verify?(args: VerifySignatureArgs): boolean | Promise<boolean>;
+    };
 
 export interface WebhookSourceCtx {
   db: Database;
   logger: Logger;
+  /**
+   * The EXACT raw request body bytes (text), populated by the route. Required by
+   * signature schemes (the signature covers these bytes) and available to any
+   * `transform()` that needs provider-specific raw access.
+   */
+  rawBody?: string;
+  /** The inbound request headers (lowercased keys), populated by the route. */
+  headers?: Record<string, string>;
 }
 
 export interface WebhookSourceMeta {
@@ -32,3 +78,9 @@ export function defineWebhookSource<T>(
 ): DefinedWebhookSource<T> {
   return def;
 }
+
+export {
+  type SignatureScheme,
+  type VerifySignatureArgs,
+  verifySignature,
+} from "./verify.js";

package/src/webhook-sources/presets/clerk.ts

@@ -0,0 +1,185 @@
+import { z } from "zod";
+import type { IngestEvent } from "../../lib/ingestion.js";
+import { defineWebhookSource } from "../define-webhook-source.js";
+
+/**
+ * Clerk webhook preset.
+ *
+ * Auth: Svix-signed (`svix-id`/`svix-timestamp`/`svix-signature`). Clerk's
+ * webhook signing secret is a `whsec_…` value — set `CLERK_WEBHOOK_SECRET` to
+ * auto-enable this source at `POST /v1/webhooks/clerk`. Signature sources FAIL
+ * CLOSED when the secret is unset.
+ *
+ * Event mapping (decision #16, normalized to the outbound vocabulary):
+ *  - `user.created`            → `contact.created`
+ *  - `user.updated`            → `contact.updated`
+ *  - `user.deleted`            → `contact.deleted` (EVENT only — decision #15)
+ *  - `waitlistEntry.created`   → `waitlist.joined`
+ *
+ * D2 split (decision, mirrors `webhook-sources/posthog.ts`): identity/profile
+ * fields → `contactProperties` ONLY; behavioral/source fields → `eventProperties`
+ * ONLY. The two bags are NEVER merged.
+ */
+
+const clerkEmailAddressSchema = z
+  .object({
+    id: z.string().optional(),
+    email_address: z.string().optional(),
+  })
+  .catchall(z.unknown());
+
+const clerkUserDataSchema = z
+  .object({
+    id: z.string().optional(),
+    primary_email_address_id: z.string().nullish(),
+    email_addresses: z.array(clerkEmailAddressSchema).nullish(),
+    first_name: z.string().nullish(),
+    last_name: z.string().nullish(),
+    image_url: z.string().nullish(),
+    profile_image_url: z.string().nullish(),
+    public_metadata: z.record(z.string(), z.unknown()).nullish(),
+  })
+  .catchall(z.unknown());
+
+const clerkWaitlistDataSchema = z
+  .object({
+    id: z.string().optional(),
+    email_address: z.string().nullish(),
+  })
+  .catchall(z.unknown());
+
+const clerkWebhookSchema = z
+  .object({
+    type: z.string(),
+    object: z.string().optional(),
+    data: z.object({}).catchall(z.unknown()).nullish(),
+  })
+  .catchall(z.unknown());
+
+type ClerkPayload = z.infer<typeof clerkWebhookSchema>;
+
+/** Resolve the primary email address from Clerk's `email_addresses` array. */
+function resolveClerkEmail(
+  data: z.infer<typeof clerkUserDataSchema>,
+): string | undefined {
+  const addresses = data.email_addresses ?? [];
+  if (addresses.length === 0) {
+    return undefined;
+  }
+  const primaryId = data.primary_email_address_id;
+  if (primaryId) {
+    const primary = addresses.find((a) => a.id === primaryId);
+    if (primary?.email_address) {
+      return primary.email_address;
+    }
+  }
+  return addresses[0]?.email_address ?? undefined;
+}
+
+export const clerkSource = defineWebhookSource({
+  meta: {
+    id: "clerk",
+    name: "Clerk",
+    description:
+      "Receives Clerk user lifecycle + waitlist webhooks (Svix-signed).",
+  },
+  auth: {
+    type: "signature",
+    scheme: "svix",
+    envKey: "CLERK_WEBHOOK_SECRET",
+    header: "svix-signature",
+  },
+  schema: clerkWebhookSchema,
+  async transform(payload: ClerkPayload): Promise<IngestEvent | null> {
+    const type = payload.type;
+
+    if (type === "waitlistEntry.created") {
+      const data = clerkWaitlistDataSchema.parse(payload.data ?? {});
+      const userEmail =
+        typeof data.email_address === "string" ? data.email_address : "";
+      const userId = data.id ?? userEmail;
+      if (!userId) {
+        return null;
+      }
+      return {
+        event: "waitlist.joined",
+        userId,
+        userEmail,
+        eventProperties: {
+          source: "clerk",
+          _clerkEvent: type,
+        },
+        contactProperties: {},
+      };
+    }
+
+    let event: string;
+    switch (type) {
+      case "user.created":
+        event = "contact.created";
+        break;
+      case "user.updated":
+        event = "contact.updated";
+        break;
+      case "user.deleted":
+        event = "contact.deleted";
+        break;
+      default:
+        return null;
+    }
+
+    const data = clerkUserDataSchema.parse(payload.data ?? {});
+    const userId = data.id;
+    if (!userId) {
+      return null;
+    }
+    const userEmail = resolveClerkEmail(data) ?? "";
+
+    // Deletes carry no profile to merge — emit the event only (decision #15).
+    if (event === "contact.deleted") {
+      return {
+        event,
+        userId,
+        userEmail,
+        eventProperties: {
+          source: "clerk",
+          clerkUserId: userId,
+          _clerkEvent: type,
+        },
+        contactProperties: {},
+      };
+    }
+
+    const avatarUrl =
+      (typeof data.image_url === "string" ? data.image_url : undefined) ??
+      (typeof data.profile_image_url === "string"
+        ? data.profile_image_url
+        : undefined);
+
+    const contactProperties: Record<string, unknown> = {
+      ...(data.public_metadata ?? {}),
+    };
+    if (typeof data.first_name === "string") {
+      contactProperties.firstName = data.first_name;
+    }
+    if (typeof data.last_name === "string") {
+      contactProperties.lastName = data.last_name;
+    }
+    if (avatarUrl) {
+      contactProperties.avatarUrl = avatarUrl;
+    }
+    contactProperties.clerkUserId = userId;
+
+    return {
+      event,
+      userId,
+      userEmail,
+      eventProperties: {
+        source: "clerk",
+        clerkUserId: userId,
+        _clerkEvent: type,
+      },
+      contactProperties,
+    };
+  },
+});

package/src/webhook-sources/presets/index.ts

@@ -0,0 +1,80 @@
+import type { env as engineEnv } from "../../env.js";
+import type { DefinedWebhookSource } from "../define-webhook-source.js";
+import { clerkSource } from "./clerk.js";
+import { segmentSource } from "./segment.js";
+import { stripeSource } from "./stripe.js";
+import { supabaseSource } from "./supabase.js";
+
+export { clerkSource } from "./clerk.js";
+export { segmentSource } from "./segment.js";
+export { stripeSource } from "./stripe.js";
+export { supabaseSource } from "./supabase.js";
+
+/**
+ * All shipped integration presets, keyed by their webhook-source id. The id is
+ * also the route segment: `PRESET_SOURCES.stripe` serves `POST /v1/webhooks/stripe`.
+ */
+export const PRESET_SOURCES = {
+  clerk: clerkSource,
+  supabase: supabaseSource,
+  stripe: stripeSource,
+  segment: segmentSource,
+} satisfies Record<string, DefinedWebhookSource>;
+
+/** The stable id of a shipped preset (`"clerk" | "supabase" | "stripe" | "segment"`). */
+export type PresetId = keyof typeof PRESET_SOURCES;
+
+/** The slice of the validated env `presetsFromEnv` reads (preset secrets + override). */
+type PresetEnv = Pick<
+  typeof engineEnv,
+  | "CLERK_WEBHOOK_SECRET"
+  | "SUPABASE_WEBHOOK_SECRET"
+  | "STRIPE_WEBHOOK_SECRET"
+  | "SEGMENT_WEBHOOK_SECRET"
+  | "ENABLED_WEBHOOK_PRESETS"
+>;
+
+/**
+ * Resolve which presets to enable from the validated env (decision #13).
+ *
+ * Resolution order:
+ *  1. `ENABLED_WEBHOOK_PRESETS === "none"` → no presets (hard off).
+ *  2. `ENABLED_WEBHOOK_PRESETS` is a csv of ids → exactly those, but ONLY when
+ *     the preset's secret (`env[auth.envKey]`) is also set (a signature source
+ *     with no secret fails closed at runtime, so enabling it is pointless).
+ *  3. `ENABLED_WEBHOOK_PRESETS === "*"` or absent → AUTO: every preset whose
+ *     secret is present.
+ *
+ * In every branch a preset is only returned when its secret is configured, so a
+ * preset can never be mounted in an always-fail-closed state by accident.
+ */
+export function presetsFromEnv(env: PresetEnv): DefinedWebhookSource[] {
+  const override = env.ENABLED_WEBHOOK_PRESETS?.trim();
+
+  if (override === "none") {
+    return [];
+  }
+
+  const hasSecret = (source: DefinedWebhookSource): boolean => {
+    const secret = env[source.auth.envKey as keyof PresetEnv];
+    return typeof secret === "string" && secret.length > 0;
+  };
+
+  // Explicit csv allow-list (anything other than "*"/empty).
+  if (override && override !== "*") {
+    const ids = new Set(
+      override
+        .split(",")
+        .map((id) => id.trim().toLowerCase())
+        .filter((id) => id.length > 0),
+    );
+    return (
+      Object.entries(PRESET_SOURCES) as [PresetId, DefinedWebhookSource][]
+    )
+      .filter(([id, source]) => ids.has(id) && hasSecret(source))
+      .map(([, source]) => source);
+  }
+
+  // AUTO ("*" or absent): every preset whose secret is set.
+  return Object.values(PRESET_SOURCES).filter(hasSecret);
+}

package/src/webhook-sources/presets/segment.ts

@@ -0,0 +1,120 @@
+import { z } from "zod";
+import type { IngestEvent } from "../../lib/ingestion.js";
+import { defineWebhookSource } from "../define-webhook-source.js";
+
+/**
+ * Segment webhook preset.
+ *
+ * Auth: generic HMAC-hex over the raw body (`x-signature`), verified with
+ * `node:crypto`. Set `SEGMENT_WEBHOOK_SECRET` to auto-enable at
+ * `POST /v1/webhooks/segment`.
+ *
+ * Event mapping (decision #16):
+ *  - `identify` → `contact.updated` (traits → `contactProperties` ONLY)
+ *  - `track`    → the literal `event` name (properties → `eventProperties` ONLY)
+ *  - `page` / `screen` / `group` / `alias` → skipped (`null`)
+ *
+ * Identity: `userId = userId ?? anonymousId`; `email` lifted from
+ * `traits.email`/`context.traits.email`. `idempotencyKey = messageId` so
+ * Segment redelivery dedupes on `user_events.idempotencyKey`.
+ */
+
+const segmentTraitsSchema = z.record(z.string(), z.unknown());
+
+const segmentWebhookSchema = z
+  .object({
+    type: z.string(),
+    event: z.string().nullish(),
+    messageId: z.string().nullish(),
+    userId: z.string().nullish(),
+    anonymousId: z.string().nullish(),
+    traits: segmentTraitsSchema.nullish(),
+    properties: z.record(z.string(), z.unknown()).nullish(),
+    context: z
+      .object({
+        traits: segmentTraitsSchema.nullish(),
+      })
+      .catchall(z.unknown())
+      .nullish(),
+  })
+  .catchall(z.unknown());
+
+type SegmentPayload = z.infer<typeof segmentWebhookSchema>;
+
+/** Pull a string email out of a traits bag, if present. */
+function emailFromTraits(
+  traits: Record<string, unknown> | null | undefined,
+): string | undefined {
+  const email = traits?.email;
+  return typeof email === "string" ? email : undefined;
+}
+
+export const segmentSource = defineWebhookSource({
+  meta: {
+    id: "segment",
+    name: "Segment",
+    description: "Receives Segment identify/track webhooks (HMAC-hex signed).",
+  },
+  auth: {
+    type: "signature",
+    scheme: "hmac-hex",
+    envKey: "SEGMENT_WEBHOOK_SECRET",
+    header: "x-signature",
+  },
+  schema: segmentWebhookSchema,
+  async transform(payload: SegmentPayload): Promise<IngestEvent | null> {
+    const userId = payload.userId ?? payload.anonymousId ?? undefined;
+    if (!userId) {
+      return null;
+    }
+
+    const traits = payload.traits ?? payload.context?.traits ?? undefined;
+    const userEmail =
+      emailFromTraits(payload.traits) ??
+      emailFromTraits(payload.context?.traits) ??
+      "";
+    const idempotencyKey = payload.messageId ?? undefined;
+
+    if (payload.type === "identify") {
+      // identify: traits are profile/identity → contactProperties ONLY.
+      const contactProperties: Record<string, unknown> = { ...(traits ?? {}) };
+      contactProperties.source = "segment";
+
+      return {
+        event: "contact.updated",
+        userId,
+        userEmail,
+        eventProperties: {
+          source: "segment",
+          _segmentType: "identify",
+        },
+        contactProperties,
+        ...(idempotencyKey ? { idempotencyKey } : {}),
+      };
+    }
+
+    if (payload.type === "track") {
+      const eventName = payload.event;
+      if (!eventName) {
+        return null;
+      }
+      // track: properties are behavioral → eventProperties ONLY.
+      const eventProperties: Record<string, unknown> = {
+        ...(payload.properties ?? {}),
+      };
+      eventProperties.source = "segment";
+
+      return {
+        event: eventName,
+        userId,
+        userEmail,
+        eventProperties,
+        contactProperties: {},
+        ...(idempotencyKey ? { idempotencyKey } : {}),
+      };
+    }
+
+    // page / screen / group / alias → skip.
+    return null;
+  },
+});