@hogsend/engine 0.6.0 → 0.8.0

package/src/webhook-sources/presets/stripe.ts

@@ -0,0 +1,147 @@
+import { z } from "zod";
+import type { IngestEvent } from "../../lib/ingestion.js";
+import { defineWebhookSource } from "../define-webhook-source.js";
+
+/**
+ * Stripe webhook preset.
+ *
+ * Auth: Stripe's `stripe-signature: t=<ts>,v1=<hex>` header, verified with
+ * `node:crypto` (NO `stripe` SDK — decision #14). Set `STRIPE_WEBHOOK_SECRET`
+ * (the `whsec_…` endpoint secret) to auto-enable at `POST /v1/webhooks/stripe`.
+ *
+ * Event mapping (decision #16, normalized to the outbound vocabulary):
+ *  - `customer.created`              → `contact.created`
+ *  - `customer.updated`              → `contact.updated`
+ *  - `customer.deleted`             → `contact.deleted` (EVENT only — decision #15)
+ *  - `customer.subscription.<action>` → `subscription.<action>`
+ *  - `invoice.<action>`              → `invoice.<action>`
+ *
+ * Identity: `userId = obj.id` for customers, `obj.customer` for subscriptions /
+ * invoices. `idempotencyKey = payload.id` (the Stripe event id) so at-least-once
+ * redelivery dedupes on `user_events.idempotencyKey`.
+ *
+ * D2 split: customer profile (`name`, `phone`, `metadata`) → `contactProperties`
+ * ONLY; everything else → `eventProperties` ONLY.
+ */
+
+const stripeObjectSchema = z
+  .object({
+    id: z.string().optional(),
+    object: z.string().optional(),
+    customer: z.string().nullish(),
+    email: z.string().nullish(),
+    name: z.string().nullish(),
+    phone: z.string().nullish(),
+    metadata: z.record(z.string(), z.unknown()).nullish(),
+  })
+  .catchall(z.unknown());
+
+const stripeWebhookSchema = z
+  .object({
+    id: z.string(),
+    type: z.string(),
+    object: z.string().optional(),
+    data: z
+      .object({
+        object: stripeObjectSchema,
+      })
+      .catchall(z.unknown()),
+  })
+  .catchall(z.unknown());
+
+type StripePayload = z.infer<typeof stripeWebhookSchema>;
+
+export const stripeSource = defineWebhookSource({
+  meta: {
+    id: "stripe",
+    name: "Stripe",
+    description:
+      "Receives Stripe customer/subscription/invoice webhooks (signature-verified).",
+  },
+  auth: {
+    type: "signature",
+    scheme: "stripe",
+    envKey: "STRIPE_WEBHOOK_SECRET",
+    header: "stripe-signature",
+  },
+  schema: stripeWebhookSchema,
+  async transform(payload: StripePayload): Promise<IngestEvent | null> {
+    const type = payload.type;
+    const obj = payload.data.object;
+
+    // Normalize the Stripe event name → Hogsend vocabulary + resolve identity.
+    let event: string;
+    let userId: string | undefined;
+    let isCustomerLifecycle = false;
+    let isDelete = false;
+
+    if (type === "customer.created" || type === "customer.updated") {
+      event =
+        type === "customer.created" ? "contact.created" : "contact.updated";
+      userId = obj.id;
+      isCustomerLifecycle = true;
+    } else if (type === "customer.deleted") {
+      event = "contact.deleted";
+      userId = obj.id;
+      isDelete = true;
+    } else if (type.startsWith("customer.subscription.")) {
+      // customer.subscription.created/updated/deleted → subscription.<action>
+      const action = type.slice("customer.subscription.".length);
+      event = `subscription.${action}`;
+      userId = typeof obj.customer === "string" ? obj.customer : undefined;
+    } else if (type.startsWith("invoice.")) {
+      const action = type.slice("invoice.".length);
+      event = `invoice.${action}`;
+      userId = typeof obj.customer === "string" ? obj.customer : undefined;
+    } else {
+      return null;
+    }
+
+    if (!userId) {
+      return null;
+    }
+
+    const userEmail = typeof obj.email === "string" ? obj.email : "";
+
+    const eventProperties: Record<string, unknown> = {
+      source: "stripe",
+      stripeCustomerId: userId,
+      stripeEventId: payload.id,
+      _stripeEvent: type,
+      stripeObject: obj.object,
+    };
+
+    // Only the customer create/update lifecycle carries a profile to merge.
+    // Deletes (decision #15) and subscription/invoice events are event-only.
+    if (!isCustomerLifecycle || isDelete) {
+      return {
+        event,
+        userId,
+        userEmail,
+        eventProperties,
+        contactProperties: {},
+        idempotencyKey: payload.id,
+      };
+    }
+
+    const contactProperties: Record<string, unknown> = {
+      ...(obj.metadata ?? {}),
+    };
+    if (typeof obj.name === "string") {
+      contactProperties.name = obj.name;
+    }
+    if (typeof obj.phone === "string") {
+      contactProperties.phone = obj.phone;
+    }
+    contactProperties.stripeCustomerId = userId;
+
+    return {
+      event,
+      userId,
+      userEmail,
+      eventProperties,
+      contactProperties,
+      idempotencyKey: payload.id,
+    };
+  },
+});

package/src/webhook-sources/presets/supabase.ts

@@ -0,0 +1,131 @@
+import { z } from "zod";
+import type { IngestEvent } from "../../lib/ingestion.js";
+import { defineWebhookSource } from "../define-webhook-source.js";
+
+/**
+ * Supabase `auth.users` webhook preset.
+ *
+ * Auth: Svix-signed when Supabase's "Send HTTP Request" hook is configured with
+ * a signing secret; falls back to the plain `x-supabase-webhook-secret` shared
+ * secret (via `fallbackMatchHeader`) for the database-webhook trigger path. Set
+ * `SUPABASE_WEBHOOK_SECRET` to auto-enable at `POST /v1/webhooks/supabase`.
+ *
+ * Only `schema === "auth" && table === "users"` rows are processed (other tables
+ * are skipped). Event mapping (decision #16, normalized):
+ *  - `INSERT` → `contact.created`
+ *  - `UPDATE` → `contact.updated`
+ *  - `DELETE` → `contact.deleted` (EVENT only — decision #15)
+ *
+ * D2 split: profile fields → `contactProperties` ONLY; behavioral/source fields
+ * → `eventProperties` ONLY.
+ */
+
+const supabaseUserRowSchema = z
+  .object({
+    id: z.string().optional(),
+    email: z.string().nullish(),
+    phone: z.string().nullish(),
+    email_confirmed_at: z.string().nullish(),
+    raw_user_meta_data: z.record(z.string(), z.unknown()).nullish(),
+  })
+  .catchall(z.unknown());
+
+const supabaseWebhookSchema = z
+  .object({
+    type: z.enum(["INSERT", "UPDATE", "DELETE"]),
+    table: z.string(),
+    schema: z.string(),
+    record: supabaseUserRowSchema.nullish(),
+    old_record: supabaseUserRowSchema.nullish(),
+  })
+  .catchall(z.unknown());
+
+type SupabasePayload = z.infer<typeof supabaseWebhookSchema>;
+
+export const supabaseSource = defineWebhookSource({
+  meta: {
+    id: "supabase",
+    name: "Supabase",
+    description:
+      "Receives Supabase auth.users INSERT/UPDATE/DELETE webhooks (Svix-signed or shared-secret).",
+  },
+  auth: {
+    type: "signature",
+    scheme: "svix",
+    envKey: "SUPABASE_WEBHOOK_SECRET",
+    header: "svix-signature",
+    fallbackMatchHeader: "x-supabase-webhook-secret",
+  },
+  schema: supabaseWebhookSchema,
+  async transform(payload: SupabasePayload): Promise<IngestEvent | null> {
+    // Only auth.users mutations map to contacts; ignore everything else.
+    if (payload.schema !== "auth" || payload.table !== "users") {
+      return null;
+    }
+
+    let event: string;
+    switch (payload.type) {
+      case "INSERT":
+        event = "contact.created";
+        break;
+      case "UPDATE":
+        event = "contact.updated";
+        break;
+      case "DELETE":
+        event = "contact.deleted";
+        break;
+      default:
+        return null;
+    }
+
+    // DELETE carries the row in `old_record`; INSERT/UPDATE in `record`.
+    const row =
+      payload.type === "DELETE"
+        ? (payload.old_record ?? payload.record)
+        : (payload.record ?? payload.old_record);
+
+    if (!row) {
+      return null;
+    }
+
+    const userId = row.id;
+    if (!userId) {
+      return null;
+    }
+    const userEmail = typeof row.email === "string" ? row.email : "";
+
+    const eventProperties: Record<string, unknown> = {
+      source: "supabase",
+      supabaseUserId: userId,
+      _supabaseEvent: payload.type,
+    };
+
+    // Deletes carry no profile to merge — emit the event only (decision #15).
+    if (event === "contact.deleted") {
+      return {
+        event,
+        userId,
+        userEmail,
+        eventProperties,
+        contactProperties: {},
+      };
+    }
+
+    const contactProperties: Record<string, unknown> = {
+      ...(row.raw_user_meta_data ?? {}),
+    };
+    if (typeof row.phone === "string") {
+      contactProperties.phone = row.phone;
+    }
+    contactProperties.emailVerified = Boolean(row.email_confirmed_at);
+    contactProperties.supabaseUserId = userId;
+
+    return {
+      event,
+      userId,
+      userEmail,
+      eventProperties,
+      contactProperties,
+    };
+  },
+});

package/src/webhook-sources/verify.ts

@@ -0,0 +1,172 @@
+import { createHmac, timingSafeEqual } from "node:crypto";
+import { Webhook } from "svix";
+
+/**
+ * The signature verification schemes understood by `defineWebhookSource`'s
+ * `auth.type: "signature"` variant. Each preset (Clerk, Supabase, Stripe,
+ * Segment) maps to one of these; the route resolves the secret from
+ * `env[auth.envKey]` and calls {@link verifySignature} BEFORE parsing/handing
+ * the payload to `transform()`.
+ *
+ *  - `"svix"`      — Standard Webhooks / Svix header set (`svix-id` /
+ *                    `svix-timestamp` / `svix-signature`). Reuses
+ *                    `svix`'s `Webhook.verify` (the same machinery `plugin-resend`
+ *                    uses for inbound Resend webhooks).
+ *  - `"stripe"`    — `stripe-signature: t=<ts>,v1=<hex>[,v1=<hex>...]`. Computes
+ *                    `HMAC_SHA256(secret, `${t}.${rawBody}`)` with `node:crypto`
+ *                    (NO `stripe` SDK), constant-time compares each `v1` candidate,
+ *                    and enforces the 5-minute timestamp tolerance.
+ *  - `"hmac-hex"`  — Generic `HMAC_SHA256(secret, rawBody)` rendered as lowercase
+ *                    hex, constant-time compared against the header value (e.g.
+ *                    Segment's `x-signature`).
+ */
+export type SignatureScheme = "svix" | "stripe" | "hmac-hex";
+
+export interface VerifySignatureArgs {
+  rawBody: string;
+  headers: Record<string, string>;
+  secret: string;
+}
+
+const STRIPE_TOLERANCE_SECONDS = 5 * 60;
+
+/**
+ * Lowercase every header key so callers can pass the raw (possibly Title-Case)
+ * header record and we still find `svix-id` / `stripe-signature` / `x-signature`.
+ */
+function lowerHeaders(headers: Record<string, string>): Record<string, string> {
+  const lowered: Record<string, string> = {};
+  for (const [key, value] of Object.entries(headers)) {
+    lowered[key.toLowerCase()] = value;
+  }
+  return lowered;
+}
+
+/**
+ * Constant-time string comparison that never short-circuits on length. Returns
+ * `false` (rather than throwing) on a length mismatch so callers fail closed.
+ */
+function safeEqual(a: string, b: string): boolean {
+  const bufA = Buffer.from(a, "utf8");
+  const bufB = Buffer.from(b, "utf8");
+  if (bufA.length !== bufB.length) {
+    return false;
+  }
+  return timingSafeEqual(bufA, bufB);
+}
+
+function verifySvix(args: VerifySignatureArgs): boolean {
+  const headers = lowerHeaders(args.headers);
+  const id = headers["svix-id"];
+  const timestamp = headers["svix-timestamp"];
+  const signature = headers["svix-signature"];
+
+  if (!id || !timestamp || !signature) {
+    return false;
+  }
+
+  try {
+    const wh = new Webhook(args.secret);
+    wh.verify(args.rawBody, {
+      "svix-id": id,
+      "svix-timestamp": timestamp,
+      "svix-signature": signature,
+    });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function verifyStripe(args: VerifySignatureArgs): boolean {
+  const headers = lowerHeaders(args.headers);
+  const header = headers["stripe-signature"];
+  if (!header) {
+    return false;
+  }
+
+  // `t=1700000000,v1=<hex>,v1=<hex>` — there may be more than one v1 candidate
+  // during a secret rotation and forward-compat `v0`/scheme fields we ignore.
+  let timestamp: string | undefined;
+  const signatures: string[] = [];
+  for (const part of header.split(",")) {
+    const eq = part.indexOf("=");
+    if (eq === -1) {
+      continue;
+    }
+    const key = part.slice(0, eq).trim();
+    const value = part.slice(eq + 1).trim();
+    if (key === "t") {
+      timestamp = value;
+    } else if (key === "v1") {
+      signatures.push(value);
+    }
+  }
+
+  if (!timestamp || signatures.length === 0) {
+    return false;
+  }
+
+  const timestampSeconds = Number(timestamp);
+  if (!Number.isFinite(timestampSeconds)) {
+    return false;
+  }
+
+  const nowSeconds = Math.floor(Date.now() / 1000);
+  if (Math.abs(nowSeconds - timestampSeconds) > STRIPE_TOLERANCE_SECONDS) {
+    return false;
+  }
+
+  const expected = createHmac("sha256", args.secret)
+    .update(`${timestamp}.${args.rawBody}`)
+    .digest("hex");
+
+  return signatures.some((candidate) => safeEqual(candidate, expected));
+}
+
+function verifyHmacHex(args: VerifySignatureArgs, headerName: string): boolean {
+  const headers = lowerHeaders(args.headers);
+  const provided = headers[headerName.toLowerCase()];
+  if (!provided) {
+    return false;
+  }
+
+  const expected = createHmac("sha256", args.secret)
+    .update(args.rawBody)
+    .digest("hex");
+
+  return safeEqual(provided.trim(), expected);
+}
+
+/**
+ * Verify an inbound provider webhook signature for the given scheme.
+ *
+ * FAILS CLOSED: returns `false` (never throws) whenever a required header is
+ * missing or the signature does not match. The route enforces that the secret
+ * itself is present before calling this — an unset signature secret is a 401,
+ * NOT an open pass-through (deliberate divergence from the `"match"` variant,
+ * which stays open when unconfigured).
+ *
+ * For `"hmac-hex"` the header carrying the hex digest is passed via `headerName`
+ * (e.g. Segment's `x-signature`); `svix`/`stripe` read their own well-known
+ * headers and ignore `headerName`.
+ */
+export function verifySignature(
+  scheme: SignatureScheme,
+  args: VerifySignatureArgs,
+  headerName?: string,
+): boolean {
+  switch (scheme) {
+    case "svix":
+      return verifySvix(args);
+    case "stripe":
+      return verifyStripe(args);
+    case "hmac-hex":
+      return verifyHmacHex(args, headerName ?? "x-signature");
+    default: {
+      // Exhaustiveness guard — an unknown scheme fails closed.
+      const _never: never = scheme;
+      return _never;
+    }
+  }
+}

package/src/worker.ts

@@ -16,7 +16,15 @@ import {
 } from "./workflows/bucket-backfill.js";
 import { bucketReconcileTask } from "./workflows/bucket-reconcile.js";
 import { checkAlertsTask } from "./workflows/check-alerts.js";
+import {
+  deliverWebhookTask,
+  reapDueWebhookDeliveriesTask,
+} from "./workflows/deliver-webhook.js";
 import { importContactsTask } from "./workflows/import-contacts.js";
+import {
+  reapStuckCampaignsTask,
+  sendCampaignTask,
+} from "./workflows/send-campaign.js";
 import { sendEmailTask } from "./workflows/send-email.js";
 
 export interface CreateWorkerOptions {
@@ -62,6 +70,10 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
   const baseWorkflows = [
     sendEmailTask,
     importContactsTask,
+    sendCampaignTask,
+    reapStuckCampaignsTask,
+    deliverWebhookTask,
+    reapDueWebhookDeliveriesTask,
     checkAlertsTask,
     bucketReconcileTask,
     bucketBackfillTask,

package/src/workflows/bucket-backfill.ts

@@ -24,6 +24,7 @@ import {
 import { getBucketRegistrySingleton } from "../buckets/registry-singleton.js";
 import { getJourneyRegistrySingleton } from "../journeys/registry-singleton.js";
 import { emitBucketTransition } from "../lib/bucket-emit.js";
+import { contactKeySql } from "../lib/contacts.js";
 import { hatchet } from "../lib/hatchet.js";
 import type { Logger } from "../lib/logger.js";
 import { createLogger } from "../lib/logger.js";
@@ -231,16 +232,20 @@ async function backfillJoins(opts: {
   for (let i = 0; i < matcherIds.length; i += BATCH_SIZE) {
     const chunk = matcherIds.slice(i, i + BATCH_SIZE);
 
-    // userEmail backfilled from the contacts row where available.
+    // userEmail backfilled from the contacts row where available. The chunk
+    // holds the RESOLVED key (coalesce(external_id, anonymous_id, id)) — for an
+    // email-only / anonymous contact that is the anonymous_id or the uuid id, NOT
+    // the (null) external_id. Looking up by `contacts.externalId` would miss
+    // those rows and write a NULL userEmail despite the contact having an email,
+    // so we key the lookup + the map by the SAME coalesce expression the chunk
+    // carries (matches reconcileBucketJoins, which reads userId + email off one
+    // contacts row).
+    const resolvedKey = contactKeySql();
     const chunkContacts = await db
-      .select({ externalId: contacts.externalId, email: contacts.email })
+      .select({ userKey: resolvedKey, email: contacts.email })
       .from(contacts)
-      .where(
-        and(inArray(contacts.externalId, chunk), isNull(contacts.deletedAt)),
-      );
-    const emailByUser = new Map(
-      chunkContacts.map((c) => [c.externalId, c.email]),
-    );
+      .where(and(inArray(resolvedKey, chunk), isNull(contacts.deletedAt)));
+    const emailByUser = new Map(chunkContacts.map((c) => [c.userKey, c.email]));
 
     // Fix A: entryCount = 1 + prior memberships for each (user, bucket), the
     // same monotonic ordinal the live join computes (check-membership.ts). On a
@@ -456,7 +461,9 @@ async function selectEventMatchers(
       .as("present");
 
     const rows = await db
-      .select({ userId: contacts.externalId })
+      .select({
+        userId: contactKeySql(),
+      })
       .from(contacts)
       .innerJoin(everFired, eq(everFired.userId, contacts.externalId))
       .leftJoin(present, eq(present.userId, contacts.externalId))
@@ -496,12 +503,15 @@ async function selectEventMatchers(
  * a per-contact `evaluateCondition` loop over live contacts. Property
  * sub-conditions evaluate against the contact's merged properties.
  *
- * KEYSET PAGINATION by `contacts.externalId` in BATCH_SIZE pages (mirrors
- * reconcileBucketJoins' `externalId asc` paging): each page selects
- * `WHERE externalId > :cursor ORDER BY externalId ASC LIMIT BATCH_SIZE`,
- * evaluates the criteria per contact, then advances the cursor to the last
- * externalId of the page — repeating until a short page ends the scan. The whole
- * contacts table is never held in memory at once.
+ * KEYSET PAGINATION by `contacts.id` in BATCH_SIZE pages: each page selects
+ * `WHERE id > :cursor ORDER BY id ASC LIMIT BATCH_SIZE`, evaluates the criteria
+ * per contact, then advances the cursor to the last `id` of the page — repeating
+ * until a short page ends the scan. The whole contacts table is never held in
+ * memory at once. Paging on `id` (the non-null unique PK) — NOT `external_id`,
+ * which is nullable (email-only / anonymous contacts) and would drop every
+ * null-external_id row and order NULLs unstably. (reconcileBucketJoins is not a
+ * keyset scan — it relies on matchers dropping out as they become active
+ * members — so this no longer mirrors it.)
  */
 async function selectCompositeMatchers(
   db: Database,
@@ -513,17 +523,18 @@ async function selectCompositeMatchers(
   for (;;) {
     const page = await db
       .select({
-        externalId: contacts.externalId,
+        id: contacts.id,
+        userId: contactKeySql(),
         properties: contacts.properties,
       })
       .from(contacts)
       .where(
         and(
           isNull(contacts.deletedAt),
-          cursor != null ? gt(contacts.externalId, cursor) : undefined,
+          cursor != null ? gt(contacts.id, cursor) : undefined,
         ),
       )
-      .orderBy(sql`${contacts.externalId} asc`)
+      .orderBy(sql`${contacts.id} asc`)
       .limit(BATCH_SIZE);
 
     for (const contact of page) {
@@ -531,17 +542,17 @@ async function selectCompositeMatchers(
         condition: criteria,
         ctx: {
           db,
-          userId: contact.externalId,
+          userId: contact.userId,
           journeyContext:
             (contact.properties as Record<string, unknown> | null) ?? {},
         },
       });
-      if (isMember) matchers.push(contact.externalId);
+      if (isMember) matchers.push(contact.userId);
     }
 
     // A short page (fewer than a full batch) means the scan is exhausted.
     if (page.length < BATCH_SIZE) break;
-    cursor = page[page.length - 1]?.externalId ?? null;
+    cursor = page[page.length - 1]?.id ?? null;
     if (cursor == null) break;
   }
 

package/src/workflows/bucket-reconcile.ts

@@ -41,6 +41,7 @@ import {
 import { getBucketRegistrySingleton } from "../buckets/registry-singleton.js";
 import { getJourneyRegistrySingleton } from "../journeys/registry-singleton.js";
 import { emitBucketTransition } from "../lib/bucket-emit.js";
+import { contactKeySql } from "../lib/contacts.js";
 import { hatchet } from "../lib/hatchet.js";
 import type { Logger } from "../lib/logger.js";
 import { createLogger } from "../lib/logger.js";
@@ -841,18 +842,27 @@ async function reconcileBucketJoins(opts: {
     ? selectPresentInAllWindows(db, absenceLegs)
     : null;
 
+  // The membership/event tables key on the RESOLVED string key (external_id ??
+  // anonymous_id ?? contact.id), NOT necessarily external_id — email-only /
+  // anonymous contacts have a NULL external_id and are keyed on their uuid /
+  // anonymous_id. Joining on contacts.externalId would force external_id NOT NULL
+  // for every candidate (the coalesce would collapse to external_id) and silently
+  // drop exactly the dormant email-only contacts this cron exists to reconcile.
+  // Join on the SAME coalesce expression so the projected key matches the join.
+  const contactKey = contactKeySql();
+
   const baseQuery = db
     .select({
-      userId: contacts.externalId,
+      userId: contactKey,
       email: contacts.email,
     })
     .from(contacts)
-    .innerJoin(everFired, eq(everFired.userId, contacts.externalId))
-    .leftJoin(activeMembers, eq(activeMembers.userId, contacts.externalId));
+    .innerJoin(everFired, eq(everFired.userId, contactKey))
+    .leftJoin(activeMembers, eq(activeMembers.userId, contactKey));
 
   const candidates = await (presentInAll
     ? baseQuery
-        .leftJoin(presentInAll, eq(presentInAll.userId, contacts.externalId))
+        .leftJoin(presentInAll, eq(presentInAll.userId, contactKey))
         .where(
           and(
             isNull(contacts.deletedAt),
@@ -864,7 +874,12 @@ async function reconcileBucketJoins(opts: {
         and(isNull(contacts.deletedAt), isNull(activeMembers.userId)),
       )
   )
-    .orderBy(sql`${contacts.externalId} asc`)
+    // Deterministic scan order for the bounded re-run (no keyset cursor; the
+    // scan advances as reconciled matchers become active members and drop out).
+    // Order by contacts.id (the non-null unique PK) so the scan is null-safe and
+    // stable even for null-external_id contacts now that the join is on the
+    // coalesce key.
+    .orderBy(sql`${contacts.id} asc`)
     .limit(BATCH_SIZE);
 
   // SET-BASED / EXACT shapes (Fix #3) — every candidate row is a true matcher,