@hogsend/engine 0.7.0 → 0.9.0

package/src/lib/outbound.ts

@@ -0,0 +1,223 @@
+import { randomUUID } from "node:crypto";
+import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
+import {
+  type Database,
+  webhookDeliveries,
+  webhookEndpoints,
+} from "@hogsend/db";
+import { and, eq, isNull, sql } from "drizzle-orm";
+import { deliverWebhookTask } from "../workflows/deliver-webhook.js";
+import type { SerializedContact } from "./contacts.js";
+import type { Logger } from "./logger.js";
+import {
+  WEBHOOK_EVENT_TYPES,
+  type WebhookEventType,
+} from "./webhook-signing.js";
+
+export {
+  type ResendEmailSendContext,
+  resolveEmailSendContextByResendId,
+} from "./tracking-events.js";
+
+/**
+ * The outbound event catalog re-exported for the emit spine. Identical to the
+ * signing lib's {@link WEBHOOK_EVENT_TYPES} — the single source of truth.
+ */
+export const OUTBOUND_EVENTS = WEBHOOK_EVENT_TYPES;
+export type OutboundEventName = WebhookEventType;
+
+interface EmailEventPayload {
+  emailSendId: string;
+  resendId: string | null;
+  templateKey: string | null;
+  userId: string | null;
+  to: string;
+  at: string;
+  // Optional enrichment (additive — older subscribers ignore absent keys).
+  category?: string;
+  subject?: string;
+}
+
+interface BucketEventPayload {
+  bucketId: string;
+  bucketName: string;
+  userId: string;
+  userEmail: string | null;
+  transition: "entered" | "left";
+  entryCount: number;
+  source: string;
+}
+
+/**
+ * The typed per-event payload map. `data` in each delivered envelope is exactly
+ * `OutboundPayloads[E]` for the emitted event `E`. Producers (the 12 hook
+ * points) construct these; subscribers receive them under `envelope.data`.
+ */
+export interface OutboundPayloads {
+  "contact.created": SerializedContact;
+  "contact.updated": SerializedContact;
+  "contact.deleted": {
+    id: string;
+    externalId: string | null;
+    email: string | null;
+  };
+  "contact.unsubscribed": {
+    externalId: string | null;
+    email: string | null;
+    category: string | null;
+    scope: "all" | "category";
+  };
+  "email.sent": {
+    emailSendId: string;
+    resendId: string;
+    templateKey: string | null;
+    to: string;
+    userId: string | null;
+    category: string | null;
+    journeyStateId: string | null;
+    subject: string;
+    sentAt: string;
+  };
+  "email.delivered": EmailEventPayload;
+  "email.opened": EmailEventPayload;
+  "email.clicked": EmailEventPayload & { linkUrl?: string; linkId?: string };
+  "email.bounced": EmailEventPayload & {
+    bounceType?: string;
+    bounceReason?: string;
+  };
+  "email.complained": EmailEventPayload & {
+    complaintType?: string;
+    reason?: string;
+  };
+  "journey.completed": {
+    journeyId: string;
+    journeyName: string;
+    stateId: string;
+    userId: string;
+    userEmail: string;
+    completedAt: string;
+  };
+  "bucket.entered": BucketEventPayload;
+  "bucket.left": BucketEventPayload & { reason?: string };
+}
+
+/**
+ * The signed envelope shape written to `webhook_deliveries.payload` and sent
+ * verbatim to subscribers. `id` is the shared `Webhook-Id`; `timestamp` is the
+ * logical-event time (ISO); `data` is the typed per-event payload.
+ */
+interface OutboundEnvelope<E extends OutboundEventName> {
+  id: string;
+  type: E;
+  timestamp: string;
+  data: OutboundPayloads[E];
+}
+
+/**
+ * THE fire-and-forget emit spine. It does NOT deliver — it selects the active,
+ * subscribed endpoints for `event`, inserts one `webhook_deliveries` row per
+ * endpoint (all sharing ONE `webhookId` = the `Webhook-Id` header), and enqueues
+ * the durable {@link deliverWebhookTask} per inserted row.
+ *
+ * Idempotency: when `dedupeKey` is provided, the unique
+ * `(endpointId, dedupeKey)` index makes a re-emit (e.g. a Hatchet retry of the
+ * producing task) a no-op via `onConflictDoNothing` — no duplicate row, no
+ * second enqueue. Events without a `dedupeKey` are never deduped (NULL keys are
+ * distinct in Postgres), which is correct for non-retryable emit points.
+ *
+ * NEVER throws to callers. Internal failures (endpoint select, insert, enqueue)
+ * are logged via `logger.warn` and swallowed so a transient outbound error can
+ * never fail a contact upsert / email send / journey step. Callers MUST STILL
+ * wrap the call as `void emitOutbound(...).catch(logger.warn)` — the `.catch` is
+ * defence-in-depth against a programming error that escapes this guard.
+ *
+ * Single-tenant: only endpoints with `organizationId IS NULL` are selected and
+ * the delivery rows are written with `organizationId ?? null`. Multi-tenant
+ * scoping is a later non-breaking change.
+ */
+export async function emitOutbound<E extends OutboundEventName>(opts: {
+  db: Database;
+  hatchet: HatchetClient;
+  logger: Logger;
+  event: E;
+  payload: OutboundPayloads[E];
+  dedupeKey?: string;
+  organizationId?: string | null;
+}): Promise<void> {
+  const { db, logger, event, payload, dedupeKey } = opts;
+  const organizationId = opts.organizationId ?? null;
+
+  try {
+    const webhookId = `msg_${randomUUID()}`;
+    const timestamp = new Date();
+
+    // (2) Active, subscribed endpoints. `event_types @> '["<event>"]'` matches
+    // the jsonb array containing this event. Single-tenant: organizationId IS
+    // NULL (NOT a hardcoded tenant — keeps the MT wiring non-breaking).
+    const endpoints = await db
+      .select({ id: webhookEndpoints.id })
+      .from(webhookEndpoints)
+      .where(
+        and(
+          eq(webhookEndpoints.disabled, false),
+          isNull(webhookEndpoints.organizationId),
+          sql`${webhookEndpoints.eventTypes} @> ${JSON.stringify([event])}::jsonb`,
+        ),
+      );
+
+    if (endpoints.length === 0) return;
+
+    // (3) The frozen envelope — signed + sent verbatim by the delivery task.
+    const envelope: OutboundEnvelope<E> = {
+      id: webhookId,
+      type: event,
+      timestamp: timestamp.toISOString(),
+      data: payload,
+    };
+
+    // (4) One delivery row per endpoint, sharing the webhookId. onConflictDoNothing
+    // on (endpointId, dedupeKey) is the producer-side fan-out idempotency guard.
+    const inserted = await db
+      .insert(webhookDeliveries)
+      .values(
+        endpoints.map((endpoint) => ({
+          endpointId: endpoint.id,
+          organizationId,
+          webhookId,
+          eventType: event,
+          dedupeKey: dedupeKey ?? null,
+          payload: envelope as unknown as Record<string, unknown>,
+          status: "pending" as const,
+          attemptCount: 0,
+          nextRetryAt: timestamp,
+        })),
+      )
+      .onConflictDoNothing({
+        target: [webhookDeliveries.endpointId, webhookDeliveries.dedupeKey],
+      })
+      .returning({ id: webhookDeliveries.id });
+
+    // (5) Enqueue the durable delivery task per freshly-inserted row,
+    // fire-and-forget. A failed enqueue is recovered by the reaper (the row is
+    // already `pending` with `nextRetryAt <= now`), so a broker hiccup here only
+    // delays — never drops — a delivery.
+    for (const row of inserted) {
+      void deliverWebhookTask
+        .runNoWait({ deliveryId: row.id })
+        .catch((error: unknown) => {
+          logger.warn("emitOutbound: deliverWebhookTask enqueue failed", {
+            deliveryId: row.id,
+            event,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        });
+    }
+  } catch (error) {
+    // FAIL-SAFE: never propagate an outbound error onto the producer's hot path.
+    logger.warn("emitOutbound failed", {
+      event,
+      dedupeKey,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+}

package/src/lib/preferences.ts

@@ -2,6 +2,11 @@ import type { Database } from "@hogsend/db";
 import { emailPreferences } from "@hogsend/db";
 import { sql } from "drizzle-orm";
 import { resolveRecipient } from "./contacts.js";
+import { hatchet } from "./hatchet.js";
+import { createLogger } from "./logger.js";
+import { emitOutbound } from "./outbound.js";
+
+const logger = createLogger(process.env.LOG_LEVEL);
 
 /**
  * Single source of truth for an `email_preferences` upsert: the `(user_id, email)`
@@ -61,6 +66,32 @@ export async function upsertEmailPreference(opts: {
       target: [emailPreferences.userId, emailPreferences.email],
       set: setClause,
     });
+
+  // OUTBOUND `contact.unsubscribed` — this is the SINGLE choke for ALL preference
+  // writes (token unsub, preference center, list-membership flips), so the emit
+  // lives here once. GATED to a genuine opt-OUT only: a full unsubscribe
+  // (`unsubscribedAll === true`) or a category flip to false. A resubscribe
+  // (`unsubscribedAll === false` / `categoryValue === true`) does NOT emit. Uses
+  // the engine `hatchet`/`logger` singletons (this lib has no request container);
+  // fire-and-forget so a transient outbound error never fails the pref write.
+  const isUnsubscribe =
+    update.unsubscribedAll === true || update.categoryValue === false;
+  if (isUnsubscribe) {
+    const scope: "all" | "category" =
+      update.unsubscribedAll === true ? "all" : "category";
+    void emitOutbound({
+      db,
+      hatchet,
+      logger,
+      event: "contact.unsubscribed",
+      payload: {
+        externalId,
+        email,
+        category: update.categoryKey ?? null,
+        scope,
+      },
+    }).catch(logger.warn);
+  }
 }
 
 /**

package/src/lib/seed-posthog-destination.ts

@@ -0,0 +1,93 @@
+import { type Database, webhookEndpoints } from "@hogsend/db";
+import { and, eq, isNull, sql } from "drizzle-orm";
+import type { Logger } from "./logger.js";
+
+/**
+ * Transaction-scoped advisory-lock key serializing the single-tenant PostHog
+ * seed across concurrent API + worker boots. An arbitrary fixed constant within
+ * int4 range (the single-arg `pg_advisory_xact_lock` overload casts to bigint).
+ */
+const SEED_ADVISORY_LOCK_KEY = 1426198835;
+
+/**
+ * The email funnel a seeded PostHog destination subscribes to — the full
+ * lifecycle that reaches PostHog on NO path before this destination existed.
+ */
+const POSTHOG_FUNNEL_EVENTS = [
+  "email.sent",
+  "email.delivered",
+  "email.opened",
+  "email.clicked",
+  "email.bounced",
+  "email.complained",
+] as const;
+
+/**
+ * Idempotently seed ONE `kind="posthog"` webhook endpoint subscribed to the
+ * email funnel, so the full email lifecycle fans out to PostHog DURABLY on the
+ * delivery spine.
+ *
+ * Guarded against duplicates: it inserts only when no single-tenant
+ * (`organization_id IS NULL`) `kind="posthog"` endpoint already exists. Safe to
+ * call from BOTH the API and worker boots (both build the client) — the second
+ * caller finds the row and no-ops. Fire-and-forget at the call site: a transient
+ * seed failure must never block boot.
+ */
+export async function seedPostHogDestination(opts: {
+  db: Database;
+  logger: Logger;
+  apiKey: string;
+  host?: string;
+}): Promise<{ seeded: boolean }> {
+  const { db, logger, apiKey, host } = opts;
+
+  // Serialize the check-then-insert across concurrent API + worker boots (both
+  // build the client) with a transaction-scoped advisory lock, so the race can
+  // never double-seed. A per-endpoint unique constraint is intentionally avoided
+  // — operators may legitimately create multiple PostHog endpoints by hand.
+  return db.transaction(async (tx) => {
+    await tx.execute(
+      sql`SELECT pg_advisory_xact_lock(${SEED_ADVISORY_LOCK_KEY})`,
+    );
+
+    const existing = await tx
+      .select({ id: webhookEndpoints.id })
+      .from(webhookEndpoints)
+      .where(
+        and(
+          isNull(webhookEndpoints.organizationId),
+          eq(webhookEndpoints.kind, "posthog"),
+        ),
+      )
+      .limit(1);
+
+    if (existing.length > 0) {
+      return { seeded: false };
+    }
+
+    await tx.insert(webhookEndpoints).values({
+      url: "posthog://capture",
+      description:
+        "Auto-seeded PostHog destination (ENABLE_POSTHOG_DESTINATION)",
+      kind: "posthog",
+      config: {
+        apiKey,
+        ...(host ? { host } : {}),
+        // Preserve continuity with the legacy fire-and-forget PostHog path,
+        // which captured clicks as "email.link_clicked"; the posthog transform
+        // remaps the canonical "email.clicked" back so existing PostHog funnels
+        // keep working after the cutover.
+        eventNames: { "email.clicked": "email.link_clicked" },
+      },
+      eventTypes: [...POSTHOG_FUNNEL_EVENTS],
+      secret: null,
+      secretPrefix: null,
+      disabled: false,
+    });
+
+    logger.info("Seeded PostHog destination on the outbound spine", {
+      events: POSTHOG_FUNNEL_EVENTS.length,
+    });
+    return { seeded: true };
+  });
+}

package/src/lib/tracked.ts

@@ -20,7 +20,15 @@ import type {
   TrackedSendResult,
 } from "./email-service-types.js";
 import { isFrequencyCapped } from "./frequency-cap.js";
-import type { Logger } from "./logger.js";
+import { hatchet } from "./hatchet.js";
+import { createLogger, type Logger } from "./logger.js";
+import { emitOutbound } from "./outbound.js";
+
+// Module-level fallback logger for the outbound emit — the tracked-mailer's
+// `logger` dep is optional, but `emitOutbound` requires one. Mirrors the
+// `createLogger(process.env.LOG_LEVEL)` singleton pattern used elsewhere in the
+// engine libs (define-journey, preferences).
+const emitLogger = createLogger(process.env.LOG_LEVEL);
 
 export type PrepareTrackedHtmlFn = (opts: {
   html: string;
@@ -270,16 +278,50 @@ export async function sendTrackedEmail<K extends TemplateName>(
       replyTo: options.replyTo,
     });
 
+    const sentAt = new Date();
     await db
       .update(emailSends)
       .set({
         resendId: result.id,
         status: "sent",
-        sentAt: new Date(),
-        updatedAt: new Date(),
+        sentAt,
+        updatedAt: sentAt,
       })
       .where(eq(emailSends.id, emailSendId));
 
+    // OUTBOUND `email.sent` — fired ONLY on a real provider-accepted send (this
+    // success branch). Suppressed/frequency-capped/failed branches and the
+    // `db === undefined` mailer fallback do NOT reach here, so they never emit.
+    // `dedupeKey` = `email.sent:<emailSendId>`: this runs inside the tracked
+    // mailer which a journey (a Hatchet-retryable durable task) invokes, so a
+    // re-execution recomputes the identical key and the unique
+    // `(endpointId, dedupeKey)` index absorbs the duplicate. STRICTLY
+    // fire-and-forget: an un-caught reject here would bubble into the catch below
+    // and wrongly re-mark the (already sent) row `failed` (risk 2).
+    void emitOutbound({
+      db,
+      hatchet,
+      logger: logger ?? emitLogger,
+      event: "email.sent",
+      dedupeKey: `email.sent:${emailSendId}`,
+      payload: {
+        emailSendId,
+        resendId: result.id,
+        templateKey: options.templateKey,
+        to: options.to,
+        userId: options.userId ?? null,
+        category: effectiveCategory ?? null,
+        journeyStateId: options.journeyStateId ?? null,
+        subject,
+        sentAt: sentAt.toISOString(),
+      },
+    }).catch((err: unknown) => {
+      (logger ?? emitLogger).warn("emitOutbound email.sent failed", {
+        emailSendId,
+        error: err instanceof Error ? err.message : String(err),
+      });
+    });
+
     return {
       emailSendId,
       resendId: result.id,

package/src/lib/tracking-events.ts

@@ -1,5 +1,4 @@
 import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
-import type { PostHogService } from "@hogsend/core";
 import type { JourneyRegistry } from "@hogsend/core/registry";
 import { type Database, emailSends, journeyStates } from "@hogsend/db";
 import { eq } from "drizzle-orm";
@@ -10,6 +9,8 @@ interface EmailSendContext {
   userId: string;
   userEmail: string;
   templateKey: string | null;
+  resendId: string | null;
+  to: string;
 }
 
 export async function resolveEmailSendContext(
@@ -20,6 +21,7 @@ export async function resolveEmailSendContext(
     .select({
       toEmail: emailSends.toEmail,
       templateKey: emailSends.templateKey,
+      resendId: emailSends.resendId,
       userId: journeyStates.userId,
       userEmail: journeyStates.userEmail,
     })
@@ -35,6 +37,58 @@ export async function resolveEmailSendContext(
     userId: row.userId ?? row.toEmail,
     userEmail: row.userEmail ?? row.toEmail,
     templateKey: row.templateKey,
+    resendId: row.resendId,
+    to: row.toEmail,
+  };
+}
+
+export interface ResendEmailSendContext {
+  emailSendId: string;
+  userId: string;
+  userEmail: string;
+  templateKey: string | null;
+  to: string;
+}
+
+/**
+ * Mirror of {@link resolveEmailSendContext} that resolves by the provider's
+ * `resendId` instead of the internal `email_sends.id`. Used by the
+ * provider-webhook enrichment path (`email.delivered`/`email.bounced`) where the
+ * only handle we hold is the Resend `email_id`.
+ *
+ * Returns the internal `emailSendId` plus the same denormalized identity
+ * (`userId`/`userEmail` fall back to the recipient address, exactly like the
+ * id-keyed resolver) and `to` recipient. Returns null when no send row carries
+ * that `resendId` yet (e.g. a webhook arriving before the send row is committed).
+ */
+export async function resolveEmailSendContextByResendId(
+  db: Database,
+  resendId: string,
+): Promise<ResendEmailSendContext | null> {
+  const rows = await db
+    .select({
+      emailSendId: emailSends.id,
+      toEmail: emailSends.toEmail,
+      templateKey: emailSends.templateKey,
+      userId: journeyStates.userId,
+      userEmail: journeyStates.userEmail,
+      sendUserId: emailSends.userId,
+      sendUserEmail: emailSends.userEmail,
+    })
+    .from(emailSends)
+    .leftJoin(journeyStates, eq(emailSends.journeyStateId, journeyStates.id))
+    .where(eq(emailSends.resendId, resendId))
+    .limit(1);
+
+  const row = rows[0];
+  if (!row) return null;
+
+  return {
+    emailSendId: row.emailSendId,
+    userId: row.userId ?? row.sendUserId ?? row.toEmail,
+    userEmail: row.userEmail ?? row.sendUserEmail ?? row.toEmail,
+    templateKey: row.templateKey,
+    to: row.toEmail,
   };
 }
 
@@ -43,18 +97,37 @@ export interface PushTrackingEventOpts {
   hatchet: HatchetClient;
   registry: JourneyRegistry;
   logger: Logger;
-  posthog?: PostHogService;
   event: string;
   emailSendId: string;
   properties?: Record<string, unknown>;
+  /**
+   * Pre-resolved send context. When provided (including `null`), the duplicate
+   * `resolveEmailSendContext` read is skipped — the tracking routes resolve once
+   * and share the result with the outbound emit on the hot path. Omit to resolve
+   * lazily.
+   */
+  resolvedContext?: EmailSendContext | null;
 }
 
+/**
+ * Re-push a first-party tracking event (open/click) back onto the INTERNAL bus
+ * (`ingestEvent`) for journey routing + `userEvents` persistence.
+ *
+ * NOTE (Phase 2): this NO LONGER fires a fire-and-forget PostHog `captureEvent`.
+ * PostHog now receives opens/clicks PER-HIT via the durable outbound spine — a
+ * `kind="posthog"` destination subscribed to `email.opened`/`email.clicked` (the
+ * tracking routes call `emitOutbound` alongside this). The legacy double-emit was
+ * removed so PostHog gets exactly one, durable copy of each hit.
+ */
 export async function pushTrackingEvent(
   opts: PushTrackingEventOpts,
 ): Promise<void> {
-  const { db, hatchet, registry, logger, posthog, event, emailSendId } = opts;
+  const { db, hatchet, registry, logger, event, emailSendId } = opts;
 
-  const ctx = await resolveEmailSendContext(db, emailSendId);
+  const ctx =
+    opts.resolvedContext !== undefined
+      ? opts.resolvedContext
+      : await resolveEmailSendContext(db, emailSendId);
   if (!ctx) return;
 
   const properties: Record<string, unknown> = {
@@ -63,12 +136,6 @@ export async function pushTrackingEvent(
     ...opts.properties,
   };
 
-  posthog?.captureEvent({
-    distinctId: ctx.userId,
-    event,
-    properties,
-  });
-
   await ingestEvent({
     db,
     registry,