@hogsend/engine 0.22.0 → 0.23.1

package/src/lib/ingestion.ts

@@ -1,9 +1,14 @@
 import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
+import type { AnalyticsProvider } from "@hogsend/core";
 import { evaluatePropertyConditions } from "@hogsend/core";
 import type { JourneyRegistry } from "@hogsend/core/registry";
 import { type Database, journeyStates, userEvents } from "@hogsend/db";
 import { and, eq, inArray, isNull } from "drizzle-orm";
 import { checkBucketMembership } from "../buckets/check-membership.js";
+import {
+  logResidualTwins,
+  mergeAnalyticsIdentities,
+} from "./analytics-identity.js";
 import { resolveOrCreateContact } from "./contacts.js";
 import type { Logger } from "./logger.js";
 
@@ -58,8 +63,17 @@ export async function ingestEvent(opts: {
   hatchet: HatchetClient;
   logger: Logger;
   event: IngestEvent;
+  /**
+   * The active analytics provider (`c.get("container").analytics`). When the
+   * identity resolve folds two keys into one (collide-MERGE or canonical-key
+   * flip), the engine fires the provider-neutral `mergeIdentities` primitive so
+   * the analytics person store stitches the same way the contact store did
+   * (§5.3). Optional: absent ⇒ DB-only resolve (no stitch), exactly as before; a
+   * provider without `identityMerge` no-ops cleanly.
+   */
+  analytics?: AnalyticsProvider;
 }): Promise<IngestResult> {
-  const { db, registry, hatchet, logger, event } = opts;
+  const { db, registry, hatchet, logger, event, analytics } = opts;
 
   // (1) Resolve identity FIRST (awaited — no longer fire-and-forget). The
   // contact-referencing tables join on a NOT NULL text key, so an email-only /
@@ -68,7 +82,13 @@ export async function ingestEvent(opts: {
   // `contacts.properties` (D2 split) and returns BOTH the canonical contact id
   // AND its resolved string key (external_id ?? anonymous_id ?? contact.id —
   // risk 1/6), so no second read-back of the contact row is needed.
-  const { resolvedKey } = await resolveOrCreateContact({
+  const {
+    id: contactId,
+    resolvedKey,
+    mergedKeys,
+    mergedIdentifiedKeys,
+    merged,
+  } = await resolveOrCreateContact({
     db,
     userId: event.userId,
     email: event.userEmail || undefined,
@@ -112,6 +132,36 @@ export async function ingestEvent(opts: {
     });
   }
 
+  // (2b) §5.3 — fire the provider-neutral identity merge at the two resolver
+  // outcomes where two keys fold into one (collide-MERGE or canonical-key flip).
+  // Placed INSIDE the idempotency-guarded block (after a FRESH insert; the
+  // duplicate path returned early above) so a Hatchet/client retry with the same
+  // idempotencyKey does NOT re-fire `alias` — honoring the "only at the moment
+  // two keys first become one" contract (PostHog `alias` is harmless on replay
+  // but firing per-retry adds queue noise). MF-2: `mergedKeys` already excludes
+  // identified `external_id`s (the resolver split them out); fire only the safe
+  // anon/uuid keys, and surface the excluded identified twins for observability.
+  if (mergedKeys?.length || mergedIdentifiedKeys?.length) {
+    if (mergedKeys?.length) {
+      mergeAnalyticsIdentities({
+        analytics,
+        survivorKey: resolvedKey,
+        loserKeys: mergedKeys,
+        reason: merged ? "collide_merge" : "key_flip",
+        contactId,
+        logger,
+      });
+    }
+    if (mergedIdentifiedKeys?.length) {
+      logResidualTwins({
+        survivorKey: resolvedKey,
+        identifiedLoserKeys: mergedIdentifiedKeys,
+        contactId,
+        logger,
+      });
+    }
+  }
+
   // (3) Build the JSON-serializable subset of eventProperties for the Hatchet
   // push payload (scalars only — the SDK serializes the envelope).
   const serializableProperties = Object.fromEntries(

package/src/lib/outbound.ts

@@ -90,6 +90,23 @@ export interface OutboundPayloads {
   "email.delivered": EmailEventPayload;
   "email.opened": EmailEventPayload;
   "email.clicked": EmailEventPayload & { linkUrl?: string; linkId?: string };
+  /**
+   * A NON-email tracked link was clicked (Discord/referral/ad-hoc
+   * `createTrackedLink`). The deliberate counterpart to `email.clicked` — a
+   * non-email click has no `email_sends` row, so it carries `emailSendId: null`
+   * and `messageId: null` and never masquerades as an email click
+   * (MF-missing #3). `userId` is the link's stitch subject (`distinct_id`) when
+   * the link is identity-bearing, else null for a broadcast link.
+   */
+  "link.clicked": {
+    linkId: string;
+    source: string | null;
+    userId: string | null;
+    emailSendId: null;
+    messageId: null;
+    linkUrl: string;
+    at: string;
+  };
   /**
    * A SEMANTIC link answered — the in-email action event (consumer-named, e.g.
    * "nps.submitted"). Emitted at most once per (send, event name): first

package/src/lib/semantic-click.ts

@@ -90,6 +90,15 @@ export async function confirmSemanticClick(
   if (!link?.event) {
     return { status: "skipped", reason: "not_semantic" };
   }
+  // The confirm path is EMAIL-semantic end to end (it claims a send's answer
+  // slot keyed on `emailSendId` and emits `email.action`). The click route only
+  // enqueues this task for links with a non-null `emailSendId`, but `emailSendId`
+  // is nullable since the identity-stitching minor — guard defensively and
+  // narrow the type for the rest of the function.
+  if (!link.emailSendId) {
+    return { status: "skipped", reason: "non_email_link" };
+  }
+  const emailSendId = link.emailSendId;
   const semanticEvent = link.event;
 
   // (1) Let the burst window close before judging the click.
@@ -109,7 +118,7 @@ export async function confirmSemanticClick(
     .innerJoin(trackedLinks, eq(linkClicks.trackedLinkId, trackedLinks.id))
     .where(
       and(
-        eq(trackedLinks.emailSendId, link.emailSendId),
+        eq(trackedLinks.emailSendId, emailSendId),
         gte(linkClicks.clickedAt, windowStart),
         lte(linkClicks.clickedAt, windowEnd),
       ),
@@ -117,7 +126,7 @@ export async function confirmSemanticClick(
   const distinctLinks = burst[0]?.n ?? 0;
   if (distinctLinks >= SEMANTIC_BURST_DISTINCT_LINKS) {
     logger.warn("Semantic answer suppressed: scanner-like click burst", {
-      emailSendId: link.emailSendId,
+      emailSendId,
       linkId: link.id,
       event: semanticEvent,
       distinctLinks,
@@ -125,21 +134,21 @@ export async function confirmSemanticClick(
     return { status: "suppressed", distinctLinks };
   }
 
-  const ctx = await resolveEmailSendContext(db, link.emailSendId);
+  const ctx = await resolveEmailSendContext(db, emailSendId);
   if (!ctx) {
     return { status: "skipped", reason: "no_send_context" };
   }
 
   // (3) Claim the answer slot. Duplicate key → stored=false BEFORE the Hatchet
   // push, so journeys/destinations see at most one answer per (send, event).
-  const semKey = `sem:${link.emailSendId}:${semanticEvent}`;
+  const semKey = `sem:${emailSendId}:${semanticEvent}`;
   const result = await pushTrackingEvent({
     db,
     hatchet,
     registry,
     logger,
     event: semanticEvent,
-    emailSendId: link.emailSendId,
+    emailSendId,
     properties: {
       ...(link.eventProperties ?? {}),
       linkId: link.id,
@@ -180,7 +189,7 @@ export async function confirmSemanticClick(
     payload: {
       event: semanticEvent,
       properties: link.eventProperties ?? null,
-      emailSendId: link.emailSendId,
+      emailSendId,
       templateKey: ctx.templateKey ?? null,
       userId: ctx.userId ?? null,
       to: ctx.to ?? ctx.userEmail ?? "",

package/src/lib/tracking-events.ts

@@ -15,8 +15,12 @@ interface EmailSendContext {
 
 export async function resolveEmailSendContext(
   db: Database,
-  emailSendId: string,
+  emailSendId: string | null,
 ): Promise<EmailSendContext | null> {
+  // A non-email tracked link (Discord/referral/ad-hoc `createTrackedLink`) has
+  // a NULL `email_send_id` — there is no send row to resolve, so short-circuit
+  // to null rather than issue a `WHERE id = NULL` query that matches nothing.
+  if (!emailSendId) return null;
   const rows = await db
     .select({
       toEmail: emailSends.toEmail,

package/src/lib/tracking.ts

@@ -240,3 +240,40 @@ export async function prepareTrackedHtml(opts: {
   });
   return result;
 }
+
+/**
+ * The mint surface for a NON-email tracked link (Discord, referral, ad-hoc).
+ * Inserts a `tracked_links` row with a NULL `emailSendId` and returns the
+ * `/v1/t/c/:id` redirect URL to use in place of the raw destination.
+ *
+ * This is the SINGLE chokepoint enforcing "broadcast links carry no subject":
+ * a link only becomes identity-bearing when the caller EXPLICITLY passes
+ * `distinctId` (the canonical contact key the click should stitch into). Per
+ * MF-4, the referral path does NOT pass `distinctId` by default (referral
+ * pages are shareable → broadcast), and the Discord destination passes
+ * `distinctId: undefined`. The `hs_t` mint at click time is still gated by
+ * `TRACKING_IDENTITY_TOKEN` (default false); a row with a NULL `distinctId`
+ * never mints a token regardless.
+ */
+export async function createTrackedLink(opts: {
+  db: Database;
+  url: string;
+  /**
+   * The canonical contact key a click should fold the visitor's anon session
+   * into. OMIT for a broadcast link (the safe default) — only an explicit,
+   * single-subject, non-shareable link should pass this.
+   */
+  distinctId?: string;
+  source: "discord" | "referral" | "link";
+  baseUrl: string;
+}): Promise<string> {
+  const id = randomUUID();
+  await opts.db.insert(trackedLinks).values({
+    id,
+    emailSendId: null,
+    distinctId: opts.distinctId ?? null,
+    source: opts.source,
+    originalUrl: opts.url,
+  });
+  return `${opts.baseUrl}/v1/t/c/${id}`;
+}

package/src/lib/webhook-signing.ts

@@ -25,9 +25,14 @@ import { Webhook } from "svix";
  */
 
 /**
- * The 14-event catalog — the SINGLE source of truth (schema, routes, client,
+ * The 15-event catalog — the SINGLE source of truth (schema, routes, client,
  * CLI all derive from this). The `webhook.test` sentinel is intentionally NOT a
  * member (it is delivered out-of-band regardless of an endpoint's `eventTypes`).
+ *
+ * `link.clicked` is the NON-email click event: a click on a tracked link that
+ * has no email send (Discord/referral/ad-hoc `createTrackedLink`). It is the
+ * deliberate counterpart to `email.clicked` so a non-email click never fires a
+ * malformed `email.clicked` (MF-missing #3).
  */
 export const WEBHOOK_EVENT_TYPES = [
   "contact.created",
@@ -44,6 +49,7 @@ export const WEBHOOK_EVENT_TYPES = [
   "journey.completed",
   "bucket.entered",
   "bucket.left",
+  "link.clicked",
 ] as const;
 
 export type WebhookEventType = (typeof WEBHOOK_EVENT_TYPES)[number];

package/src/routes/admin/bulk.ts

@@ -144,6 +144,12 @@ const replayRoute = createRoute({
       },
       description: "Replay results",
     },
+    400: {
+      content: {
+        "application/json": { schema: errorSchema },
+      },
+      description: "No replay selection (eventIds or filter) provided",
+    },
   },
 });
 
@@ -362,14 +368,26 @@ export const bulkRouter = new OpenAPIHono<AppEnv>()
         conditions.push(lte(userEvents.occurredAt, new Date(body.filter.to)));
       }
 
-      const where = conditions.length > 0 ? and(...conditions) : undefined;
+      // Refuse an unscoped replay. With no `eventIds` and no filter the WHERE
+      // would collapse to `undefined`, silently re-pushing the most-recent
+      // `limit` events back through the full ingestion pipeline (re-triggering
+      // journeys, re-evaluating exits). Require an explicit selection.
+      if (conditions.length === 0) {
+        return c.json(
+          {
+            error:
+              "Replay requires `eventIds` or at least one `filter` field (event, userId, from, to).",
+          },
+          400,
+        );
+      }
 
       events = await db
         .select()
         .from(userEvents)
-        .where(where)
+        .where(and(...conditions))
         .orderBy(desc(userEvents.occurredAt))
-        .limit(body.limit ?? 100);
+        .limit(body.limit);
     }
 
     let replayed = 0;

package/src/routes/admin/preferences.ts

@@ -147,6 +147,14 @@ export const preferencesRouter = new OpenAPIHono<AppEnv>()
             ? {
                 suppressed: body.suppressed,
                 suppressedAt: body.suppressed ? new Date() : null,
+                // Un-suppressing clears the bounce slate. `bounceCount` only
+                // drives the auto-suppress threshold (the send-gate keys off
+                // `suppressed`/`unsubscribedAll`), so a leftover count would
+                // otherwise keep a bounced recipient pinned to the suppression
+                // list with no way to remove them.
+                ...(body.suppressed
+                  ? {}
+                  : { bounceCount: 0, lastBounceAt: null }),
               }
             : {}),
           ...(body.categories !== undefined

package/src/routes/admin/reporting.ts

@@ -362,8 +362,19 @@ reportingRouter.get("/sends/export", async (c) => {
     );
   }
 
+  // The export intentionally returns all matching sends, but is hard-capped at
+  // MAX_EXPORT_ROWS. Signal when the result was truncated so a caller never
+  // mistakes a partial CSV for the complete history.
+  const truncated = rows.length >= MAX_EXPORT_ROWS;
+
   return c.body(lines.join("\n"), 200, {
     "Content-Type": "text/csv; charset=utf-8",
     "Content-Disposition": 'attachment; filename="email-sends.csv"',
+    ...(truncated
+      ? {
+          "X-Hogsend-Export-Truncated": "true",
+          "X-Hogsend-Export-Limit": String(MAX_EXPORT_ROWS),
+        }
+      : {}),
   });
 });

package/src/routes/admin/suppressions.ts

@@ -1,6 +1,6 @@
 import { emailPreferences } from "@hogsend/db";
 import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
-import { and, count, desc, eq, gt, type SQL } from "drizzle-orm";
+import { and, count, desc, eq, gt, or, type SQL } from "drizzle-orm";
 import type { AppEnv } from "../../app.js";
 import { serializePrefs } from "../../lib/contacts.js";
 
@@ -8,6 +8,11 @@ import { serializePrefs } from "../../lib/contacts.js";
 // `complained` has no dedicated column — a complaint sets `suppressed` without
 // incrementing `bounceCount` (see mailer `handleComplaint`), so we identify it
 // as suppressed-but-not-bounced.
+//
+// IMPORTANT: the `email_preferences` table holds a row for (nearly) every
+// contact, most of whom are NOT suppressed. The "All" view must therefore
+// restrict to recipients suppressed in *some* way — returning `undefined`
+// here would drop the WHERE clause entirely and list every contact.
 function typeFilter(
   type: "bounced" | "unsubscribed" | "complained" | undefined,
 ): SQL | undefined {
@@ -22,7 +27,12 @@ function typeFilter(
         eq(emailPreferences.bounceCount, 0),
       );
     default:
-      return undefined;
+      // "All" = the union of every suppression reason.
+      return or(
+        eq(emailPreferences.suppressed, true),
+        eq(emailPreferences.unsubscribedAll, true),
+        gt(emailPreferences.bounceCount, 0),
+      );
   }
 }
 

package/src/routes/contacts/index.ts

@@ -39,6 +39,10 @@ const upsertRoute = createRoute({
           schema: z.object({
             email: z.string().email().optional(),
             userId: z.string().min(1).optional(),
+            // §4: caller's analytics anon id — the resolver's 2nd-precedence
+            // key. An EXTRA, never a third identity arm: `requireIdentity`
+            // still requires email or userId below.
+            anonymousId: z.string().min(1).max(200).optional(),
             properties: z.record(z.string(), z.unknown()).optional(),
             lists: z.record(z.string(), z.boolean()).optional(),
           }),
@@ -142,6 +146,9 @@ export const contactsRouter = new OpenAPIHono<AppEnv>()
       db,
       userId: body.userId,
       email: body.email,
+      // §4: 2nd-precedence resolver key (zero-merge stitch). Identity is still
+      // enforced via `requireIdentity` (email/userId) above.
+      anonymousId: body.anonymousId,
       contactProperties: body.properties,
     });
 

package/src/routes/events/index.ts

@@ -9,6 +9,14 @@ const eventRequestSchema = z.object({
   name: z.string().min(1),
   email: z.string().email().optional(),
   userId: z.string().min(1).optional(),
+  // §4: the caller's analytics anon id (e.g. posthog-js `get_distinct_id()`).
+  // 2nd in the resolver's key precedence (`external → email → anonymous →
+  // discord`), so when no `external_id` is attached the contact's canonical key
+  // BECOMES this value — the browser's own anon events and the server's captures
+  // then land on ONE analytics person with zero merge calls. An EXTRA, never a
+  // third identity arm: `requireIdentity` still requires email or userId
+  // (anon-only public ingest is an abuse vector).
+  anonymousId: z.string().min(1).max(200).optional(),
   eventProperties: z.record(z.string(), z.unknown()).optional(),
   contactProperties: z.record(z.string(), z.unknown()).optional(),
   lists: z.record(z.string(), z.boolean()).optional(),
@@ -68,7 +76,7 @@ const eventRoute = createRoute({
 export const eventsRouter = new OpenAPIHono<AppEnv>().openapi(
   eventRoute,
   async (c) => {
-    const { db, registry, hatchet, logger } = c.get("container");
+    const { db, registry, hatchet, logger, analytics } = c.get("container");
     const body = c.req.valid("json");
 
     const guard = requireIdentity(c, body);
@@ -83,10 +91,17 @@ export const eventsRouter = new OpenAPIHono<AppEnv>().openapi(
       registry,
       hatchet,
       logger,
+      // §5.3: thread the active analytics provider so a collide-MERGE / key-flip
+      // fires the provider-neutral `mergeIdentities` stitch. Absent ⇒ no-op.
+      analytics,
       event: {
         event: body.name,
         userId: body.userId,
         userEmail: body.email,
+        // §4: 2nd-precedence resolver key — lets the contact's canonical key
+        // equal the browser anon id (zero-merge stitch). Identity is still
+        // enforced via `requireIdentity` (email/userId) above.
+        anonymousId: body.anonymousId,
         eventProperties: body.eventProperties ?? {},
         contactProperties: body.contactProperties,
         idempotencyKey,

package/src/routes/tracking/answer.ts

@@ -150,8 +150,15 @@ export const answerRouter = new OpenAPIHono<AppEnv>()
       );
     }
 
-    const ctx = await resolveEmailSendContext(db, link.emailSendId);
-    if (ctx) {
+    // The answer/comment flow is EMAIL-semantic (it re-ingests a
+    // `<event>.comment` keyed on the send). A non-email semantic link has no
+    // send to attribute the comment to — `emailSendId` is nullable since the
+    // identity-stitching minor, so narrow it here.
+    const emailSendId = link.emailSendId;
+    const ctx = emailSendId
+      ? await resolveEmailSendContext(db, emailSendId)
+      : null;
+    if (ctx && emailSendId) {
       // `<event>.comment` is a consumer-namespace event — journeys can wait
       // on it and destinations receive it like any other. First comment per
       // (send, event) wins; repeats are no-ops.
@@ -161,7 +168,7 @@ export const answerRouter = new OpenAPIHono<AppEnv>()
         registry,
         logger,
         event: `${link.event}.comment`,
-        emailSendId: link.emailSendId,
+        emailSendId,
         properties: {
           comment,
           parentEvent: link.event,
@@ -169,7 +176,7 @@ export const answerRouter = new OpenAPIHono<AppEnv>()
           linkId: link.id,
         },
         resolvedContext: ctx,
-        idempotencyKey: `semc:${link.emailSendId}:${link.event}`,
+        idempotencyKey: `semc:${emailSendId}:${link.event}`,
       }).catch((err) => {
         logger.warn("Failed to ingest answer comment", {
           linkId: link.id,