@hogsend/engine 0.39.0 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/package.json +9 -7
  2. package/src/buckets/check-membership.ts +6 -1
  3. package/src/campaigns/cohort-sql.ts +198 -0
  4. package/src/campaigns/define-campaign.ts +301 -0
  5. package/src/campaigns/reconcile.ts +391 -0
  6. package/src/campaigns/steps.ts +48 -0
  7. package/src/connectors/define-action.ts +70 -0
  8. package/src/container.ts +144 -42
  9. package/src/destinations/presets/posthog.ts +7 -1
  10. package/src/env.ts +14 -0
  11. package/src/index.ts +98 -3
  12. package/src/journeys/constants.ts +10 -0
  13. package/src/journeys/define-journey.ts +14 -405
  14. package/src/journeys/execute-journey-run.ts +608 -0
  15. package/src/journeys/graph/build-graph.ts +212 -56
  16. package/src/journeys/journey-boundary.ts +54 -0
  17. package/src/journeys/journey-context.ts +374 -59
  18. package/src/journeys/record-once.ts +110 -0
  19. package/src/lib/auth.ts +33 -1
  20. package/src/lib/blueprints.ts +576 -0
  21. package/src/lib/campaign-send-key.ts +58 -0
  22. package/src/lib/connector-actions.ts +203 -19
  23. package/src/lib/contacts.ts +54 -0
  24. package/src/lib/email-service-types.ts +4 -1
  25. package/src/lib/email.ts +13 -2
  26. package/src/lib/feed.ts +77 -57
  27. package/src/lib/frequency-cap.ts +45 -15
  28. package/src/lib/hatchet-duration.ts +22 -0
  29. package/src/lib/html.ts +1 -0
  30. package/src/lib/ingestion.ts +145 -4
  31. package/src/lib/journey-suppress.ts +87 -0
  32. package/src/lib/links.ts +204 -18
  33. package/src/lib/outbound.ts +20 -0
  34. package/src/lib/preferences.ts +80 -1
  35. package/src/lib/tracked.ts +44 -29
  36. package/src/lib/tracking-event-names.ts +8 -0
  37. package/src/lib/webhook-signing.ts +6 -1
  38. package/src/lists/channels.ts +75 -0
  39. package/src/lists/define-list.ts +34 -0
  40. package/src/lists/registry.ts +29 -2
  41. package/src/mcp/blueprint-tools.ts +460 -0
  42. package/src/routes/admin/blueprints.ts +711 -0
  43. package/src/routes/admin/campaigns.ts +341 -0
  44. package/src/routes/admin/emails.ts +13 -0
  45. package/src/routes/admin/index.ts +6 -0
  46. package/src/routes/admin/journeys.ts +4 -2
  47. package/src/routes/admin/links.ts +303 -36
  48. package/src/routes/admin/lists.ts +51 -0
  49. package/src/routes/campaigns/index.ts +297 -50
  50. package/src/routes/email/preferences.ts +50 -13
  51. package/src/routes/feed/recipient.ts +2 -43
  52. package/src/routes/index.ts +6 -0
  53. package/src/routes/lists/index.ts +138 -18
  54. package/src/routes/tracking/arrive.ts +260 -0
  55. package/src/routes/tracking/click-pipeline.ts +329 -0
  56. package/src/routes/tracking/click.ts +5 -270
  57. package/src/routes/tracking/index.ts +2 -0
  58. package/src/routes/tracking/vanity.ts +64 -0
  59. package/src/worker.ts +27 -0
  60. package/src/workflows/bucket-backfill.ts +4 -2
  61. package/src/workflows/bucket-reconcile.ts +11 -4
  62. package/src/workflows/check-alerts.ts +130 -1
  63. package/src/workflows/journey-blueprint-interpreter.ts +561 -0
  64. package/src/workflows/send-campaign.ts +673 -127
@@ -1,6 +1,6 @@
1
1
  import type { EmailProvider } from "@hogsend/core";
2
2
  import type { Database } from "@hogsend/db";
3
- import { emailPreferences, emailSends } from "@hogsend/db";
3
+ import { emailSends } from "@hogsend/db";
4
4
  import type {
5
5
  EmailSuppressionError,
6
6
  RetryOptions,
@@ -30,8 +30,10 @@ import {
30
30
  } from "./email-service-types.js";
31
31
  import { isFrequencyCapped } from "./frequency-cap.js";
32
32
  import { hatchet } from "./hatchet.js";
33
+ import { checkJourneySuppress } from "./journey-suppress.js";
33
34
  import { createLogger, type Logger } from "./logger.js";
34
35
  import { emitOutbound } from "./outbound.js";
36
+ import { readRecipientPreferences } from "./preferences.js";
35
37
  import {
36
38
  buildRedirect,
37
39
  isUnaddressable,
@@ -257,6 +259,38 @@ async function sendTrackedEmailInner<K extends TemplateName>(
257
259
  });
258
260
  }
259
261
  }
262
+
263
+ // Journey suppress (`meta.suppress`) — a per-recipient min-gap flooding
264
+ // guard, enforced ONLY for journey-bound sends (a JourneyBoundary is
265
+ // present) whose journey sets suppress > 0. Slotted ADJACENT to the
266
+ // frequency cap so the skip mirrors it exactly: on a hit there is no
267
+ // provider call, no `email_sends` row, and no throw — the journey run
268
+ // continues. It MUST run AFTER the idempotency short-circuit above: a replay
269
+ // of an already-dispatched send has to return that dup (from its prior
270
+ // `sent` row) BEFORE any suppress evaluation, or the second drive would
271
+ // wrongly re-evaluate suppression for a send that already went out. The
272
+ // verdict itself is RECORDED set-once (see checkJourneySuppress) so it is
273
+ // replay-stable. Non-journey sends (no boundary: transactional API /
274
+ // password-reset flows) are inert here.
275
+ const boundary = getJourneyBoundary();
276
+ const suppress = await checkJourneySuppress({
277
+ db,
278
+ boundary,
279
+ to: options.to,
280
+ idempotencyKey: options.idempotencyKey,
281
+ });
282
+ if (suppress.suppressed) {
283
+ logger?.info("send skipped: journey_suppressed", {
284
+ to: options.to,
285
+ journeyId: boundary?.journeyId,
286
+ });
287
+ return trackedSendResult({
288
+ emailSendId: "",
289
+ messageId: "",
290
+ status: "skipped",
291
+ reason: "journey_suppressed",
292
+ });
293
+ }
260
294
  }
261
295
 
262
296
  // Unsubscribe surface (RFC 8058 / CAN-SPAM): generate the per-recipient
@@ -553,19 +587,13 @@ async function checkSuppression(
553
587
  email: string,
554
588
  category?: string,
555
589
  ): Promise<SuppressionReason> {
556
- // An address can legitimately have MORE THAN ONE `email_preferences` row:
557
- // the PK is (user_id, email), and a suppression imported before the contact
558
- // existed is keyed (email, email) while later interactive writes key
559
- // (external_id, email). Aggregate across ALL rows — any suppression signal
560
- // on any row blocks the send — so an imported unsubscribe/bounce can never
561
- // be shadowed by a newer clean row for the same address.
562
- const rows = await db
563
- .select()
564
- .from(emailPreferences)
565
- .where(eq(emailPreferences.email, email));
566
-
567
- if (rows.some((prefs) => prefs.suppressed)) return "suppressed";
568
- if (rows.some((prefs) => prefs.unsubscribedAll)) return "unsubscribed";
590
+ // Aggregated across ALL `email_preferences` rows for this address by the shared
591
+ // reader — an address can legitimately have MORE THAN ONE row (multi-row
592
+ // (user_id, email) aggregation rationale lives in readRecipientPreferences).
593
+ const prefs = await readRecipientPreferences(db, { email });
594
+
595
+ if (prefs.suppressed) return "suppressed";
596
+ if (prefs.unsubscribedAll) return "unsubscribed";
569
597
 
570
598
  // Registry-aware polarity (§2.6, D3) — applied through the SINGLE source of
571
599
  // truth `ListRegistry.isSubscribed` so it matches the preference center EXACTLY
@@ -575,21 +603,8 @@ async function checkSuppression(
575
603
  // so absence-of-true (the common "never opted in" case) MUST block — otherwise
576
604
  // a contact the preference center shows as "Unsubscribed" would still receive
577
605
  // the mail (the two surfaces would disagree, which §2.6 forbids).
578
- // Category maps are merged across rows with explicit FALSE winning: an
579
- // opt-out recorded on any row blocks, matching the conservative aggregation
580
- // above. (In the common single-row case this is identical to reading the
581
- // row's map directly.)
582
- if (category) {
583
- const categories: Record<string, boolean> = {};
584
- for (const prefs of rows) {
585
- const map = (prefs.categories ?? {}) as Record<string, boolean>;
586
- for (const [key, value] of Object.entries(map)) {
587
- categories[key] = (categories[key] ?? true) && value;
588
- }
589
- }
590
- if (!getListRegistry().isSubscribed(categories, category)) {
591
- return "category_unsubscribed";
592
- }
606
+ if (category && !getListRegistry().isSubscribed(prefs.categories, category)) {
607
+ return "category_unsubscribed";
593
608
  }
594
609
 
595
610
  return null;
@@ -12,3 +12,11 @@ export const EMAIL_LINK_CLICKED = "email.link_clicked" as const;
12
12
  // raw mint distinctId; the BUS event carries the MANAGED `links.id` as `linkId`
13
13
  // + the RESOLVED survivor contact key (see `pushLinkClickEvent`).
14
14
  export const LINK_CLICKED = "link.clicked" as const;
15
+
16
+ // A visitor CONFIRMED landing from a tracked hit (POST /v1/t/arrive, opt-in
17
+ // `hs_ref`). Subset of `link.clicked`: fires only when the link opts in AND
18
+ // the landing page integrates. Unlike `link.clicked`, it carries the VISITOR's
19
+ // identity (token-verified userId or clamped anon id) — which is what answers
20
+ // "did a known user scan this QR?". Bus + outbound both define `linkId` as the
21
+ // managed `links.id` and carry `trackedLinkId` separately (no legacy split).
22
+ export const LINK_ARRIVED = "link.arrived" as const;
@@ -25,7 +25,7 @@ import { Webhook } from "svix";
25
25
  */
26
26
 
27
27
  /**
28
- * The 15-event catalog — the SINGLE source of truth (schema, routes, client,
28
+ * The 16-event catalog — the SINGLE source of truth (schema, routes, client,
29
29
  * CLI all derive from this). The `webhook.test` sentinel is intentionally NOT a
30
30
  * member (it is delivered out-of-band regardless of an endpoint's `eventTypes`).
31
31
  *
@@ -33,6 +33,10 @@ import { Webhook } from "svix";
33
33
  * has no email send (Discord/referral/ad-hoc `createTrackedLink`). It is the
34
34
  * deliberate counterpart to `email.clicked` so a non-email click never fires a
35
35
  * malformed `email.clicked` (MF-missing #3).
36
+ *
37
+ * `link.arrived` is the landing-confirmed subset of `link.clicked`: the
38
+ * visitor reported back from the destination (opt-in `hs_ref` +
39
+ * POST /v1/t/arrive) with identity evidence.
36
40
  */
37
41
  export const WEBHOOK_EVENT_TYPES = [
38
42
  "contact.created",
@@ -50,6 +54,7 @@ export const WEBHOOK_EVENT_TYPES = [
50
54
  "bucket.entered",
51
55
  "bucket.left",
52
56
  "link.clicked",
57
+ "link.arrived",
53
58
  ] as const;
54
59
 
55
60
  export type WebhookEventType = (typeof WEBHOOK_EVENT_TYPES)[number];
@@ -0,0 +1,75 @@
1
+ /**
2
+ * Channel lists — engine-synthesized delivery channels layered on the SAME
3
+ * `email_preferences.categories` JSONB namespace as author-defined lists (D3).
4
+ * A channel is an auto-registered opt-out list: `in_app` (the notification
5
+ * feed) plus one per connector that exposes member-directed actions (discord,
6
+ * telegram). There is no new table and no authoring surface — channels are
7
+ * derived from the registered connector actions at container build time.
8
+ *
9
+ * POLARITY (zero behaviour flip): every channel is `defaultOptIn: true`, so
10
+ * `ListRegistry.isSubscribed(categories, id)` returns `categories[id] !== false`
11
+ * — subscribed unless the recipient explicitly set the category to `false`.
12
+ * That is EXACTLY the unknown-id fallback an UNregistered channel already
13
+ * resolved to (`isSubscribedByDefault` → `?? true`), so registering these lists
14
+ * flips zero existing behaviour; it only makes the channels visible in the
15
+ * catalog and (in a later phase) enforceable in `sendConnectorAction`.
16
+ */
17
+ import type { DefinedConnectorAction } from "../connectors/define-action.js";
18
+ import type { ListMeta } from "./define-list.js";
19
+
20
+ /**
21
+ * Reserved list id for the engine's in-app channel (the notification feed).
22
+ * Always synthesized first by {@link synthesizeChannelLists}; also the
23
+ * suppression key consulted by `sendFeedItem`.
24
+ */
25
+ export const IN_APP_LIST_ID = "in_app";
26
+
27
+ /** Uppercase the first character of a connector id for the channel's display name. */
28
+ function capitalize(id: string): string {
29
+ return id.length === 0 ? id : id.charAt(0).toUpperCase() + id.slice(1);
30
+ }
31
+
32
+ /**
33
+ * Synthesize the engine-owned channel {@link ListMeta}s from the registered
34
+ * connector actions.
35
+ *
36
+ * Always yields the `in_app` channel first. Then one channel per DISTINCT
37
+ * `connectorId` among actions declaring a member audience
38
+ * (`action.audience?.kind === "member"`), de-duplicated via a `Set` in stable
39
+ * first-appearance order. Actions WITHOUT a member audience (ops/channel-
40
+ * directed) never mint a channel. Channel display name = the connector id with
41
+ * its first letter capitalized ("discord" → "Discord").
42
+ *
43
+ * Every channel shares the opt-out polarity shape `{ defaultOptIn: true,
44
+ * enabled: true, kind: "channel" }` — see the module docblock for why this
45
+ * flips no existing behaviour.
46
+ */
47
+ export function synthesizeChannelLists(
48
+ actions: DefinedConnectorAction[],
49
+ ): ListMeta[] {
50
+ const channels: ListMeta[] = [
51
+ {
52
+ id: IN_APP_LIST_ID,
53
+ name: "In-app feed",
54
+ defaultOptIn: true,
55
+ enabled: true,
56
+ kind: "channel",
57
+ },
58
+ ];
59
+
60
+ const seen = new Set<string>();
61
+ for (const action of actions) {
62
+ if (action.audience?.kind !== "member") continue;
63
+ if (seen.has(action.connectorId)) continue;
64
+ seen.add(action.connectorId);
65
+ channels.push({
66
+ id: action.connectorId,
67
+ name: capitalize(action.connectorId),
68
+ defaultOptIn: true,
69
+ enabled: true,
70
+ kind: "channel",
71
+ });
72
+ }
73
+
74
+ return channels;
75
+ }
@@ -32,6 +32,27 @@ export function isReservedListId(id: string): boolean {
32
32
  /** Allowed list-id shape: alphanumerics, dash, underscore (case-insensitive). */
33
33
  const LIST_ID_PATTERN = /^[a-z0-9_-]+$/i;
34
34
 
35
+ /**
36
+ * Reserved list id for the engine's in-app channel list (the notification
37
+ * feed). Unlike {@link RESERVED_LIST_IDS} — which doubles as the template
38
+ * category allowlist in `container.ts` — this id is a channel that IS
39
+ * auto-registered by the engine (`synthesizeChannelLists`), so it gets its own
40
+ * dedicated {@link defineList} guard instead of joining that set (adding it
41
+ * there would change the allowlist's meaning). Kept lowercase; the check is
42
+ * case-insensitive.
43
+ */
44
+ const IN_APP_RESERVED_LIST_ID = "in_app";
45
+
46
+ /**
47
+ * The kind of list. `"topic"` is an author-defined subscription category (the
48
+ * `defineList` output — every manual list is a topic). `"channel"` is an
49
+ * engine-synthesized delivery channel (the in-app feed + one per connector that
50
+ * exposes member-directed actions), minted by `synthesizeChannelLists`, never
51
+ * authored. Read sites treat an absent `kind` as `"topic"` (`meta.kind ??
52
+ * "topic"`) so pre-existing manual `ListMeta` constructions keep compiling.
53
+ */
54
+ export type ListKind = "channel" | "topic";
55
+
35
56
  /**
36
57
  * The validated, fully-defaulted list metadata. `enabled` is always present
37
58
  * after `defineList` (defaults to `true`); authoring input leaves it optional.
@@ -42,6 +63,12 @@ export interface ListMeta<Id extends string = string> {
42
63
  description?: string;
43
64
  defaultOptIn: boolean;
44
65
  enabled: boolean;
66
+ /**
67
+ * Whether this is an author-defined `"topic"` or an engine-synthesized
68
+ * delivery `"channel"`. Optional so existing manual constructions compile;
69
+ * read as `meta.kind ?? "topic"`. `defineList` always stamps `"topic"`.
70
+ */
71
+ kind?: ListKind;
45
72
  }
46
73
 
47
74
  /**
@@ -82,12 +109,19 @@ export function defineList<const Id extends string>(meta: {
82
109
  );
83
110
  }
84
111
 
112
+ if (id.toLowerCase() === IN_APP_RESERVED_LIST_ID) {
113
+ throw new Error(
114
+ `Reserved list id "${id}": it is the engine's in-app channel list, auto-registered for the notification feed.`,
115
+ );
116
+ }
117
+
85
118
  const resolvedMeta: ListMeta<Id> = {
86
119
  id,
87
120
  name,
88
121
  ...(description !== undefined ? { description } : {}),
89
122
  defaultOptIn,
90
123
  enabled: enabled ?? true,
124
+ kind: "topic",
91
125
  };
92
126
 
93
127
  return {
@@ -64,6 +64,15 @@ export class ListRegistry {
64
64
  const defaultOptIn = this.isSubscribedByDefault(id);
65
65
  return defaultOptIn ? categories[id] !== false : categories[id] === true;
66
66
  }
67
+
68
+ /**
69
+ * Is `id` a registered channel (an engine-synthesized delivery channel — the
70
+ * in-app feed or a per-connector opt-out list) rather than an author-defined
71
+ * topic? False for topics, unknown ids, and the built-in categories.
72
+ */
73
+ isChannel(id: string): boolean {
74
+ return this.get(id)?.kind === "channel";
75
+ }
67
76
  }
68
77
 
69
78
  /**
@@ -74,12 +83,21 @@ export class ListRegistry {
74
83
  *
75
84
  * `parseEnabledFilter` (journeys/registry.ts) is reused as-is — `ENABLED_LISTS`
76
85
  * honours the same `"*"`-or-csv contract as `ENABLED_JOURNEYS` /
77
- * `ENABLED_BUCKETS`. Disabled lists (filtered out OR `enabled: false`) are NOT
78
- * registered, so an unknown id resolves to legacy opt-in.
86
+ * `ENABLED_BUCKETS`. Disabled user lists (filtered out OR `enabled: false`) are
87
+ * NOT registered, so an unknown id resolves to legacy opt-in.
88
+ *
89
+ * `channels` (engine-synthesized channel metas from `synthesizeChannelLists`)
90
+ * are registered AFTER the user lists, UNCONDITIONALLY — the `ENABLED_LISTS`
91
+ * filter does NOT apply. A "disabled" channel would hide the catalog entry while
92
+ * opt-out-polarity enforcement silently continued (a confusing half-state), so
93
+ * channels are always on. If a filtered-IN user list id collides with a channel
94
+ * id, we throw an actionable error rather than let one silently shadow the
95
+ * other.
79
96
  */
80
97
  export function buildListRegistry(
81
98
  lists: DefinedList[],
82
99
  enabledFilter?: string,
100
+ channels: ListMeta[] = [],
83
101
  ): ListRegistry {
84
102
  const registry = new ListRegistry();
85
103
  const enabled = parseEnabledFilter(enabledFilter);
@@ -90,6 +108,15 @@ export function buildListRegistry(
90
108
  }
91
109
  }
92
110
 
111
+ for (const channel of channels) {
112
+ if (registry.has(channel.id)) {
113
+ throw new Error(
114
+ `List id "${channel.id}" collides with the auto-registered channel list for the "${channel.id}" connector. Rename your defineList id.`,
115
+ );
116
+ }
117
+ registry.register(channel);
118
+ }
119
+
93
120
  setListRegistry(registry);
94
121
  return registry;
95
122
  }