@hogsend/engine 0.39.0 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/package.json +9 -7
  2. package/src/buckets/check-membership.ts +6 -1
  3. package/src/campaigns/cohort-sql.ts +198 -0
  4. package/src/campaigns/define-campaign.ts +301 -0
  5. package/src/campaigns/reconcile.ts +391 -0
  6. package/src/campaigns/steps.ts +48 -0
  7. package/src/connectors/define-action.ts +70 -0
  8. package/src/container.ts +144 -42
  9. package/src/destinations/presets/posthog.ts +7 -1
  10. package/src/env.ts +14 -0
  11. package/src/index.ts +98 -3
  12. package/src/journeys/constants.ts +10 -0
  13. package/src/journeys/define-journey.ts +14 -405
  14. package/src/journeys/execute-journey-run.ts +608 -0
  15. package/src/journeys/graph/build-graph.ts +212 -56
  16. package/src/journeys/journey-boundary.ts +54 -0
  17. package/src/journeys/journey-context.ts +374 -59
  18. package/src/journeys/record-once.ts +110 -0
  19. package/src/lib/auth.ts +33 -1
  20. package/src/lib/blueprints.ts +576 -0
  21. package/src/lib/campaign-send-key.ts +58 -0
  22. package/src/lib/connector-actions.ts +203 -19
  23. package/src/lib/contacts.ts +54 -0
  24. package/src/lib/email-service-types.ts +4 -1
  25. package/src/lib/email.ts +13 -2
  26. package/src/lib/feed.ts +77 -57
  27. package/src/lib/frequency-cap.ts +45 -15
  28. package/src/lib/hatchet-duration.ts +22 -0
  29. package/src/lib/html.ts +1 -0
  30. package/src/lib/ingestion.ts +145 -4
  31. package/src/lib/journey-suppress.ts +87 -0
  32. package/src/lib/links.ts +204 -18
  33. package/src/lib/outbound.ts +20 -0
  34. package/src/lib/preferences.ts +80 -1
  35. package/src/lib/tracked.ts +44 -29
  36. package/src/lib/tracking-event-names.ts +8 -0
  37. package/src/lib/webhook-signing.ts +6 -1
  38. package/src/lists/channels.ts +75 -0
  39. package/src/lists/define-list.ts +34 -0
  40. package/src/lists/registry.ts +29 -2
  41. package/src/mcp/blueprint-tools.ts +460 -0
  42. package/src/routes/admin/blueprints.ts +711 -0
  43. package/src/routes/admin/campaigns.ts +341 -0
  44. package/src/routes/admin/emails.ts +13 -0
  45. package/src/routes/admin/index.ts +6 -0
  46. package/src/routes/admin/journeys.ts +4 -2
  47. package/src/routes/admin/links.ts +303 -36
  48. package/src/routes/admin/lists.ts +51 -0
  49. package/src/routes/campaigns/index.ts +297 -50
  50. package/src/routes/email/preferences.ts +50 -13
  51. package/src/routes/feed/recipient.ts +2 -43
  52. package/src/routes/index.ts +6 -0
  53. package/src/routes/lists/index.ts +138 -18
  54. package/src/routes/tracking/arrive.ts +260 -0
  55. package/src/routes/tracking/click-pipeline.ts +329 -0
  56. package/src/routes/tracking/click.ts +5 -270
  57. package/src/routes/tracking/index.ts +2 -0
  58. package/src/routes/tracking/vanity.ts +64 -0
  59. package/src/worker.ts +27 -0
  60. package/src/workflows/bucket-backfill.ts +4 -2
  61. package/src/workflows/bucket-reconcile.ts +11 -4
  62. package/src/workflows/check-alerts.ts +130 -1
  63. package/src/workflows/journey-blueprint-interpreter.ts +561 -0
  64. package/src/workflows/send-campaign.ts +673 -127
@@ -12,7 +12,10 @@ import {
12
12
  } from "../../lib/contacts.js";
13
13
  import type { Logger } from "../../lib/logger.js";
14
14
  import { emitOutbound } from "../../lib/outbound.js";
15
- import { applyListMembership } from "../../lib/preferences.js";
15
+ import {
16
+ applyListMembership,
17
+ upsertEmailPreference,
18
+ } from "../../lib/preferences.js";
16
19
  import { errorSchema } from "../../lib/schemas.js";
17
20
  import { getListRegistry } from "../../lists/registry-singleton.js";
18
21
  import {
@@ -26,6 +29,11 @@ const listSummarySchema = z.object({
26
29
  name: z.string(),
27
30
  description: z.string().optional(),
28
31
  defaultOptIn: z.boolean(),
32
+ // Whether this is an author-defined `"topic"` or an engine-synthesized
33
+ // delivery `"channel"` (the in-app feed + one per member-directed connector).
34
+ // Always emitted by the server (`meta.kind ?? "topic"`); additive for
35
+ // consumers that ignore it.
36
+ kind: z.enum(["channel", "topic"]),
29
37
  // The resolved opt-in state for the identity in the query (`categories[id] ??
30
38
  // defaultOptIn`). An anon / publishable / no-identity read degrades to
31
39
  // `defaultOptIn` (never leaks another contact's state — mirrors the
@@ -108,6 +116,54 @@ const preferencesRoute = createRoute({
108
116
  },
109
117
  });
110
118
 
119
+ // NEW public data-plane write for the email/global master opt-out
120
+ // (`unsubscribedAll`). Until now the ONLY writers of `unsubscribed_all` were the
121
+ // HMAC unsubscribe-token flow and the admin PUT — a browser had no first-party
122
+ // way to set the master toggle. This is the email/global master; per-category
123
+ // channels + topics are managed via the `/{id}/(un)subscribe` endpoints. Static
124
+ // `/preferences` POST is registered adjacent to its GET; it never collides with
125
+ // `/{id}/subscribe` (distinct path shape) so match order is safe.
126
+ const setPreferencesBodySchema = z.object({
127
+ unsubscribedAll: z.boolean(),
128
+ email: z.string().optional(),
129
+ userId: z.string().optional(),
130
+ // Publishable-key identity assertion (§Phase 1). Ignored on the secret path.
131
+ userToken: z.string().optional(),
132
+ });
133
+
134
+ const setPreferencesRoute = createRoute({
135
+ method: "post",
136
+ path: "/preferences",
137
+ tags: ["Lists"],
138
+ summary: "Set a contact's global opt-out (`unsubscribedAll`)",
139
+ description:
140
+ "Writes the email/global master toggle (`unsubscribed_all`) for the resolved contact — the account-wide email opt-out. Per-category channels + topics are managed via `POST /v1/lists/{id}/(un)subscribe`. Behind the publishable gate: a pk_ key must present a verified userToken to act on a concrete identity.",
141
+ request: {
142
+ body: {
143
+ content: { "application/json": { schema: setPreferencesBodySchema } },
144
+ },
145
+ },
146
+ responses: {
147
+ 200: {
148
+ content: {
149
+ "application/json": {
150
+ schema: z.object({ unsubscribedAll: z.boolean() }),
151
+ },
152
+ },
153
+ description: "Global opt-out updated",
154
+ },
155
+ 400: {
156
+ content: { "application/json": { schema: errorSchema } },
157
+ description: "Missing identity or no resolvable email",
158
+ },
159
+ 403: {
160
+ content: { "application/json": { schema: errorSchema } },
161
+ description:
162
+ "Publishable key attempted to act on another identity without a verified userToken",
163
+ },
164
+ },
165
+ });
166
+
111
167
  const subscribeRoute = createRoute({
112
168
  method: "post",
113
169
  path: "/{id}/subscribe",
@@ -186,6 +242,36 @@ const unsubscribeRoute = createRoute({
186
242
  },
187
243
  });
188
244
 
245
+ /**
246
+ * INTENT-LAYER `contact.created` emit (decision #3), shared by the list
247
+ * subscribe/unsubscribe path and the master-toggle write. A preference flip is
248
+ * NOT a contact-property delta (so no `contact.updated`); we emit ONLY on the
249
+ * first creation, after a read-back, fire-and-forget so a transient outbound
250
+ * error never fails the preference write. A no-op when `created` is false.
251
+ */
252
+ function emitContactCreatedIfNew(opts: {
253
+ db: Database;
254
+ hatchet: HatchetClient;
255
+ logger: Logger;
256
+ contactId: string;
257
+ created: boolean;
258
+ }): void {
259
+ const { db, hatchet, logger, contactId, created } = opts;
260
+ if (!created) return;
261
+ void resolveContact({ db, id: contactId })
262
+ .then((row) => {
263
+ if (!row) return;
264
+ return emitOutbound({
265
+ db,
266
+ hatchet,
267
+ logger,
268
+ event: "contact.created",
269
+ payload: serializeContact(row),
270
+ });
271
+ })
272
+ .catch(logger.warn);
273
+ }
274
+
189
275
  /**
190
276
  * The shared side-effect of subscribe + unsubscribe (identical apart from the
191
277
  * boolean polarity): validate the list id, guard identity, then resolve/create
@@ -231,23 +317,7 @@ async function applyListSubscription(opts: {
231
317
  email,
232
318
  });
233
319
 
234
- // INTENT-LAYER outbound emit (decision #3): the lists route emits
235
- // `contact.created` ONLY on first creation (a list flip is not a contact
236
- // property delta, so no `contact.updated`). Fire-and-forget after a read-back.
237
- if (created) {
238
- void resolveContact({ db, id: contactId })
239
- .then((row) => {
240
- if (!row) return;
241
- return emitOutbound({
242
- db,
243
- hatchet,
244
- logger,
245
- event: "contact.created",
246
- payload: serializeContact(row),
247
- });
248
- })
249
- .catch(logger.warn);
250
- }
320
+ emitContactCreatedIfNew({ db, hatchet, logger, contactId, created });
251
321
 
252
322
  await applyListMembership({
253
323
  db,
@@ -303,6 +373,7 @@ export const listsRouter = new OpenAPIHono<AppEnv>()
303
373
  name: l.name,
304
374
  ...(l.description !== undefined ? { description: l.description } : {}),
305
375
  defaultOptIn: l.defaultOptIn,
376
+ kind: l.kind ?? "topic",
306
377
  subscribed: categories[l.id] ?? l.defaultOptIn,
307
378
  }));
308
379
 
@@ -357,6 +428,55 @@ export const listsRouter = new OpenAPIHono<AppEnv>()
357
428
  200,
358
429
  );
359
430
  })
431
+ .openapi(setPreferencesRoute, async (c) => {
432
+ const { db, hatchet, logger, env } = c.get("container");
433
+ const { unsubscribedAll, email, userId, userToken } = c.req.valid("json");
434
+
435
+ // Identical publishable gate to the subscribe handlers: a secret key defers
436
+ // to requireIdentity; a pk_ key needs a verified userToken to act on a
437
+ // concrete identity.
438
+ const guard = gatePublishableIdentity(
439
+ c,
440
+ { email, userId, userToken },
441
+ env.BETTER_AUTH_SECRET,
442
+ );
443
+ if (guard) return guard;
444
+
445
+ if (!email && !userId) {
446
+ return c.json({ error: "email or userId is required" }, 400);
447
+ }
448
+
449
+ // Resolve/create the contact FIRST (mirroring the subscribe path) so a real
450
+ // row + uuid exists before the preference write keys on `external_id ?? id`.
451
+ const { id: contactId, created } = await resolveOrCreateContact({
452
+ db,
453
+ userId,
454
+ email,
455
+ });
456
+
457
+ emitContactCreatedIfNew({ db, hatchet, logger, contactId, created });
458
+
459
+ // Resolve the deterministic `(external_id ?? contact.id, email)` pair the
460
+ // same way `applyListMembership` does — `email_preferences.email` is NOT
461
+ // NULL, so a contact with no resolvable email 400s (mirrors its error).
462
+ const recipient = await resolveRecipient({ db, userId, email });
463
+ if (!recipient) {
464
+ return c.json(
465
+ { error: "Contact has no email; cannot manage list membership" },
466
+ 400,
467
+ );
468
+ }
469
+ const externalId = recipient.externalId ?? recipient.contactId;
470
+
471
+ await upsertEmailPreference({
472
+ db,
473
+ externalId,
474
+ email: recipient.email,
475
+ update: { unsubscribedAll },
476
+ });
477
+
478
+ return c.json({ unsubscribedAll }, 200);
479
+ })
360
480
  .openapi(subscribeRoute, async (c) => {
361
481
  const { db, hatchet, logger, env } = c.get("container");
362
482
  const { id } = c.req.valid("param");
@@ -0,0 +1,260 @@
1
+ import { linkClicks, links, trackedLinks } from "@hogsend/db";
2
+ import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
3
+ import { and, eq, isNull } from "drizzle-orm";
4
+ import type { AppEnv } from "../../app.js";
5
+ import {
6
+ collidesWithIdentified,
7
+ PublishableAnonymousMergeError,
8
+ } from "../../lib/contacts.js";
9
+ import { ingestEvent } from "../../lib/ingestion.js";
10
+ import { emitOutbound } from "../../lib/outbound.js";
11
+ import { LINK_ARRIVED } from "../../lib/tracking-event-names.js";
12
+ import {
13
+ InvalidUserTokenError,
14
+ verifyUserToken,
15
+ } from "../../lib/user-token.js";
16
+
17
+ /**
18
+ * Arrival attribution: the landing page reports "a visitor arrived from
19
+ * tracked hit `ref`" with the visitor's identity evidence. `ref` is the
20
+ * `hs_ref=<link_clicks.id>` the redirect appended (opt-in per link,
21
+ * `links.append_ref`). Joins visitor → that specific scan/click, stamps the
22
+ * click row (first-write-wins), and emits the journey-triggerable
23
+ * `link.arrived` event.
24
+ *
25
+ * TRUST MODEL (mirrors the events route + feed recipient — see
26
+ * `gatePublishableIdentity` and `resolveFeedRecipient`):
27
+ * - `userToken` (HMAC, BETTER_AUTH_SECRET) is the ONLY way a browser asserts
28
+ * a concrete userId. `visitor_kind = 'token'`.
29
+ * - a raw `anonymousId` is PROVENANCE-ONLY: rejected if it collides with an
30
+ * identified contact's canonical key (`collidesWithIdentified`), and the
31
+ * bus ingest runs under `restrictToAnonymous` — it can never attach to /
32
+ * merge into an identified contact. `visitor_kind = 'anon'`.
33
+ * - a bare asserted email/userId is not even in the schema.
34
+ * - INVARIANT: nothing the ref resolves to (above all `links.distinct_id`)
35
+ * ever enters the contact resolver as a subject — identity inputs come
36
+ * ONLY from the token or the clamped anon id.
37
+ *
38
+ * NOTE: with a valid userToken this is the first `/v1/t/*` route that writes
39
+ * journey-triggering events for an identified subject. That authority is the
40
+ * token itself (the same holder could call `/v1/events`); possession of `ref`
41
+ * only selects WHICH click row gets attributed.
42
+ *
43
+ * IDEMPOTENCY: the ingest subject is ALWAYS read from the stamped row, never
44
+ * the current request — so a replayed ref is a self-healing retry (the
45
+ * `link:arrived:<ref>` idempotencyKey caps it at one event ever, for the
46
+ * originally stamped visitor), not a way to re-attribute or double-fire.
47
+ *
48
+ * ANTI-ORACLE: every outcome — success, unknown ref, non-participating link,
49
+ * already stamped, invalid token, collision skip, no identity — returns the
50
+ * SAME `200 {"ok":true}`. An unauthenticated caller learns nothing about
51
+ * contacts or refs. 400 only for a malformed body.
52
+ */
53
+ const arriveRoute = createRoute({
54
+ method: "post",
55
+ path: "/arrive",
56
+ tags: ["Tracking"],
57
+ summary: "Report a landing-page arrival from a tracked link hit",
58
+ request: {
59
+ body: {
60
+ content: {
61
+ "application/json": {
62
+ schema: z.object({
63
+ // The `hs_ref` value from the landing URL.
64
+ ref: z.string().uuid(),
65
+ // The visitor's OWN browser anon id (`hs_anon_id`) — provenance
66
+ // only, clamped.
67
+ anonymousId: z.string().min(1).max(200).optional(),
68
+ // Server-minted HMAC token binding a userId — the only way a
69
+ // browser asserts an identified visitor. Wins over anonymousId.
70
+ userToken: z.string().min(1).max(2048).optional(),
71
+ }),
72
+ },
73
+ },
74
+ },
75
+ },
76
+ responses: {
77
+ 200: {
78
+ description: "Acknowledged (uniform across all outcomes)",
79
+ content: {
80
+ "application/json": { schema: z.object({ ok: z.literal(true) }) },
81
+ },
82
+ },
83
+ 400: { description: "Malformed body" },
84
+ },
85
+ });
86
+
87
+ export const arriveRouter = new OpenAPIHono<AppEnv>().openapi(
88
+ arriveRoute,
89
+ async (c) => {
90
+ const { ref, anonymousId, userToken } = c.req.valid("json");
91
+ const { db, env, logger, hatchet, registry, analytics } =
92
+ c.get("container");
93
+ const ok = () => c.json({ ok: true as const }, 200);
94
+
95
+ // Resolve the hit + its link, join-gated on the opt-in: a ref for a
96
+ // non-participating link is only obtainable out-of-band → no-op.
97
+ const rows = await db
98
+ .select({
99
+ visitorDistinctId: linkClicks.visitorDistinctId,
100
+ visitorKind: linkClicks.visitorKind,
101
+ destinationUrl: linkClicks.destinationUrl,
102
+ trackedLinkId: trackedLinks.id,
103
+ source: trackedLinks.source,
104
+ linkId: links.id,
105
+ campaign: links.campaign,
106
+ })
107
+ .from(linkClicks)
108
+ .innerJoin(trackedLinks, eq(linkClicks.trackedLinkId, trackedLinks.id))
109
+ .innerJoin(
110
+ links,
111
+ and(eq(trackedLinks.linkId, links.id), eq(links.appendRef, true)),
112
+ )
113
+ .where(eq(linkClicks.id, ref))
114
+ .limit(1);
115
+ const hit = rows[0];
116
+ if (!hit) return ok();
117
+
118
+ // Resolve the visitor evidence. INVARIANT: identity comes ONLY from here —
119
+ // never from anything the ref resolved to.
120
+ let claimId: string | null = null;
121
+ let claimKind: "token" | "anon" | null = null;
122
+ if (userToken) {
123
+ try {
124
+ claimId = verifyUserToken({
125
+ token: userToken,
126
+ secret: env.BETTER_AUTH_SECRET,
127
+ }).userId;
128
+ claimKind = "token";
129
+ } catch (err) {
130
+ if (err instanceof InvalidUserTokenError) {
131
+ // Background beacon — nothing for the caller to do differently, and
132
+ // a distinguishable failure would add oracle surface.
133
+ logger.info("arrive: invalid userToken (no-op)", { ref });
134
+ return ok();
135
+ }
136
+ throw err;
137
+ }
138
+ } else if (anonymousId) {
139
+ // The stamp guard: an "anon id" that is really an identified contact's
140
+ // canonical key would forge "that person arrived here" in the stamp
141
+ // even though the clamped ingest would suppress the event.
142
+ if (await collidesWithIdentified(db, anonymousId)) {
143
+ logger.info("arrive: anonymousId collides with identified (no-op)", {
144
+ ref,
145
+ });
146
+ return ok();
147
+ }
148
+ claimId = anonymousId;
149
+ claimKind = "anon";
150
+ } else {
151
+ return ok();
152
+ }
153
+
154
+ // First-write-wins stamp; on a replay, read back the ORIGINAL stamp — the
155
+ // ingest below always runs with the stamped identity, never the current
156
+ // request's, so a replay is a self-healing retry, not a re-attribution.
157
+ const [fresh] = await db
158
+ .update(linkClicks)
159
+ .set({
160
+ visitorDistinctId: claimId,
161
+ visitorKind: claimKind,
162
+ arrivedAt: new Date(),
163
+ })
164
+ .where(and(eq(linkClicks.id, ref), isNull(linkClicks.visitorDistinctId)))
165
+ .returning({
166
+ visitorDistinctId: linkClicks.visitorDistinctId,
167
+ visitorKind: linkClicks.visitorKind,
168
+ });
169
+
170
+ const stamp =
171
+ fresh ??
172
+ (hit.visitorDistinctId
173
+ ? {
174
+ visitorDistinctId: hit.visitorDistinctId,
175
+ visitorKind: hit.visitorKind,
176
+ }
177
+ : // Raced another arrive between our select and update — re-read.
178
+ (
179
+ await db
180
+ .select({
181
+ visitorDistinctId: linkClicks.visitorDistinctId,
182
+ visitorKind: linkClicks.visitorKind,
183
+ })
184
+ .from(linkClicks)
185
+ .where(eq(linkClicks.id, ref))
186
+ .limit(1)
187
+ )[0]);
188
+ if (!stamp?.visitorDistinctId) return ok();
189
+ const isToken = stamp.visitorKind === "token";
190
+
191
+ // Bus ingest from the STAMPED identity. `link:arrived:<ref>` (no time
192
+ // bucket — once per ref forever): the user_events unique index +
193
+ // compensating delete make replays converge on exactly one event.
194
+ try {
195
+ const result = await ingestEvent({
196
+ db,
197
+ registry,
198
+ hatchet,
199
+ logger,
200
+ analytics,
201
+ restrictToAnonymous: !isToken,
202
+ event: {
203
+ event: LINK_ARRIVED,
204
+ ...(isToken
205
+ ? { userId: stamp.visitorDistinctId }
206
+ : { anonymousId: stamp.visitorDistinctId }),
207
+ eventProperties: {
208
+ linkId: hit.linkId,
209
+ trackedLinkId: hit.trackedLinkId,
210
+ ref,
211
+ source: hit.source,
212
+ campaign: hit.campaign,
213
+ linkUrl: hit.destinationUrl,
214
+ visitorKind: stamp.visitorKind,
215
+ },
216
+ source: "tracking",
217
+ idempotencyKey: `link:arrived:${ref}`,
218
+ },
219
+ });
220
+
221
+ // Outbound only on the FRESH store — outbound has no dedupe of its own,
222
+ // so this is what keeps webhooks at one delivery per ref.
223
+ if (result.stored) {
224
+ void emitOutbound({
225
+ db,
226
+ hatchet,
227
+ logger,
228
+ event: "link.arrived",
229
+ payload: {
230
+ linkId: hit.linkId,
231
+ trackedLinkId: hit.trackedLinkId,
232
+ ref,
233
+ source: hit.source,
234
+ campaign: hit.campaign,
235
+ userId: isToken ? stamp.visitorDistinctId : null,
236
+ anonymousId: isToken ? null : stamp.visitorDistinctId,
237
+ visitorKind: isToken ? "token" : "anon",
238
+ linkUrl: hit.destinationUrl,
239
+ at: new Date().toISOString(),
240
+ },
241
+ }).catch(logger.warn);
242
+ }
243
+ } catch (err) {
244
+ if (err instanceof PublishableAnonymousMergeError) {
245
+ // The anon id became identified between the stamp guard and the
246
+ // clamped resolve (race) — suppress the event, keep the stamp, 200.
247
+ logger.info("arrive: clamped anon ingest suppressed", { ref });
248
+ return ok();
249
+ }
250
+ // Transient ingest failure — the stamp is durable and a replayed ref
251
+ // self-heals, so never fail the beacon.
252
+ logger.warn("arrive: ingest failed (stamp kept; replay heals)", {
253
+ ref,
254
+ error: err instanceof Error ? err.message : String(err),
255
+ });
256
+ }
257
+
258
+ return ok();
259
+ },
260
+ );