@hogsend/engine 0.43.0 → 0.45.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/package.json +12 -9
  2. package/src/container.ts +289 -13
  3. package/src/destinations/define-destination.ts +7 -3
  4. package/src/destinations/presets/posthog.ts +31 -0
  5. package/src/index.ts +99 -0
  6. package/src/journeys/execute-journey-run.ts +93 -0
  7. package/src/journeys/journey-context.ts +4 -0
  8. package/src/lib/analytics-providers-from-env.ts +106 -43
  9. package/src/lib/attribution-backfill.ts +285 -0
  10. package/src/lib/attribution.ts +239 -0
  11. package/src/lib/contacts.ts +71 -0
  12. package/src/lib/conversion-dispatch.ts +326 -0
  13. package/src/lib/conversions.ts +197 -0
  14. package/src/lib/crm-deals.ts +320 -0
  15. package/src/lib/crm-ingest.ts +341 -0
  16. package/src/lib/crm-provider-registry.ts +41 -0
  17. package/src/lib/crm-registry-singleton.ts +22 -0
  18. package/src/lib/deal-money-events.ts +85 -0
  19. package/src/lib/email-service-types.ts +16 -1
  20. package/src/lib/event-names.ts +1 -1
  21. package/src/lib/funnel-progress.ts +76 -0
  22. package/src/lib/funnel-registry.ts +108 -0
  23. package/src/lib/funnel-transitions.ts +280 -0
  24. package/src/lib/holdout.ts +62 -0
  25. package/src/lib/ingestion.ts +218 -9
  26. package/src/lib/lift-stats.ts +125 -0
  27. package/src/lib/mailer.ts +1 -0
  28. package/src/lib/operator.ts +31 -0
  29. package/src/lib/outbound.ts +51 -0
  30. package/src/lib/posthog-scopes.ts +30 -6
  31. package/src/lib/revenue.ts +116 -0
  32. package/src/lib/sms-service-types.ts +7 -1
  33. package/src/lib/sms-tracked.ts +35 -0
  34. package/src/lib/tracked.ts +40 -0
  35. package/src/lib/tracking-events.ts +26 -0
  36. package/src/lib/unconfigured-email-provider.ts +45 -0
  37. package/src/lib/webhook-signing.ts +7 -0
  38. package/src/routes/admin/analytics.ts +14 -5
  39. package/src/routes/admin/attribution.ts +414 -0
  40. package/src/routes/admin/contacts.ts +71 -11
  41. package/src/routes/admin/conversions.ts +247 -0
  42. package/src/routes/admin/deals.ts +472 -0
  43. package/src/routes/admin/funnels.ts +228 -0
  44. package/src/routes/admin/index.ts +8 -0
  45. package/src/routes/admin/journeys.ts +158 -1
  46. package/src/routes/admin/readiness.ts +97 -2
  47. package/src/routes/admin/templates.ts +28 -1
  48. package/src/routes/events/index.ts +12 -0
  49. package/src/routes/tracking/click-pipeline.ts +26 -18
  50. package/src/routes/webhooks/crm-provider.ts +117 -0
  51. package/src/routes/webhooks/index.ts +5 -1
  52. package/src/sources/define-contact-source.ts +159 -0
  53. package/src/sources/registry.ts +71 -0
  54. package/src/sources/webhook-source.ts +109 -0
  55. package/src/worker.ts +4 -0
  56. package/src/workflows/crm-reconcile.ts +129 -0
  57. package/src/workflows/dispatch-conversion.ts +31 -0
  58. package/src/workflows/send-campaign.ts +5 -0
@@ -59,8 +59,14 @@ export interface SmsTrackedSendResult {
59
59
  * recent SMS to this recipient from the same journey.
60
60
  * - `"test_mode_blocked"` — test mode active but no `HOGSEND_TEST_PHONE`, so
61
61
  * the send was blocked rather than delivered to the real recipient.
62
+ * - `"control_group"` — the recipient is in the global control group
63
+ * (impact plan §4.3); non-transactional sends are withheld.
62
64
  */
63
- reason?: "frequency_capped" | "journey_suppressed" | "test_mode_blocked";
65
+ reason?:
66
+ | "frequency_capped"
67
+ | "journey_suppressed"
68
+ | "test_mode_blocked"
69
+ | "control_group";
64
70
  }
65
71
 
66
72
  // ---------------------------------------------------------------------------
@@ -19,6 +19,7 @@ import { logTransition } from "../journeys/journey-log.js";
19
19
  import { getListRegistry } from "../lists/registry-singleton.js";
20
20
  import type { FrequencyCapConfig } from "./email-service-types.js";
21
21
  import { hatchet } from "./hatchet.js";
22
+ import { isGlobalControl } from "./holdout.js";
22
23
  import { checkJourneySuppress } from "./journey-suppress.js";
23
24
  import { createLogger, type Logger } from "./logger.js";
24
25
  import { emitOutbound } from "./outbound.js";
@@ -215,6 +216,40 @@ async function sendTrackedSmsInner<K extends SmsTemplateName>(
215
216
  }
216
217
 
217
218
  if (!options.skipPreferenceCheck) {
219
+ // Global control group (impact plan §4.3) — the SMS leg of the email
220
+ // gate in tracked.ts: same deterministic bucket, same key precedence
221
+ // (userId when known, else the recipient address), same marker emit
222
+ // (per-endpoint deduped on the contact key), so a contact with a stable
223
+ // userId is controlled consistently across channels.
224
+ if (effectiveCategory !== "transactional") {
225
+ const controlKey = options.userId ?? options.to;
226
+ if (isGlobalControl(controlKey)) {
227
+ logger?.info("sms skipped: control_group", { to: options.to });
228
+ void emitOutbound({
229
+ db,
230
+ hatchet,
231
+ logger: logger ?? emitLogger,
232
+ event: "contact.control_group",
233
+ dedupeKey: `contact.control_group:${controlKey.toLowerCase()}`,
234
+ payload: {
235
+ userId: options.userId ?? null,
236
+ email: null,
237
+ at: new Date().toISOString(),
238
+ },
239
+ }).catch((err: unknown) => {
240
+ (logger ?? emitLogger).warn("control_group marker emit failed", {
241
+ error: err instanceof Error ? err.message : String(err),
242
+ });
243
+ });
244
+ return {
245
+ smsSendId: "",
246
+ messageId: "",
247
+ status: "skipped",
248
+ reason: "control_group",
249
+ };
250
+ }
251
+ }
252
+
218
253
  if (frequencyCap) {
219
254
  const capped = await isSmsFrequencyCapped({
220
255
  db,
@@ -30,6 +30,7 @@ import {
30
30
  } from "./email-service-types.js";
31
31
  import { isFrequencyCapped } from "./frequency-cap.js";
32
32
  import { hatchet } from "./hatchet.js";
33
+ import { isGlobalControl } from "./holdout.js";
33
34
  import { checkJourneySuppress } from "./journey-suppress.js";
34
35
  import { createLogger, type Logger } from "./logger.js";
35
36
  import { emitOutbound } from "./outbound.js";
@@ -209,6 +210,7 @@ async function sendTrackedEmailInner<K extends TemplateName>(
209
210
  subject: options.subject ?? "",
210
211
  category: effectiveCategory,
211
212
  journeyStateId: options.journeyStateId,
213
+ campaignId: options.campaignId,
212
214
  userId: options.userId,
213
215
  userEmail: options.userEmail ?? options.to,
214
216
  status: "failed",
@@ -232,6 +234,42 @@ async function sendTrackedEmailInner<K extends TemplateName>(
232
234
  });
233
235
  }
234
236
 
237
+ // Global control group (impact plan §4.3) — a deterministic contact-level
238
+ // bucket excluded from every non-transactional send so program-level lift
239
+ // is measurable. Runs INSIDE the non-system branch and additionally
240
+ // exempts the transactional category (control contacts still get
241
+ // receipts/resets). No provider call, no row — mirrors the frequency-cap
242
+ // skip. The outbound marker is per-endpoint deduped on the contact key,
243
+ // so the first suppressed send publishes membership exactly once.
244
+ if (effectiveCategory !== "transactional") {
245
+ const controlKey = options.userId ?? options.to;
246
+ if (isGlobalControl(controlKey)) {
247
+ logger?.info("send skipped: control_group", { to: options.to });
248
+ void emitOutbound({
249
+ db,
250
+ hatchet,
251
+ logger: logger ?? emitLogger,
252
+ event: "contact.control_group",
253
+ dedupeKey: `contact.control_group:${controlKey.toLowerCase()}`,
254
+ payload: {
255
+ userId: options.userId ?? null,
256
+ email: options.to,
257
+ at: new Date().toISOString(),
258
+ },
259
+ }).catch((err: unknown) => {
260
+ (logger ?? emitLogger).warn("control_group marker emit failed", {
261
+ error: err instanceof Error ? err.message : String(err),
262
+ });
263
+ });
264
+ return trackedSendResult({
265
+ emailSendId: "",
266
+ messageId: "",
267
+ status: "skipped",
268
+ reason: "control_group",
269
+ });
270
+ }
271
+ }
272
+
235
273
  // Frequency cap — consulted only for non-system sends (system mail sets
236
274
  // skipPreferenceCheck and bypasses both suppression and the cap). On a cap
237
275
  // hit: no provider call, no row inserted, no throw — the journey continues.
@@ -357,6 +395,7 @@ async function sendTrackedEmailInner<K extends TemplateName>(
357
395
  subject: options.subject ?? "",
358
396
  category: effectiveCategory,
359
397
  journeyStateId: options.journeyStateId,
398
+ campaignId: options.campaignId,
360
399
  userId: options.userId,
361
400
  userEmail: options.userEmail ?? options.to,
362
401
  status: "failed",
@@ -411,6 +450,7 @@ async function sendTrackedEmailInner<K extends TemplateName>(
411
450
  subject: wireSubject,
412
451
  category: effectiveCategory,
413
452
  journeyStateId: options.journeyStateId,
453
+ campaignId: options.campaignId,
414
454
  userId: options.userId,
415
455
  userEmail: options.userEmail ?? options.to,
416
456
  status: "queued",
@@ -17,6 +17,14 @@ interface EmailSendContext {
17
17
  templateKey: string | null;
18
18
  messageId: string | null;
19
19
  to: string;
20
+ /**
21
+ * Attribution scope (plan §1.2): the enrollment's journey id and the send's
22
+ * campaign id, stamped into every tracking event's properties so the credit
23
+ * ledger can slice by journey/campaign without a join back through
24
+ * email_sends. Null for transactional/raw sends.
25
+ */
26
+ journeyId: string | null;
27
+ campaignId: string | null;
20
28
  }
21
29
 
22
30
  export async function resolveEmailSendContext(
@@ -32,8 +40,10 @@ export async function resolveEmailSendContext(
32
40
  toEmail: emailSends.toEmail,
33
41
  templateKey: emailSends.templateKey,
34
42
  messageId: emailSends.messageId,
43
+ campaignId: emailSends.campaignId,
35
44
  userId: journeyStates.userId,
36
45
  userEmail: journeyStates.userEmail,
46
+ journeyId: journeyStates.journeyId,
37
47
  })
38
48
  .from(emailSends)
39
49
  .leftJoin(journeyStates, eq(emailSends.journeyStateId, journeyStates.id))
@@ -49,6 +59,8 @@ export async function resolveEmailSendContext(
49
59
  templateKey: row.templateKey,
50
60
  messageId: row.messageId,
51
61
  to: row.toEmail,
62
+ journeyId: row.journeyId,
63
+ campaignId: row.campaignId,
52
64
  };
53
65
  }
54
66
 
@@ -165,6 +177,10 @@ export async function pushTrackingEvent(
165
177
  const properties: Record<string, unknown> = {
166
178
  emailSendId,
167
179
  templateKey: ctx.templateKey,
180
+ // Attribution scope (plan §1.2) — journey/campaign identity rides the
181
+ // spine on every touch event so credits can be sliced without re-joining.
182
+ journeyId: ctx.journeyId,
183
+ campaignId: ctx.campaignId,
168
184
  ...opts.properties,
169
185
  };
170
186
 
@@ -270,6 +286,12 @@ export interface SmsSendContext {
270
286
  templateKey: string | null;
271
287
  messageId: string | null;
272
288
  to: string;
289
+ /**
290
+ * Attribution scope (plan §1.2): the enrollment's journey id. No campaign
291
+ * leg — campaign steps are email-only today, so `sms_sends` carries no
292
+ * campaign column (plan §5 Q3).
293
+ */
294
+ journeyId: string | null;
273
295
  }
274
296
 
275
297
  /**
@@ -290,6 +312,7 @@ export async function resolveSmsSendContext(
290
312
  sendUserId: smsSends.userId,
291
313
  userId: journeyStates.userId,
292
314
  userEmail: journeyStates.userEmail,
315
+ journeyId: journeyStates.journeyId,
293
316
  })
294
317
  .from(smsSends)
295
318
  .leftJoin(journeyStates, eq(smsSends.journeyStateId, journeyStates.id))
@@ -305,6 +328,7 @@ export async function resolveSmsSendContext(
305
328
  templateKey: row.templateKey,
306
329
  messageId: row.messageId,
307
330
  to: row.toPhone,
331
+ journeyId: row.journeyId,
308
332
  };
309
333
  }
310
334
 
@@ -351,6 +375,8 @@ export async function pushSmsTrackingEvent(
351
375
  eventProperties: {
352
376
  smsSendId,
353
377
  templateKey: ctx.templateKey,
378
+ // Attribution scope (plan §1.2) — see pushTrackingEvent.
379
+ journeyId: ctx.journeyId,
354
380
  ...opts.properties,
355
381
  },
356
382
  source: "tracking",
@@ -0,0 +1,45 @@
1
+ import { defineEmailProvider } from "@hogsend/core";
2
+
3
+ /**
4
+ * The inert email provider a container boots with when NO provider is
5
+ * configured and none was explicitly requested (mirrors the SMS channel's
6
+ * operator-opt-in posture: unconfigured ⇒ inert stub, not a boot crash).
7
+ *
8
+ * A fresh scaffold has no RESEND_API_KEY yet — it should still boot, serve
9
+ * Studio, ingest events and run non-email journeys. Every email send fails
10
+ * per-call with this actionable message instead; the container logs one boot
11
+ * warning. Never registered in the provider registry, so no webhook route
12
+ * resolves it.
13
+ *
14
+ * DELIBERATE seam choice (vs SMS's service-level stub): the stub is the WIRE
15
+ * inside the real tracked mailer, so an unconfigured send still runs the
16
+ * pipeline (preferences → render → tracking → `email_sends` write) and fails
17
+ * at dispatch — leaving a failed-send row visible in Studio, which is exactly
18
+ * the operator-facing breadcrumb a "why didn't my email go out?" moment needs.
19
+ */
20
+ export const UNCONFIGURED_EMAIL_MESSAGE =
21
+ "no email provider configured — set RESEND_API_KEY (or POSTMARK_SERVER_TOKEN " +
22
+ "with EMAIL_PROVIDER=postmark), or pass `email: { provider }` to " +
23
+ "createHogsendClient().";
24
+
25
+ export function createUnconfiguredEmailProvider() {
26
+ return defineEmailProvider({
27
+ meta: {
28
+ id: "unconfigured",
29
+ name: "No email provider configured",
30
+ },
31
+ capabilities: {},
32
+ async send() {
33
+ throw new Error(UNCONFIGURED_EMAIL_MESSAGE);
34
+ },
35
+ async sendBatch() {
36
+ throw new Error(UNCONFIGURED_EMAIL_MESSAGE);
37
+ },
38
+ verifyWebhook() {
39
+ throw new Error(UNCONFIGURED_EMAIL_MESSAGE);
40
+ },
41
+ parseWebhook() {
42
+ throw new Error(UNCONFIGURED_EMAIL_MESSAGE);
43
+ },
44
+ });
45
+ }
@@ -55,6 +55,9 @@ export const WEBHOOK_EVENT_TYPES = [
55
55
  "contact.deleted",
56
56
  "contact.unsubscribed",
57
57
  "contact.subscribed",
58
+ // Global control group membership (impact plan §4.3) — emitted once per
59
+ // contact key on the first withheld send.
60
+ "contact.control_group",
58
61
  "email.sent",
59
62
  "email.delivered",
60
63
  "email.opened",
@@ -67,10 +70,14 @@ export const WEBHOOK_EVENT_TYPES = [
67
70
  "sms.failed",
68
71
  "sms.clicked",
69
72
  "journey.completed",
73
+ "journey.heldout",
70
74
  "bucket.entered",
71
75
  "bucket.left",
72
76
  "link.clicked",
73
77
  "link.arrived",
78
+ "funnel.stage_changed",
79
+ "deal.quoted",
80
+ "deal.sold",
74
81
  ] as const;
75
82
 
76
83
  export type WebhookEventType = (typeof WEBHOOK_EVENT_TYPES)[number];
@@ -3,7 +3,10 @@ import { DEFAULT_HOST, derivePrivateHost } from "@hogsend/plugin-posthog";
3
3
  import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
4
4
  import type { AppEnv } from "../../app.js";
5
5
  import { createTokenManager } from "../../lib/oauth-token-manager.js";
6
- import { EXPECTED_POSTHOG_SCOPES } from "../../lib/posthog-scopes.js";
6
+ import {
7
+ EXPECTED_POSTHOG_SCOPES,
8
+ posthogScopeSatisfied,
9
+ } from "../../lib/posthog-scopes.js";
7
10
  import {
8
11
  getDerivedCredential,
9
12
  getProviderCredential,
@@ -202,8 +205,12 @@ export const analyticsAdminRouter = new OpenAPIHono<AppEnv>()
202
205
  try {
203
206
  const oauth = await getProviderCredential(db, "posthog");
204
207
  if (oauth) {
208
+ // Hierarchical scopes: a granted `X:write` satisfies `X:read` (the
209
+ // grant normalizes both-halves requests down to write-only).
205
210
  const granted = oauth.payload.scopes;
206
- scopeGap = EXPECTED_POSTHOG_SCOPES.filter((s) => !granted.includes(s));
211
+ scopeGap = EXPECTED_POSTHOG_SCOPES.filter(
212
+ (s) => !posthogScopeSatisfied(granted, s),
213
+ );
207
214
  }
208
215
  } catch {
209
216
  scopeGap = [];
@@ -293,9 +300,11 @@ export const analyticsAdminRouter = new OpenAPIHono<AppEnv>()
293
300
  });
294
301
 
295
302
  // Opportunistically persist the phc_ (project api_token) the provisioner
296
- // read on its way through — it powers the OPTIONAL outbound capture path
297
- // and activates on the next deploy (no lazy boot-time seam). Re-read the
298
- // stored payload to merge over the just-persisted webhook secret.
303
+ // read on its way through — it powers the OPTIONAL outbound capture path.
304
+ // `activateStoredPosthogAnalytics` (container boot) reads it back, so
305
+ // capture comes alive on the next boot with NO hand-pasted
306
+ // POSTHOG_API_KEY. Re-read the stored payload to merge over the
307
+ // just-persisted webhook secret.
299
308
  if (result.projectApiKey) {
300
309
  const cur = (await getDerivedCredential(db, "posthog")) ?? {};
301
310
  await saveDerivedCredential(db, "posthog", {