@hogsend/engine 0.42.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +10 -7
- package/src/container.ts +419 -4
- package/src/destinations/define-destination.ts +7 -3
- package/src/destinations/presets/posthog.ts +31 -0
- package/src/env.ts +28 -0
- package/src/index.ts +138 -0
- package/src/journeys/execute-journey-run.ts +93 -0
- package/src/journeys/journey-boundary.ts +5 -1
- package/src/journeys/journey-context.ts +30 -0
- package/src/lib/attribution-backfill.ts +285 -0
- package/src/lib/attribution.ts +239 -0
- package/src/lib/contacts.ts +71 -0
- package/src/lib/conversion-dispatch.ts +326 -0
- package/src/lib/conversions.ts +197 -0
- package/src/lib/crm-deals.ts +320 -0
- package/src/lib/crm-ingest.ts +341 -0
- package/src/lib/crm-provider-registry.ts +41 -0
- package/src/lib/crm-registry-singleton.ts +22 -0
- package/src/lib/deal-money-events.ts +85 -0
- package/src/lib/email-service-types.ts +16 -1
- package/src/lib/event-names.ts +1 -1
- package/src/lib/funnel-progress.ts +76 -0
- package/src/lib/funnel-registry.ts +108 -0
- package/src/lib/funnel-transitions.ts +280 -0
- package/src/lib/holdout.ts +62 -0
- package/src/lib/ingestion.ts +218 -9
- package/src/lib/journey-suppress.ts +33 -2
- package/src/lib/lift-stats.ts +125 -0
- package/src/lib/mailer.ts +1 -0
- package/src/lib/operator.ts +31 -0
- package/src/lib/outbound.ts +109 -0
- package/src/lib/phone.ts +19 -0
- package/src/lib/preferences.ts +32 -0
- package/src/lib/revenue.ts +116 -0
- package/src/lib/sms-frequency-cap.ts +47 -0
- package/src/lib/sms-inbound.ts +200 -0
- package/src/lib/sms-link-tracking.ts +160 -0
- package/src/lib/sms-mailer.ts +338 -0
- package/src/lib/sms-provider-registry.ts +35 -0
- package/src/lib/sms-providers-from-env.ts +100 -0
- package/src/lib/sms-service-types.ts +175 -0
- package/src/lib/sms-tracked.ts +625 -0
- package/src/lib/sms.ts +117 -0
- package/src/lib/tracked.ts +40 -0
- package/src/lib/tracking-event-names.ts +6 -0
- package/src/lib/tracking-events.ts +133 -1
- package/src/lib/tracking.ts +2 -1
- package/src/lib/webhook-signing.ts +24 -1
- package/src/lists/channels.ts +21 -0
- package/src/routes/admin/attribution.ts +414 -0
- package/src/routes/admin/contacts.ts +71 -11
- package/src/routes/admin/conversions.ts +247 -0
- package/src/routes/admin/deals.ts +472 -0
- package/src/routes/admin/funnels.ts +228 -0
- package/src/routes/admin/index.ts +8 -0
- package/src/routes/admin/journeys.ts +158 -1
- package/src/routes/admin/readiness.ts +97 -2
- package/src/routes/admin/templates.ts +28 -1
- package/src/routes/email/unsubscribe.ts +1 -0
- package/src/routes/events/index.ts +12 -0
- package/src/routes/index.ts +6 -0
- package/src/routes/lists/index.ts +54 -7
- package/src/routes/tracking/click-pipeline.ts +105 -16
- package/src/routes/tracking/short.ts +61 -0
- package/src/routes/webhooks/crm-provider.ts +117 -0
- package/src/routes/webhooks/index.ts +9 -1
- package/src/routes/webhooks/sms-provider.ts +110 -0
- package/src/sources/define-contact-source.ts +159 -0
- package/src/sources/registry.ts +71 -0
- package/src/sources/webhook-source.ts +109 -0
- package/src/worker.ts +4 -0
- package/src/workflows/crm-reconcile.ts +129 -0
- package/src/workflows/dispatch-conversion.ts +31 -0
- package/src/workflows/import-contacts.ts +61 -4
- package/src/workflows/send-campaign.ts +5 -0
package/src/lib/ingestion.ts
CHANGED
|
@@ -25,10 +25,22 @@ import {
|
|
|
25
25
|
getAnalytics,
|
|
26
26
|
getAnalyticsEventMirror,
|
|
27
27
|
} from "./analytics-singleton.js";
|
|
28
|
+
import { recordAttributionCredits } from "./attribution.js";
|
|
28
29
|
import {
|
|
29
30
|
ContactProvenanceLostError,
|
|
30
31
|
resolveOrCreateContact,
|
|
31
32
|
} from "./contacts.js";
|
|
33
|
+
import {
|
|
34
|
+
conversionEventId,
|
|
35
|
+
enqueueConversionDispatches,
|
|
36
|
+
} from "./conversion-dispatch.js";
|
|
37
|
+
import {
|
|
38
|
+
evaluateConversionsAtIngest,
|
|
39
|
+
getConversionRegistry,
|
|
40
|
+
} from "./conversions.js";
|
|
41
|
+
import { getCrmSyncConfig } from "./crm-registry-singleton.js";
|
|
42
|
+
import { recordFunnelProgressAtIngest } from "./funnel-progress.js";
|
|
43
|
+
import { applyFunnelTransitionsAtIngest } from "./funnel-transitions.js";
|
|
32
44
|
import type { Logger } from "./logger.js";
|
|
33
45
|
|
|
34
46
|
export interface IngestEvent {
|
|
@@ -57,6 +69,15 @@ export interface IngestEvent {
|
|
|
57
69
|
eventProperties: Record<string, unknown>;
|
|
58
70
|
/** D2: → `contacts.properties` merge ONLY. */
|
|
59
71
|
contactProperties?: Record<string, unknown>;
|
|
72
|
+
/**
|
|
73
|
+
* The event's own monetary worth (deal value, order total). Stored on the
|
|
74
|
+
* first-class `user_events.value` column — the revenue spine every rollup,
|
|
75
|
+
* conversion definition, and attribution credit reads from. Non-finite
|
|
76
|
+
* numbers are dropped (with a warn) rather than failing the ingest.
|
|
77
|
+
*/
|
|
78
|
+
value?: number;
|
|
79
|
+
/** ISO-4217 alpha code for `value`; uppercased here. Defaults to null. */
|
|
80
|
+
currency?: string;
|
|
60
81
|
idempotencyKey?: string;
|
|
61
82
|
/**
|
|
62
83
|
* Caller-supplied event time (§2.5 `timestamp`). When set, `user_events`
|
|
@@ -295,6 +316,10 @@ export async function ingestEvent(opts: {
|
|
|
295
316
|
contactId: event.contactId,
|
|
296
317
|
contactProperties: event.contactProperties,
|
|
297
318
|
restrictToAnonymous: opts.restrictToAnonymous,
|
|
319
|
+
// First-touch contact provenance from the event's pipeline origin — a
|
|
320
|
+
// Contact Source ("clay"/"attio") or "api"/"posthog"/…; only stamped when
|
|
321
|
+
// the contact is created (or first fill-in-linked) with no prior source.
|
|
322
|
+
source: event.source,
|
|
298
323
|
});
|
|
299
324
|
} catch (err) {
|
|
300
325
|
// Provenance pin pointed at a hard-deleted/unfollowable subject: drop the
|
|
@@ -326,12 +351,41 @@ export async function ingestEvent(opts: {
|
|
|
326
351
|
// falls back to the `occurred_at` DB default (ingest instant).
|
|
327
352
|
const occurredAt = event.occurredAt ? new Date(event.occurredAt) : undefined;
|
|
328
353
|
|
|
354
|
+
// Money normalization — permissive at the spine (webhook sources feed this
|
|
355
|
+
// path with arbitrary payloads): a non-finite value or malformed currency is
|
|
356
|
+
// dropped with a warn, never a failed ingest. Currency without value is
|
|
357
|
+
// meaningless and dropped; value without currency stores with null currency
|
|
358
|
+
// (single-currency deploys omit it everywhere).
|
|
359
|
+
let value: number | null = null;
|
|
360
|
+
let currency: string | null = null;
|
|
361
|
+
if (event.value !== undefined) {
|
|
362
|
+
if (typeof event.value === "number" && Number.isFinite(event.value)) {
|
|
363
|
+
value = event.value;
|
|
364
|
+
if (event.currency !== undefined) {
|
|
365
|
+
const code = event.currency.trim().toUpperCase();
|
|
366
|
+
if (/^[A-Z]{3}$/.test(code)) {
|
|
367
|
+
currency = code;
|
|
368
|
+
} else {
|
|
369
|
+
logger.warn("ingestEvent: dropping malformed currency", {
|
|
370
|
+
event: event.event,
|
|
371
|
+
currency: event.currency,
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
} else {
|
|
376
|
+
logger.warn("ingestEvent: dropping non-finite value", {
|
|
377
|
+
event: event.event,
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
329
382
|
// (2) Idempotency dedup + `user_events` insert keyed on the resolved key, with
|
|
330
383
|
// ONLY eventProperties in the properties bag (D2). `ctx.trigger` now supplies a
|
|
331
384
|
// deterministic key (`journeyTrigger:<runAnchor>:<site>:<event>`), so a journey
|
|
332
385
|
// replay re-pushing the same trigger hits the onConflictDoNothing early-return
|
|
333
386
|
// below — the push, checkExits, contact upsert, and alias never re-fire.
|
|
334
387
|
let idempotentInsertId: string | undefined;
|
|
388
|
+
let insertedRow: { id: string; occurredAt: Date } | undefined;
|
|
335
389
|
if (event.idempotencyKey) {
|
|
336
390
|
const result = await db
|
|
337
391
|
.insert(userEvents)
|
|
@@ -339,6 +393,8 @@ export async function ingestEvent(opts: {
|
|
|
339
393
|
userId: resolvedKey,
|
|
340
394
|
event: event.event,
|
|
341
395
|
properties: event.eventProperties,
|
|
396
|
+
value,
|
|
397
|
+
currency,
|
|
342
398
|
source: event.source ?? null,
|
|
343
399
|
idempotencyKey: event.idempotencyKey,
|
|
344
400
|
...(occurredAt ? { occurredAt } : {}),
|
|
@@ -346,20 +402,27 @@ export async function ingestEvent(opts: {
|
|
|
346
402
|
.onConflictDoNothing({
|
|
347
403
|
target: userEvents.idempotencyKey,
|
|
348
404
|
})
|
|
349
|
-
.returning({ id: userEvents.id });
|
|
405
|
+
.returning({ id: userEvents.id, occurredAt: userEvents.occurredAt });
|
|
350
406
|
|
|
351
407
|
if (result.length === 0) {
|
|
352
408
|
return { stored: false, exits: [], contactKey: resolvedKey };
|
|
353
409
|
}
|
|
354
410
|
idempotentInsertId = result[0]?.id;
|
|
411
|
+
insertedRow = result[0];
|
|
355
412
|
} else {
|
|
356
|
-
await db
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
413
|
+
const result = await db
|
|
414
|
+
.insert(userEvents)
|
|
415
|
+
.values({
|
|
416
|
+
userId: resolvedKey,
|
|
417
|
+
event: event.event,
|
|
418
|
+
properties: event.eventProperties,
|
|
419
|
+
value,
|
|
420
|
+
currency,
|
|
421
|
+
source: event.source ?? null,
|
|
422
|
+
...(occurredAt ? { occurredAt } : {}),
|
|
423
|
+
})
|
|
424
|
+
.returning({ id: userEvents.id, occurredAt: userEvents.occurredAt });
|
|
425
|
+
insertedRow = result[0];
|
|
363
426
|
}
|
|
364
427
|
|
|
365
428
|
// (2b) §5.3 — fire the provider-neutral identity merge at the two resolver
|
|
@@ -413,7 +476,16 @@ export async function ingestEvent(opts: {
|
|
|
413
476
|
mirrorProvider.capture({
|
|
414
477
|
distinctId: resolvedKey,
|
|
415
478
|
event: event.event,
|
|
416
|
-
|
|
479
|
+
// The revenue spine fans out with the event: `value`/`currency` ride as
|
|
480
|
+
// plain properties so the analytics tool (PostHog et al.) can sum
|
|
481
|
+
// revenue without a Hogsend round-trip. Property-bag keys of the same
|
|
482
|
+
// name lose to the first-class columns.
|
|
483
|
+
properties: {
|
|
484
|
+
...event.eventProperties,
|
|
485
|
+
...(value !== null
|
|
486
|
+
? { value, ...(currency ? { currency } : {}) }
|
|
487
|
+
: {}),
|
|
488
|
+
},
|
|
417
489
|
});
|
|
418
490
|
} catch (err) {
|
|
419
491
|
logger.debug("event mirror capture failed (non-fatal)", {
|
|
@@ -512,6 +584,143 @@ export async function ingestEvent(opts: {
|
|
|
512
584
|
});
|
|
513
585
|
}
|
|
514
586
|
|
|
587
|
+
// (5c)+(5d) — the post-store hooks. Both fire only AFTER the push settled
|
|
588
|
+
// (a failed publish compensating-deletes the event row, so a rolled-back
|
|
589
|
+
// event never leaves a fired conversion or a moved deal behind) and only
|
|
590
|
+
// on a FRESH insert (replays early-returned at (2)). Each is best-effort
|
|
591
|
+
// in its own try/catch: the event is already durable, so a hook failure
|
|
592
|
+
// warns rather than failing the caller.
|
|
593
|
+
const hookEvent = insertedRow
|
|
594
|
+
? {
|
|
595
|
+
name: event.event,
|
|
596
|
+
source: event.source ?? null,
|
|
597
|
+
properties: event.eventProperties,
|
|
598
|
+
value,
|
|
599
|
+
currency,
|
|
600
|
+
occurredAt: insertedRow.occurredAt,
|
|
601
|
+
}
|
|
602
|
+
: null;
|
|
603
|
+
|
|
604
|
+
// (5c) Conversion-point evaluation (plan §5.1) — the unique
|
|
605
|
+
// (definition, event) index makes any replay a no-op.
|
|
606
|
+
if (insertedRow && hookEvent) {
|
|
607
|
+
try {
|
|
608
|
+
const fired = await evaluateConversionsAtIngest({
|
|
609
|
+
db,
|
|
610
|
+
logger,
|
|
611
|
+
registry: getConversionRegistry(),
|
|
612
|
+
event: hookEvent,
|
|
613
|
+
eventRowId: insertedRow.id,
|
|
614
|
+
contactId,
|
|
615
|
+
userKey: resolvedKey,
|
|
616
|
+
});
|
|
617
|
+
for (const firedConversion of fired) {
|
|
618
|
+
// Fan fired conversions out to their ad-platform destinations
|
|
619
|
+
// (§5.2) FIRST — the pending dispatch row is the recoverable
|
|
620
|
+
// anchor, and the conversion row won't re-fire on replay, so
|
|
621
|
+
// nothing may run ahead of it and throw. The deterministic
|
|
622
|
+
// event_id is stable across retries AND re-evaluations.
|
|
623
|
+
const destinationIds = firedConversion.definition.meta.destinations;
|
|
624
|
+
if (destinationIds && destinationIds.length > 0) {
|
|
625
|
+
await enqueueConversionDispatches({
|
|
626
|
+
db,
|
|
627
|
+
logger,
|
|
628
|
+
conversionId: firedConversion.conversionId,
|
|
629
|
+
eventId: conversionEventId({
|
|
630
|
+
contactId,
|
|
631
|
+
definitionId: firedConversion.definition.meta.id,
|
|
632
|
+
eventRowId: insertedRow.id,
|
|
633
|
+
}),
|
|
634
|
+
destinationIds,
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
// The attribution ledger (§6.1): every model's credits over the
|
|
639
|
+
// contact's touchpoint path, written once at conversion time.
|
|
640
|
+
// Pure reporting — individually isolated so a transient ledger
|
|
641
|
+
// failure can never cost a dispatch or a later conversion.
|
|
642
|
+
try {
|
|
643
|
+
await recordAttributionCredits({
|
|
644
|
+
db,
|
|
645
|
+
logger,
|
|
646
|
+
conversionId: firedConversion.conversionId,
|
|
647
|
+
userKey: resolvedKey,
|
|
648
|
+
value: firedConversion.value,
|
|
649
|
+
currency: firedConversion.currency,
|
|
650
|
+
occurredAt: insertedRow.occurredAt,
|
|
651
|
+
windowDays:
|
|
652
|
+
firedConversion.definition.meta.attributionWindowDays ?? 90,
|
|
653
|
+
windows: firedConversion.definition.meta.windows,
|
|
654
|
+
});
|
|
655
|
+
} catch (err) {
|
|
656
|
+
logger.warn("attribution credit write failed", {
|
|
657
|
+
conversionId: firedConversion.conversionId,
|
|
658
|
+
error: err instanceof Error ? err.message : String(err),
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
} catch (err) {
|
|
663
|
+
logger.warn("Conversion evaluation failed", {
|
|
664
|
+
event: event.event,
|
|
665
|
+
userId: resolvedKey,
|
|
666
|
+
error: err instanceof Error ? err.message : String(err),
|
|
667
|
+
});
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
// (5d) Event-driven funnel stage transitions (event-native funnels). The
|
|
672
|
+
// money events it mints (`deal.quoted`/`deal.sold`) recurse through
|
|
673
|
+
// ingestEvent, bounded at define time: `deal.`/`funnel.`/`crm.` events
|
|
674
|
+
// cannot be stage triggers.
|
|
675
|
+
if (insertedRow && hookEvent) {
|
|
676
|
+
const funnelRegistry = getCrmSyncConfig()?.funnels;
|
|
677
|
+
if (funnelRegistry) {
|
|
678
|
+
try {
|
|
679
|
+
await applyFunnelTransitionsAtIngest({
|
|
680
|
+
db,
|
|
681
|
+
registry,
|
|
682
|
+
hatchet,
|
|
683
|
+
logger,
|
|
684
|
+
analytics,
|
|
685
|
+
funnels: funnelRegistry,
|
|
686
|
+
event: hookEvent,
|
|
687
|
+
eventRowId: insertedRow.id,
|
|
688
|
+
contactId,
|
|
689
|
+
userKey: resolvedKey,
|
|
690
|
+
});
|
|
691
|
+
} catch (err) {
|
|
692
|
+
logger.warn("Funnel transition evaluation failed", {
|
|
693
|
+
event: event.event,
|
|
694
|
+
userId: resolvedKey,
|
|
695
|
+
error: err instanceof Error ? err.message : String(err),
|
|
696
|
+
});
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
// (5e) Funnel progression REPORTING projection (impact plan §3.3) —
|
|
700
|
+
// first-reach `funnel_progress` rows over the SAME transition rules
|
|
701
|
+
// and gates as the deal mover above, so the two projections never
|
|
702
|
+
// disagree. Best-effort like every hook; the unique (contact, funnel,
|
|
703
|
+
// stage) index makes any replay a no-op.
|
|
704
|
+
try {
|
|
705
|
+
await recordFunnelProgressAtIngest({
|
|
706
|
+
db,
|
|
707
|
+
logger,
|
|
708
|
+
funnels: funnelRegistry,
|
|
709
|
+
event: hookEvent,
|
|
710
|
+
eventRowId: insertedRow.id,
|
|
711
|
+
contactId,
|
|
712
|
+
userKey: resolvedKey,
|
|
713
|
+
});
|
|
714
|
+
} catch (err) {
|
|
715
|
+
logger.warn("Funnel progression write failed", {
|
|
716
|
+
event: event.event,
|
|
717
|
+
userId: resolvedKey,
|
|
718
|
+
error: err instanceof Error ? err.message : String(err),
|
|
719
|
+
});
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
|
|
515
724
|
// (6) Real-time bucket membership re-evaluation (Section 6.1). NOT part of the
|
|
516
725
|
// Promise.all above: its property eval reads contact state ⊕ this-ingest
|
|
517
726
|
// contactProperties patch, and its bucket:entered/left emissions recurse back
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { Database } from "@hogsend/db";
|
|
2
|
-
import { emailSends, journeyStates } from "@hogsend/db";
|
|
2
|
+
import { emailSends, journeyStates, smsSends } from "@hogsend/db";
|
|
3
3
|
import { and, eq, gte, ne } from "drizzle-orm";
|
|
4
4
|
import type { JourneyBoundary } from "../journeys/journey-boundary.js";
|
|
5
5
|
import { recordOnce } from "../journeys/record-once.js";
|
|
@@ -36,6 +36,33 @@ export async function hasRecentJourneySend(opts: {
|
|
|
36
36
|
return rows.length > 0;
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
+
/**
|
|
40
|
+
* The SMS leg of {@link hasRecentJourneySend}: same cross-enrollment EXISTS
|
|
41
|
+
* over `sms_sends` keyed by recipient phone. `meta.suppress` is a per-journey,
|
|
42
|
+
* per-recipient min-gap — it must gap SMS exactly as it gaps email.
|
|
43
|
+
*/
|
|
44
|
+
export async function hasRecentJourneySmsSend(opts: {
|
|
45
|
+
db: Database;
|
|
46
|
+
journeyId: string;
|
|
47
|
+
to: string;
|
|
48
|
+
since: Date;
|
|
49
|
+
}): Promise<boolean> {
|
|
50
|
+
const rows = await opts.db
|
|
51
|
+
.select({ id: smsSends.id })
|
|
52
|
+
.from(smsSends)
|
|
53
|
+
.innerJoin(journeyStates, eq(smsSends.journeyStateId, journeyStates.id))
|
|
54
|
+
.where(
|
|
55
|
+
and(
|
|
56
|
+
eq(journeyStates.journeyId, opts.journeyId),
|
|
57
|
+
eq(smsSends.toPhone, opts.to),
|
|
58
|
+
gte(smsSends.createdAt, opts.since),
|
|
59
|
+
ne(smsSends.status, "failed"),
|
|
60
|
+
),
|
|
61
|
+
)
|
|
62
|
+
.limit(1);
|
|
63
|
+
return rows.length > 0;
|
|
64
|
+
}
|
|
65
|
+
|
|
39
66
|
/**
|
|
40
67
|
* The RECORDED-once journey-suppress verdict for a boundary send. Returns
|
|
41
68
|
* `{ suppressed }` — `true` means the tracked mailer must skip the send
|
|
@@ -58,6 +85,8 @@ export async function checkJourneySuppress(opts: {
|
|
|
58
85
|
boundary: JourneyBoundary | undefined;
|
|
59
86
|
to: string;
|
|
60
87
|
idempotencyKey: string | undefined;
|
|
88
|
+
/** Which channel's send history the min-gap EXISTS runs against. */
|
|
89
|
+
channel?: "email" | "sms";
|
|
61
90
|
/** Injectable clock for tests; defaults to the live wall clock. */
|
|
62
91
|
now?: number;
|
|
63
92
|
}): Promise<{ suppressed: boolean }> {
|
|
@@ -79,8 +108,10 @@ export async function checkJourneySuppress(opts: {
|
|
|
79
108
|
key: `suppress:${idempotencyKey}`,
|
|
80
109
|
compute: async () => {
|
|
81
110
|
const since = new Date((opts.now ?? Date.now()) - suppressMs);
|
|
111
|
+
const hasRecent =
|
|
112
|
+
opts.channel === "sms" ? hasRecentJourneySmsSend : hasRecentJourneySend;
|
|
82
113
|
return {
|
|
83
|
-
suppressed: await
|
|
114
|
+
suppressed: await hasRecent({ db, journeyId, to, since }),
|
|
84
115
|
};
|
|
85
116
|
},
|
|
86
117
|
});
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Holdout lift statistics (docs/attribution-impact-plan.md §4.2, open Q1
|
|
3
|
+
* resolved): Bayesian beta-binomial with a uniform Beta(1,1) prior — the
|
|
4
|
+
* lean, honest method at self-hosted volumes. Win probability is
|
|
5
|
+
* P(treatment rate > control rate) under independent posteriors
|
|
6
|
+
* Beta(k+1, n−k+1), computed by DETERMINISTIC numeric integration (no
|
|
7
|
+
* sampling — stable across runs and trivially testable).
|
|
8
|
+
*
|
|
9
|
+
* Presentation guards, per the plan's honesty stance:
|
|
10
|
+
* - suppression floor: fewer than {@link MIN_COMBINED_CONVERSIONS} combined
|
|
11
|
+
* conversions ⇒ no win probability at all (Klaviyo suppresses at the same
|
|
12
|
+
* threshold; below it the number is noise wearing a percentage sign);
|
|
13
|
+
* - small-sample flag: either cohort under {@link SMALL_SAMPLE_FLOOR}
|
|
14
|
+
* contacts ⇒ `smallSample: true` so surfaces can warn loudly (Braze's
|
|
15
|
+
* rule of thumb is ~1,000; we present rather than lock out).
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
export const MIN_COMBINED_CONVERSIONS = 10;
|
|
19
|
+
export const SMALL_SAMPLE_FLOOR = 100;
|
|
20
|
+
|
|
21
|
+
/** Integration grid — 2001 points is ample at any realistic cohort size. */
|
|
22
|
+
const GRID = 2001;
|
|
23
|
+
|
|
24
|
+
function logGamma(x: number): number {
|
|
25
|
+
// Lanczos approximation (g=7, n=9) — standard coefficients.
|
|
26
|
+
const coefficients = [
|
|
27
|
+
0.99999999999980993, 676.5203681218851, -1259.1392167224028,
|
|
28
|
+
771.32342877765313, -176.61502916214059, 12.507343278686905,
|
|
29
|
+
-0.13857109526572012, 9.9843695780195716e-6, 1.5056327351493116e-7,
|
|
30
|
+
];
|
|
31
|
+
if (x < 0.5) {
|
|
32
|
+
return Math.log(Math.PI / Math.sin(Math.PI * x)) - logGamma(1 - x);
|
|
33
|
+
}
|
|
34
|
+
const xShifted = x - 1;
|
|
35
|
+
let sum = coefficients[0] as number;
|
|
36
|
+
for (let i = 1; i < coefficients.length; i++) {
|
|
37
|
+
sum += (coefficients[i] as number) / (xShifted + i);
|
|
38
|
+
}
|
|
39
|
+
const t = xShifted + 7.5;
|
|
40
|
+
return (
|
|
41
|
+
0.5 * Math.log(2 * Math.PI) +
|
|
42
|
+
(xShifted + 0.5) * Math.log(t) -
|
|
43
|
+
t +
|
|
44
|
+
Math.log(sum)
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function logBetaPdf(x: number, a: number, b: number): number {
|
|
49
|
+
if (x <= 0 || x >= 1) return Number.NEGATIVE_INFINITY;
|
|
50
|
+
return (
|
|
51
|
+
(a - 1) * Math.log(x) +
|
|
52
|
+
(b - 1) * Math.log(1 - x) +
|
|
53
|
+
logGamma(a + b) -
|
|
54
|
+
logGamma(a) -
|
|
55
|
+
logGamma(b)
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* P(X > Y) for X ~ Beta(a1,b1), Y ~ Beta(a2,b2): midpoint-rule integration
|
|
61
|
+
* of ∫ f_X(x)·F_Y(x) dx. Midpoints (never 0 or 1) sidestep the boundary —
|
|
62
|
+
* Beta pdfs are singular or discontinuous there for shape params ≤ 1, which
|
|
63
|
+
* a trapezoid pass silently truncates.
|
|
64
|
+
*/
|
|
65
|
+
export function betaWinProbability(
|
|
66
|
+
a1: number,
|
|
67
|
+
b1: number,
|
|
68
|
+
a2: number,
|
|
69
|
+
b2: number,
|
|
70
|
+
): number {
|
|
71
|
+
const dx = 1 / GRID;
|
|
72
|
+
let cdfY = 0;
|
|
73
|
+
let p = 0;
|
|
74
|
+
for (let i = 0; i < GRID; i++) {
|
|
75
|
+
const x = (i + 0.5) * dx;
|
|
76
|
+
const massY = Math.exp(logBetaPdf(x, a2, b2)) * dx;
|
|
77
|
+
// F_Y at this midpoint ≈ mass through the previous cells + half this one.
|
|
78
|
+
const cdfAtX = cdfY + massY / 2;
|
|
79
|
+
p += Math.exp(logBetaPdf(x, a1, b1)) * cdfAtX * dx;
|
|
80
|
+
cdfY += massY;
|
|
81
|
+
}
|
|
82
|
+
// Clamp tiny numeric drift.
|
|
83
|
+
return Math.min(1, Math.max(0, p));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export interface LiftVerdict {
|
|
87
|
+
/** (treatmentRate − controlRate) / controlRate; null when control is 0. */
|
|
88
|
+
liftPercent: number | null;
|
|
89
|
+
/** P(treatment > control); null when suppressed. */
|
|
90
|
+
winProbability: number | null;
|
|
91
|
+
/** True when combined conversions are under the suppression floor. */
|
|
92
|
+
suppressed: boolean;
|
|
93
|
+
/** True when either cohort is under the small-sample floor. */
|
|
94
|
+
smallSample: boolean;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function computeLift(opts: {
|
|
98
|
+
treatment: { contacts: number; converters: number };
|
|
99
|
+
control: { contacts: number; converters: number };
|
|
100
|
+
}): LiftVerdict {
|
|
101
|
+
const { treatment, control } = opts;
|
|
102
|
+
const treatmentRate =
|
|
103
|
+
treatment.contacts > 0 ? treatment.converters / treatment.contacts : 0;
|
|
104
|
+
const controlRate =
|
|
105
|
+
control.contacts > 0 ? control.converters / control.contacts : 0;
|
|
106
|
+
const suppressed =
|
|
107
|
+
treatment.converters + control.converters < MIN_COMBINED_CONVERSIONS;
|
|
108
|
+
return {
|
|
109
|
+
liftPercent:
|
|
110
|
+
controlRate > 0
|
|
111
|
+
? ((treatmentRate - controlRate) / controlRate) * 100
|
|
112
|
+
: null,
|
|
113
|
+
winProbability: suppressed
|
|
114
|
+
? null
|
|
115
|
+
: betaWinProbability(
|
|
116
|
+
treatment.converters + 1,
|
|
117
|
+
treatment.contacts - treatment.converters + 1,
|
|
118
|
+
control.converters + 1,
|
|
119
|
+
control.contacts - control.converters + 1,
|
|
120
|
+
),
|
|
121
|
+
suppressed,
|
|
122
|
+
smallSample:
|
|
123
|
+
Math.min(treatment.contacts, control.contacts) < SMALL_SAMPLE_FLOOR,
|
|
124
|
+
};
|
|
125
|
+
}
|
package/src/lib/mailer.ts
CHANGED
|
@@ -148,6 +148,7 @@ export function createTrackedMailer(
|
|
|
148
148
|
to: options.to,
|
|
149
149
|
subject: options.subject,
|
|
150
150
|
journeyStateId: options.journeyStateId,
|
|
151
|
+
campaignId: options.campaignId,
|
|
151
152
|
userId: options.userId,
|
|
152
153
|
userEmail: options.userEmail,
|
|
153
154
|
category: options.category,
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { type Database, user } from "@hogsend/db";
|
|
2
|
+
import { normalizeEmail, normalizeEmailOrNull } from "./contacts.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* True iff `email` is a member of the operator set: a row in the `user` table,
|
|
6
|
+
* or the env-configured operator inbox (`HOGSEND_TEST_EMAIL` /
|
|
7
|
+
* `STUDIO_ADMIN_EMAIL`). Comparison is case-insensitive (`normalizeEmail`).
|
|
8
|
+
*
|
|
9
|
+
* The `user` table IS the operator set: admins are minted only by the CLI or the
|
|
10
|
+
* boot bootstrap (there is no unauthenticated path to create one), so a
|
|
11
|
+
* closed-signup deploy has a handful of rows. The two env addresses are the same
|
|
12
|
+
* operator inboxes test-mode redirects to.
|
|
13
|
+
*/
|
|
14
|
+
export async function isOperatorAddress(opts: {
|
|
15
|
+
db: Database;
|
|
16
|
+
env: { HOGSEND_TEST_EMAIL?: string; STUDIO_ADMIN_EMAIL?: string };
|
|
17
|
+
email: string;
|
|
18
|
+
}): Promise<boolean> {
|
|
19
|
+
const target = normalizeEmail(opts.email);
|
|
20
|
+
if (!target) return false;
|
|
21
|
+
|
|
22
|
+
// Match how addresses are stored/compared everywhere else (contacts.ts) so
|
|
23
|
+
// this security gate can't silently diverge from the canonical normalization.
|
|
24
|
+
if (normalizeEmailOrNull(opts.env.HOGSEND_TEST_EMAIL) === target) return true;
|
|
25
|
+
if (normalizeEmailOrNull(opts.env.STUDIO_ADMIN_EMAIL) === target) return true;
|
|
26
|
+
|
|
27
|
+
// The admin team is a handful of rows (closed signup), so a full scan +
|
|
28
|
+
// normalized compare is cheaper and simpler than a lower() predicate.
|
|
29
|
+
const rows = await opts.db.select({ email: user.email }).from(user);
|
|
30
|
+
return rows.some((r) => normalizeEmail(r.email) === target);
|
|
31
|
+
}
|
package/src/lib/outbound.ts
CHANGED
|
@@ -76,6 +76,19 @@ export interface OutboundPayloads {
|
|
|
76
76
|
category: string | null;
|
|
77
77
|
scope: "all" | "category";
|
|
78
78
|
};
|
|
79
|
+
/**
|
|
80
|
+
* A genuine opt-IN (resubscribe-all or a category/channel grant) — the
|
|
81
|
+
* mirror of `contact.unsubscribed`, emitted from the same preference-write
|
|
82
|
+
* choke. `source` is the grant provenance: `"api"` (default),
|
|
83
|
+
* `"preference_center"`, `"started_keyword"`, `"import"`.
|
|
84
|
+
*/
|
|
85
|
+
"contact.subscribed": {
|
|
86
|
+
externalId: string | null;
|
|
87
|
+
email: string | null;
|
|
88
|
+
category: string | null;
|
|
89
|
+
scope: "all" | "category";
|
|
90
|
+
source: string;
|
|
91
|
+
};
|
|
79
92
|
"email.sent": {
|
|
80
93
|
emailSendId: string;
|
|
81
94
|
messageId: string;
|
|
@@ -152,6 +165,51 @@ export interface OutboundPayloads {
|
|
|
152
165
|
complaintType?: string;
|
|
153
166
|
reason?: string;
|
|
154
167
|
};
|
|
168
|
+
"sms.sent": {
|
|
169
|
+
smsSendId: string;
|
|
170
|
+
messageId: string;
|
|
171
|
+
templateKey: string | null;
|
|
172
|
+
to: string;
|
|
173
|
+
userId: string | null;
|
|
174
|
+
category: string | null;
|
|
175
|
+
journeyStateId: string | null;
|
|
176
|
+
segments: number | null;
|
|
177
|
+
sentAt: string;
|
|
178
|
+
};
|
|
179
|
+
"sms.delivered": {
|
|
180
|
+
smsSendId: string;
|
|
181
|
+
messageId: string;
|
|
182
|
+
templateKey: string | null;
|
|
183
|
+
userId: string | null;
|
|
184
|
+
to: string;
|
|
185
|
+
at: string;
|
|
186
|
+
};
|
|
187
|
+
"sms.failed": {
|
|
188
|
+
smsSendId: string;
|
|
189
|
+
messageId: string;
|
|
190
|
+
templateKey: string | null;
|
|
191
|
+
userId: string | null;
|
|
192
|
+
to: string;
|
|
193
|
+
at: string;
|
|
194
|
+
errorCode?: string;
|
|
195
|
+
errorReason?: string;
|
|
196
|
+
};
|
|
197
|
+
/**
|
|
198
|
+
* A tracked SMS short link was clicked (first-party, PER-HIT — the SMS
|
|
199
|
+
* sibling of `email.clicked`). `linkId` is the `tracked_links.id`,
|
|
200
|
+
* `linkUrl` the original destination. `userId` is null for a raw send with
|
|
201
|
+
* no resolvable contact.
|
|
202
|
+
*/
|
|
203
|
+
"sms.clicked": {
|
|
204
|
+
smsSendId: string;
|
|
205
|
+
messageId: string | null;
|
|
206
|
+
templateKey: string | null;
|
|
207
|
+
userId: string | null;
|
|
208
|
+
to: string;
|
|
209
|
+
at: string;
|
|
210
|
+
linkUrl: string;
|
|
211
|
+
linkId: string;
|
|
212
|
+
};
|
|
155
213
|
"journey.completed": {
|
|
156
214
|
journeyId: string;
|
|
157
215
|
journeyName: string;
|
|
@@ -160,8 +218,59 @@ export interface OutboundPayloads {
|
|
|
160
218
|
userEmail: string;
|
|
161
219
|
completedAt: string;
|
|
162
220
|
};
|
|
221
|
+
/**
|
|
222
|
+
* Global control group membership (impact plan §4.3) — emitted once per
|
|
223
|
+
* contact key (per-endpoint dedupe) on the first withheld send. `userId`
|
|
224
|
+
* is null for identity-less sends; `email` null on the SMS leg.
|
|
225
|
+
*/
|
|
226
|
+
"contact.control_group": {
|
|
227
|
+
userId: string | null;
|
|
228
|
+
email: string | null;
|
|
229
|
+
at: string;
|
|
230
|
+
};
|
|
231
|
+
/** Holdout diversion (impact plan §4.1) — the counterfactual as data. */
|
|
232
|
+
"journey.heldout": {
|
|
233
|
+
journeyId: string;
|
|
234
|
+
journeyName: string;
|
|
235
|
+
stateId: string;
|
|
236
|
+
userId: string;
|
|
237
|
+
userEmail: string;
|
|
238
|
+
heldOutAt: string;
|
|
239
|
+
};
|
|
163
240
|
"bucket.entered": BucketEventPayload;
|
|
164
241
|
"bucket.left": BucketEventPayload & { reason?: string };
|
|
242
|
+
/**
|
|
243
|
+
* A CRM pipeline change landed on the spine (webhook or reconciliation
|
|
244
|
+
* poll). Native identifiers — the canonical stage may be null when the
|
|
245
|
+
* per-provider stage map has no entry (surfaced, never dropped).
|
|
246
|
+
*/
|
|
247
|
+
"funnel.stage_changed": CrmDealEventPayload & {
|
|
248
|
+
stageId: string;
|
|
249
|
+
stageName: string | null;
|
|
250
|
+
status: string | null;
|
|
251
|
+
};
|
|
252
|
+
/** A deal FIRST reached canonical `quoted`. Value = quote value. */
|
|
253
|
+
"deal.quoted": CrmDealEventPayload;
|
|
254
|
+
/** A deal FIRST reached canonical `sold`. Value = deal value. */
|
|
255
|
+
"deal.sold": CrmDealEventPayload;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Shared payload for the `crm.*` outbound family. */
|
|
259
|
+
export interface CrmDealEventPayload {
|
|
260
|
+
provider: string;
|
|
261
|
+
dealId: string;
|
|
262
|
+
pipelineId: string | null;
|
|
263
|
+
/**
|
|
264
|
+
* The code-defined funnel claiming this deal's pipeline (defineFunnel).
|
|
265
|
+
* The emit site always sent it; the type lagged behind (impact plan §2
|
|
266
|
+
* seam 5) — now part of the contract.
|
|
267
|
+
*/
|
|
268
|
+
funnelId: string | null;
|
|
269
|
+
canonicalStage: string | null;
|
|
270
|
+
value: number | null;
|
|
271
|
+
currency: string | null;
|
|
272
|
+
userId: string | null;
|
|
273
|
+
at: string;
|
|
165
274
|
}
|
|
166
275
|
|
|
167
276
|
/**
|
package/src/lib/phone.ts
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Strict E.164 normalization. Strips common human separators (spaces, dashes,
|
|
3
|
+
* parens, dots) then requires a leading `+`, a non-zero country digit, and
|
|
4
|
+
* 7–15 total digits (the E.164 max is 15). Fail-closed: anything that doesn't
|
|
5
|
+
* match returns `null` — the caller rejects it rather than guessing a country
|
|
6
|
+
* code (no libphonenumber dependency, no implicit `+1`).
|
|
7
|
+
*/
|
|
8
|
+
export function normalizePhone(
|
|
9
|
+
input: string | null | undefined,
|
|
10
|
+
): string | null {
|
|
11
|
+
if (!input) return null;
|
|
12
|
+
const stripped = input.trim().replace(/[\s\-().]/g, "");
|
|
13
|
+
return /^\+[1-9]\d{6,14}$/.test(stripped) ? stripped : null;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
/** Whether `input` is already a valid E.164 number (post-normalization). */
|
|
17
|
+
export function isE164(input: string | null | undefined): boolean {
|
|
18
|
+
return normalizePhone(input) !== null;
|
|
19
|
+
}
|