@hogsend/engine 0.43.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +12 -9
- package/src/container.ts +289 -13
- package/src/destinations/define-destination.ts +7 -3
- package/src/destinations/presets/posthog.ts +31 -0
- package/src/index.ts +99 -0
- package/src/journeys/execute-journey-run.ts +93 -0
- package/src/journeys/journey-context.ts +4 -0
- package/src/lib/analytics-providers-from-env.ts +106 -43
- package/src/lib/attribution-backfill.ts +285 -0
- package/src/lib/attribution.ts +239 -0
- package/src/lib/contacts.ts +71 -0
- package/src/lib/conversion-dispatch.ts +326 -0
- package/src/lib/conversions.ts +197 -0
- package/src/lib/crm-deals.ts +320 -0
- package/src/lib/crm-ingest.ts +341 -0
- package/src/lib/crm-provider-registry.ts +41 -0
- package/src/lib/crm-registry-singleton.ts +22 -0
- package/src/lib/deal-money-events.ts +85 -0
- package/src/lib/email-service-types.ts +16 -1
- package/src/lib/event-names.ts +1 -1
- package/src/lib/funnel-progress.ts +76 -0
- package/src/lib/funnel-registry.ts +108 -0
- package/src/lib/funnel-transitions.ts +280 -0
- package/src/lib/holdout.ts +62 -0
- package/src/lib/ingestion.ts +218 -9
- package/src/lib/lift-stats.ts +125 -0
- package/src/lib/mailer.ts +1 -0
- package/src/lib/operator.ts +31 -0
- package/src/lib/outbound.ts +51 -0
- package/src/lib/posthog-scopes.ts +30 -6
- package/src/lib/revenue.ts +116 -0
- package/src/lib/sms-service-types.ts +7 -1
- package/src/lib/sms-tracked.ts +35 -0
- package/src/lib/tracked.ts +40 -0
- package/src/lib/tracking-events.ts +26 -0
- package/src/lib/unconfigured-email-provider.ts +45 -0
- package/src/lib/webhook-signing.ts +7 -0
- package/src/routes/admin/analytics.ts +14 -5
- package/src/routes/admin/attribution.ts +414 -0
- package/src/routes/admin/contacts.ts +71 -11
- package/src/routes/admin/conversions.ts +247 -0
- package/src/routes/admin/deals.ts +472 -0
- package/src/routes/admin/funnels.ts +228 -0
- package/src/routes/admin/index.ts +8 -0
- package/src/routes/admin/journeys.ts +158 -1
- package/src/routes/admin/readiness.ts +97 -2
- package/src/routes/admin/templates.ts +28 -1
- package/src/routes/events/index.ts +12 -0
- package/src/routes/tracking/click-pipeline.ts +26 -18
- package/src/routes/webhooks/crm-provider.ts +117 -0
- package/src/routes/webhooks/index.ts +5 -1
- package/src/sources/define-contact-source.ts +159 -0
- package/src/sources/registry.ts +71 -0
- package/src/sources/webhook-source.ts +109 -0
- package/src/worker.ts +4 -0
- package/src/workflows/crm-reconcile.ts +129 -0
- package/src/workflows/dispatch-conversion.ts +31 -0
- package/src/workflows/send-campaign.ts +5 -0
package/src/lib/ingestion.ts
CHANGED
|
@@ -25,10 +25,22 @@ import {
|
|
|
25
25
|
getAnalytics,
|
|
26
26
|
getAnalyticsEventMirror,
|
|
27
27
|
} from "./analytics-singleton.js";
|
|
28
|
+
import { recordAttributionCredits } from "./attribution.js";
|
|
28
29
|
import {
|
|
29
30
|
ContactProvenanceLostError,
|
|
30
31
|
resolveOrCreateContact,
|
|
31
32
|
} from "./contacts.js";
|
|
33
|
+
import {
|
|
34
|
+
conversionEventId,
|
|
35
|
+
enqueueConversionDispatches,
|
|
36
|
+
} from "./conversion-dispatch.js";
|
|
37
|
+
import {
|
|
38
|
+
evaluateConversionsAtIngest,
|
|
39
|
+
getConversionRegistry,
|
|
40
|
+
} from "./conversions.js";
|
|
41
|
+
import { getCrmSyncConfig } from "./crm-registry-singleton.js";
|
|
42
|
+
import { recordFunnelProgressAtIngest } from "./funnel-progress.js";
|
|
43
|
+
import { applyFunnelTransitionsAtIngest } from "./funnel-transitions.js";
|
|
32
44
|
import type { Logger } from "./logger.js";
|
|
33
45
|
|
|
34
46
|
export interface IngestEvent {
|
|
@@ -57,6 +69,15 @@ export interface IngestEvent {
|
|
|
57
69
|
eventProperties: Record<string, unknown>;
|
|
58
70
|
/** D2: → `contacts.properties` merge ONLY. */
|
|
59
71
|
contactProperties?: Record<string, unknown>;
|
|
72
|
+
/**
|
|
73
|
+
* The event's own monetary worth (deal value, order total). Stored on the
|
|
74
|
+
* first-class `user_events.value` column — the revenue spine every rollup,
|
|
75
|
+
* conversion definition, and attribution credit reads from. Non-finite
|
|
76
|
+
* numbers are dropped (with a warn) rather than failing the ingest.
|
|
77
|
+
*/
|
|
78
|
+
value?: number;
|
|
79
|
+
/** ISO-4217 alpha code for `value`; uppercased here. Defaults to null. */
|
|
80
|
+
currency?: string;
|
|
60
81
|
idempotencyKey?: string;
|
|
61
82
|
/**
|
|
62
83
|
* Caller-supplied event time (§2.5 `timestamp`). When set, `user_events`
|
|
@@ -295,6 +316,10 @@ export async function ingestEvent(opts: {
|
|
|
295
316
|
contactId: event.contactId,
|
|
296
317
|
contactProperties: event.contactProperties,
|
|
297
318
|
restrictToAnonymous: opts.restrictToAnonymous,
|
|
319
|
+
// First-touch contact provenance from the event's pipeline origin — a
|
|
320
|
+
// Contact Source ("clay"/"attio") or "api"/"posthog"/…; only stamped when
|
|
321
|
+
// the contact is created (or first fill-in-linked) with no prior source.
|
|
322
|
+
source: event.source,
|
|
298
323
|
});
|
|
299
324
|
} catch (err) {
|
|
300
325
|
// Provenance pin pointed at a hard-deleted/unfollowable subject: drop the
|
|
@@ -326,12 +351,41 @@ export async function ingestEvent(opts: {
|
|
|
326
351
|
// falls back to the `occurred_at` DB default (ingest instant).
|
|
327
352
|
const occurredAt = event.occurredAt ? new Date(event.occurredAt) : undefined;
|
|
328
353
|
|
|
354
|
+
// Money normalization — permissive at the spine (webhook sources feed this
|
|
355
|
+
// path with arbitrary payloads): a non-finite value or malformed currency is
|
|
356
|
+
// dropped with a warn, never a failed ingest. Currency without value is
|
|
357
|
+
// meaningless and dropped; value without currency stores with null currency
|
|
358
|
+
// (single-currency deploys omit it everywhere).
|
|
359
|
+
let value: number | null = null;
|
|
360
|
+
let currency: string | null = null;
|
|
361
|
+
if (event.value !== undefined) {
|
|
362
|
+
if (typeof event.value === "number" && Number.isFinite(event.value)) {
|
|
363
|
+
value = event.value;
|
|
364
|
+
if (event.currency !== undefined) {
|
|
365
|
+
const code = event.currency.trim().toUpperCase();
|
|
366
|
+
if (/^[A-Z]{3}$/.test(code)) {
|
|
367
|
+
currency = code;
|
|
368
|
+
} else {
|
|
369
|
+
logger.warn("ingestEvent: dropping malformed currency", {
|
|
370
|
+
event: event.event,
|
|
371
|
+
currency: event.currency,
|
|
372
|
+
});
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
} else {
|
|
376
|
+
logger.warn("ingestEvent: dropping non-finite value", {
|
|
377
|
+
event: event.event,
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
|
|
329
382
|
// (2) Idempotency dedup + `user_events` insert keyed on the resolved key, with
|
|
330
383
|
// ONLY eventProperties in the properties bag (D2). `ctx.trigger` now supplies a
|
|
331
384
|
// deterministic key (`journeyTrigger:<runAnchor>:<site>:<event>`), so a journey
|
|
332
385
|
// replay re-pushing the same trigger hits the onConflictDoNothing early-return
|
|
333
386
|
// below — the push, checkExits, contact upsert, and alias never re-fire.
|
|
334
387
|
let idempotentInsertId: string | undefined;
|
|
388
|
+
let insertedRow: { id: string; occurredAt: Date } | undefined;
|
|
335
389
|
if (event.idempotencyKey) {
|
|
336
390
|
const result = await db
|
|
337
391
|
.insert(userEvents)
|
|
@@ -339,6 +393,8 @@ export async function ingestEvent(opts: {
|
|
|
339
393
|
userId: resolvedKey,
|
|
340
394
|
event: event.event,
|
|
341
395
|
properties: event.eventProperties,
|
|
396
|
+
value,
|
|
397
|
+
currency,
|
|
342
398
|
source: event.source ?? null,
|
|
343
399
|
idempotencyKey: event.idempotencyKey,
|
|
344
400
|
...(occurredAt ? { occurredAt } : {}),
|
|
@@ -346,20 +402,27 @@ export async function ingestEvent(opts: {
|
|
|
346
402
|
.onConflictDoNothing({
|
|
347
403
|
target: userEvents.idempotencyKey,
|
|
348
404
|
})
|
|
349
|
-
.returning({ id: userEvents.id });
|
|
405
|
+
.returning({ id: userEvents.id, occurredAt: userEvents.occurredAt });
|
|
350
406
|
|
|
351
407
|
if (result.length === 0) {
|
|
352
408
|
return { stored: false, exits: [], contactKey: resolvedKey };
|
|
353
409
|
}
|
|
354
410
|
idempotentInsertId = result[0]?.id;
|
|
411
|
+
insertedRow = result[0];
|
|
355
412
|
} else {
|
|
356
|
-
await db
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
413
|
+
const result = await db
|
|
414
|
+
.insert(userEvents)
|
|
415
|
+
.values({
|
|
416
|
+
userId: resolvedKey,
|
|
417
|
+
event: event.event,
|
|
418
|
+
properties: event.eventProperties,
|
|
419
|
+
value,
|
|
420
|
+
currency,
|
|
421
|
+
source: event.source ?? null,
|
|
422
|
+
...(occurredAt ? { occurredAt } : {}),
|
|
423
|
+
})
|
|
424
|
+
.returning({ id: userEvents.id, occurredAt: userEvents.occurredAt });
|
|
425
|
+
insertedRow = result[0];
|
|
363
426
|
}
|
|
364
427
|
|
|
365
428
|
// (2b) §5.3 — fire the provider-neutral identity merge at the two resolver
|
|
@@ -413,7 +476,16 @@ export async function ingestEvent(opts: {
|
|
|
413
476
|
mirrorProvider.capture({
|
|
414
477
|
distinctId: resolvedKey,
|
|
415
478
|
event: event.event,
|
|
416
|
-
|
|
479
|
+
// The revenue spine fans out with the event: `value`/`currency` ride as
|
|
480
|
+
// plain properties so the analytics tool (PostHog et al.) can sum
|
|
481
|
+
// revenue without a Hogsend round-trip. Property-bag keys of the same
|
|
482
|
+
// name lose to the first-class columns.
|
|
483
|
+
properties: {
|
|
484
|
+
...event.eventProperties,
|
|
485
|
+
...(value !== null
|
|
486
|
+
? { value, ...(currency ? { currency } : {}) }
|
|
487
|
+
: {}),
|
|
488
|
+
},
|
|
417
489
|
});
|
|
418
490
|
} catch (err) {
|
|
419
491
|
logger.debug("event mirror capture failed (non-fatal)", {
|
|
@@ -512,6 +584,143 @@ export async function ingestEvent(opts: {
|
|
|
512
584
|
});
|
|
513
585
|
}
|
|
514
586
|
|
|
587
|
+
// (5c)+(5d) — the post-store hooks. Both fire only AFTER the push settled
|
|
588
|
+
// (a failed publish compensating-deletes the event row, so a rolled-back
|
|
589
|
+
// event never leaves a fired conversion or a moved deal behind) and only
|
|
590
|
+
// on a FRESH insert (replays early-returned at (2)). Each is best-effort
|
|
591
|
+
// in its own try/catch: the event is already durable, so a hook failure
|
|
592
|
+
// warns rather than failing the caller.
|
|
593
|
+
const hookEvent = insertedRow
|
|
594
|
+
? {
|
|
595
|
+
name: event.event,
|
|
596
|
+
source: event.source ?? null,
|
|
597
|
+
properties: event.eventProperties,
|
|
598
|
+
value,
|
|
599
|
+
currency,
|
|
600
|
+
occurredAt: insertedRow.occurredAt,
|
|
601
|
+
}
|
|
602
|
+
: null;
|
|
603
|
+
|
|
604
|
+
// (5c) Conversion-point evaluation (plan §5.1) — the unique
|
|
605
|
+
// (definition, event) index makes any replay a no-op.
|
|
606
|
+
if (insertedRow && hookEvent) {
|
|
607
|
+
try {
|
|
608
|
+
const fired = await evaluateConversionsAtIngest({
|
|
609
|
+
db,
|
|
610
|
+
logger,
|
|
611
|
+
registry: getConversionRegistry(),
|
|
612
|
+
event: hookEvent,
|
|
613
|
+
eventRowId: insertedRow.id,
|
|
614
|
+
contactId,
|
|
615
|
+
userKey: resolvedKey,
|
|
616
|
+
});
|
|
617
|
+
for (const firedConversion of fired) {
|
|
618
|
+
// Fan fired conversions out to their ad-platform destinations
|
|
619
|
+
// (§5.2) FIRST — the pending dispatch row is the recoverable
|
|
620
|
+
// anchor, and the conversion row won't re-fire on replay, so
|
|
621
|
+
// nothing may run ahead of it and throw. The deterministic
|
|
622
|
+
// event_id is stable across retries AND re-evaluations.
|
|
623
|
+
const destinationIds = firedConversion.definition.meta.destinations;
|
|
624
|
+
if (destinationIds && destinationIds.length > 0) {
|
|
625
|
+
await enqueueConversionDispatches({
|
|
626
|
+
db,
|
|
627
|
+
logger,
|
|
628
|
+
conversionId: firedConversion.conversionId,
|
|
629
|
+
eventId: conversionEventId({
|
|
630
|
+
contactId,
|
|
631
|
+
definitionId: firedConversion.definition.meta.id,
|
|
632
|
+
eventRowId: insertedRow.id,
|
|
633
|
+
}),
|
|
634
|
+
destinationIds,
|
|
635
|
+
});
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
// The attribution ledger (§6.1): every model's credits over the
|
|
639
|
+
// contact's touchpoint path, written once at conversion time.
|
|
640
|
+
// Pure reporting — individually isolated so a transient ledger
|
|
641
|
+
// failure can never cost a dispatch or a later conversion.
|
|
642
|
+
try {
|
|
643
|
+
await recordAttributionCredits({
|
|
644
|
+
db,
|
|
645
|
+
logger,
|
|
646
|
+
conversionId: firedConversion.conversionId,
|
|
647
|
+
userKey: resolvedKey,
|
|
648
|
+
value: firedConversion.value,
|
|
649
|
+
currency: firedConversion.currency,
|
|
650
|
+
occurredAt: insertedRow.occurredAt,
|
|
651
|
+
windowDays:
|
|
652
|
+
firedConversion.definition.meta.attributionWindowDays ?? 90,
|
|
653
|
+
windows: firedConversion.definition.meta.windows,
|
|
654
|
+
});
|
|
655
|
+
} catch (err) {
|
|
656
|
+
logger.warn("attribution credit write failed", {
|
|
657
|
+
conversionId: firedConversion.conversionId,
|
|
658
|
+
error: err instanceof Error ? err.message : String(err),
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
} catch (err) {
|
|
663
|
+
logger.warn("Conversion evaluation failed", {
|
|
664
|
+
event: event.event,
|
|
665
|
+
userId: resolvedKey,
|
|
666
|
+
error: err instanceof Error ? err.message : String(err),
|
|
667
|
+
});
|
|
668
|
+
}
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
// (5d) Event-driven funnel stage transitions (event-native funnels). The
|
|
672
|
+
// money events it mints (`deal.quoted`/`deal.sold`) recurse through
|
|
673
|
+
// ingestEvent, bounded at define time: `deal.`/`funnel.`/`crm.` events
|
|
674
|
+
// cannot be stage triggers.
|
|
675
|
+
if (insertedRow && hookEvent) {
|
|
676
|
+
const funnelRegistry = getCrmSyncConfig()?.funnels;
|
|
677
|
+
if (funnelRegistry) {
|
|
678
|
+
try {
|
|
679
|
+
await applyFunnelTransitionsAtIngest({
|
|
680
|
+
db,
|
|
681
|
+
registry,
|
|
682
|
+
hatchet,
|
|
683
|
+
logger,
|
|
684
|
+
analytics,
|
|
685
|
+
funnels: funnelRegistry,
|
|
686
|
+
event: hookEvent,
|
|
687
|
+
eventRowId: insertedRow.id,
|
|
688
|
+
contactId,
|
|
689
|
+
userKey: resolvedKey,
|
|
690
|
+
});
|
|
691
|
+
} catch (err) {
|
|
692
|
+
logger.warn("Funnel transition evaluation failed", {
|
|
693
|
+
event: event.event,
|
|
694
|
+
userId: resolvedKey,
|
|
695
|
+
error: err instanceof Error ? err.message : String(err),
|
|
696
|
+
});
|
|
697
|
+
}
|
|
698
|
+
|
|
699
|
+
// (5e) Funnel progression REPORTING projection (impact plan §3.3) —
|
|
700
|
+
// first-reach `funnel_progress` rows over the SAME transition rules
|
|
701
|
+
// and gates as the deal mover above, so the two projections never
|
|
702
|
+
// disagree. Best-effort like every hook; the unique (contact, funnel,
|
|
703
|
+
// stage) index makes any replay a no-op.
|
|
704
|
+
try {
|
|
705
|
+
await recordFunnelProgressAtIngest({
|
|
706
|
+
db,
|
|
707
|
+
logger,
|
|
708
|
+
funnels: funnelRegistry,
|
|
709
|
+
event: hookEvent,
|
|
710
|
+
eventRowId: insertedRow.id,
|
|
711
|
+
contactId,
|
|
712
|
+
userKey: resolvedKey,
|
|
713
|
+
});
|
|
714
|
+
} catch (err) {
|
|
715
|
+
logger.warn("Funnel progression write failed", {
|
|
716
|
+
event: event.event,
|
|
717
|
+
userId: resolvedKey,
|
|
718
|
+
error: err instanceof Error ? err.message : String(err),
|
|
719
|
+
});
|
|
720
|
+
}
|
|
721
|
+
}
|
|
722
|
+
}
|
|
723
|
+
|
|
515
724
|
// (6) Real-time bucket membership re-evaluation (Section 6.1). NOT part of the
|
|
516
725
|
// Promise.all above: its property eval reads contact state ⊕ this-ingest
|
|
517
726
|
// contactProperties patch, and its bucket:entered/left emissions recurse back
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Holdout lift statistics (docs/attribution-impact-plan.md §4.2, open Q1
|
|
3
|
+
* resolved): Bayesian beta-binomial with a uniform Beta(1,1) prior — the
|
|
4
|
+
* lean, honest method at self-hosted volumes. Win probability is
|
|
5
|
+
* P(treatment rate > control rate) under independent posteriors
|
|
6
|
+
* Beta(k+1, n−k+1), computed by DETERMINISTIC numeric integration (no
|
|
7
|
+
* sampling — stable across runs and trivially testable).
|
|
8
|
+
*
|
|
9
|
+
* Presentation guards, per the plan's honesty stance:
|
|
10
|
+
* - suppression floor: fewer than {@link MIN_COMBINED_CONVERSIONS} combined
|
|
11
|
+
* conversions ⇒ no win probability at all (Klaviyo suppresses at the same
|
|
12
|
+
* threshold; below it the number is noise wearing a percentage sign);
|
|
13
|
+
* - small-sample flag: either cohort under {@link SMALL_SAMPLE_FLOOR}
|
|
14
|
+
* contacts ⇒ `smallSample: true` so surfaces can warn loudly (Braze's
|
|
15
|
+
* rule of thumb is ~1,000; we present rather than lock out).
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
export const MIN_COMBINED_CONVERSIONS = 10;
|
|
19
|
+
export const SMALL_SAMPLE_FLOOR = 100;
|
|
20
|
+
|
|
21
|
+
/** Integration grid — 2001 points is ample at any realistic cohort size. */
|
|
22
|
+
const GRID = 2001;
|
|
23
|
+
|
|
24
|
+
function logGamma(x: number): number {
|
|
25
|
+
// Lanczos approximation (g=7, n=9) — standard coefficients.
|
|
26
|
+
const coefficients = [
|
|
27
|
+
0.99999999999980993, 676.5203681218851, -1259.1392167224028,
|
|
28
|
+
771.32342877765313, -176.61502916214059, 12.507343278686905,
|
|
29
|
+
-0.13857109526572012, 9.9843695780195716e-6, 1.5056327351493116e-7,
|
|
30
|
+
];
|
|
31
|
+
if (x < 0.5) {
|
|
32
|
+
return Math.log(Math.PI / Math.sin(Math.PI * x)) - logGamma(1 - x);
|
|
33
|
+
}
|
|
34
|
+
const xShifted = x - 1;
|
|
35
|
+
let sum = coefficients[0] as number;
|
|
36
|
+
for (let i = 1; i < coefficients.length; i++) {
|
|
37
|
+
sum += (coefficients[i] as number) / (xShifted + i);
|
|
38
|
+
}
|
|
39
|
+
const t = xShifted + 7.5;
|
|
40
|
+
return (
|
|
41
|
+
0.5 * Math.log(2 * Math.PI) +
|
|
42
|
+
(xShifted + 0.5) * Math.log(t) -
|
|
43
|
+
t +
|
|
44
|
+
Math.log(sum)
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function logBetaPdf(x: number, a: number, b: number): number {
|
|
49
|
+
if (x <= 0 || x >= 1) return Number.NEGATIVE_INFINITY;
|
|
50
|
+
return (
|
|
51
|
+
(a - 1) * Math.log(x) +
|
|
52
|
+
(b - 1) * Math.log(1 - x) +
|
|
53
|
+
logGamma(a + b) -
|
|
54
|
+
logGamma(a) -
|
|
55
|
+
logGamma(b)
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* P(X > Y) for X ~ Beta(a1,b1), Y ~ Beta(a2,b2): midpoint-rule integration
|
|
61
|
+
* of ∫ f_X(x)·F_Y(x) dx. Midpoints (never 0 or 1) sidestep the boundary —
|
|
62
|
+
* Beta pdfs are singular or discontinuous there for shape params ≤ 1, which
|
|
63
|
+
* a trapezoid pass silently truncates.
|
|
64
|
+
*/
|
|
65
|
+
export function betaWinProbability(
|
|
66
|
+
a1: number,
|
|
67
|
+
b1: number,
|
|
68
|
+
a2: number,
|
|
69
|
+
b2: number,
|
|
70
|
+
): number {
|
|
71
|
+
const dx = 1 / GRID;
|
|
72
|
+
let cdfY = 0;
|
|
73
|
+
let p = 0;
|
|
74
|
+
for (let i = 0; i < GRID; i++) {
|
|
75
|
+
const x = (i + 0.5) * dx;
|
|
76
|
+
const massY = Math.exp(logBetaPdf(x, a2, b2)) * dx;
|
|
77
|
+
// F_Y at this midpoint ≈ mass through the previous cells + half this one.
|
|
78
|
+
const cdfAtX = cdfY + massY / 2;
|
|
79
|
+
p += Math.exp(logBetaPdf(x, a1, b1)) * cdfAtX * dx;
|
|
80
|
+
cdfY += massY;
|
|
81
|
+
}
|
|
82
|
+
// Clamp tiny numeric drift.
|
|
83
|
+
return Math.min(1, Math.max(0, p));
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
export interface LiftVerdict {
|
|
87
|
+
/** (treatmentRate − controlRate) / controlRate; null when control is 0. */
|
|
88
|
+
liftPercent: number | null;
|
|
89
|
+
/** P(treatment > control); null when suppressed. */
|
|
90
|
+
winProbability: number | null;
|
|
91
|
+
/** True when combined conversions are under the suppression floor. */
|
|
92
|
+
suppressed: boolean;
|
|
93
|
+
/** True when either cohort is under the small-sample floor. */
|
|
94
|
+
smallSample: boolean;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export function computeLift(opts: {
|
|
98
|
+
treatment: { contacts: number; converters: number };
|
|
99
|
+
control: { contacts: number; converters: number };
|
|
100
|
+
}): LiftVerdict {
|
|
101
|
+
const { treatment, control } = opts;
|
|
102
|
+
const treatmentRate =
|
|
103
|
+
treatment.contacts > 0 ? treatment.converters / treatment.contacts : 0;
|
|
104
|
+
const controlRate =
|
|
105
|
+
control.contacts > 0 ? control.converters / control.contacts : 0;
|
|
106
|
+
const suppressed =
|
|
107
|
+
treatment.converters + control.converters < MIN_COMBINED_CONVERSIONS;
|
|
108
|
+
return {
|
|
109
|
+
liftPercent:
|
|
110
|
+
controlRate > 0
|
|
111
|
+
? ((treatmentRate - controlRate) / controlRate) * 100
|
|
112
|
+
: null,
|
|
113
|
+
winProbability: suppressed
|
|
114
|
+
? null
|
|
115
|
+
: betaWinProbability(
|
|
116
|
+
treatment.converters + 1,
|
|
117
|
+
treatment.contacts - treatment.converters + 1,
|
|
118
|
+
control.converters + 1,
|
|
119
|
+
control.contacts - control.converters + 1,
|
|
120
|
+
),
|
|
121
|
+
suppressed,
|
|
122
|
+
smallSample:
|
|
123
|
+
Math.min(treatment.contacts, control.contacts) < SMALL_SAMPLE_FLOOR,
|
|
124
|
+
};
|
|
125
|
+
}
|
package/src/lib/mailer.ts
CHANGED
|
@@ -148,6 +148,7 @@ export function createTrackedMailer(
|
|
|
148
148
|
to: options.to,
|
|
149
149
|
subject: options.subject,
|
|
150
150
|
journeyStateId: options.journeyStateId,
|
|
151
|
+
campaignId: options.campaignId,
|
|
151
152
|
userId: options.userId,
|
|
152
153
|
userEmail: options.userEmail,
|
|
153
154
|
category: options.category,
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { type Database, user } from "@hogsend/db";
|
|
2
|
+
import { normalizeEmail, normalizeEmailOrNull } from "./contacts.js";
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* True iff `email` is a member of the operator set: a row in the `user` table,
|
|
6
|
+
* or the env-configured operator inbox (`HOGSEND_TEST_EMAIL` /
|
|
7
|
+
* `STUDIO_ADMIN_EMAIL`). Comparison is case-insensitive (`normalizeEmail`).
|
|
8
|
+
*
|
|
9
|
+
* The `user` table IS the operator set: admins are minted only by the CLI or the
|
|
10
|
+
* boot bootstrap (there is no unauthenticated path to create one), so a
|
|
11
|
+
* closed-signup deploy has a handful of rows. The two env addresses are the same
|
|
12
|
+
* operator inboxes test-mode redirects to.
|
|
13
|
+
*/
|
|
14
|
+
export async function isOperatorAddress(opts: {
|
|
15
|
+
db: Database;
|
|
16
|
+
env: { HOGSEND_TEST_EMAIL?: string; STUDIO_ADMIN_EMAIL?: string };
|
|
17
|
+
email: string;
|
|
18
|
+
}): Promise<boolean> {
|
|
19
|
+
const target = normalizeEmail(opts.email);
|
|
20
|
+
if (!target) return false;
|
|
21
|
+
|
|
22
|
+
// Match how addresses are stored/compared everywhere else (contacts.ts) so
|
|
23
|
+
// this security gate can't silently diverge from the canonical normalization.
|
|
24
|
+
if (normalizeEmailOrNull(opts.env.HOGSEND_TEST_EMAIL) === target) return true;
|
|
25
|
+
if (normalizeEmailOrNull(opts.env.STUDIO_ADMIN_EMAIL) === target) return true;
|
|
26
|
+
|
|
27
|
+
// The admin team is a handful of rows (closed signup), so a full scan +
|
|
28
|
+
// normalized compare is cheaper and simpler than a lower() predicate.
|
|
29
|
+
const rows = await opts.db.select({ email: user.email }).from(user);
|
|
30
|
+
return rows.some((r) => normalizeEmail(r.email) === target);
|
|
31
|
+
}
|
package/src/lib/outbound.ts
CHANGED
|
@@ -218,8 +218,59 @@ export interface OutboundPayloads {
|
|
|
218
218
|
userEmail: string;
|
|
219
219
|
completedAt: string;
|
|
220
220
|
};
|
|
221
|
+
/**
|
|
222
|
+
* Global control group membership (impact plan §4.3) — emitted once per
|
|
223
|
+
* contact key (per-endpoint dedupe) on the first withheld send. `userId`
|
|
224
|
+
* is null for identity-less sends; `email` null on the SMS leg.
|
|
225
|
+
*/
|
|
226
|
+
"contact.control_group": {
|
|
227
|
+
userId: string | null;
|
|
228
|
+
email: string | null;
|
|
229
|
+
at: string;
|
|
230
|
+
};
|
|
231
|
+
/** Holdout diversion (impact plan §4.1) — the counterfactual as data. */
|
|
232
|
+
"journey.heldout": {
|
|
233
|
+
journeyId: string;
|
|
234
|
+
journeyName: string;
|
|
235
|
+
stateId: string;
|
|
236
|
+
userId: string;
|
|
237
|
+
userEmail: string;
|
|
238
|
+
heldOutAt: string;
|
|
239
|
+
};
|
|
221
240
|
"bucket.entered": BucketEventPayload;
|
|
222
241
|
"bucket.left": BucketEventPayload & { reason?: string };
|
|
242
|
+
/**
|
|
243
|
+
* A CRM pipeline change landed on the spine (webhook or reconciliation
|
|
244
|
+
* poll). Native identifiers — the canonical stage may be null when the
|
|
245
|
+
* per-provider stage map has no entry (surfaced, never dropped).
|
|
246
|
+
*/
|
|
247
|
+
"funnel.stage_changed": CrmDealEventPayload & {
|
|
248
|
+
stageId: string;
|
|
249
|
+
stageName: string | null;
|
|
250
|
+
status: string | null;
|
|
251
|
+
};
|
|
252
|
+
/** A deal FIRST reached canonical `quoted`. Value = quote value. */
|
|
253
|
+
"deal.quoted": CrmDealEventPayload;
|
|
254
|
+
/** A deal FIRST reached canonical `sold`. Value = deal value. */
|
|
255
|
+
"deal.sold": CrmDealEventPayload;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Shared payload for the `crm.*` outbound family. */
|
|
259
|
+
export interface CrmDealEventPayload {
|
|
260
|
+
provider: string;
|
|
261
|
+
dealId: string;
|
|
262
|
+
pipelineId: string | null;
|
|
263
|
+
/**
|
|
264
|
+
* The code-defined funnel claiming this deal's pipeline (defineFunnel).
|
|
265
|
+
* The emit site always sent it; the type lagged behind (impact plan §2
|
|
266
|
+
* seam 5) — now part of the contract.
|
|
267
|
+
*/
|
|
268
|
+
funnelId: string | null;
|
|
269
|
+
canonicalStage: string | null;
|
|
270
|
+
value: number | null;
|
|
271
|
+
currency: string | null;
|
|
272
|
+
userId: string | null;
|
|
273
|
+
at: string;
|
|
223
274
|
}
|
|
224
275
|
|
|
225
276
|
/**
|
|
@@ -5,12 +5,14 @@
|
|
|
5
5
|
* so future read/write features activate without forcing a reconnect. Every
|
|
6
6
|
* name is validated against PostHog's published `scopes_supported`.
|
|
7
7
|
*
|
|
8
|
-
* LOCKSTEP (3 places
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
8
|
+
* LOCKSTEP (3 places): this constant, the CLI's `POSTHOG_SCOPES`
|
|
9
|
+
* (`packages/cli/src/lib/oauth.ts` — the CLI avoids the engine ROOT import
|
|
10
|
+
* because the barrel does import-time env validation, but this module is
|
|
11
|
+
* dependency-free and exported as the `@hogsend/engine/posthog-scopes`
|
|
12
|
+
* subpath), and the `scope` field of the hosted CIMD document
|
|
13
|
+
* (`apps/docs/public/.well-known/hogsend-posthog-client.json`) — that one is
|
|
14
|
+
* genuinely un-importable. PostHog gates the scopes it will grant by the CIMD
|
|
15
|
+
* doc, so grep all three and keep them identical before changing any of them.
|
|
14
16
|
*/
|
|
15
17
|
export const EXPECTED_POSTHOG_SCOPES: string[] = [
|
|
16
18
|
"person:read",
|
|
@@ -27,3 +29,25 @@ export const EXPECTED_POSTHOG_SCOPES: string[] = [
|
|
|
27
29
|
"event_definition:read",
|
|
28
30
|
"property_definition:read",
|
|
29
31
|
];
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* PostHog's scope model is hierarchical — `X:write` implies `X:read` — and
|
|
35
|
+
* the grant NORMALIZES a request for both halves down to `X:write` alone. A
|
|
36
|
+
* literal set-difference against the requested list therefore reports the
|
|
37
|
+
* implied `:read` halves as "missing" on every fully-successful consent
|
|
38
|
+
* (e.g. person:read, hog_function:read, cohort:read). Treat a requested
|
|
39
|
+
* `X:read` as satisfied when the grant carries `X:write`.
|
|
40
|
+
*
|
|
41
|
+
* The CLI imports this via the `@hogsend/engine/posthog-scopes` subpath —
|
|
42
|
+
* single implementation, no mirror.
|
|
43
|
+
*/
|
|
44
|
+
export function posthogScopeSatisfied(
|
|
45
|
+
granted: string[],
|
|
46
|
+
scope: string,
|
|
47
|
+
): boolean {
|
|
48
|
+
return (
|
|
49
|
+
granted.includes(scope) ||
|
|
50
|
+
(scope.endsWith(":read") &&
|
|
51
|
+
granted.includes(scope.replace(/:read$/, ":write")))
|
|
52
|
+
);
|
|
53
|
+
}
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { type Database, userEvents } from "@hogsend/db";
|
|
2
|
+
import {
|
|
3
|
+
and,
|
|
4
|
+
count,
|
|
5
|
+
eq,
|
|
6
|
+
isNotNull,
|
|
7
|
+
isNull,
|
|
8
|
+
max,
|
|
9
|
+
ne,
|
|
10
|
+
notInArray,
|
|
11
|
+
or,
|
|
12
|
+
sql,
|
|
13
|
+
} from "drizzle-orm";
|
|
14
|
+
import { getCrmSyncConfig } from "./crm-registry-singleton.js";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Revenue rollups over the event spine — pure reads of the first-class
|
|
18
|
+
* `user_events.value`/`currency` columns (served by the partial
|
|
19
|
+
* `user_events_valued_user_idx`). Totals are grouped per currency and NEVER
|
|
20
|
+
* summed across currencies (a GBP deal and a USD deal don't add).
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Events whose `value` must NOT count toward revenue: one CRM deal's value
|
|
25
|
+
* rides `funnel.stage_changed` on EVERY stage change plus the once-per-stage
|
|
26
|
+
* money events, so summing them all counts one sale several times over —
|
|
27
|
+
* and a quote is unrealized money, not revenue. Only `deal.sold` (and
|
|
28
|
+
* any other valued event) contributes.
|
|
29
|
+
*/
|
|
30
|
+
export const REVENUE_EXCLUDED_EVENTS = [
|
|
31
|
+
"funnel.stage_changed",
|
|
32
|
+
"deal.quoted",
|
|
33
|
+
] as const;
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* The full exclusion list at query time: the static machinery events plus
|
|
37
|
+
* every funnel MILESTONE trigger event. An event that triggers a
|
|
38
|
+
* quoted/won stage hands its value to the minted money event (`deal.sold`
|
|
39
|
+
* counts; `deal.quoted` is unrealized), so counting the raw trigger row too
|
|
40
|
+
* would double-count one sale — the event-native twin of the
|
|
41
|
+
* `funnel.stage_changed` exclusion. Non-milestone triggers never mint and
|
|
42
|
+
* stay counted.
|
|
43
|
+
*/
|
|
44
|
+
export function revenueExcludedEvents(): string[] {
|
|
45
|
+
return [
|
|
46
|
+
...REVENUE_EXCLUDED_EVENTS,
|
|
47
|
+
...(getCrmSyncConfig()?.funnels.milestoneTriggerEvents() ?? []),
|
|
48
|
+
];
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Rollup trust gate: browser (`pk_`/`inapp`) events can carry any value
|
|
53
|
+
* anyone mints, so they never count toward revenue — the same trust tier
|
|
54
|
+
* the conversion-point forged-value guard enforces. `source` null =
|
|
55
|
+
* engine-written, trusted.
|
|
56
|
+
*/
|
|
57
|
+
export function trustedValuedEventFilter() {
|
|
58
|
+
return and(
|
|
59
|
+
isNotNull(userEvents.value),
|
|
60
|
+
notInArray(userEvents.event, revenueExcludedEvents()),
|
|
61
|
+
or(isNull(userEvents.source), ne(userEvents.source, "inapp")),
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export interface ContactRevenueTotal {
|
|
66
|
+
/** ISO-4217 code, or null for valued events ingested without a currency. */
|
|
67
|
+
currency: string | null;
|
|
68
|
+
total: number;
|
|
69
|
+
/** Number of valued events contributing to this total. */
|
|
70
|
+
count: number;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface ContactRevenue {
|
|
74
|
+
totals: ContactRevenueTotal[];
|
|
75
|
+
/** `occurredAt` of the most recent valued event, across all currencies. */
|
|
76
|
+
lastValuedAt: string | null;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Sum a contact's valued events, per currency. `key` is the contact's
|
|
81
|
+
* canonical event key (`external_id ?? anonymous_id ?? id`) — the same key
|
|
82
|
+
* `ingestEvent` stamps on `user_events.user_id`.
|
|
83
|
+
*/
|
|
84
|
+
export async function getContactRevenue(opts: {
|
|
85
|
+
db: Database;
|
|
86
|
+
key: string;
|
|
87
|
+
}): Promise<ContactRevenue> {
|
|
88
|
+
const rows = await opts.db
|
|
89
|
+
.select({
|
|
90
|
+
currency: userEvents.currency,
|
|
91
|
+
total: sql<number>`sum(${userEvents.value})::float8`,
|
|
92
|
+
count: count(),
|
|
93
|
+
lastAt: max(userEvents.occurredAt),
|
|
94
|
+
})
|
|
95
|
+
.from(userEvents)
|
|
96
|
+
.where(and(eq(userEvents.userId, opts.key), trustedValuedEventFilter()))
|
|
97
|
+
.groupBy(userEvents.currency);
|
|
98
|
+
|
|
99
|
+
let lastValuedAt: Date | null = null;
|
|
100
|
+
for (const row of rows) {
|
|
101
|
+
if (row.lastAt && (!lastValuedAt || row.lastAt > lastValuedAt)) {
|
|
102
|
+
lastValuedAt = row.lastAt;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
return {
|
|
107
|
+
totals: rows
|
|
108
|
+
.map((row) => ({
|
|
109
|
+
currency: row.currency,
|
|
110
|
+
total: row.total ?? 0,
|
|
111
|
+
count: Number(row.count),
|
|
112
|
+
}))
|
|
113
|
+
.sort((a, b) => b.total - a.total),
|
|
114
|
+
lastValuedAt: lastValuedAt ? lastValuedAt.toISOString() : null,
|
|
115
|
+
};
|
|
116
|
+
}
|