@hogsend/engine 0.43.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +12 -9
- package/src/container.ts +289 -13
- package/src/destinations/define-destination.ts +7 -3
- package/src/destinations/presets/posthog.ts +31 -0
- package/src/index.ts +99 -0
- package/src/journeys/execute-journey-run.ts +93 -0
- package/src/journeys/journey-context.ts +4 -0
- package/src/lib/analytics-providers-from-env.ts +106 -43
- package/src/lib/attribution-backfill.ts +285 -0
- package/src/lib/attribution.ts +239 -0
- package/src/lib/contacts.ts +71 -0
- package/src/lib/conversion-dispatch.ts +326 -0
- package/src/lib/conversions.ts +197 -0
- package/src/lib/crm-deals.ts +320 -0
- package/src/lib/crm-ingest.ts +341 -0
- package/src/lib/crm-provider-registry.ts +41 -0
- package/src/lib/crm-registry-singleton.ts +22 -0
- package/src/lib/deal-money-events.ts +85 -0
- package/src/lib/email-service-types.ts +16 -1
- package/src/lib/event-names.ts +1 -1
- package/src/lib/funnel-progress.ts +76 -0
- package/src/lib/funnel-registry.ts +108 -0
- package/src/lib/funnel-transitions.ts +280 -0
- package/src/lib/holdout.ts +62 -0
- package/src/lib/ingestion.ts +218 -9
- package/src/lib/lift-stats.ts +125 -0
- package/src/lib/mailer.ts +1 -0
- package/src/lib/operator.ts +31 -0
- package/src/lib/outbound.ts +51 -0
- package/src/lib/posthog-scopes.ts +30 -6
- package/src/lib/revenue.ts +116 -0
- package/src/lib/sms-service-types.ts +7 -1
- package/src/lib/sms-tracked.ts +35 -0
- package/src/lib/tracked.ts +40 -0
- package/src/lib/tracking-events.ts +26 -0
- package/src/lib/unconfigured-email-provider.ts +45 -0
- package/src/lib/webhook-signing.ts +7 -0
- package/src/routes/admin/analytics.ts +14 -5
- package/src/routes/admin/attribution.ts +414 -0
- package/src/routes/admin/contacts.ts +71 -11
- package/src/routes/admin/conversions.ts +247 -0
- package/src/routes/admin/deals.ts +472 -0
- package/src/routes/admin/funnels.ts +228 -0
- package/src/routes/admin/index.ts +8 -0
- package/src/routes/admin/journeys.ts +158 -1
- package/src/routes/admin/readiness.ts +97 -2
- package/src/routes/admin/templates.ts +28 -1
- package/src/routes/events/index.ts +12 -0
- package/src/routes/tracking/click-pipeline.ts +26 -18
- package/src/routes/webhooks/crm-provider.ts +117 -0
- package/src/routes/webhooks/index.ts +5 -1
- package/src/sources/define-contact-source.ts +159 -0
- package/src/sources/registry.ts +71 -0
- package/src/sources/webhook-source.ts +109 -0
- package/src/worker.ts +4 -0
- package/src/workflows/crm-reconcile.ts +129 -0
- package/src/workflows/dispatch-conversion.ts +31 -0
- package/src/workflows/send-campaign.ts +5 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { CrmProviderRegistry } from "./crm-provider-registry.js";
|
|
2
|
+
import type { FunnelRegistry } from "./funnel-registry.js";
|
|
3
|
+
import { createOptionalSingleton } from "./singleton.js";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Process singleton for the CRM sync layer — set by `createHogsendClient`
|
|
7
|
+
* (both API and worker call it), read by the `crm-reconcile` cron task which
|
|
8
|
+
* has no client reference. Optional: a deploy with no CRM providers leaves it
|
|
9
|
+
* unset and the cron no-ops.
|
|
10
|
+
*/
|
|
11
|
+
export interface CrmSyncConfig {
|
|
12
|
+
registry: CrmProviderRegistry;
|
|
13
|
+
/** Funnel routing — includes the synthesized "default" (crm sugar). */
|
|
14
|
+
funnels: FunnelRegistry;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const singleton = createOptionalSingleton<CrmSyncConfig>();
|
|
18
|
+
|
|
19
|
+
export const setCrmSyncConfig = singleton.set;
|
|
20
|
+
export const getCrmSyncConfig = singleton.get;
|
|
21
|
+
/** Reset the singleton — only for test cleanup. */
|
|
22
|
+
export const resetCrmSyncConfig = singleton.reset;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
|
|
2
|
+
import type { AnalyticsProvider, PipelineLadder } from "@hogsend/core";
|
|
3
|
+
import type { JourneyRegistry } from "@hogsend/core/registry";
|
|
4
|
+
import type { Database } from "@hogsend/db";
|
|
5
|
+
import type { AppliedStageChange } from "./crm-deals.js";
|
|
6
|
+
import { DEAL_QUOTED, DEAL_SOLD } from "./crm-ingest.js";
|
|
7
|
+
import { ingestEvent } from "./ingestion.js";
|
|
8
|
+
import type { Logger } from "./logger.js";
|
|
9
|
+
import { type CrmDealEventPayload, emitOutbound } from "./outbound.js";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Mint the once-per-deal money events (`deal.quoted`/`deal.sold`) after a
|
|
13
|
+
* projection apply — the shared tail of BOTH stage-change producers (the
|
|
14
|
+
* CRM ingest sink and the event-trigger path). Freshness is the projection
|
|
15
|
+
* row's fact (`freshQuoted`/`freshSold`), so whichever producer reaches a
|
|
16
|
+
* milestone first mints it; the idempotency key only guards replays of the
|
|
17
|
+
* same mint.
|
|
18
|
+
*/
|
|
19
|
+
export async function mintDealMoneyEvents(opts: {
|
|
20
|
+
db: Database;
|
|
21
|
+
registry: JourneyRegistry;
|
|
22
|
+
hatchet: HatchetClient;
|
|
23
|
+
logger: Logger;
|
|
24
|
+
analytics?: AnalyticsProvider;
|
|
25
|
+
applied: AppliedStageChange;
|
|
26
|
+
ladder: PipelineLadder;
|
|
27
|
+
/** `crm-canonical:${provider}:${dealId}` | `funnel-canonical:${funnelId}:${externalId}` */
|
|
28
|
+
idempotencyPrefix: string;
|
|
29
|
+
source: string;
|
|
30
|
+
userId: string;
|
|
31
|
+
userEmail?: string;
|
|
32
|
+
contactId: string;
|
|
33
|
+
/** Path-specific extras; the helper adds `canonical_stage` itself. */
|
|
34
|
+
baseProperties: Record<string, unknown>;
|
|
35
|
+
occurredAt: string;
|
|
36
|
+
outboundPayload: CrmDealEventPayload;
|
|
37
|
+
}): Promise<void> {
|
|
38
|
+
const { applied, ladder } = opts;
|
|
39
|
+
const moneyEvents: Array<typeof DEAL_QUOTED | typeof DEAL_SOLD> = [
|
|
40
|
+
...(applied.freshQuoted ? [DEAL_QUOTED] : []),
|
|
41
|
+
...(applied.freshSold ? [DEAL_SOLD] : []),
|
|
42
|
+
];
|
|
43
|
+
for (const moneyEvent of moneyEvents) {
|
|
44
|
+
// The idempotency key keeps the SEMANTIC literal (stable across ladder
|
|
45
|
+
// edits); the event property carries the ladder's actual stage id.
|
|
46
|
+
const semantic = moneyEvent === DEAL_QUOTED ? "quoted" : "sold";
|
|
47
|
+
const stage =
|
|
48
|
+
(moneyEvent === DEAL_QUOTED ? ladder.quotedStage : ladder.soldStage) ??
|
|
49
|
+
semantic;
|
|
50
|
+
const idempotencyKey = `${opts.idempotencyPrefix}:${semantic}`;
|
|
51
|
+
await ingestEvent({
|
|
52
|
+
db: opts.db,
|
|
53
|
+
registry: opts.registry,
|
|
54
|
+
hatchet: opts.hatchet,
|
|
55
|
+
logger: opts.logger,
|
|
56
|
+
analytics: opts.analytics,
|
|
57
|
+
event: {
|
|
58
|
+
event: moneyEvent,
|
|
59
|
+
userId: opts.userId,
|
|
60
|
+
...(opts.userEmail ? { userEmail: opts.userEmail } : {}),
|
|
61
|
+
contactId: opts.contactId,
|
|
62
|
+
eventProperties: { ...opts.baseProperties, canonical_stage: stage },
|
|
63
|
+
...(applied.value !== null
|
|
64
|
+
? {
|
|
65
|
+
value: applied.value,
|
|
66
|
+
...(applied.currency ? { currency: applied.currency } : {}),
|
|
67
|
+
}
|
|
68
|
+
: {}),
|
|
69
|
+
occurredAt: opts.occurredAt,
|
|
70
|
+
idempotencyKey,
|
|
71
|
+
source: opts.source,
|
|
72
|
+
},
|
|
73
|
+
});
|
|
74
|
+
void emitOutbound({
|
|
75
|
+
db: opts.db,
|
|
76
|
+
hatchet: opts.hatchet,
|
|
77
|
+
logger: opts.logger,
|
|
78
|
+
event: moneyEvent,
|
|
79
|
+
payload: opts.outboundPayload,
|
|
80
|
+
dedupeKey: idempotencyKey,
|
|
81
|
+
}).catch((err) =>
|
|
82
|
+
opts.logger.warn("deal money outbound emit failed", { err }),
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
@@ -45,6 +45,12 @@ export interface SendTrackedEmailOptions<
|
|
|
45
45
|
to: string;
|
|
46
46
|
subject?: string;
|
|
47
47
|
journeyStateId?: string;
|
|
48
|
+
/**
|
|
49
|
+
* Campaign identity for campaign-dispatched sends, persisted on the
|
|
50
|
+
* email_sends row (attribution scope + stats). NULL for journey /
|
|
51
|
+
* transactional sends.
|
|
52
|
+
*/
|
|
53
|
+
campaignId?: string;
|
|
48
54
|
/** Denormalized recipient identity, persisted on the email_sends row for reporting. */
|
|
49
55
|
userId?: string;
|
|
50
56
|
userEmail?: string;
|
|
@@ -82,8 +88,15 @@ export interface TrackedSendResult {
|
|
|
82
88
|
* - `"test_mode_blocked"` — test mode was active but no redirect address
|
|
83
89
|
* resolved (no `HOGSEND_TEST_EMAIL` / `STUDIO_ADMIN_EMAIL`), so the send was
|
|
84
90
|
* blocked rather than delivered to the real recipient.
|
|
91
|
+
* - `"control_group"` — the recipient is in the global control group
|
|
92
|
+
* (impact plan §4.3): non-transactional sends are withheld so
|
|
93
|
+
* program-level lift is measurable. Transactional mail still delivers.
|
|
85
94
|
*/
|
|
86
|
-
reason?:
|
|
95
|
+
reason?:
|
|
96
|
+
| "frequency_capped"
|
|
97
|
+
| "journey_suppressed"
|
|
98
|
+
| "test_mode_blocked"
|
|
99
|
+
| "control_group";
|
|
87
100
|
}
|
|
88
101
|
|
|
89
102
|
/**
|
|
@@ -159,6 +172,8 @@ export interface EmailServiceSendOptions<
|
|
|
159
172
|
from?: string;
|
|
160
173
|
subject?: string;
|
|
161
174
|
journeyStateId?: string;
|
|
175
|
+
/** Campaign identity for campaign-dispatched sends (see SendTrackedEmailOptions). */
|
|
176
|
+
campaignId?: string;
|
|
162
177
|
/** Denormalized recipient identity, persisted on the email_sends row for reporting. */
|
|
163
178
|
userId?: string;
|
|
164
179
|
userEmail?: string;
|
package/src/lib/event-names.ts
CHANGED
|
@@ -35,7 +35,7 @@ export type EventNameEntry = z.infer<typeof eventNameEntrySchema>;
|
|
|
35
35
|
|
|
36
36
|
const EVENT_NAMES_NOTE =
|
|
37
37
|
"Event names are an open vocabulary — this is observed + declared usage, not a closed registry. " +
|
|
38
|
-
"Reserved namespaces (email.*, journey.*, bucket.*, contact.*) are engine-emitted; don't use them as blueprint trigger events.";
|
|
38
|
+
"Reserved namespaces (email.*, journey.*, bucket.*, contact.*, deal.*, funnel.*) are engine-emitted; don't use them as blueprint trigger events.";
|
|
39
39
|
|
|
40
40
|
export async function listEventNameVocabulary(opts: {
|
|
41
41
|
db: HogsendClient["db"];
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { canonicalStageRank } from "@hogsend/core";
|
|
2
|
+
import { type Database, funnelProgress } from "@hogsend/db";
|
|
3
|
+
import type { FunnelRegistry } from "./funnel-registry.js";
|
|
4
|
+
import { resolveFunnelTargets } from "./funnel-transitions.js";
|
|
5
|
+
import type { Logger } from "./logger.js";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Event-funnel REPORTING projection (docs/attribution-impact-plan.md §3.3) —
|
|
9
|
+
* one first-reach row per (contact, funnel, stage), the raw material for
|
|
10
|
+
* progression counts ("how many reached activated") and velocity ("median
|
|
11
|
+
* time signed_up → activated"). Complements the deals projection the
|
|
12
|
+
* transition machinery (funnel-transitions.ts) maintains: deals hold ONE
|
|
13
|
+
* current stage + money timestamps; this holds every stage's first-reach
|
|
14
|
+
* instant, which is what progression rates and velocity medians read.
|
|
15
|
+
*
|
|
16
|
+
* Targets come from {@link resolveFunnelTargets} — the SAME gates and
|
|
17
|
+
* winner selection as the deal mover, so the two projections can never
|
|
18
|
+
* disagree about what counts as a stage event. `lost` is a deal outcome,
|
|
19
|
+
* not progress, and is skipped; the unique (contact, funnel, stage) index
|
|
20
|
+
* absorbs replays and repeats — `reachedAt` is FIRST reach by construction.
|
|
21
|
+
*/
|
|
22
|
+
export async function recordFunnelProgressAtIngest(opts: {
|
|
23
|
+
db: Database;
|
|
24
|
+
logger: Logger;
|
|
25
|
+
funnels: FunnelRegistry | undefined;
|
|
26
|
+
event: {
|
|
27
|
+
name: string;
|
|
28
|
+
source: string | null;
|
|
29
|
+
properties: Record<string, unknown>;
|
|
30
|
+
value: number | null;
|
|
31
|
+
currency: string | null;
|
|
32
|
+
occurredAt: Date;
|
|
33
|
+
};
|
|
34
|
+
eventRowId: string;
|
|
35
|
+
contactId: string;
|
|
36
|
+
userKey: string;
|
|
37
|
+
}): Promise<{ reached: number }> {
|
|
38
|
+
const { db, logger, funnels, event, eventRowId, contactId, userKey } = opts;
|
|
39
|
+
if (!funnels) return { reached: 0 };
|
|
40
|
+
const targets = resolveFunnelTargets({ funnels, event });
|
|
41
|
+
|
|
42
|
+
let reached = 0;
|
|
43
|
+
for (const { funnel, target } of targets) {
|
|
44
|
+
if (target === "lost") continue;
|
|
45
|
+
const rank = canonicalStageRank(target, funnel.ladder);
|
|
46
|
+
if (rank === null || rank < 0) continue;
|
|
47
|
+
const inserted = await db
|
|
48
|
+
.insert(funnelProgress)
|
|
49
|
+
.values({
|
|
50
|
+
contactId,
|
|
51
|
+
userKey,
|
|
52
|
+
funnelId: funnel.meta.id,
|
|
53
|
+
stage: target,
|
|
54
|
+
stageRank: rank,
|
|
55
|
+
reachedAt: event.occurredAt,
|
|
56
|
+
eventId: eventRowId,
|
|
57
|
+
})
|
|
58
|
+
.onConflictDoNothing({
|
|
59
|
+
target: [
|
|
60
|
+
funnelProgress.contactId,
|
|
61
|
+
funnelProgress.funnelId,
|
|
62
|
+
funnelProgress.stage,
|
|
63
|
+
],
|
|
64
|
+
})
|
|
65
|
+
.returning({ id: funnelProgress.id });
|
|
66
|
+
if (inserted[0]) {
|
|
67
|
+
reached++;
|
|
68
|
+
logger.debug("funnel stage reached", {
|
|
69
|
+
funnel: funnel.meta.id,
|
|
70
|
+
stage: target,
|
|
71
|
+
userKey,
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
return { reached };
|
|
76
|
+
}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
DefinedFunnel,
|
|
3
|
+
FunnelBinding,
|
|
4
|
+
FunnelTransition,
|
|
5
|
+
} from "@hogsend/core";
|
|
6
|
+
import { DEFAULT_FUNNEL_ID } from "@hogsend/core";
|
|
7
|
+
|
|
8
|
+
/** A resolved claim: the owning funnel + the binding that matched (absent
|
|
9
|
+
* when traffic merely fell through to the default funnel). */
|
|
10
|
+
export interface ResolvedFunnelClaim {
|
|
11
|
+
funnel: DefinedFunnel;
|
|
12
|
+
binding?: FunnelBinding;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** One event→stage rule bound to its owning funnel. */
|
|
16
|
+
export interface FunnelTransitionMatch {
|
|
17
|
+
funnel: DefinedFunnel;
|
|
18
|
+
transition: FunnelTransition;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Funnel routing (plan §5b.4): which funnel claims a stage event's
|
|
23
|
+
* (provider, pipeline). Claims come straight from each funnel's CRM
|
|
24
|
+
* `bindings` — an exact pipeline claim beats a provider-wide `"*"`.
|
|
25
|
+
* Overlapping claims are a boot error, not a runtime coin-flip. Events
|
|
26
|
+
* nobody claims fall back to the `"default"` funnel when one is registered
|
|
27
|
+
* (the `crm.{stages,stageMaps}` sugar synthesizes it).
|
|
28
|
+
*/
|
|
29
|
+
export class FunnelRegistry {
|
|
30
|
+
private byId = new Map<string, DefinedFunnel>();
|
|
31
|
+
/** `${provider}:${pipelineId}` (or `${provider}:*`) → funnel + binding. */
|
|
32
|
+
private claims = new Map<string, ResolvedFunnelClaim>();
|
|
33
|
+
/** Event name → the transitions (across funnels) it can fire. */
|
|
34
|
+
private transitions = new Map<string, FunnelTransitionMatch[]>();
|
|
35
|
+
/** Trigger events wired to a quoted/won milestone stage. */
|
|
36
|
+
private milestoneTriggers = new Set<string>();
|
|
37
|
+
|
|
38
|
+
constructor(funnels: DefinedFunnel[] = []) {
|
|
39
|
+
for (const funnel of funnels) {
|
|
40
|
+
if (this.byId.has(funnel.meta.id)) {
|
|
41
|
+
throw new Error(`duplicate funnel id "${funnel.meta.id}"`);
|
|
42
|
+
}
|
|
43
|
+
this.byId.set(funnel.meta.id, funnel);
|
|
44
|
+
for (const binding of funnel.meta.bindings ?? []) {
|
|
45
|
+
const key = `${binding.provider}:${binding.pipeline}`;
|
|
46
|
+
const existing = this.claims.get(key);
|
|
47
|
+
if (existing) {
|
|
48
|
+
throw new Error(
|
|
49
|
+
`funnels "${existing.funnel.meta.id}" and "${funnel.meta.id}" both claim ` +
|
|
50
|
+
`${binding.provider} pipeline "${binding.pipeline}" — one pipeline feeds one funnel`,
|
|
51
|
+
);
|
|
52
|
+
}
|
|
53
|
+
this.claims.set(key, { funnel, binding });
|
|
54
|
+
}
|
|
55
|
+
for (const transition of funnel.transitions) {
|
|
56
|
+
const list = this.transitions.get(transition.event) ?? [];
|
|
57
|
+
list.push({ funnel, transition });
|
|
58
|
+
this.transitions.set(transition.event, list);
|
|
59
|
+
if (
|
|
60
|
+
transition.stageId === funnel.ladder.quotedStage ||
|
|
61
|
+
transition.stageId === funnel.ladder.soldStage
|
|
62
|
+
) {
|
|
63
|
+
this.milestoneTriggers.add(transition.event);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Event→stage rules listening on this event name (empty when none). */
|
|
70
|
+
transitionsFor(event: string): FunnelTransitionMatch[] {
|
|
71
|
+
return this.transitions.get(event) ?? [];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Trigger events wired to a money-milestone stage in ANY funnel. Their
|
|
76
|
+
* value is handed to the minted `deal.quoted`/`deal.sold`, so revenue
|
|
77
|
+
* rollups exclude the raw trigger rows (else one sale counts twice).
|
|
78
|
+
*/
|
|
79
|
+
milestoneTriggerEvents(): string[] {
|
|
80
|
+
return [...this.milestoneTriggers];
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/** Route one stage event: exact pipeline claim → provider `"*"` → default. */
|
|
84
|
+
resolve(
|
|
85
|
+
providerId: string,
|
|
86
|
+
pipelineId: string | undefined,
|
|
87
|
+
): ResolvedFunnelClaim | undefined {
|
|
88
|
+
const claimed =
|
|
89
|
+
(pipelineId
|
|
90
|
+
? this.claims.get(`${providerId}:${pipelineId}`)
|
|
91
|
+
: undefined) ?? this.claims.get(`${providerId}:*`);
|
|
92
|
+
if (claimed) return claimed;
|
|
93
|
+
const fallback = this.byId.get(DEFAULT_FUNNEL_ID);
|
|
94
|
+
return fallback ? { funnel: fallback } : undefined;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
get(id: string): DefinedFunnel | undefined {
|
|
98
|
+
return this.byId.get(id);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
getAll(): DefinedFunnel[] {
|
|
102
|
+
return [...this.byId.values()];
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
count(): number {
|
|
106
|
+
return this.byId.size;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
|
|
2
|
+
import type { AnalyticsProvider, DefinedFunnel } from "@hogsend/core";
|
|
3
|
+
import {
|
|
4
|
+
canonicalStageRank,
|
|
5
|
+
evaluatePropertyConditions,
|
|
6
|
+
overlayEventMoney,
|
|
7
|
+
sourceAllowed,
|
|
8
|
+
} from "@hogsend/core";
|
|
9
|
+
import type { JourneyRegistry } from "@hogsend/core/registry";
|
|
10
|
+
import { type Database, deals } from "@hogsend/db";
|
|
11
|
+
import { and, desc, eq, isNull, sql } from "drizzle-orm";
|
|
12
|
+
import { applyCrmStageEvent, EVENTS_DEAL_PROVIDER } from "./crm-deals.js";
|
|
13
|
+
import { FUNNEL_STAGE_CHANGED } from "./crm-ingest.js";
|
|
14
|
+
import { mintDealMoneyEvents } from "./deal-money-events.js";
|
|
15
|
+
import type { FunnelRegistry } from "./funnel-registry.js";
|
|
16
|
+
import type { Logger } from "./logger.js";
|
|
17
|
+
import { emitOutbound } from "./outbound.js";
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Event-driven funnel stage transitions — the step-(5d) ingest hook. For
|
|
21
|
+
* each funnel whose `on`/`lostOn` triggers match the just-stored event:
|
|
22
|
+
* trust-gate the source, evaluate the `where` conditions, pick the winning
|
|
23
|
+
* stage, and apply it to the contact's open deal in that funnel (one deal
|
|
24
|
+
* per contact per funnel, across producers — an open CRM-born row is moved
|
|
25
|
+
* in place; no open deal mints a synthetic `events`-provider row). Money
|
|
26
|
+
* milestones ride the same projection freshness as the CRM path, so
|
|
27
|
+
* whichever producer reaches a milestone first mints it.
|
|
28
|
+
*
|
|
29
|
+
* The triggering event IS already on the spine, so this path never ingests
|
|
30
|
+
* a `funnel.stage_changed` twin — it emits outbound only. Loop safety is
|
|
31
|
+
* define-time: `deal.`/`funnel.`/`crm.` events are rejected as triggers, so
|
|
32
|
+
* the money events this hook mints can never re-enter it.
|
|
33
|
+
*/
|
|
34
|
+
/**
|
|
35
|
+
* The gated, per-funnel WINNING stage targets for one event — source
|
|
36
|
+
* allowlist, `where` over money-overlaid properties, then the highest-ranked
|
|
37
|
+
* matching positive stage per funnel (`lost` only wins unopposed). Shared by
|
|
38
|
+
* the deal mover below and the `funnel_progress` reporting projection
|
|
39
|
+
* (funnel-progress.ts) so the two projections apply IDENTICAL rules and can
|
|
40
|
+
* never disagree about what counts as a stage event.
|
|
41
|
+
*/
|
|
42
|
+
export function resolveFunnelTargets(opts: {
|
|
43
|
+
funnels: FunnelRegistry;
|
|
44
|
+
event: {
|
|
45
|
+
name: string;
|
|
46
|
+
source: string | null;
|
|
47
|
+
properties: Record<string, unknown>;
|
|
48
|
+
value: number | null;
|
|
49
|
+
currency: string | null;
|
|
50
|
+
};
|
|
51
|
+
logger?: Logger;
|
|
52
|
+
}): Array<{ funnel: DefinedFunnel; target: string }> {
|
|
53
|
+
const { funnels, event, logger } = opts;
|
|
54
|
+
const matches = funnels.transitionsFor(event.name);
|
|
55
|
+
if (matches.length === 0) return [];
|
|
56
|
+
|
|
57
|
+
const whereProperties = overlayEventMoney(
|
|
58
|
+
event.properties,
|
|
59
|
+
event.value,
|
|
60
|
+
event.currency,
|
|
61
|
+
);
|
|
62
|
+
|
|
63
|
+
const byFunnel = new Map<
|
|
64
|
+
string,
|
|
65
|
+
{ funnel: DefinedFunnel; stages: string[] }
|
|
66
|
+
>();
|
|
67
|
+
for (const { funnel, transition } of matches) {
|
|
68
|
+
if (!sourceAllowed(funnel.meta.sources, event.source)) {
|
|
69
|
+
logger?.debug("funnel transition skipped: source not allowed", {
|
|
70
|
+
funnel: funnel.meta.id,
|
|
71
|
+
event: event.name,
|
|
72
|
+
source: event.source,
|
|
73
|
+
});
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
if (
|
|
77
|
+
transition.where &&
|
|
78
|
+
!evaluatePropertyConditions({
|
|
79
|
+
conditions: transition.where,
|
|
80
|
+
properties: whereProperties,
|
|
81
|
+
})
|
|
82
|
+
) {
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
const entry = byFunnel.get(funnel.meta.id) ?? { funnel, stages: [] };
|
|
86
|
+
entry.stages.push(transition.stageId);
|
|
87
|
+
byFunnel.set(funnel.meta.id, entry);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
return [...byFunnel.values()].map(({ funnel, stages }) => {
|
|
91
|
+
const positive = stages
|
|
92
|
+
.filter((s) => s !== "lost")
|
|
93
|
+
.sort(
|
|
94
|
+
(a, b) =>
|
|
95
|
+
(canonicalStageRank(b, funnel.ladder) ?? -1) -
|
|
96
|
+
(canonicalStageRank(a, funnel.ladder) ?? -1),
|
|
97
|
+
);
|
|
98
|
+
return { funnel, target: positive[0] ?? "lost" };
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export async function applyFunnelTransitionsAtIngest(opts: {
|
|
103
|
+
db: Database;
|
|
104
|
+
registry: JourneyRegistry;
|
|
105
|
+
hatchet: HatchetClient;
|
|
106
|
+
logger: Logger;
|
|
107
|
+
analytics?: AnalyticsProvider;
|
|
108
|
+
funnels: FunnelRegistry;
|
|
109
|
+
event: {
|
|
110
|
+
name: string;
|
|
111
|
+
source: string | null;
|
|
112
|
+
properties: Record<string, unknown>;
|
|
113
|
+
value: number | null;
|
|
114
|
+
currency: string | null;
|
|
115
|
+
occurredAt: Date;
|
|
116
|
+
};
|
|
117
|
+
eventRowId: string;
|
|
118
|
+
contactId: string;
|
|
119
|
+
userKey: string;
|
|
120
|
+
}): Promise<void> {
|
|
121
|
+
const {
|
|
122
|
+
db,
|
|
123
|
+
registry,
|
|
124
|
+
hatchet,
|
|
125
|
+
logger,
|
|
126
|
+
analytics,
|
|
127
|
+
funnels,
|
|
128
|
+
event,
|
|
129
|
+
eventRowId,
|
|
130
|
+
contactId,
|
|
131
|
+
userKey,
|
|
132
|
+
} = opts;
|
|
133
|
+
|
|
134
|
+
const targets = resolveFunnelTargets({ funnels, event, logger });
|
|
135
|
+
if (targets.length === 0) return;
|
|
136
|
+
|
|
137
|
+
for (const { funnel, target } of targets) {
|
|
138
|
+
try {
|
|
139
|
+
const funnelId = funnel.meta.id;
|
|
140
|
+
|
|
141
|
+
// Explicit multi-deal: a `deal_id` event property addresses one deal
|
|
142
|
+
// directly. Otherwise: the contact's open deal in THIS funnel,
|
|
143
|
+
// whichever producer created it. A `lost` trigger only ever CLOSES an
|
|
144
|
+
// existing deal — with none (open, or seen under the explicit id) it
|
|
145
|
+
// is a no-op, never a row born lost.
|
|
146
|
+
const rawDealId = event.properties.deal_id;
|
|
147
|
+
const explicitDealId =
|
|
148
|
+
typeof rawDealId === "string" || typeof rawDealId === "number"
|
|
149
|
+
? String(rawDealId)
|
|
150
|
+
: null;
|
|
151
|
+
let provider = EVENTS_DEAL_PROVIDER;
|
|
152
|
+
let externalId = explicitDealId
|
|
153
|
+
? `${funnelId}:${contactId}:${explicitDealId}`
|
|
154
|
+
: `${funnelId}:${contactId}`;
|
|
155
|
+
let pipelineId: string | undefined;
|
|
156
|
+
if (explicitDealId) {
|
|
157
|
+
if (target === "lost") {
|
|
158
|
+
const exists = await db
|
|
159
|
+
.select({ id: deals.id })
|
|
160
|
+
.from(deals)
|
|
161
|
+
.where(
|
|
162
|
+
and(
|
|
163
|
+
eq(deals.provider, provider),
|
|
164
|
+
eq(deals.externalId, externalId),
|
|
165
|
+
),
|
|
166
|
+
)
|
|
167
|
+
.limit(1);
|
|
168
|
+
if (!exists[0]) continue;
|
|
169
|
+
}
|
|
170
|
+
} else {
|
|
171
|
+
const open = await db
|
|
172
|
+
.select({
|
|
173
|
+
provider: deals.provider,
|
|
174
|
+
externalId: deals.externalId,
|
|
175
|
+
pipelineId: deals.pipelineId,
|
|
176
|
+
})
|
|
177
|
+
.from(deals)
|
|
178
|
+
.where(
|
|
179
|
+
and(
|
|
180
|
+
eq(deals.funnelId, funnelId),
|
|
181
|
+
eq(deals.contactId, contactId),
|
|
182
|
+
isNull(deals.soldAt),
|
|
183
|
+
isNull(deals.lostAt),
|
|
184
|
+
),
|
|
185
|
+
)
|
|
186
|
+
.orderBy(
|
|
187
|
+
sql`${deals.lastStageAt} desc nulls last`,
|
|
188
|
+
desc(deals.createdAt),
|
|
189
|
+
)
|
|
190
|
+
.limit(1);
|
|
191
|
+
if (open[0]) {
|
|
192
|
+
provider = open[0].provider;
|
|
193
|
+
externalId = open[0].externalId;
|
|
194
|
+
pipelineId = open[0].pipelineId ?? undefined;
|
|
195
|
+
} else if (target === "lost") {
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
const applied = await applyCrmStageEvent({
|
|
201
|
+
db,
|
|
202
|
+
logger,
|
|
203
|
+
providerId: provider,
|
|
204
|
+
contactId,
|
|
205
|
+
event: {
|
|
206
|
+
dealId: externalId,
|
|
207
|
+
stageId: target,
|
|
208
|
+
...(pipelineId ? { pipelineId } : {}),
|
|
209
|
+
...(event.value !== null
|
|
210
|
+
? {
|
|
211
|
+
value: {
|
|
212
|
+
amount: event.value,
|
|
213
|
+
...(event.currency ? { currency: event.currency } : {}),
|
|
214
|
+
},
|
|
215
|
+
}
|
|
216
|
+
: {}),
|
|
217
|
+
occurredAt: event.occurredAt.toISOString(),
|
|
218
|
+
},
|
|
219
|
+
canonicalStage: target,
|
|
220
|
+
ladder: funnel.ladder,
|
|
221
|
+
funnelId,
|
|
222
|
+
// A new-cycle trigger colliding with a closed synthetic row must not
|
|
223
|
+
// clobber its realized value (re-entry is deferred, not destructive).
|
|
224
|
+
protectTerminalValue: true,
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
const outboundPayload = {
|
|
228
|
+
provider,
|
|
229
|
+
dealId: externalId,
|
|
230
|
+
pipelineId: pipelineId ?? null,
|
|
231
|
+
funnelId,
|
|
232
|
+
canonicalStage: target,
|
|
233
|
+
value: applied.value,
|
|
234
|
+
currency: applied.currency,
|
|
235
|
+
userId: userKey,
|
|
236
|
+
at: event.occurredAt.toISOString(),
|
|
237
|
+
};
|
|
238
|
+
void emitOutbound({
|
|
239
|
+
db,
|
|
240
|
+
hatchet,
|
|
241
|
+
logger,
|
|
242
|
+
event: FUNNEL_STAGE_CHANGED,
|
|
243
|
+
payload: {
|
|
244
|
+
...outboundPayload,
|
|
245
|
+
stageId: target,
|
|
246
|
+
stageName: null,
|
|
247
|
+
status: null,
|
|
248
|
+
},
|
|
249
|
+
dedupeKey: `funnel-stage:${funnelId}:${externalId}:${target}:${eventRowId}`,
|
|
250
|
+
}).catch((err) => logger.warn("funnel outbound emit failed", { err }));
|
|
251
|
+
|
|
252
|
+
await mintDealMoneyEvents({
|
|
253
|
+
db,
|
|
254
|
+
registry,
|
|
255
|
+
hatchet,
|
|
256
|
+
logger,
|
|
257
|
+
analytics,
|
|
258
|
+
applied,
|
|
259
|
+
ladder: funnel.ladder,
|
|
260
|
+
idempotencyPrefix: `funnel-canonical:${funnelId}:${externalId}`,
|
|
261
|
+
source: "funnel",
|
|
262
|
+
userId: userKey,
|
|
263
|
+
contactId,
|
|
264
|
+
baseProperties: {
|
|
265
|
+
deal_id: externalId,
|
|
266
|
+
funnel_id: funnelId,
|
|
267
|
+
trigger_event: event.name,
|
|
268
|
+
},
|
|
269
|
+
occurredAt: event.occurredAt.toISOString(),
|
|
270
|
+
outboundPayload,
|
|
271
|
+
});
|
|
272
|
+
} catch (err) {
|
|
273
|
+
logger.warn("funnel transition failed", {
|
|
274
|
+
funnel: funnel.meta.id,
|
|
275
|
+
event: event.name,
|
|
276
|
+
error: err instanceof Error ? err.message : String(err),
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Deterministic holdout assignment (docs/attribution-impact-plan.md §4.1).
|
|
5
|
+
* NO RNG and NO clock — the replay law: journey tasks replay-from-top on
|
|
6
|
+
* eviction/crash, so the same (userId, journeyId, salt) must bucket
|
|
7
|
+
* identically on every evaluation, forever. sha256's first 4 bytes give a
|
|
8
|
+
* uniform 0–9999 bucket; a contact is held out when their bucket falls
|
|
9
|
+
* below percent × 100.
|
|
10
|
+
*/
|
|
11
|
+
export function holdoutBucket(opts: {
|
|
12
|
+
userId: string;
|
|
13
|
+
journeyId: string;
|
|
14
|
+
salt?: string;
|
|
15
|
+
}): number {
|
|
16
|
+
const digest = createHash("sha256")
|
|
17
|
+
.update(`${opts.salt ?? opts.journeyId}:${opts.journeyId}:${opts.userId}`)
|
|
18
|
+
.digest();
|
|
19
|
+
return digest.readUInt32BE(0) % 10000;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function isHeldOut(opts: {
|
|
23
|
+
userId: string;
|
|
24
|
+
journeyId: string;
|
|
25
|
+
/** 0–50; values outside are clamped (a holdout is never the majority). */
|
|
26
|
+
percent: number;
|
|
27
|
+
salt?: string;
|
|
28
|
+
}): boolean {
|
|
29
|
+
const percent = Math.min(50, Math.max(0, opts.percent));
|
|
30
|
+
if (percent === 0) return false;
|
|
31
|
+
return holdoutBucket(opts) < percent * 100;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Global control group (impact plan §4.3) — a contact-level bucket excluded
|
|
36
|
+
* from ALL non-transactional email/SMS sends, for program-level lift.
|
|
37
|
+
* Operator opt-in via `GLOBAL_CONTROL_PERCENT` (0 = off, clamped to 15 —
|
|
38
|
+
* the industry ceiling; Braze allows 1–15%). Explicitly optional: at SMB
|
|
39
|
+
* volume the per-journey holdout is the workhorse. `GLOBAL_CONTROL_SALT`
|
|
40
|
+
* rotation re-buckets the population.
|
|
41
|
+
*/
|
|
42
|
+
export function globalControlPercent(): number {
|
|
43
|
+
const raw = Number(process.env.GLOBAL_CONTROL_PERCENT ?? 0);
|
|
44
|
+
if (!Number.isFinite(raw) || raw <= 0) return 0;
|
|
45
|
+
return Math.min(15, raw);
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Whether the send identity (`userId` when known, else the recipient
|
|
50
|
+
* address) is in the global control group. Same determinism contract as
|
|
51
|
+
* {@link holdoutBucket}; case-normalized so an email key is stable.
|
|
52
|
+
*/
|
|
53
|
+
export function isGlobalControl(key: string): boolean {
|
|
54
|
+
const percent = globalControlPercent();
|
|
55
|
+
if (percent === 0) return false;
|
|
56
|
+
const digest = createHash("sha256")
|
|
57
|
+
.update(
|
|
58
|
+
`${process.env.GLOBAL_CONTROL_SALT ?? "global-control"}:GLOBAL:${key.toLowerCase()}`,
|
|
59
|
+
)
|
|
60
|
+
.digest();
|
|
61
|
+
return digest.readUInt32BE(0) % 10000 < percent * 100;
|
|
62
|
+
}
|