@hogsend/engine 0.43.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +12 -9
- package/src/container.ts +289 -13
- package/src/destinations/define-destination.ts +7 -3
- package/src/destinations/presets/posthog.ts +31 -0
- package/src/index.ts +99 -0
- package/src/journeys/execute-journey-run.ts +93 -0
- package/src/journeys/journey-context.ts +4 -0
- package/src/lib/analytics-providers-from-env.ts +106 -43
- package/src/lib/attribution-backfill.ts +285 -0
- package/src/lib/attribution.ts +239 -0
- package/src/lib/contacts.ts +71 -0
- package/src/lib/conversion-dispatch.ts +326 -0
- package/src/lib/conversions.ts +197 -0
- package/src/lib/crm-deals.ts +320 -0
- package/src/lib/crm-ingest.ts +341 -0
- package/src/lib/crm-provider-registry.ts +41 -0
- package/src/lib/crm-registry-singleton.ts +22 -0
- package/src/lib/deal-money-events.ts +85 -0
- package/src/lib/email-service-types.ts +16 -1
- package/src/lib/event-names.ts +1 -1
- package/src/lib/funnel-progress.ts +76 -0
- package/src/lib/funnel-registry.ts +108 -0
- package/src/lib/funnel-transitions.ts +280 -0
- package/src/lib/holdout.ts +62 -0
- package/src/lib/ingestion.ts +218 -9
- package/src/lib/lift-stats.ts +125 -0
- package/src/lib/mailer.ts +1 -0
- package/src/lib/operator.ts +31 -0
- package/src/lib/outbound.ts +51 -0
- package/src/lib/posthog-scopes.ts +30 -6
- package/src/lib/revenue.ts +116 -0
- package/src/lib/sms-service-types.ts +7 -1
- package/src/lib/sms-tracked.ts +35 -0
- package/src/lib/tracked.ts +40 -0
- package/src/lib/tracking-events.ts +26 -0
- package/src/lib/unconfigured-email-provider.ts +45 -0
- package/src/lib/webhook-signing.ts +7 -0
- package/src/routes/admin/analytics.ts +14 -5
- package/src/routes/admin/attribution.ts +414 -0
- package/src/routes/admin/contacts.ts +71 -11
- package/src/routes/admin/conversions.ts +247 -0
- package/src/routes/admin/deals.ts +472 -0
- package/src/routes/admin/funnels.ts +228 -0
- package/src/routes/admin/index.ts +8 -0
- package/src/routes/admin/journeys.ts +158 -1
- package/src/routes/admin/readiness.ts +97 -2
- package/src/routes/admin/templates.ts +28 -1
- package/src/routes/events/index.ts +12 -0
- package/src/routes/tracking/click-pipeline.ts +26 -18
- package/src/routes/webhooks/crm-provider.ts +117 -0
- package/src/routes/webhooks/index.ts +5 -1
- package/src/sources/define-contact-source.ts +159 -0
- package/src/sources/registry.ts +71 -0
- package/src/sources/webhook-source.ts +109 -0
- package/src/worker.ts +4 -0
- package/src/workflows/crm-reconcile.ts +129 -0
- package/src/workflows/dispatch-conversion.ts +31 -0
- package/src/workflows/send-campaign.ts +5 -0
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
import {
|
|
2
|
+
attributionCredits,
|
|
3
|
+
contacts,
|
|
4
|
+
conversions,
|
|
5
|
+
type Database,
|
|
6
|
+
userEvents,
|
|
7
|
+
} from "@hogsend/db";
|
|
8
|
+
import {
|
|
9
|
+
and,
|
|
10
|
+
asc,
|
|
11
|
+
eq,
|
|
12
|
+
gt,
|
|
13
|
+
gte,
|
|
14
|
+
inArray,
|
|
15
|
+
isNotNull,
|
|
16
|
+
or,
|
|
17
|
+
sql,
|
|
18
|
+
} from "drizzle-orm";
|
|
19
|
+
import { recordAttributionCredits } from "./attribution.js";
|
|
20
|
+
import type { ConversionRegistry } from "./conversions.js";
|
|
21
|
+
import { evaluateConversionsAtIngest } from "./conversions.js";
|
|
22
|
+
import type { Logger } from "./logger.js";
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Attribution backfill (impact plan §5.1) — replays HISTORY through the
|
|
26
|
+
* same idempotent machinery ingest uses, so an existing deploy that
|
|
27
|
+
* upgrades gets its whole event history credited, and a definition/window
|
|
28
|
+
* change gets a deliberate, logged recompute path.
|
|
29
|
+
*
|
|
30
|
+
* Two stages behind one cursor (`events:<id>` → `credits:<id>`):
|
|
31
|
+
*
|
|
32
|
+
* 1. `events` — scan historical `user_events` matching the registry's
|
|
33
|
+
* trigger set and run conversion evaluation. The unique
|
|
34
|
+
* (definition, event) index makes replays no-ops; only genuinely new
|
|
35
|
+
* conversions are minted. Ad-platform dispatches are deliberately NOT
|
|
36
|
+
* enqueued — historical conversions must never re-fire to destinations.
|
|
37
|
+
* 2. `credits` — scan conversions rows (of the scoped definitions) that
|
|
38
|
+
* have NO credit rows yet and write their ledger entries under the
|
|
39
|
+
* definition's CURRENT window config.
|
|
40
|
+
*
|
|
41
|
+
* `recompute: true` (requires `definitionId`) deletes the definition's
|
|
42
|
+
* existing credit rows up front, then stage 2 re-credits everything — the
|
|
43
|
+
* ONE sanctioned exception to the ledger's write-once philosophy, guarded
|
|
44
|
+
* and logged. Conversion rows themselves are never deleted.
|
|
45
|
+
*
|
|
46
|
+
* Batch-shaped by design: each call processes up to `limit` rows of the
|
|
47
|
+
* current stage and returns `nextCursor` (null = done). The CLI loops it;
|
|
48
|
+
* an admin can curl it. Contacts are matched only — a historical event
|
|
49
|
+
* whose user never became a contact is skipped, never minted.
|
|
50
|
+
*/
|
|
51
|
+
export interface BackfillBatchResult {
|
|
52
|
+
stage: "events" | "credits";
|
|
53
|
+
processed: number;
|
|
54
|
+
conversionsFired: number;
|
|
55
|
+
creditsWritten: number;
|
|
56
|
+
nextCursor: string | null;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const UUID_RE =
|
|
60
|
+
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
61
|
+
|
|
62
|
+
export async function backfillAttributionBatch(opts: {
|
|
63
|
+
db: Database;
|
|
64
|
+
logger: Logger;
|
|
65
|
+
registry: ConversionRegistry | undefined;
|
|
66
|
+
definitionId?: string;
|
|
67
|
+
since?: Date;
|
|
68
|
+
cursor?: string;
|
|
69
|
+
limit?: number;
|
|
70
|
+
recompute?: boolean;
|
|
71
|
+
}): Promise<BackfillBatchResult> {
|
|
72
|
+
const { db, logger, registry, definitionId, since } = opts;
|
|
73
|
+
const limit = Math.min(2000, Math.max(1, opts.limit ?? 500));
|
|
74
|
+
|
|
75
|
+
const defs = (registry?.getAll() ?? []).filter(
|
|
76
|
+
(def) => !definitionId || def.meta.id === definitionId,
|
|
77
|
+
);
|
|
78
|
+
if (defs.length === 0) {
|
|
79
|
+
return {
|
|
80
|
+
stage: "credits",
|
|
81
|
+
processed: 0,
|
|
82
|
+
conversionsFired: 0,
|
|
83
|
+
creditsWritten: 0,
|
|
84
|
+
nextCursor: null,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
const defIds = defs.map((def) => def.meta.id);
|
|
88
|
+
const defById = new Map(defs.map((def) => [def.meta.id, def]));
|
|
89
|
+
const namedEvents = [
|
|
90
|
+
...new Set(
|
|
91
|
+
defs
|
|
92
|
+
.map((def) => def.meta.trigger.event)
|
|
93
|
+
.filter((event) => event !== "*"),
|
|
94
|
+
),
|
|
95
|
+
];
|
|
96
|
+
const hasWildcard = defs.some((def) => def.meta.trigger.event === "*");
|
|
97
|
+
|
|
98
|
+
const [stage, lastId] = opts.cursor
|
|
99
|
+
? (opts.cursor.split(":", 2) as ["events" | "credits", string])
|
|
100
|
+
: (["events", ""] as const);
|
|
101
|
+
|
|
102
|
+
// Recompute (guarded): first call only, one definition only.
|
|
103
|
+
if (opts.recompute && !opts.cursor) {
|
|
104
|
+
if (!definitionId) {
|
|
105
|
+
throw new Error("recompute requires a definitionId — never blanket");
|
|
106
|
+
}
|
|
107
|
+
const deleted = await db
|
|
108
|
+
.delete(attributionCredits)
|
|
109
|
+
.where(
|
|
110
|
+
inArray(
|
|
111
|
+
attributionCredits.conversionId,
|
|
112
|
+
db
|
|
113
|
+
.select({ id: conversions.id })
|
|
114
|
+
.from(conversions)
|
|
115
|
+
.where(eq(conversions.definitionId, definitionId)),
|
|
116
|
+
),
|
|
117
|
+
)
|
|
118
|
+
.returning({ id: attributionCredits.id });
|
|
119
|
+
logger.warn("attribution recompute: credit rows deleted for refill", {
|
|
120
|
+
definitionId,
|
|
121
|
+
deleted: deleted.length,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
if (stage === "events") {
|
|
126
|
+
const eventMatch = or(
|
|
127
|
+
namedEvents.length > 0
|
|
128
|
+
? inArray(userEvents.event, namedEvents)
|
|
129
|
+
: sql`false`,
|
|
130
|
+
hasWildcard ? isNotNull(userEvents.value) : sql`false`,
|
|
131
|
+
);
|
|
132
|
+
const rows = await db
|
|
133
|
+
.select({
|
|
134
|
+
id: userEvents.id,
|
|
135
|
+
event: userEvents.event,
|
|
136
|
+
userId: userEvents.userId,
|
|
137
|
+
properties: userEvents.properties,
|
|
138
|
+
value: userEvents.value,
|
|
139
|
+
currency: userEvents.currency,
|
|
140
|
+
source: userEvents.source,
|
|
141
|
+
occurredAt: userEvents.occurredAt,
|
|
142
|
+
})
|
|
143
|
+
.from(userEvents)
|
|
144
|
+
.where(
|
|
145
|
+
and(
|
|
146
|
+
eventMatch,
|
|
147
|
+
...(since ? [gte(userEvents.occurredAt, since)] : []),
|
|
148
|
+
...(lastId ? [gt(userEvents.id, lastId)] : []),
|
|
149
|
+
),
|
|
150
|
+
)
|
|
151
|
+
.orderBy(asc(userEvents.id))
|
|
152
|
+
.limit(limit);
|
|
153
|
+
|
|
154
|
+
// Batch contact resolution: canonical keys are externalId or the
|
|
155
|
+
// contact row uuid. MATCH ONLY — backfill never mints contacts.
|
|
156
|
+
const keys = [...new Set(rows.map((row) => row.userId))];
|
|
157
|
+
const uuidKeys = keys.filter((key) => UUID_RE.test(key));
|
|
158
|
+
const contactRows =
|
|
159
|
+
keys.length > 0
|
|
160
|
+
? await db
|
|
161
|
+
.select({ id: contacts.id, externalId: contacts.externalId })
|
|
162
|
+
.from(contacts)
|
|
163
|
+
.where(
|
|
164
|
+
or(
|
|
165
|
+
inArray(contacts.externalId, keys),
|
|
166
|
+
uuidKeys.length > 0
|
|
167
|
+
? inArray(contacts.id, uuidKeys)
|
|
168
|
+
: sql`false`,
|
|
169
|
+
),
|
|
170
|
+
)
|
|
171
|
+
: [];
|
|
172
|
+
const contactByKey = new Map<string, string>();
|
|
173
|
+
for (const contact of contactRows) {
|
|
174
|
+
contactByKey.set(contact.id, contact.id);
|
|
175
|
+
if (contact.externalId) contactByKey.set(contact.externalId, contact.id);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
let conversionsFired = 0;
|
|
179
|
+
let creditsWritten = 0;
|
|
180
|
+
for (const row of rows) {
|
|
181
|
+
const contactId = contactByKey.get(row.userId);
|
|
182
|
+
if (!contactId) continue;
|
|
183
|
+
const fired = await evaluateConversionsAtIngest({
|
|
184
|
+
db,
|
|
185
|
+
logger,
|
|
186
|
+
registry,
|
|
187
|
+
event: {
|
|
188
|
+
name: row.event,
|
|
189
|
+
source: row.source ?? null,
|
|
190
|
+
properties: (row.properties ?? {}) as Record<string, unknown>,
|
|
191
|
+
value: row.value,
|
|
192
|
+
currency: row.currency,
|
|
193
|
+
occurredAt: row.occurredAt,
|
|
194
|
+
},
|
|
195
|
+
eventRowId: row.id,
|
|
196
|
+
contactId,
|
|
197
|
+
userKey: row.userId,
|
|
198
|
+
});
|
|
199
|
+
for (const conversion of fired) {
|
|
200
|
+
if (definitionId && conversion.definition.meta.id !== definitionId) {
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
conversionsFired++;
|
|
204
|
+
const { touchpoints } = await recordAttributionCredits({
|
|
205
|
+
db,
|
|
206
|
+
logger,
|
|
207
|
+
conversionId: conversion.conversionId,
|
|
208
|
+
userKey: row.userId,
|
|
209
|
+
value: conversion.value,
|
|
210
|
+
currency: conversion.currency,
|
|
211
|
+
occurredAt: row.occurredAt,
|
|
212
|
+
windowDays: conversion.definition.meta.attributionWindowDays ?? 90,
|
|
213
|
+
windows: conversion.definition.meta.windows,
|
|
214
|
+
});
|
|
215
|
+
if (touchpoints > 0) creditsWritten++;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const nextCursor =
|
|
220
|
+
rows.length < limit
|
|
221
|
+
? "credits:"
|
|
222
|
+
: `events:${rows[rows.length - 1]?.id ?? ""}`;
|
|
223
|
+
return {
|
|
224
|
+
stage: "events",
|
|
225
|
+
processed: rows.length,
|
|
226
|
+
conversionsFired,
|
|
227
|
+
creditsWritten,
|
|
228
|
+
nextCursor,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// Stage 2: credit conversions that have no ledger rows yet (after a
|
|
233
|
+
// recompute delete, that is all of the definition's conversions).
|
|
234
|
+
const uncredited = sql`not exists (
|
|
235
|
+
select 1 from ${attributionCredits}
|
|
236
|
+
where ${attributionCredits.conversionId} = ${conversions.id}
|
|
237
|
+
)`;
|
|
238
|
+
const rows = await db
|
|
239
|
+
.select({
|
|
240
|
+
id: conversions.id,
|
|
241
|
+
definitionId: conversions.definitionId,
|
|
242
|
+
userKey: conversions.userKey,
|
|
243
|
+
value: conversions.value,
|
|
244
|
+
currency: conversions.currency,
|
|
245
|
+
occurredAt: conversions.occurredAt,
|
|
246
|
+
})
|
|
247
|
+
.from(conversions)
|
|
248
|
+
.where(
|
|
249
|
+
and(
|
|
250
|
+
inArray(conversions.definitionId, defIds),
|
|
251
|
+
uncredited,
|
|
252
|
+
...(since ? [gte(conversions.occurredAt, since)] : []),
|
|
253
|
+
...(lastId ? [gt(conversions.id, lastId)] : []),
|
|
254
|
+
),
|
|
255
|
+
)
|
|
256
|
+
.orderBy(asc(conversions.id))
|
|
257
|
+
.limit(limit);
|
|
258
|
+
|
|
259
|
+
let creditsWritten = 0;
|
|
260
|
+
for (const row of rows) {
|
|
261
|
+
const def = defById.get(row.definitionId);
|
|
262
|
+
if (!def) continue;
|
|
263
|
+
const { touchpoints } = await recordAttributionCredits({
|
|
264
|
+
db,
|
|
265
|
+
logger,
|
|
266
|
+
conversionId: row.id,
|
|
267
|
+
userKey: row.userKey,
|
|
268
|
+
value: row.value,
|
|
269
|
+
currency: row.currency,
|
|
270
|
+
occurredAt: row.occurredAt,
|
|
271
|
+
windowDays: def.meta.attributionWindowDays ?? 90,
|
|
272
|
+
windows: def.meta.windows,
|
|
273
|
+
});
|
|
274
|
+
if (touchpoints > 0) creditsWritten++;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
return {
|
|
278
|
+
stage: "credits",
|
|
279
|
+
processed: rows.length,
|
|
280
|
+
conversionsFired: 0,
|
|
281
|
+
creditsWritten,
|
|
282
|
+
nextCursor:
|
|
283
|
+
rows.length < limit ? null : `credits:${rows[rows.length - 1]?.id ?? ""}`,
|
|
284
|
+
};
|
|
285
|
+
}
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type AttributionTouchpoint,
|
|
3
|
+
computeAllModels,
|
|
4
|
+
} from "@hogsend/attribution";
|
|
5
|
+
import {
|
|
6
|
+
type DurationObject,
|
|
7
|
+
durationToMs,
|
|
8
|
+
TOUCHPOINT_EVENTS,
|
|
9
|
+
type TouchpointChannel,
|
|
10
|
+
touchpointChannel,
|
|
11
|
+
} from "@hogsend/core";
|
|
12
|
+
import {
|
|
13
|
+
attributionCredits,
|
|
14
|
+
type Database,
|
|
15
|
+
emailSends,
|
|
16
|
+
journeyStates,
|
|
17
|
+
userEvents,
|
|
18
|
+
} from "@hogsend/db";
|
|
19
|
+
import { and, asc, eq, gte, inArray, lte } from "drizzle-orm";
|
|
20
|
+
import type { Logger } from "./logger.js";
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Attribution scope carried per touchpoint into the ledger row
|
|
24
|
+
* (docs/attribution-impact-plan.md §1.3). All nullable — a touch outside any
|
|
25
|
+
* journey/campaign has none.
|
|
26
|
+
*/
|
|
27
|
+
interface TouchScope {
|
|
28
|
+
journeyId: string | null;
|
|
29
|
+
campaignId: string | null;
|
|
30
|
+
templateKey: string | null;
|
|
31
|
+
funnelId: string | null;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const UUID_RE =
|
|
35
|
+
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
36
|
+
|
|
37
|
+
/** A property value usable as a scope id (scalar string, non-empty). */
|
|
38
|
+
function scopeString(value: unknown): string | null {
|
|
39
|
+
return typeof value === "string" && value.length > 0 ? value : null;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/** Like {@link scopeString} but must be a uuid (campaign_id column is uuid). */
|
|
43
|
+
function scopeUuid(value: unknown): string | null {
|
|
44
|
+
const str = scopeString(value);
|
|
45
|
+
return str && UUID_RE.test(str) ? str : null;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Resolve each touchpoint's attribution scope from its stamped properties
|
|
50
|
+
* (`pushTrackingEvent` stamps journeyId/campaignId/templateKey — plan §1.2).
|
|
51
|
+
* Events ingested BEFORE stamping carry an `emailSendId` but no
|
|
52
|
+
* `journeyId`/`campaignId` keys at all (post-stamp events carry explicit
|
|
53
|
+
* nulls) — those fall back to one batched email_sends → journey_states join.
|
|
54
|
+
* SMS pre-stamp events are left unscoped: scope stamping landed days after
|
|
55
|
+
* the SMS channel itself, so the unscoped window is negligible and not worth
|
|
56
|
+
* a second join path.
|
|
57
|
+
*/
|
|
58
|
+
async function resolveTouchScopes(
|
|
59
|
+
db: Database,
|
|
60
|
+
rows: Array<{ id: string; properties: Record<string, unknown> | null }>,
|
|
61
|
+
): Promise<Map<string, TouchScope>> {
|
|
62
|
+
const scopes = new Map<string, TouchScope>();
|
|
63
|
+
/** Pre-stamp email touches: eventRowId → emailSendId, backfilled via join. */
|
|
64
|
+
const fallback = new Map<string, string>();
|
|
65
|
+
|
|
66
|
+
for (const row of rows) {
|
|
67
|
+
const props = row.properties ?? {};
|
|
68
|
+
const stamped = "journeyId" in props || "campaignId" in props;
|
|
69
|
+
const emailSendId = scopeString(props.emailSendId);
|
|
70
|
+
if (!stamped && emailSendId) fallback.set(row.id, emailSendId);
|
|
71
|
+
scopes.set(row.id, {
|
|
72
|
+
journeyId: scopeString(props.journeyId),
|
|
73
|
+
campaignId: scopeUuid(props.campaignId),
|
|
74
|
+
templateKey: scopeString(props.templateKey),
|
|
75
|
+
// No touch event stamps funnel scope today; accept either spelling if
|
|
76
|
+
// one ever does (money/deal events use snake_case on the spine).
|
|
77
|
+
funnelId: scopeString(props.funnelId) ?? scopeString(props.funnel_id),
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (fallback.size > 0) {
|
|
82
|
+
const sendRows = await db
|
|
83
|
+
.select({
|
|
84
|
+
id: emailSends.id,
|
|
85
|
+
campaignId: emailSends.campaignId,
|
|
86
|
+
templateKey: emailSends.templateKey,
|
|
87
|
+
journeyId: journeyStates.journeyId,
|
|
88
|
+
})
|
|
89
|
+
.from(emailSends)
|
|
90
|
+
.leftJoin(journeyStates, eq(emailSends.journeyStateId, journeyStates.id))
|
|
91
|
+
.where(inArray(emailSends.id, [...new Set(fallback.values())]));
|
|
92
|
+
const bySendId = new Map(sendRows.map((send) => [send.id, send]));
|
|
93
|
+
for (const [eventRowId, emailSendId] of fallback) {
|
|
94
|
+
const send = bySendId.get(emailSendId);
|
|
95
|
+
if (!send) continue;
|
|
96
|
+
const scope = scopes.get(eventRowId);
|
|
97
|
+
if (!scope) continue;
|
|
98
|
+
scope.journeyId = send.journeyId;
|
|
99
|
+
scope.campaignId = send.campaignId;
|
|
100
|
+
scope.templateKey ??= send.templateKey;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
return scopes;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* The attribution ledger writer (plan §6.1): when a conversion fires, read
|
|
109
|
+
* the contact's touchpoint path inside the definition's lookback window and
|
|
110
|
+
* persist EVERY model's credit allocation into `attribution_credits`.
|
|
111
|
+
* Computing all models up front makes "switch the reporting model" a WHERE
|
|
112
|
+
* clause instead of a historical re-derivation.
|
|
113
|
+
*
|
|
114
|
+
* Idempotent: the unique (conversion, model, touchpoint) index turns any
|
|
115
|
+
* replay into a no-op — same inheritance as the conversion row itself.
|
|
116
|
+
*/
|
|
117
|
+
export async function recordAttributionCredits(opts: {
|
|
118
|
+
db: Database;
|
|
119
|
+
logger: Logger;
|
|
120
|
+
conversionId: string;
|
|
121
|
+
/** The contact's canonical event key (`user_events.user_id`). */
|
|
122
|
+
userKey: string;
|
|
123
|
+
value: number | null;
|
|
124
|
+
currency: string | null;
|
|
125
|
+
/** When the conversion happened. */
|
|
126
|
+
occurredAt: Date;
|
|
127
|
+
/** Lookback window in days (defineConversion attributionWindowDays). */
|
|
128
|
+
windowDays: number;
|
|
129
|
+
/**
|
|
130
|
+
* Per-channel window overrides (plan §2.1). A listed channel's touches
|
|
131
|
+
* qualify only inside its own window; unlisted channels use `windowDays`.
|
|
132
|
+
*/
|
|
133
|
+
windows?: Partial<Record<TouchpointChannel, DurationObject>>;
|
|
134
|
+
}): Promise<{ touchpoints: number }> {
|
|
135
|
+
const { db, logger, conversionId, userKey, value, currency, occurredAt } =
|
|
136
|
+
opts;
|
|
137
|
+
const defaultWindowMs = opts.windowDays * 24 * 60 * 60 * 1000;
|
|
138
|
+
const channelWindowMs = new Map<TouchpointChannel, number>(
|
|
139
|
+
Object.entries(opts.windows ?? {}).map(([channel, duration]) => [
|
|
140
|
+
channel as TouchpointChannel,
|
|
141
|
+
durationToMs(duration as DurationObject),
|
|
142
|
+
]),
|
|
143
|
+
);
|
|
144
|
+
// One query at the widest window; the per-channel narrowing happens in the
|
|
145
|
+
// classification loop below (channels are a query-time concept anyway).
|
|
146
|
+
const windowStart = new Date(
|
|
147
|
+
occurredAt.getTime() -
|
|
148
|
+
Math.max(defaultWindowMs, ...channelWindowMs.values()),
|
|
149
|
+
);
|
|
150
|
+
|
|
151
|
+
const rows = await db
|
|
152
|
+
.select({
|
|
153
|
+
id: userEvents.id,
|
|
154
|
+
event: userEvents.event,
|
|
155
|
+
occurredAt: userEvents.occurredAt,
|
|
156
|
+
properties: userEvents.properties,
|
|
157
|
+
})
|
|
158
|
+
.from(userEvents)
|
|
159
|
+
.where(
|
|
160
|
+
and(
|
|
161
|
+
eq(userEvents.userId, userKey),
|
|
162
|
+
inArray(userEvents.event, [...TOUCHPOINT_EVENTS]),
|
|
163
|
+
gte(userEvents.occurredAt, windowStart),
|
|
164
|
+
lte(userEvents.occurredAt, occurredAt),
|
|
165
|
+
),
|
|
166
|
+
)
|
|
167
|
+
.orderBy(asc(userEvents.occurredAt));
|
|
168
|
+
|
|
169
|
+
const touchpoints: AttributionTouchpoint[] = [];
|
|
170
|
+
for (const row of rows) {
|
|
171
|
+
const channel = touchpointChannel(row.event);
|
|
172
|
+
if (!channel) continue; // unreachable: the IN filter is the class list
|
|
173
|
+
const windowMs = channelWindowMs.get(channel) ?? defaultWindowMs;
|
|
174
|
+
if (row.occurredAt.getTime() < occurredAt.getTime() - windowMs) continue;
|
|
175
|
+
touchpoints.push({
|
|
176
|
+
id: row.id,
|
|
177
|
+
event: row.event,
|
|
178
|
+
channel,
|
|
179
|
+
occurredAt: row.occurredAt.getTime(),
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
if (touchpoints.length === 0) {
|
|
183
|
+
// No path — the conversion stays unattributed (reporting reads the gap
|
|
184
|
+
// as conversion value minus credited value; we never invent a touch).
|
|
185
|
+
return { touchpoints: 0 };
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
const byId = new Map(touchpoints.map((t) => [t.id, t]));
|
|
189
|
+
// Only qualifying touches need scope — a window-dropped row must not cost
|
|
190
|
+
// a fallback join.
|
|
191
|
+
const scopes = await resolveTouchScopes(
|
|
192
|
+
db,
|
|
193
|
+
rows.filter((row) => byId.has(row.id)),
|
|
194
|
+
);
|
|
195
|
+
const allModels = computeAllModels(touchpoints, {
|
|
196
|
+
conversionAt: occurredAt.getTime(),
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
const values = Object.entries(allModels).flatMap(([model, credits]) =>
|
|
200
|
+
credits.map((credit) => {
|
|
201
|
+
const touch = byId.get(credit.touchpointId) as AttributionTouchpoint;
|
|
202
|
+
const scope = scopes.get(credit.touchpointId);
|
|
203
|
+
return {
|
|
204
|
+
conversionId,
|
|
205
|
+
model,
|
|
206
|
+
touchpointEventId: credit.touchpointId,
|
|
207
|
+
touchpointEvent: touch.event,
|
|
208
|
+
channel: touch.channel,
|
|
209
|
+
touchpointAt: new Date(touch.occurredAt),
|
|
210
|
+
journeyId: scope?.journeyId ?? null,
|
|
211
|
+
campaignId: scope?.campaignId ?? null,
|
|
212
|
+
templateKey: scope?.templateKey ?? null,
|
|
213
|
+
funnelId: scope?.funnelId ?? null,
|
|
214
|
+
weight: credit.weight,
|
|
215
|
+
value:
|
|
216
|
+
value !== null ? Math.round(credit.weight * value * 100) / 100 : null,
|
|
217
|
+
currency: value !== null ? currency : null,
|
|
218
|
+
convertedAt: occurredAt,
|
|
219
|
+
};
|
|
220
|
+
}),
|
|
221
|
+
);
|
|
222
|
+
|
|
223
|
+
await db
|
|
224
|
+
.insert(attributionCredits)
|
|
225
|
+
.values(values)
|
|
226
|
+
.onConflictDoNothing({
|
|
227
|
+
target: [
|
|
228
|
+
attributionCredits.conversionId,
|
|
229
|
+
attributionCredits.model,
|
|
230
|
+
attributionCredits.touchpointEventId,
|
|
231
|
+
],
|
|
232
|
+
});
|
|
233
|
+
logger.debug("attribution credits recorded", {
|
|
234
|
+
conversionId,
|
|
235
|
+
touchpoints: touchpoints.length,
|
|
236
|
+
rows: values.length,
|
|
237
|
+
});
|
|
238
|
+
return { touchpoints: touchpoints.length };
|
|
239
|
+
}
|
package/src/lib/contacts.ts
CHANGED
|
@@ -2,7 +2,9 @@ import {
|
|
|
2
2
|
bucketMemberships,
|
|
3
3
|
contactAliases,
|
|
4
4
|
contacts,
|
|
5
|
+
crmLinks,
|
|
5
6
|
type Database,
|
|
7
|
+
deals,
|
|
6
8
|
emailPreferences,
|
|
7
9
|
emailSends,
|
|
8
10
|
journeyStates,
|
|
@@ -547,6 +549,17 @@ export async function resolveOrCreateContact(opts: {
|
|
|
547
549
|
* `restrictToAnonymous`.
|
|
548
550
|
*/
|
|
549
551
|
contactId?: string;
|
|
552
|
+
/**
|
|
553
|
+
* PROVENANCE (best-effort metadata, NOT an identity key): the Source id that
|
|
554
|
+
* created this contact — a Contact Source id ("clay"/"attio") or the ingest
|
|
555
|
+
* `source`. First-touch: written on create, and on a fill-in-link/merge that
|
|
556
|
+
* supplies one ONLY when the resolved row has none; never overwrites an
|
|
557
|
+
* existing value. NEVER participates in key resolution or survivor selection,
|
|
558
|
+
* so it cannot steer identity — a safe non-identity column threaded alongside.
|
|
559
|
+
*/
|
|
560
|
+
source?: string;
|
|
561
|
+
/** Timestamp paired with {@link source}; defaults to now() at create time. */
|
|
562
|
+
sourcedAt?: Date;
|
|
550
563
|
}): Promise<{
|
|
551
564
|
id: string;
|
|
552
565
|
/**
|
|
@@ -583,6 +596,8 @@ export async function resolveOrCreateContact(opts: {
|
|
|
583
596
|
const anonymousId = opts.anonymousId?.trim() || undefined;
|
|
584
597
|
const discordId = opts.discordId?.trim() || undefined;
|
|
585
598
|
const contactId = opts.contactId?.trim() || undefined;
|
|
599
|
+
const source = opts.source?.trim() || undefined;
|
|
600
|
+
const sourcedAt = opts.sourcedAt;
|
|
586
601
|
// §Phase 1 GAP-1: the publishable clamp only bites an ANON-ONLY write (the
|
|
587
602
|
// only shape a token-less pk_ key can produce — the gate 403s any
|
|
588
603
|
// email/userId without a verified userToken before we get here). An identified
|
|
@@ -651,6 +666,10 @@ export async function resolveOrCreateContact(opts: {
|
|
|
651
666
|
email: email ?? null,
|
|
652
667
|
anonymousId: anonymousId ?? null,
|
|
653
668
|
discordId: discordId ?? null,
|
|
669
|
+
// First-touch provenance: stamp source (+ paired sourcedAt) on the
|
|
670
|
+
// brand-new row; both stay null when no source was supplied.
|
|
671
|
+
source: source ?? null,
|
|
672
|
+
sourcedAt: source ? (sourcedAt ?? new Date()) : null,
|
|
654
673
|
// §2.1: explicit null clears a key — never persist a null-valued prop.
|
|
655
674
|
properties: stripNulls(patch),
|
|
656
675
|
})
|
|
@@ -685,6 +704,8 @@ export async function resolveOrCreateContact(opts: {
|
|
|
685
704
|
discordId,
|
|
686
705
|
patch,
|
|
687
706
|
hasPatch,
|
|
707
|
+
source,
|
|
708
|
+
sourcedAt,
|
|
688
709
|
});
|
|
689
710
|
return {
|
|
690
711
|
id,
|
|
@@ -712,6 +733,8 @@ export async function resolveOrCreateContact(opts: {
|
|
|
712
733
|
discordId,
|
|
713
734
|
patch,
|
|
714
735
|
hasPatch,
|
|
736
|
+
source,
|
|
737
|
+
sourcedAt,
|
|
715
738
|
});
|
|
716
739
|
return {
|
|
717
740
|
id,
|
|
@@ -732,6 +755,9 @@ interface ResolveCtx {
|
|
|
732
755
|
discordId?: string;
|
|
733
756
|
patch: Record<string, unknown>;
|
|
734
757
|
hasPatch: boolean;
|
|
758
|
+
/** First-touch provenance (see {@link resolveOrCreateContact} `source`). */
|
|
759
|
+
source?: string;
|
|
760
|
+
sourcedAt?: Date;
|
|
735
761
|
}
|
|
736
762
|
|
|
737
763
|
/**
|
|
@@ -786,6 +812,12 @@ async function fillInLink(
|
|
|
786
812
|
nextAnonymousId = ctx.anonymousId;
|
|
787
813
|
promoted.push({ kind: "anonymous", value: ctx.anonymousId });
|
|
788
814
|
}
|
|
815
|
+
// First-touch provenance: only stamp when the row has none, so an inbound
|
|
816
|
+
// contact that a Source later re-touches keeps its original origin.
|
|
817
|
+
if (ctx.source && !row.source) {
|
|
818
|
+
set.source = ctx.source;
|
|
819
|
+
set.sourcedAt = ctx.sourcedAt ?? new Date();
|
|
820
|
+
}
|
|
789
821
|
if (ctx.hasPatch) {
|
|
790
822
|
set.properties = mergePropertiesSql(ctx.patch);
|
|
791
823
|
}
|
|
@@ -916,6 +948,21 @@ async function mergeContacts(
|
|
|
916
948
|
// (vi) email_preferences FOLD (never blind-rewrite — risk 6).
|
|
917
949
|
await foldEmailPreferences(tx, loserKeysToRewrite, survivorKey);
|
|
918
950
|
|
|
951
|
+
// (vi-b) deals + crm_links re-point: these carry contact_id uuid FKs
|
|
952
|
+
// (not user keys), which the key rewrites above never touch. Without
|
|
953
|
+
// this, the loser's open deal is orphaned on a soft-deleted row — the
|
|
954
|
+
// survivor's next stage event/trigger would mint a SECOND deal (and a
|
|
955
|
+
// duplicate deal.sold). No unique index involves contact_id, so plain
|
|
956
|
+
// UPDATEs suffice.
|
|
957
|
+
await tx
|
|
958
|
+
.update(deals)
|
|
959
|
+
.set({ contactId: survivor.id })
|
|
960
|
+
.where(eq(deals.contactId, loser.id));
|
|
961
|
+
await tx
|
|
962
|
+
.update(crmLinks)
|
|
963
|
+
.set({ contactId: survivor.id })
|
|
964
|
+
.where(eq(crmLinks.contactId, loser.id));
|
|
965
|
+
|
|
919
966
|
// (ix) RECORD aliases for each loser key → survivor.
|
|
920
967
|
await recordMergeAliases(tx, survivor.id, loser);
|
|
921
968
|
}
|
|
@@ -984,6 +1031,25 @@ async function mergeContacts(
|
|
|
984
1031
|
const next = ctx.discordId ?? fromLoser;
|
|
985
1032
|
if (next) survivorSet.discordId = next;
|
|
986
1033
|
}
|
|
1034
|
+
// Provenance (best-effort): the survivor keeps its own source; only when it
|
|
1035
|
+
// has none does it adopt the call's, else the earliest-sourced loser's — so a
|
|
1036
|
+
// merge never erases a recorded origin but also never invents survivor state.
|
|
1037
|
+
if (!survivor.source) {
|
|
1038
|
+
const sourcedLoser = losers
|
|
1039
|
+
.filter((l) => l.source)
|
|
1040
|
+
.sort(
|
|
1041
|
+
(a, b) =>
|
|
1042
|
+
(a.sourcedAt?.getTime() ?? Number.POSITIVE_INFINITY) -
|
|
1043
|
+
(b.sourcedAt?.getTime() ?? Number.POSITIVE_INFINITY),
|
|
1044
|
+
)[0];
|
|
1045
|
+
const nextSource = ctx.source ?? sourcedLoser?.source ?? undefined;
|
|
1046
|
+
if (nextSource) {
|
|
1047
|
+
survivorSet.source = nextSource;
|
|
1048
|
+
survivorSet.sourcedAt = ctx.source
|
|
1049
|
+
? (ctx.sourcedAt ?? new Date())
|
|
1050
|
+
: (sourcedLoser?.sourcedAt ?? new Date());
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
987
1053
|
|
|
988
1054
|
// (viii) Soft-delete the losers FIRST — frees their external_id/email/
|
|
989
1055
|
// anonymous_id from the partial-unique indexes (WHERE deleted_at IS NULL) —
|
|
@@ -1450,6 +1516,9 @@ export async function upsertContact(opts: {
|
|
|
1450
1516
|
anonymousId?: string;
|
|
1451
1517
|
discordId?: string;
|
|
1452
1518
|
properties?: Record<string, unknown>;
|
|
1519
|
+
/** First-touch provenance (see {@link resolveOrCreateContact} `source`). */
|
|
1520
|
+
source?: string;
|
|
1521
|
+
sourcedAt?: Date;
|
|
1453
1522
|
}): Promise<{
|
|
1454
1523
|
id: string;
|
|
1455
1524
|
resolvedKey: string;
|
|
@@ -1468,6 +1537,8 @@ export async function upsertContact(opts: {
|
|
|
1468
1537
|
anonymousId: opts.anonymousId,
|
|
1469
1538
|
discordId: opts.discordId,
|
|
1470
1539
|
contactProperties: opts.properties,
|
|
1540
|
+
source: opts.source,
|
|
1541
|
+
sourcedAt: opts.sourcedAt,
|
|
1471
1542
|
});
|
|
1472
1543
|
}
|
|
1473
1544
|
|