@hogsend/engine 0.43.0 → 0.45.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/package.json +12 -9
  2. package/src/container.ts +289 -13
  3. package/src/destinations/define-destination.ts +7 -3
  4. package/src/destinations/presets/posthog.ts +31 -0
  5. package/src/index.ts +99 -0
  6. package/src/journeys/execute-journey-run.ts +93 -0
  7. package/src/journeys/journey-context.ts +4 -0
  8. package/src/lib/analytics-providers-from-env.ts +106 -43
  9. package/src/lib/attribution-backfill.ts +285 -0
  10. package/src/lib/attribution.ts +239 -0
  11. package/src/lib/contacts.ts +71 -0
  12. package/src/lib/conversion-dispatch.ts +326 -0
  13. package/src/lib/conversions.ts +197 -0
  14. package/src/lib/crm-deals.ts +320 -0
  15. package/src/lib/crm-ingest.ts +341 -0
  16. package/src/lib/crm-provider-registry.ts +41 -0
  17. package/src/lib/crm-registry-singleton.ts +22 -0
  18. package/src/lib/deal-money-events.ts +85 -0
  19. package/src/lib/email-service-types.ts +16 -1
  20. package/src/lib/event-names.ts +1 -1
  21. package/src/lib/funnel-progress.ts +76 -0
  22. package/src/lib/funnel-registry.ts +108 -0
  23. package/src/lib/funnel-transitions.ts +280 -0
  24. package/src/lib/holdout.ts +62 -0
  25. package/src/lib/ingestion.ts +218 -9
  26. package/src/lib/lift-stats.ts +125 -0
  27. package/src/lib/mailer.ts +1 -0
  28. package/src/lib/operator.ts +31 -0
  29. package/src/lib/outbound.ts +51 -0
  30. package/src/lib/posthog-scopes.ts +30 -6
  31. package/src/lib/revenue.ts +116 -0
  32. package/src/lib/sms-service-types.ts +7 -1
  33. package/src/lib/sms-tracked.ts +35 -0
  34. package/src/lib/tracked.ts +40 -0
  35. package/src/lib/tracking-events.ts +26 -0
  36. package/src/lib/unconfigured-email-provider.ts +45 -0
  37. package/src/lib/webhook-signing.ts +7 -0
  38. package/src/routes/admin/analytics.ts +14 -5
  39. package/src/routes/admin/attribution.ts +414 -0
  40. package/src/routes/admin/contacts.ts +71 -11
  41. package/src/routes/admin/conversions.ts +247 -0
  42. package/src/routes/admin/deals.ts +472 -0
  43. package/src/routes/admin/funnels.ts +228 -0
  44. package/src/routes/admin/index.ts +8 -0
  45. package/src/routes/admin/journeys.ts +158 -1
  46. package/src/routes/admin/readiness.ts +97 -2
  47. package/src/routes/admin/templates.ts +28 -1
  48. package/src/routes/events/index.ts +12 -0
  49. package/src/routes/tracking/click-pipeline.ts +26 -18
  50. package/src/routes/webhooks/crm-provider.ts +117 -0
  51. package/src/routes/webhooks/index.ts +5 -1
  52. package/src/sources/define-contact-source.ts +159 -0
  53. package/src/sources/registry.ts +71 -0
  54. package/src/sources/webhook-source.ts +109 -0
  55. package/src/worker.ts +4 -0
  56. package/src/workflows/crm-reconcile.ts +129 -0
  57. package/src/workflows/dispatch-conversion.ts +31 -0
  58. package/src/workflows/send-campaign.ts +5 -0
@@ -0,0 +1,285 @@
1
+ import {
2
+ attributionCredits,
3
+ contacts,
4
+ conversions,
5
+ type Database,
6
+ userEvents,
7
+ } from "@hogsend/db";
8
+ import {
9
+ and,
10
+ asc,
11
+ eq,
12
+ gt,
13
+ gte,
14
+ inArray,
15
+ isNotNull,
16
+ or,
17
+ sql,
18
+ } from "drizzle-orm";
19
+ import { recordAttributionCredits } from "./attribution.js";
20
+ import type { ConversionRegistry } from "./conversions.js";
21
+ import { evaluateConversionsAtIngest } from "./conversions.js";
22
+ import type { Logger } from "./logger.js";
23
+
24
+ /**
25
+ * Attribution backfill (impact plan §5.1) — replays HISTORY through the
26
+ * same idempotent machinery ingest uses, so an existing deploy that
27
+ * upgrades gets its whole event history credited, and a definition/window
28
+ * change gets a deliberate, logged recompute path.
29
+ *
30
+ * Two stages behind one cursor (`events:<id>` → `credits:<id>`):
31
+ *
32
+ * 1. `events` — scan historical `user_events` matching the registry's
33
+ * trigger set and run conversion evaluation. The unique
34
+ * (definition, event) index makes replays no-ops; only genuinely new
35
+ * conversions are minted. Ad-platform dispatches are deliberately NOT
36
+ * enqueued — historical conversions must never re-fire to destinations.
37
+ * 2. `credits` — scan conversions rows (of the scoped definitions) that
38
+ * have NO credit rows yet and write their ledger entries under the
39
+ * definition's CURRENT window config.
40
+ *
41
+ * `recompute: true` (requires `definitionId`) deletes the definition's
42
+ * existing credit rows up front, then stage 2 re-credits everything — the
43
+ * ONE sanctioned exception to the ledger's write-once philosophy, guarded
44
+ * and logged. Conversion rows themselves are never deleted.
45
+ *
46
+ * Batch-shaped by design: each call processes up to `limit` rows of the
47
+ * current stage and returns `nextCursor` (null = done). The CLI loops it;
48
+ * an admin can curl it. Contacts are matched only — a historical event
49
+ * whose user never became a contact is skipped, never minted.
50
+ */
51
+ export interface BackfillBatchResult {
52
+ stage: "events" | "credits";
53
+ processed: number;
54
+ conversionsFired: number;
55
+ creditsWritten: number;
56
+ nextCursor: string | null;
57
+ }
58
+
59
+ const UUID_RE =
60
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
61
+
62
+ export async function backfillAttributionBatch(opts: {
63
+ db: Database;
64
+ logger: Logger;
65
+ registry: ConversionRegistry | undefined;
66
+ definitionId?: string;
67
+ since?: Date;
68
+ cursor?: string;
69
+ limit?: number;
70
+ recompute?: boolean;
71
+ }): Promise<BackfillBatchResult> {
72
+ const { db, logger, registry, definitionId, since } = opts;
73
+ const limit = Math.min(2000, Math.max(1, opts.limit ?? 500));
74
+
75
+ const defs = (registry?.getAll() ?? []).filter(
76
+ (def) => !definitionId || def.meta.id === definitionId,
77
+ );
78
+ if (defs.length === 0) {
79
+ return {
80
+ stage: "credits",
81
+ processed: 0,
82
+ conversionsFired: 0,
83
+ creditsWritten: 0,
84
+ nextCursor: null,
85
+ };
86
+ }
87
+ const defIds = defs.map((def) => def.meta.id);
88
+ const defById = new Map(defs.map((def) => [def.meta.id, def]));
89
+ const namedEvents = [
90
+ ...new Set(
91
+ defs
92
+ .map((def) => def.meta.trigger.event)
93
+ .filter((event) => event !== "*"),
94
+ ),
95
+ ];
96
+ const hasWildcard = defs.some((def) => def.meta.trigger.event === "*");
97
+
98
+ const [stage, lastId] = opts.cursor
99
+ ? (opts.cursor.split(":", 2) as ["events" | "credits", string])
100
+ : (["events", ""] as const);
101
+
102
+ // Recompute (guarded): first call only, one definition only.
103
+ if (opts.recompute && !opts.cursor) {
104
+ if (!definitionId) {
105
+ throw new Error("recompute requires a definitionId — never blanket");
106
+ }
107
+ const deleted = await db
108
+ .delete(attributionCredits)
109
+ .where(
110
+ inArray(
111
+ attributionCredits.conversionId,
112
+ db
113
+ .select({ id: conversions.id })
114
+ .from(conversions)
115
+ .where(eq(conversions.definitionId, definitionId)),
116
+ ),
117
+ )
118
+ .returning({ id: attributionCredits.id });
119
+ logger.warn("attribution recompute: credit rows deleted for refill", {
120
+ definitionId,
121
+ deleted: deleted.length,
122
+ });
123
+ }
124
+
125
+ if (stage === "events") {
126
+ const eventMatch = or(
127
+ namedEvents.length > 0
128
+ ? inArray(userEvents.event, namedEvents)
129
+ : sql`false`,
130
+ hasWildcard ? isNotNull(userEvents.value) : sql`false`,
131
+ );
132
+ const rows = await db
133
+ .select({
134
+ id: userEvents.id,
135
+ event: userEvents.event,
136
+ userId: userEvents.userId,
137
+ properties: userEvents.properties,
138
+ value: userEvents.value,
139
+ currency: userEvents.currency,
140
+ source: userEvents.source,
141
+ occurredAt: userEvents.occurredAt,
142
+ })
143
+ .from(userEvents)
144
+ .where(
145
+ and(
146
+ eventMatch,
147
+ ...(since ? [gte(userEvents.occurredAt, since)] : []),
148
+ ...(lastId ? [gt(userEvents.id, lastId)] : []),
149
+ ),
150
+ )
151
+ .orderBy(asc(userEvents.id))
152
+ .limit(limit);
153
+
154
+ // Batch contact resolution: canonical keys are externalId or the
155
+ // contact row uuid. MATCH ONLY — backfill never mints contacts.
156
+ const keys = [...new Set(rows.map((row) => row.userId))];
157
+ const uuidKeys = keys.filter((key) => UUID_RE.test(key));
158
+ const contactRows =
159
+ keys.length > 0
160
+ ? await db
161
+ .select({ id: contacts.id, externalId: contacts.externalId })
162
+ .from(contacts)
163
+ .where(
164
+ or(
165
+ inArray(contacts.externalId, keys),
166
+ uuidKeys.length > 0
167
+ ? inArray(contacts.id, uuidKeys)
168
+ : sql`false`,
169
+ ),
170
+ )
171
+ : [];
172
+ const contactByKey = new Map<string, string>();
173
+ for (const contact of contactRows) {
174
+ contactByKey.set(contact.id, contact.id);
175
+ if (contact.externalId) contactByKey.set(contact.externalId, contact.id);
176
+ }
177
+
178
+ let conversionsFired = 0;
179
+ let creditsWritten = 0;
180
+ for (const row of rows) {
181
+ const contactId = contactByKey.get(row.userId);
182
+ if (!contactId) continue;
183
+ const fired = await evaluateConversionsAtIngest({
184
+ db,
185
+ logger,
186
+ registry,
187
+ event: {
188
+ name: row.event,
189
+ source: row.source ?? null,
190
+ properties: (row.properties ?? {}) as Record<string, unknown>,
191
+ value: row.value,
192
+ currency: row.currency,
193
+ occurredAt: row.occurredAt,
194
+ },
195
+ eventRowId: row.id,
196
+ contactId,
197
+ userKey: row.userId,
198
+ });
199
+ for (const conversion of fired) {
200
+ if (definitionId && conversion.definition.meta.id !== definitionId) {
201
+ continue;
202
+ }
203
+ conversionsFired++;
204
+ const { touchpoints } = await recordAttributionCredits({
205
+ db,
206
+ logger,
207
+ conversionId: conversion.conversionId,
208
+ userKey: row.userId,
209
+ value: conversion.value,
210
+ currency: conversion.currency,
211
+ occurredAt: row.occurredAt,
212
+ windowDays: conversion.definition.meta.attributionWindowDays ?? 90,
213
+ windows: conversion.definition.meta.windows,
214
+ });
215
+ if (touchpoints > 0) creditsWritten++;
216
+ }
217
+ }
218
+
219
+ const nextCursor =
220
+ rows.length < limit
221
+ ? "credits:"
222
+ : `events:${rows[rows.length - 1]?.id ?? ""}`;
223
+ return {
224
+ stage: "events",
225
+ processed: rows.length,
226
+ conversionsFired,
227
+ creditsWritten,
228
+ nextCursor,
229
+ };
230
+ }
231
+
232
+ // Stage 2: credit conversions that have no ledger rows yet (after a
233
+ // recompute delete, that is all of the definition's conversions).
234
+ const uncredited = sql`not exists (
235
+ select 1 from ${attributionCredits}
236
+ where ${attributionCredits.conversionId} = ${conversions.id}
237
+ )`;
238
+ const rows = await db
239
+ .select({
240
+ id: conversions.id,
241
+ definitionId: conversions.definitionId,
242
+ userKey: conversions.userKey,
243
+ value: conversions.value,
244
+ currency: conversions.currency,
245
+ occurredAt: conversions.occurredAt,
246
+ })
247
+ .from(conversions)
248
+ .where(
249
+ and(
250
+ inArray(conversions.definitionId, defIds),
251
+ uncredited,
252
+ ...(since ? [gte(conversions.occurredAt, since)] : []),
253
+ ...(lastId ? [gt(conversions.id, lastId)] : []),
254
+ ),
255
+ )
256
+ .orderBy(asc(conversions.id))
257
+ .limit(limit);
258
+
259
+ let creditsWritten = 0;
260
+ for (const row of rows) {
261
+ const def = defById.get(row.definitionId);
262
+ if (!def) continue;
263
+ const { touchpoints } = await recordAttributionCredits({
264
+ db,
265
+ logger,
266
+ conversionId: row.id,
267
+ userKey: row.userKey,
268
+ value: row.value,
269
+ currency: row.currency,
270
+ occurredAt: row.occurredAt,
271
+ windowDays: def.meta.attributionWindowDays ?? 90,
272
+ windows: def.meta.windows,
273
+ });
274
+ if (touchpoints > 0) creditsWritten++;
275
+ }
276
+
277
+ return {
278
+ stage: "credits",
279
+ processed: rows.length,
280
+ conversionsFired: 0,
281
+ creditsWritten,
282
+ nextCursor:
283
+ rows.length < limit ? null : `credits:${rows[rows.length - 1]?.id ?? ""}`,
284
+ };
285
+ }
@@ -0,0 +1,239 @@
1
+ import {
2
+ type AttributionTouchpoint,
3
+ computeAllModels,
4
+ } from "@hogsend/attribution";
5
+ import {
6
+ type DurationObject,
7
+ durationToMs,
8
+ TOUCHPOINT_EVENTS,
9
+ type TouchpointChannel,
10
+ touchpointChannel,
11
+ } from "@hogsend/core";
12
+ import {
13
+ attributionCredits,
14
+ type Database,
15
+ emailSends,
16
+ journeyStates,
17
+ userEvents,
18
+ } from "@hogsend/db";
19
+ import { and, asc, eq, gte, inArray, lte } from "drizzle-orm";
20
+ import type { Logger } from "./logger.js";
21
+
22
+ /**
23
+ * Attribution scope carried per touchpoint into the ledger row
24
+ * (docs/attribution-impact-plan.md §1.3). All nullable — a touch outside any
25
+ * journey/campaign has none.
26
+ */
27
+ interface TouchScope {
28
+ journeyId: string | null;
29
+ campaignId: string | null;
30
+ templateKey: string | null;
31
+ funnelId: string | null;
32
+ }
33
+
34
+ const UUID_RE =
35
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
36
+
37
+ /** A property value usable as a scope id (scalar string, non-empty). */
38
+ function scopeString(value: unknown): string | null {
39
+ return typeof value === "string" && value.length > 0 ? value : null;
40
+ }
41
+
42
+ /** Like {@link scopeString} but must be a uuid (campaign_id column is uuid). */
43
+ function scopeUuid(value: unknown): string | null {
44
+ const str = scopeString(value);
45
+ return str && UUID_RE.test(str) ? str : null;
46
+ }
47
+
48
+ /**
49
+ * Resolve each touchpoint's attribution scope from its stamped properties
50
+ * (`pushTrackingEvent` stamps journeyId/campaignId/templateKey — plan §1.2).
51
+ * Events ingested BEFORE stamping carry an `emailSendId` but no
52
+ * `journeyId`/`campaignId` keys at all (post-stamp events carry explicit
53
+ * nulls) — those fall back to one batched email_sends → journey_states join.
54
+ * SMS pre-stamp events are left unscoped: scope stamping landed days after
55
+ * the SMS channel itself, so the unscoped window is negligible and not worth
56
+ * a second join path.
57
+ */
58
+ async function resolveTouchScopes(
59
+ db: Database,
60
+ rows: Array<{ id: string; properties: Record<string, unknown> | null }>,
61
+ ): Promise<Map<string, TouchScope>> {
62
+ const scopes = new Map<string, TouchScope>();
63
+ /** Pre-stamp email touches: eventRowId → emailSendId, backfilled via join. */
64
+ const fallback = new Map<string, string>();
65
+
66
+ for (const row of rows) {
67
+ const props = row.properties ?? {};
68
+ const stamped = "journeyId" in props || "campaignId" in props;
69
+ const emailSendId = scopeString(props.emailSendId);
70
+ if (!stamped && emailSendId) fallback.set(row.id, emailSendId);
71
+ scopes.set(row.id, {
72
+ journeyId: scopeString(props.journeyId),
73
+ campaignId: scopeUuid(props.campaignId),
74
+ templateKey: scopeString(props.templateKey),
75
+ // No touch event stamps funnel scope today; accept either spelling if
76
+ // one ever does (money/deal events use snake_case on the spine).
77
+ funnelId: scopeString(props.funnelId) ?? scopeString(props.funnel_id),
78
+ });
79
+ }
80
+
81
+ if (fallback.size > 0) {
82
+ const sendRows = await db
83
+ .select({
84
+ id: emailSends.id,
85
+ campaignId: emailSends.campaignId,
86
+ templateKey: emailSends.templateKey,
87
+ journeyId: journeyStates.journeyId,
88
+ })
89
+ .from(emailSends)
90
+ .leftJoin(journeyStates, eq(emailSends.journeyStateId, journeyStates.id))
91
+ .where(inArray(emailSends.id, [...new Set(fallback.values())]));
92
+ const bySendId = new Map(sendRows.map((send) => [send.id, send]));
93
+ for (const [eventRowId, emailSendId] of fallback) {
94
+ const send = bySendId.get(emailSendId);
95
+ if (!send) continue;
96
+ const scope = scopes.get(eventRowId);
97
+ if (!scope) continue;
98
+ scope.journeyId = send.journeyId;
99
+ scope.campaignId = send.campaignId;
100
+ scope.templateKey ??= send.templateKey;
101
+ }
102
+ }
103
+
104
+ return scopes;
105
+ }
106
+
107
+ /**
108
+ * The attribution ledger writer (plan §6.1): when a conversion fires, read
109
+ * the contact's touchpoint path inside the definition's lookback window and
110
+ * persist EVERY model's credit allocation into `attribution_credits`.
111
+ * Computing all models up front makes "switch the reporting model" a WHERE
112
+ * clause instead of a historical re-derivation.
113
+ *
114
+ * Idempotent: the unique (conversion, model, touchpoint) index turns any
115
+ * replay into a no-op — same inheritance as the conversion row itself.
116
+ */
117
+ export async function recordAttributionCredits(opts: {
118
+ db: Database;
119
+ logger: Logger;
120
+ conversionId: string;
121
+ /** The contact's canonical event key (`user_events.user_id`). */
122
+ userKey: string;
123
+ value: number | null;
124
+ currency: string | null;
125
+ /** When the conversion happened. */
126
+ occurredAt: Date;
127
+ /** Lookback window in days (defineConversion attributionWindowDays). */
128
+ windowDays: number;
129
+ /**
130
+ * Per-channel window overrides (plan §2.1). A listed channel's touches
131
+ * qualify only inside its own window; unlisted channels use `windowDays`.
132
+ */
133
+ windows?: Partial<Record<TouchpointChannel, DurationObject>>;
134
+ }): Promise<{ touchpoints: number }> {
135
+ const { db, logger, conversionId, userKey, value, currency, occurredAt } =
136
+ opts;
137
+ const defaultWindowMs = opts.windowDays * 24 * 60 * 60 * 1000;
138
+ const channelWindowMs = new Map<TouchpointChannel, number>(
139
+ Object.entries(opts.windows ?? {}).map(([channel, duration]) => [
140
+ channel as TouchpointChannel,
141
+ durationToMs(duration as DurationObject),
142
+ ]),
143
+ );
144
+ // One query at the widest window; the per-channel narrowing happens in the
145
+ // classification loop below (channels are a query-time concept anyway).
146
+ const windowStart = new Date(
147
+ occurredAt.getTime() -
148
+ Math.max(defaultWindowMs, ...channelWindowMs.values()),
149
+ );
150
+
151
+ const rows = await db
152
+ .select({
153
+ id: userEvents.id,
154
+ event: userEvents.event,
155
+ occurredAt: userEvents.occurredAt,
156
+ properties: userEvents.properties,
157
+ })
158
+ .from(userEvents)
159
+ .where(
160
+ and(
161
+ eq(userEvents.userId, userKey),
162
+ inArray(userEvents.event, [...TOUCHPOINT_EVENTS]),
163
+ gte(userEvents.occurredAt, windowStart),
164
+ lte(userEvents.occurredAt, occurredAt),
165
+ ),
166
+ )
167
+ .orderBy(asc(userEvents.occurredAt));
168
+
169
+ const touchpoints: AttributionTouchpoint[] = [];
170
+ for (const row of rows) {
171
+ const channel = touchpointChannel(row.event);
172
+ if (!channel) continue; // unreachable: the IN filter is the class list
173
+ const windowMs = channelWindowMs.get(channel) ?? defaultWindowMs;
174
+ if (row.occurredAt.getTime() < occurredAt.getTime() - windowMs) continue;
175
+ touchpoints.push({
176
+ id: row.id,
177
+ event: row.event,
178
+ channel,
179
+ occurredAt: row.occurredAt.getTime(),
180
+ });
181
+ }
182
+ if (touchpoints.length === 0) {
183
+ // No path — the conversion stays unattributed (reporting reads the gap
184
+ // as conversion value minus credited value; we never invent a touch).
185
+ return { touchpoints: 0 };
186
+ }
187
+
188
+ const byId = new Map(touchpoints.map((t) => [t.id, t]));
189
+ // Only qualifying touches need scope — a window-dropped row must not cost
190
+ // a fallback join.
191
+ const scopes = await resolveTouchScopes(
192
+ db,
193
+ rows.filter((row) => byId.has(row.id)),
194
+ );
195
+ const allModels = computeAllModels(touchpoints, {
196
+ conversionAt: occurredAt.getTime(),
197
+ });
198
+
199
+ const values = Object.entries(allModels).flatMap(([model, credits]) =>
200
+ credits.map((credit) => {
201
+ const touch = byId.get(credit.touchpointId) as AttributionTouchpoint;
202
+ const scope = scopes.get(credit.touchpointId);
203
+ return {
204
+ conversionId,
205
+ model,
206
+ touchpointEventId: credit.touchpointId,
207
+ touchpointEvent: touch.event,
208
+ channel: touch.channel,
209
+ touchpointAt: new Date(touch.occurredAt),
210
+ journeyId: scope?.journeyId ?? null,
211
+ campaignId: scope?.campaignId ?? null,
212
+ templateKey: scope?.templateKey ?? null,
213
+ funnelId: scope?.funnelId ?? null,
214
+ weight: credit.weight,
215
+ value:
216
+ value !== null ? Math.round(credit.weight * value * 100) / 100 : null,
217
+ currency: value !== null ? currency : null,
218
+ convertedAt: occurredAt,
219
+ };
220
+ }),
221
+ );
222
+
223
+ await db
224
+ .insert(attributionCredits)
225
+ .values(values)
226
+ .onConflictDoNothing({
227
+ target: [
228
+ attributionCredits.conversionId,
229
+ attributionCredits.model,
230
+ attributionCredits.touchpointEventId,
231
+ ],
232
+ });
233
+ logger.debug("attribution credits recorded", {
234
+ conversionId,
235
+ touchpoints: touchpoints.length,
236
+ rows: values.length,
237
+ });
238
+ return { touchpoints: touchpoints.length };
239
+ }
@@ -2,7 +2,9 @@ import {
2
2
  bucketMemberships,
3
3
  contactAliases,
4
4
  contacts,
5
+ crmLinks,
5
6
  type Database,
7
+ deals,
6
8
  emailPreferences,
7
9
  emailSends,
8
10
  journeyStates,
@@ -547,6 +549,17 @@ export async function resolveOrCreateContact(opts: {
547
549
  * `restrictToAnonymous`.
548
550
  */
549
551
  contactId?: string;
552
+ /**
553
+ * PROVENANCE (best-effort metadata, NOT an identity key): the Source id that
554
+ * created this contact — a Contact Source id ("clay"/"attio") or the ingest
555
+ * `source`. First-touch: written on create, and on a fill-in-link/merge that
556
+ * supplies one ONLY when the resolved row has none; never overwrites an
557
+ * existing value. NEVER participates in key resolution or survivor selection,
558
+ * so it cannot steer identity — a safe non-identity column threaded alongside.
559
+ */
560
+ source?: string;
561
+ /** Timestamp paired with {@link source}; defaults to now() at create time. */
562
+ sourcedAt?: Date;
550
563
  }): Promise<{
551
564
  id: string;
552
565
  /**
@@ -583,6 +596,8 @@ export async function resolveOrCreateContact(opts: {
583
596
  const anonymousId = opts.anonymousId?.trim() || undefined;
584
597
  const discordId = opts.discordId?.trim() || undefined;
585
598
  const contactId = opts.contactId?.trim() || undefined;
599
+ const source = opts.source?.trim() || undefined;
600
+ const sourcedAt = opts.sourcedAt;
586
601
  // §Phase 1 GAP-1: the publishable clamp only bites an ANON-ONLY write (the
587
602
  // only shape a token-less pk_ key can produce — the gate 403s any
588
603
  // email/userId without a verified userToken before we get here). An identified
@@ -651,6 +666,10 @@ export async function resolveOrCreateContact(opts: {
651
666
  email: email ?? null,
652
667
  anonymousId: anonymousId ?? null,
653
668
  discordId: discordId ?? null,
669
+ // First-touch provenance: stamp source (+ paired sourcedAt) on the
670
+ // brand-new row; both stay null when no source was supplied.
671
+ source: source ?? null,
672
+ sourcedAt: source ? (sourcedAt ?? new Date()) : null,
654
673
  // §2.1: explicit null clears a key — never persist a null-valued prop.
655
674
  properties: stripNulls(patch),
656
675
  })
@@ -685,6 +704,8 @@ export async function resolveOrCreateContact(opts: {
685
704
  discordId,
686
705
  patch,
687
706
  hasPatch,
707
+ source,
708
+ sourcedAt,
688
709
  });
689
710
  return {
690
711
  id,
@@ -712,6 +733,8 @@ export async function resolveOrCreateContact(opts: {
712
733
  discordId,
713
734
  patch,
714
735
  hasPatch,
736
+ source,
737
+ sourcedAt,
715
738
  });
716
739
  return {
717
740
  id,
@@ -732,6 +755,9 @@ interface ResolveCtx {
732
755
  discordId?: string;
733
756
  patch: Record<string, unknown>;
734
757
  hasPatch: boolean;
758
+ /** First-touch provenance (see {@link resolveOrCreateContact} `source`). */
759
+ source?: string;
760
+ sourcedAt?: Date;
735
761
  }
736
762
 
737
763
  /**
@@ -786,6 +812,12 @@ async function fillInLink(
786
812
  nextAnonymousId = ctx.anonymousId;
787
813
  promoted.push({ kind: "anonymous", value: ctx.anonymousId });
788
814
  }
815
+ // First-touch provenance: only stamp when the row has none, so an inbound
816
+ // contact that a Source later re-touches keeps its original origin.
817
+ if (ctx.source && !row.source) {
818
+ set.source = ctx.source;
819
+ set.sourcedAt = ctx.sourcedAt ?? new Date();
820
+ }
789
821
  if (ctx.hasPatch) {
790
822
  set.properties = mergePropertiesSql(ctx.patch);
791
823
  }
@@ -916,6 +948,21 @@ async function mergeContacts(
916
948
  // (vi) email_preferences FOLD (never blind-rewrite — risk 6).
917
949
  await foldEmailPreferences(tx, loserKeysToRewrite, survivorKey);
918
950
 
951
+ // (vi-b) deals + crm_links re-point: these carry contact_id uuid FKs
952
+ // (not user keys), which the key rewrites above never touch. Without
953
+ // this, the loser's open deal is orphaned on a soft-deleted row — the
954
+ // survivor's next stage event/trigger would mint a SECOND deal (and a
955
+ // duplicate deal.sold). No unique index involves contact_id, so plain
956
+ // UPDATEs suffice.
957
+ await tx
958
+ .update(deals)
959
+ .set({ contactId: survivor.id })
960
+ .where(eq(deals.contactId, loser.id));
961
+ await tx
962
+ .update(crmLinks)
963
+ .set({ contactId: survivor.id })
964
+ .where(eq(crmLinks.contactId, loser.id));
965
+
919
966
  // (ix) RECORD aliases for each loser key → survivor.
920
967
  await recordMergeAliases(tx, survivor.id, loser);
921
968
  }
@@ -984,6 +1031,25 @@ async function mergeContacts(
984
1031
  const next = ctx.discordId ?? fromLoser;
985
1032
  if (next) survivorSet.discordId = next;
986
1033
  }
1034
+ // Provenance (best-effort): the survivor keeps its own source; only when it
1035
+ // has none does it adopt the call's, else the earliest-sourced loser's — so a
1036
+ // merge never erases a recorded origin but also never invents survivor state.
1037
+ if (!survivor.source) {
1038
+ const sourcedLoser = losers
1039
+ .filter((l) => l.source)
1040
+ .sort(
1041
+ (a, b) =>
1042
+ (a.sourcedAt?.getTime() ?? Number.POSITIVE_INFINITY) -
1043
+ (b.sourcedAt?.getTime() ?? Number.POSITIVE_INFINITY),
1044
+ )[0];
1045
+ const nextSource = ctx.source ?? sourcedLoser?.source ?? undefined;
1046
+ if (nextSource) {
1047
+ survivorSet.source = nextSource;
1048
+ survivorSet.sourcedAt = ctx.source
1049
+ ? (ctx.sourcedAt ?? new Date())
1050
+ : (sourcedLoser?.sourcedAt ?? new Date());
1051
+ }
1052
+ }
987
1053
 
988
1054
  // (viii) Soft-delete the losers FIRST — frees their external_id/email/
989
1055
  // anonymous_id from the partial-unique indexes (WHERE deleted_at IS NULL) —
@@ -1450,6 +1516,9 @@ export async function upsertContact(opts: {
1450
1516
  anonymousId?: string;
1451
1517
  discordId?: string;
1452
1518
  properties?: Record<string, unknown>;
1519
+ /** First-touch provenance (see {@link resolveOrCreateContact} `source`). */
1520
+ source?: string;
1521
+ sourcedAt?: Date;
1453
1522
  }): Promise<{
1454
1523
  id: string;
1455
1524
  resolvedKey: string;
@@ -1468,6 +1537,8 @@ export async function upsertContact(opts: {
1468
1537
  anonymousId: opts.anonymousId,
1469
1538
  discordId: opts.discordId,
1470
1539
  contactProperties: opts.properties,
1540
+ source: opts.source,
1541
+ sourcedAt: opts.sourcedAt,
1471
1542
  });
1472
1543
  }
1473
1544