@hogsend/engine 0.43.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +10 -9
- package/src/container.ts +229 -2
- package/src/destinations/define-destination.ts +7 -3
- package/src/destinations/presets/posthog.ts +31 -0
- package/src/index.ts +99 -0
- package/src/journeys/execute-journey-run.ts +93 -0
- package/src/journeys/journey-context.ts +4 -0
- package/src/lib/attribution-backfill.ts +285 -0
- package/src/lib/attribution.ts +239 -0
- package/src/lib/contacts.ts +71 -0
- package/src/lib/conversion-dispatch.ts +326 -0
- package/src/lib/conversions.ts +197 -0
- package/src/lib/crm-deals.ts +320 -0
- package/src/lib/crm-ingest.ts +341 -0
- package/src/lib/crm-provider-registry.ts +41 -0
- package/src/lib/crm-registry-singleton.ts +22 -0
- package/src/lib/deal-money-events.ts +85 -0
- package/src/lib/email-service-types.ts +16 -1
- package/src/lib/event-names.ts +1 -1
- package/src/lib/funnel-progress.ts +76 -0
- package/src/lib/funnel-registry.ts +108 -0
- package/src/lib/funnel-transitions.ts +280 -0
- package/src/lib/holdout.ts +62 -0
- package/src/lib/ingestion.ts +218 -9
- package/src/lib/lift-stats.ts +125 -0
- package/src/lib/mailer.ts +1 -0
- package/src/lib/operator.ts +31 -0
- package/src/lib/outbound.ts +51 -0
- package/src/lib/revenue.ts +116 -0
- package/src/lib/sms-service-types.ts +7 -1
- package/src/lib/sms-tracked.ts +35 -0
- package/src/lib/tracked.ts +40 -0
- package/src/lib/tracking-events.ts +26 -0
- package/src/lib/webhook-signing.ts +7 -0
- package/src/routes/admin/attribution.ts +414 -0
- package/src/routes/admin/contacts.ts +71 -11
- package/src/routes/admin/conversions.ts +247 -0
- package/src/routes/admin/deals.ts +472 -0
- package/src/routes/admin/funnels.ts +228 -0
- package/src/routes/admin/index.ts +8 -0
- package/src/routes/admin/journeys.ts +158 -1
- package/src/routes/admin/readiness.ts +97 -2
- package/src/routes/admin/templates.ts +28 -1
- package/src/routes/events/index.ts +12 -0
- package/src/routes/tracking/click-pipeline.ts +26 -18
- package/src/routes/webhooks/crm-provider.ts +117 -0
- package/src/routes/webhooks/index.ts +5 -1
- package/src/sources/define-contact-source.ts +159 -0
- package/src/sources/registry.ts +71 -0
- package/src/sources/webhook-source.ts +109 -0
- package/src/worker.ts +4 -0
- package/src/workflows/crm-reconcile.ts +129 -0
- package/src/workflows/dispatch-conversion.ts +31 -0
- package/src/workflows/send-campaign.ts +5 -0
package/src/lib/sms-tracked.ts
CHANGED
|
@@ -19,6 +19,7 @@ import { logTransition } from "../journeys/journey-log.js";
|
|
|
19
19
|
import { getListRegistry } from "../lists/registry-singleton.js";
|
|
20
20
|
import type { FrequencyCapConfig } from "./email-service-types.js";
|
|
21
21
|
import { hatchet } from "./hatchet.js";
|
|
22
|
+
import { isGlobalControl } from "./holdout.js";
|
|
22
23
|
import { checkJourneySuppress } from "./journey-suppress.js";
|
|
23
24
|
import { createLogger, type Logger } from "./logger.js";
|
|
24
25
|
import { emitOutbound } from "./outbound.js";
|
|
@@ -215,6 +216,40 @@ async function sendTrackedSmsInner<K extends SmsTemplateName>(
|
|
|
215
216
|
}
|
|
216
217
|
|
|
217
218
|
if (!options.skipPreferenceCheck) {
|
|
219
|
+
// Global control group (impact plan §4.3) — the SMS leg of the email
|
|
220
|
+
// gate in tracked.ts: same deterministic bucket, same key precedence
|
|
221
|
+
// (userId when known, else the recipient address), same marker emit
|
|
222
|
+
// (per-endpoint deduped on the contact key), so a contact with a stable
|
|
223
|
+
// userId is controlled consistently across channels.
|
|
224
|
+
if (effectiveCategory !== "transactional") {
|
|
225
|
+
const controlKey = options.userId ?? options.to;
|
|
226
|
+
if (isGlobalControl(controlKey)) {
|
|
227
|
+
logger?.info("sms skipped: control_group", { to: options.to });
|
|
228
|
+
void emitOutbound({
|
|
229
|
+
db,
|
|
230
|
+
hatchet,
|
|
231
|
+
logger: logger ?? emitLogger,
|
|
232
|
+
event: "contact.control_group",
|
|
233
|
+
dedupeKey: `contact.control_group:${controlKey.toLowerCase()}`,
|
|
234
|
+
payload: {
|
|
235
|
+
userId: options.userId ?? null,
|
|
236
|
+
email: null,
|
|
237
|
+
at: new Date().toISOString(),
|
|
238
|
+
},
|
|
239
|
+
}).catch((err: unknown) => {
|
|
240
|
+
(logger ?? emitLogger).warn("control_group marker emit failed", {
|
|
241
|
+
error: err instanceof Error ? err.message : String(err),
|
|
242
|
+
});
|
|
243
|
+
});
|
|
244
|
+
return {
|
|
245
|
+
smsSendId: "",
|
|
246
|
+
messageId: "",
|
|
247
|
+
status: "skipped",
|
|
248
|
+
reason: "control_group",
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
218
253
|
if (frequencyCap) {
|
|
219
254
|
const capped = await isSmsFrequencyCapped({
|
|
220
255
|
db,
|
package/src/lib/tracked.ts
CHANGED
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
} from "./email-service-types.js";
|
|
31
31
|
import { isFrequencyCapped } from "./frequency-cap.js";
|
|
32
32
|
import { hatchet } from "./hatchet.js";
|
|
33
|
+
import { isGlobalControl } from "./holdout.js";
|
|
33
34
|
import { checkJourneySuppress } from "./journey-suppress.js";
|
|
34
35
|
import { createLogger, type Logger } from "./logger.js";
|
|
35
36
|
import { emitOutbound } from "./outbound.js";
|
|
@@ -209,6 +210,7 @@ async function sendTrackedEmailInner<K extends TemplateName>(
|
|
|
209
210
|
subject: options.subject ?? "",
|
|
210
211
|
category: effectiveCategory,
|
|
211
212
|
journeyStateId: options.journeyStateId,
|
|
213
|
+
campaignId: options.campaignId,
|
|
212
214
|
userId: options.userId,
|
|
213
215
|
userEmail: options.userEmail ?? options.to,
|
|
214
216
|
status: "failed",
|
|
@@ -232,6 +234,42 @@ async function sendTrackedEmailInner<K extends TemplateName>(
|
|
|
232
234
|
});
|
|
233
235
|
}
|
|
234
236
|
|
|
237
|
+
// Global control group (impact plan §4.3) — a deterministic contact-level
|
|
238
|
+
// bucket excluded from every non-transactional send so program-level lift
|
|
239
|
+
// is measurable. Runs INSIDE the non-system branch and additionally
|
|
240
|
+
// exempts the transactional category (control contacts still get
|
|
241
|
+
// receipts/resets). No provider call, no row — mirrors the frequency-cap
|
|
242
|
+
// skip. The outbound marker is per-endpoint deduped on the contact key,
|
|
243
|
+
// so the first suppressed send publishes membership exactly once.
|
|
244
|
+
if (effectiveCategory !== "transactional") {
|
|
245
|
+
const controlKey = options.userId ?? options.to;
|
|
246
|
+
if (isGlobalControl(controlKey)) {
|
|
247
|
+
logger?.info("send skipped: control_group", { to: options.to });
|
|
248
|
+
void emitOutbound({
|
|
249
|
+
db,
|
|
250
|
+
hatchet,
|
|
251
|
+
logger: logger ?? emitLogger,
|
|
252
|
+
event: "contact.control_group",
|
|
253
|
+
dedupeKey: `contact.control_group:${controlKey.toLowerCase()}`,
|
|
254
|
+
payload: {
|
|
255
|
+
userId: options.userId ?? null,
|
|
256
|
+
email: options.to,
|
|
257
|
+
at: new Date().toISOString(),
|
|
258
|
+
},
|
|
259
|
+
}).catch((err: unknown) => {
|
|
260
|
+
(logger ?? emitLogger).warn("control_group marker emit failed", {
|
|
261
|
+
error: err instanceof Error ? err.message : String(err),
|
|
262
|
+
});
|
|
263
|
+
});
|
|
264
|
+
return trackedSendResult({
|
|
265
|
+
emailSendId: "",
|
|
266
|
+
messageId: "",
|
|
267
|
+
status: "skipped",
|
|
268
|
+
reason: "control_group",
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
235
273
|
// Frequency cap — consulted only for non-system sends (system mail sets
|
|
236
274
|
// skipPreferenceCheck and bypasses both suppression and the cap). On a cap
|
|
237
275
|
// hit: no provider call, no row inserted, no throw — the journey continues.
|
|
@@ -357,6 +395,7 @@ async function sendTrackedEmailInner<K extends TemplateName>(
|
|
|
357
395
|
subject: options.subject ?? "",
|
|
358
396
|
category: effectiveCategory,
|
|
359
397
|
journeyStateId: options.journeyStateId,
|
|
398
|
+
campaignId: options.campaignId,
|
|
360
399
|
userId: options.userId,
|
|
361
400
|
userEmail: options.userEmail ?? options.to,
|
|
362
401
|
status: "failed",
|
|
@@ -411,6 +450,7 @@ async function sendTrackedEmailInner<K extends TemplateName>(
|
|
|
411
450
|
subject: wireSubject,
|
|
412
451
|
category: effectiveCategory,
|
|
413
452
|
journeyStateId: options.journeyStateId,
|
|
453
|
+
campaignId: options.campaignId,
|
|
414
454
|
userId: options.userId,
|
|
415
455
|
userEmail: options.userEmail ?? options.to,
|
|
416
456
|
status: "queued",
|
|
@@ -17,6 +17,14 @@ interface EmailSendContext {
|
|
|
17
17
|
templateKey: string | null;
|
|
18
18
|
messageId: string | null;
|
|
19
19
|
to: string;
|
|
20
|
+
/**
|
|
21
|
+
* Attribution scope (plan §1.2): the enrollment's journey id and the send's
|
|
22
|
+
* campaign id, stamped into every tracking event's properties so the credit
|
|
23
|
+
* ledger can slice by journey/campaign without a join back through
|
|
24
|
+
* email_sends. Null for transactional/raw sends.
|
|
25
|
+
*/
|
|
26
|
+
journeyId: string | null;
|
|
27
|
+
campaignId: string | null;
|
|
20
28
|
}
|
|
21
29
|
|
|
22
30
|
export async function resolveEmailSendContext(
|
|
@@ -32,8 +40,10 @@ export async function resolveEmailSendContext(
|
|
|
32
40
|
toEmail: emailSends.toEmail,
|
|
33
41
|
templateKey: emailSends.templateKey,
|
|
34
42
|
messageId: emailSends.messageId,
|
|
43
|
+
campaignId: emailSends.campaignId,
|
|
35
44
|
userId: journeyStates.userId,
|
|
36
45
|
userEmail: journeyStates.userEmail,
|
|
46
|
+
journeyId: journeyStates.journeyId,
|
|
37
47
|
})
|
|
38
48
|
.from(emailSends)
|
|
39
49
|
.leftJoin(journeyStates, eq(emailSends.journeyStateId, journeyStates.id))
|
|
@@ -49,6 +59,8 @@ export async function resolveEmailSendContext(
|
|
|
49
59
|
templateKey: row.templateKey,
|
|
50
60
|
messageId: row.messageId,
|
|
51
61
|
to: row.toEmail,
|
|
62
|
+
journeyId: row.journeyId,
|
|
63
|
+
campaignId: row.campaignId,
|
|
52
64
|
};
|
|
53
65
|
}
|
|
54
66
|
|
|
@@ -165,6 +177,10 @@ export async function pushTrackingEvent(
|
|
|
165
177
|
const properties: Record<string, unknown> = {
|
|
166
178
|
emailSendId,
|
|
167
179
|
templateKey: ctx.templateKey,
|
|
180
|
+
// Attribution scope (plan §1.2) — journey/campaign identity rides the
|
|
181
|
+
// spine on every touch event so credits can be sliced without re-joining.
|
|
182
|
+
journeyId: ctx.journeyId,
|
|
183
|
+
campaignId: ctx.campaignId,
|
|
168
184
|
...opts.properties,
|
|
169
185
|
};
|
|
170
186
|
|
|
@@ -270,6 +286,12 @@ export interface SmsSendContext {
|
|
|
270
286
|
templateKey: string | null;
|
|
271
287
|
messageId: string | null;
|
|
272
288
|
to: string;
|
|
289
|
+
/**
|
|
290
|
+
* Attribution scope (plan §1.2): the enrollment's journey id. No campaign
|
|
291
|
+
* leg — campaign steps are email-only today, so `sms_sends` carries no
|
|
292
|
+
* campaign column (plan §5 Q3).
|
|
293
|
+
*/
|
|
294
|
+
journeyId: string | null;
|
|
273
295
|
}
|
|
274
296
|
|
|
275
297
|
/**
|
|
@@ -290,6 +312,7 @@ export async function resolveSmsSendContext(
|
|
|
290
312
|
sendUserId: smsSends.userId,
|
|
291
313
|
userId: journeyStates.userId,
|
|
292
314
|
userEmail: journeyStates.userEmail,
|
|
315
|
+
journeyId: journeyStates.journeyId,
|
|
293
316
|
})
|
|
294
317
|
.from(smsSends)
|
|
295
318
|
.leftJoin(journeyStates, eq(smsSends.journeyStateId, journeyStates.id))
|
|
@@ -305,6 +328,7 @@ export async function resolveSmsSendContext(
|
|
|
305
328
|
templateKey: row.templateKey,
|
|
306
329
|
messageId: row.messageId,
|
|
307
330
|
to: row.toPhone,
|
|
331
|
+
journeyId: row.journeyId,
|
|
308
332
|
};
|
|
309
333
|
}
|
|
310
334
|
|
|
@@ -351,6 +375,8 @@ export async function pushSmsTrackingEvent(
|
|
|
351
375
|
eventProperties: {
|
|
352
376
|
smsSendId,
|
|
353
377
|
templateKey: ctx.templateKey,
|
|
378
|
+
// Attribution scope (plan §1.2) — see pushTrackingEvent.
|
|
379
|
+
journeyId: ctx.journeyId,
|
|
354
380
|
...opts.properties,
|
|
355
381
|
},
|
|
356
382
|
source: "tracking",
|
|
@@ -55,6 +55,9 @@ export const WEBHOOK_EVENT_TYPES = [
|
|
|
55
55
|
"contact.deleted",
|
|
56
56
|
"contact.unsubscribed",
|
|
57
57
|
"contact.subscribed",
|
|
58
|
+
// Global control group membership (impact plan §4.3) — emitted once per
|
|
59
|
+
// contact key on the first withheld send.
|
|
60
|
+
"contact.control_group",
|
|
58
61
|
"email.sent",
|
|
59
62
|
"email.delivered",
|
|
60
63
|
"email.opened",
|
|
@@ -67,10 +70,14 @@ export const WEBHOOK_EVENT_TYPES = [
|
|
|
67
70
|
"sms.failed",
|
|
68
71
|
"sms.clicked",
|
|
69
72
|
"journey.completed",
|
|
73
|
+
"journey.heldout",
|
|
70
74
|
"bucket.entered",
|
|
71
75
|
"bucket.left",
|
|
72
76
|
"link.clicked",
|
|
73
77
|
"link.arrived",
|
|
78
|
+
"funnel.stage_changed",
|
|
79
|
+
"deal.quoted",
|
|
80
|
+
"deal.sold",
|
|
74
81
|
] as const;
|
|
75
82
|
|
|
76
83
|
export type WebhookEventType = (typeof WEBHOOK_EVENT_TYPES)[number];
|
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
import { attributionCredits, campaigns, conversions } from "@hogsend/db";
|
|
2
|
+
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
+
import { and, count, eq, gte, isNotNull, sql } from "drizzle-orm";
|
|
4
|
+
import type { AppEnv } from "../../app.js";
|
|
5
|
+
import { backfillAttributionBatch } from "../../lib/attribution-backfill.js";
|
|
6
|
+
import { getConversionRegistry } from "../../lib/conversions.js";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Admin attribution API (docs/revenue-attribution-plan.md §6.2, extended by
|
|
10
|
+
* docs/attribution-impact-plan.md §1.5) — reads over the credit ledger.
|
|
11
|
+
* Every model was persisted at conversion time, so model comparison is a
|
|
12
|
+
* GROUP BY, not a re-computation — and since the ledger carries
|
|
13
|
+
* journey/campaign/template scope, "revenue by journey" is too.
|
|
14
|
+
*/
|
|
15
|
+
|
|
16
|
+
const GROUP_DIMENSIONS = [
|
|
17
|
+
"channel",
|
|
18
|
+
"journey",
|
|
19
|
+
"campaign",
|
|
20
|
+
"template",
|
|
21
|
+
] as const;
|
|
22
|
+
type GroupDimension = (typeof GROUP_DIMENSIONS)[number];
|
|
23
|
+
|
|
24
|
+
const rowSchema = z.object({
|
|
25
|
+
model: z.string(),
|
|
26
|
+
/**
|
|
27
|
+
* The grouped dimension's value — channel name, journey id, campaign id,
|
|
28
|
+
* or template key. Null = credits carrying no scope on that dimension
|
|
29
|
+
* (e.g. a transactional-send click under groupBy=journey).
|
|
30
|
+
*/
|
|
31
|
+
key: z.string().nullable(),
|
|
32
|
+
/** Human label where the key is opaque (campaign name); else null. */
|
|
33
|
+
label: z.string().nullable(),
|
|
34
|
+
/** Back-compat mirror of `key`, present only when groupBy=channel. */
|
|
35
|
+
channel: z.string().optional(),
|
|
36
|
+
currency: z.string().nullable(),
|
|
37
|
+
/** Sum of credited value (weight × conversion value), this currency. */
|
|
38
|
+
value: z.number(),
|
|
39
|
+
/** Sum of weights — "how many conversions" this key earned. */
|
|
40
|
+
conversions: z.number(),
|
|
41
|
+
/** Number of credited touchpoints. */
|
|
42
|
+
touches: z.number(),
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Coverage — the ledger only divides conversions that HAD a touchpoint path,
|
|
47
|
+
* so credited value is a subset of fired conversion value. Reporting both
|
|
48
|
+
* keeps the delta (direct / imported / pre-tracking conversions) explicit
|
|
49
|
+
* instead of silently missing. Scoped by `definitionId` only — the
|
|
50
|
+
* journeyId/campaignId row filters do NOT narrow totals (a conversion is not
|
|
51
|
+
* "inside" a journey; its touches are).
|
|
52
|
+
*/
|
|
53
|
+
const totalsSchema = z.object({
|
|
54
|
+
currency: z.string().nullable(),
|
|
55
|
+
/** Total fired conversion value in the window (this currency). */
|
|
56
|
+
value: z.number(),
|
|
57
|
+
conversions: z.number(),
|
|
58
|
+
/** Subset with at least one credit row — i.e. a touchpoint path. */
|
|
59
|
+
attributedValue: z.number(),
|
|
60
|
+
attributedConversions: z.number(),
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Influenced (impact plan §3.1) — the model-invariant COVERAGE number
|
|
65
|
+
* (Dreamdata semantics): a conversion is influenced by a scope when the
|
|
66
|
+
* contact had ≥1 touchpoint from it inside the window. Deliberately
|
|
67
|
+
* multi-counted across scopes — it answers "which conversions did this
|
|
68
|
+
* journey touch at all", never sums to total, and is presented as reach,
|
|
69
|
+
* not credit.
|
|
70
|
+
*/
|
|
71
|
+
const influencedSchema = z.object({
|
|
72
|
+
key: z.string(),
|
|
73
|
+
/** Human label where the key is opaque (campaign name); else null. */
|
|
74
|
+
label: z.string().nullable(),
|
|
75
|
+
currency: z.string().nullable(),
|
|
76
|
+
/** Conversions with ≥1 touch from this scope (full count, not weighted). */
|
|
77
|
+
conversions: z.number(),
|
|
78
|
+
/** FULL value of those conversions (not fractional credit). */
|
|
79
|
+
value: z.number(),
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Overlap transparency (impact plan §2.3, the anti-Braze): no incumbent
|
|
84
|
+
* dedupes credit across journeys/campaigns — Braze documents conversion
|
|
85
|
+
* rates over 100%. Our fractional models already sum to 1 per conversion
|
|
86
|
+
* (that IS the dedup); this block additionally SHOWS the overlap: how many
|
|
87
|
+
* scope-claimed conversions were claimed by more than one value of the
|
|
88
|
+
* grouped dimension, and what per-scope single-credit totals would add up
|
|
89
|
+
* to (`scopeSummedValue` vs the real `value`).
|
|
90
|
+
*/
|
|
91
|
+
const overlapSchema = z.object({
|
|
92
|
+
currency: z.string().nullable(),
|
|
93
|
+
/** Conversions claimed by ≥1 non-null scope on this dimension. */
|
|
94
|
+
conversions: z.number(),
|
|
95
|
+
/** Subset claimed by MORE THAN ONE distinct scope. */
|
|
96
|
+
multiScopeConversions: z.number(),
|
|
97
|
+
/** Real conversion value across the claimed set. */
|
|
98
|
+
value: z.number(),
|
|
99
|
+
/** What "each scope takes full credit" reporting would sum to. */
|
|
100
|
+
scopeSummedValue: z.number(),
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
const summaryRoute = createRoute({
|
|
104
|
+
method: "get",
|
|
105
|
+
path: "/",
|
|
106
|
+
tags: ["Admin"],
|
|
107
|
+
summary: "Attribution credit rollup (model × dimension × currency)",
|
|
108
|
+
request: {
|
|
109
|
+
query: z.object({
|
|
110
|
+
days: z.coerce.number().min(1).max(365).default(90),
|
|
111
|
+
/** Scope to one conversion point. */
|
|
112
|
+
definitionId: z.string().optional(),
|
|
113
|
+
/** The dimension credits are grouped by (impact plan §1.5). */
|
|
114
|
+
groupBy: z.enum(GROUP_DIMENSIONS).default("channel"),
|
|
115
|
+
/** Only credits whose touchpoint belongs to this journey. */
|
|
116
|
+
journeyId: z.string().optional(),
|
|
117
|
+
/** Only credits whose touchpoint belongs to this campaign. */
|
|
118
|
+
campaignId: z.string().optional(),
|
|
119
|
+
}),
|
|
120
|
+
},
|
|
121
|
+
responses: {
|
|
122
|
+
200: {
|
|
123
|
+
content: {
|
|
124
|
+
"application/json": {
|
|
125
|
+
schema: z.object({
|
|
126
|
+
days: z.number(),
|
|
127
|
+
groupBy: z.enum(GROUP_DIMENSIONS),
|
|
128
|
+
rows: z.array(rowSchema),
|
|
129
|
+
totals: z.array(totalsSchema),
|
|
130
|
+
overlap: z.array(overlapSchema),
|
|
131
|
+
influenced: z.array(influencedSchema),
|
|
132
|
+
}),
|
|
133
|
+
},
|
|
134
|
+
},
|
|
135
|
+
description:
|
|
136
|
+
"Flat rollup rows — clients pivot into per-model breakdowns and model-comparison matrices",
|
|
137
|
+
},
|
|
138
|
+
},
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
/**
|
|
142
|
+
* Backfill (impact plan §5.1) — batch-shaped replay of history through the
|
|
143
|
+
* idempotent conversion + ledger machinery. Loop until `nextCursor` is null
|
|
144
|
+
* (`hogsend attribution backfill` does exactly that). Lives here because
|
|
145
|
+
* the code-first conversion registry only exists inside the API process.
|
|
146
|
+
*/
|
|
147
|
+
const backfillRoute = createRoute({
|
|
148
|
+
method: "post",
|
|
149
|
+
path: "/backfill",
|
|
150
|
+
tags: ["Admin"],
|
|
151
|
+
summary: "Backfill conversions + attribution credits from event history",
|
|
152
|
+
request: {
|
|
153
|
+
body: {
|
|
154
|
+
content: {
|
|
155
|
+
"application/json": {
|
|
156
|
+
schema: z.object({
|
|
157
|
+
definitionId: z.string().optional(),
|
|
158
|
+
/** Only events/conversions at or after this instant. */
|
|
159
|
+
since: z.string().datetime().optional(),
|
|
160
|
+
/** Opaque batch cursor from the previous response. */
|
|
161
|
+
cursor: z.string().optional(),
|
|
162
|
+
limit: z.number().min(1).max(2000).default(500),
|
|
163
|
+
/**
|
|
164
|
+
* Delete-then-refill the definition's credits under CURRENT
|
|
165
|
+
* window config. Requires definitionId; logged loudly.
|
|
166
|
+
*/
|
|
167
|
+
recompute: z.boolean().default(false),
|
|
168
|
+
}),
|
|
169
|
+
},
|
|
170
|
+
},
|
|
171
|
+
},
|
|
172
|
+
},
|
|
173
|
+
responses: {
|
|
174
|
+
200: {
|
|
175
|
+
content: {
|
|
176
|
+
"application/json": {
|
|
177
|
+
schema: z.object({
|
|
178
|
+
stage: z.enum(["events", "credits"]),
|
|
179
|
+
processed: z.number(),
|
|
180
|
+
conversionsFired: z.number(),
|
|
181
|
+
creditsWritten: z.number(),
|
|
182
|
+
/** Null = backfill complete. */
|
|
183
|
+
nextCursor: z.string().nullable(),
|
|
184
|
+
}),
|
|
185
|
+
},
|
|
186
|
+
},
|
|
187
|
+
description: "One processed batch; loop until nextCursor is null",
|
|
188
|
+
},
|
|
189
|
+
400: {
|
|
190
|
+
content: {
|
|
191
|
+
"application/json": { schema: z.object({ error: z.string() }) },
|
|
192
|
+
},
|
|
193
|
+
description: "recompute without definitionId",
|
|
194
|
+
},
|
|
195
|
+
},
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
export const adminAttributionRouter = new OpenAPIHono<AppEnv>()
|
|
199
|
+
.openapi(summaryRoute, async (c) => {
|
|
200
|
+
const { db } = c.get("container");
|
|
201
|
+
const { days, definitionId, groupBy, journeyId, campaignId } =
|
|
202
|
+
c.req.valid("query");
|
|
203
|
+
const since = new Date(
|
|
204
|
+
Date.now() - days * 24 * 60 * 60 * 1000,
|
|
205
|
+
).toISOString();
|
|
206
|
+
const sinceTs = sql`${since}::timestamptz`;
|
|
207
|
+
const definitionFilter = definitionId
|
|
208
|
+
? [eq(conversions.definitionId, definitionId)]
|
|
209
|
+
: [];
|
|
210
|
+
const scopeFilters = [
|
|
211
|
+
...(journeyId ? [eq(attributionCredits.journeyId, journeyId)] : []),
|
|
212
|
+
...(campaignId ? [eq(attributionCredits.campaignId, campaignId)] : []),
|
|
213
|
+
];
|
|
214
|
+
|
|
215
|
+
const groupColumns = {
|
|
216
|
+
channel: attributionCredits.channel,
|
|
217
|
+
journey: attributionCredits.journeyId,
|
|
218
|
+
campaign: attributionCredits.campaignId,
|
|
219
|
+
template: attributionCredits.templateKey,
|
|
220
|
+
} satisfies Record<GroupDimension, unknown>;
|
|
221
|
+
const groupCol = groupColumns[groupBy];
|
|
222
|
+
|
|
223
|
+
const rows = await db
|
|
224
|
+
.select({
|
|
225
|
+
model: attributionCredits.model,
|
|
226
|
+
key: sql<string | null>`${groupCol}::text`,
|
|
227
|
+
currency: attributionCredits.currency,
|
|
228
|
+
value: sql<number>`coalesce(sum(${attributionCredits.value}), 0)::float8`,
|
|
229
|
+
conversions: sql<number>`coalesce(sum(${attributionCredits.weight}), 0)::float8`,
|
|
230
|
+
touches: count(),
|
|
231
|
+
})
|
|
232
|
+
.from(attributionCredits)
|
|
233
|
+
.innerJoin(
|
|
234
|
+
conversions,
|
|
235
|
+
eq(attributionCredits.conversionId, conversions.id),
|
|
236
|
+
)
|
|
237
|
+
.where(
|
|
238
|
+
and(
|
|
239
|
+
gte(attributionCredits.convertedAt, sinceTs),
|
|
240
|
+
...definitionFilter,
|
|
241
|
+
...scopeFilters,
|
|
242
|
+
),
|
|
243
|
+
)
|
|
244
|
+
.groupBy(
|
|
245
|
+
attributionCredits.model,
|
|
246
|
+
sql`${groupCol}`,
|
|
247
|
+
attributionCredits.currency,
|
|
248
|
+
);
|
|
249
|
+
|
|
250
|
+
// Influenced (§3.1): one row per (scope key, conversion) via DISTINCT,
|
|
251
|
+
// then full conversion count + value per key. Any single model
|
|
252
|
+
// enumerates the credited path; weight is ignored by construction.
|
|
253
|
+
const influencedClaims = db.$with("influenced_claims").as(
|
|
254
|
+
db
|
|
255
|
+
.selectDistinct({
|
|
256
|
+
conversionId: attributionCredits.conversionId,
|
|
257
|
+
key: sql<string>`${groupCol}::text`.as("key"),
|
|
258
|
+
})
|
|
259
|
+
.from(attributionCredits)
|
|
260
|
+
.where(
|
|
261
|
+
and(
|
|
262
|
+
eq(attributionCredits.model, "linear"),
|
|
263
|
+
gte(attributionCredits.convertedAt, sinceTs),
|
|
264
|
+
isNotNull(groupCol),
|
|
265
|
+
),
|
|
266
|
+
),
|
|
267
|
+
);
|
|
268
|
+
const influenced = await db
|
|
269
|
+
.with(influencedClaims)
|
|
270
|
+
.select({
|
|
271
|
+
key: influencedClaims.key,
|
|
272
|
+
currency: conversions.currency,
|
|
273
|
+
conversions: count(),
|
|
274
|
+
value: sql<number>`coalesce(sum(${conversions.value}), 0)::float8`,
|
|
275
|
+
})
|
|
276
|
+
.from(influencedClaims)
|
|
277
|
+
.innerJoin(conversions, eq(conversions.id, influencedClaims.conversionId))
|
|
278
|
+
.where(and(...definitionFilter))
|
|
279
|
+
.groupBy(influencedClaims.key, conversions.currency);
|
|
280
|
+
|
|
281
|
+
// Campaign keys are opaque uuids — resolve names server-side so every
|
|
282
|
+
// client renders the same labels. Other dimensions are already readable.
|
|
283
|
+
const labels = new Map<string, string>();
|
|
284
|
+
if (groupBy === "campaign") {
|
|
285
|
+
const ids = [
|
|
286
|
+
...new Set(
|
|
287
|
+
[...rows.map((r) => r.key), ...influenced.map((r) => r.key)].filter(
|
|
288
|
+
(k): k is string => !!k,
|
|
289
|
+
),
|
|
290
|
+
),
|
|
291
|
+
];
|
|
292
|
+
if (ids.length > 0) {
|
|
293
|
+
const named = await db
|
|
294
|
+
.select({ id: campaigns.id, name: campaigns.name })
|
|
295
|
+
.from(campaigns)
|
|
296
|
+
.where(
|
|
297
|
+
sql`${campaigns.id} in (${sql.join(
|
|
298
|
+
ids.map((id) => sql`${id}::uuid`),
|
|
299
|
+
sql`, `,
|
|
300
|
+
)})`,
|
|
301
|
+
);
|
|
302
|
+
for (const row of named) labels.set(row.id, row.name);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// Overlap (§2.3): per conversion, how many distinct non-null values of
|
|
307
|
+
// the grouped dimension claimed it (any single model enumerates the
|
|
308
|
+
// credited path — `linear` includes every in-window touch). Unfiltered
|
|
309
|
+
// by journeyId/campaignId: overlap is about the whole claiming set.
|
|
310
|
+
const claims = db.$with("claims").as(
|
|
311
|
+
db
|
|
312
|
+
.select({
|
|
313
|
+
conversionId: attributionCredits.conversionId,
|
|
314
|
+
scopeCount: sql<number>`count(distinct ${groupCol})`.as(
|
|
315
|
+
"scope_count",
|
|
316
|
+
),
|
|
317
|
+
})
|
|
318
|
+
.from(attributionCredits)
|
|
319
|
+
.where(
|
|
320
|
+
and(
|
|
321
|
+
eq(attributionCredits.model, "linear"),
|
|
322
|
+
gte(attributionCredits.convertedAt, sinceTs),
|
|
323
|
+
isNotNull(groupCol),
|
|
324
|
+
),
|
|
325
|
+
)
|
|
326
|
+
.groupBy(attributionCredits.conversionId),
|
|
327
|
+
);
|
|
328
|
+
const overlap = await db
|
|
329
|
+
.with(claims)
|
|
330
|
+
.select({
|
|
331
|
+
currency: conversions.currency,
|
|
332
|
+
conversions: count(),
|
|
333
|
+
multiScopeConversions: sql<number>`(count(*) filter (where ${claims.scopeCount} > 1))::int`,
|
|
334
|
+
value: sql<number>`coalesce(sum(${conversions.value}), 0)::float8`,
|
|
335
|
+
scopeSummedValue: sql<number>`coalesce(sum(${conversions.value} * ${claims.scopeCount}), 0)::float8`,
|
|
336
|
+
})
|
|
337
|
+
.from(claims)
|
|
338
|
+
.innerJoin(conversions, eq(conversions.id, claims.conversionId))
|
|
339
|
+
.where(and(...definitionFilter))
|
|
340
|
+
.groupBy(conversions.currency);
|
|
341
|
+
|
|
342
|
+
const credited = sql`exists (select 1 from ${attributionCredits} where ${attributionCredits.conversionId} = ${conversions.id})`;
|
|
343
|
+
const totals = await db
|
|
344
|
+
.select({
|
|
345
|
+
currency: conversions.currency,
|
|
346
|
+
value: sql<number>`coalesce(sum(${conversions.value}), 0)::float8`,
|
|
347
|
+
conversions: count(),
|
|
348
|
+
attributedValue: sql<number>`coalesce(sum(${conversions.value}) filter (where ${credited}), 0)::float8`,
|
|
349
|
+
attributedConversions: sql<number>`(count(*) filter (where ${credited}))::int`,
|
|
350
|
+
})
|
|
351
|
+
.from(conversions)
|
|
352
|
+
.where(and(gte(conversions.occurredAt, sinceTs), ...definitionFilter))
|
|
353
|
+
.groupBy(conversions.currency);
|
|
354
|
+
|
|
355
|
+
return c.json(
|
|
356
|
+
{
|
|
357
|
+
days,
|
|
358
|
+
groupBy,
|
|
359
|
+
rows: rows.map((row) => ({
|
|
360
|
+
model: row.model,
|
|
361
|
+
key: row.key,
|
|
362
|
+
label: (row.key ? labels.get(row.key) : undefined) ?? null,
|
|
363
|
+
...(groupBy === "channel" && row.key ? { channel: row.key } : {}),
|
|
364
|
+
currency: row.currency,
|
|
365
|
+
value: row.value,
|
|
366
|
+
conversions: row.conversions,
|
|
367
|
+
touches: Number(row.touches),
|
|
368
|
+
})),
|
|
369
|
+
totals: totals.map((row) => ({
|
|
370
|
+
currency: row.currency,
|
|
371
|
+
value: row.value,
|
|
372
|
+
conversions: Number(row.conversions),
|
|
373
|
+
attributedValue: row.attributedValue,
|
|
374
|
+
attributedConversions: Number(row.attributedConversions),
|
|
375
|
+
})),
|
|
376
|
+
overlap: overlap.map((row) => ({
|
|
377
|
+
currency: row.currency,
|
|
378
|
+
conversions: Number(row.conversions),
|
|
379
|
+
multiScopeConversions: row.multiScopeConversions,
|
|
380
|
+
value: row.value,
|
|
381
|
+
scopeSummedValue: row.scopeSummedValue,
|
|
382
|
+
})),
|
|
383
|
+
influenced: influenced.map((row) => ({
|
|
384
|
+
key: row.key,
|
|
385
|
+
label: labels.get(row.key) ?? null,
|
|
386
|
+
currency: row.currency,
|
|
387
|
+
conversions: Number(row.conversions),
|
|
388
|
+
value: row.value,
|
|
389
|
+
})),
|
|
390
|
+
},
|
|
391
|
+
200,
|
|
392
|
+
);
|
|
393
|
+
})
|
|
394
|
+
.openapi(backfillRoute, async (c) => {
|
|
395
|
+
const { db, logger } = c.get("container");
|
|
396
|
+
const body = c.req.valid("json");
|
|
397
|
+
if (body.recompute && !body.definitionId) {
|
|
398
|
+
return c.json(
|
|
399
|
+
{ error: "recompute requires a definitionId — never blanket" },
|
|
400
|
+
400,
|
|
401
|
+
);
|
|
402
|
+
}
|
|
403
|
+
const result = await backfillAttributionBatch({
|
|
404
|
+
db,
|
|
405
|
+
logger,
|
|
406
|
+
registry: getConversionRegistry(),
|
|
407
|
+
definitionId: body.definitionId,
|
|
408
|
+
since: body.since ? new Date(body.since) : undefined,
|
|
409
|
+
cursor: body.cursor,
|
|
410
|
+
limit: body.limit,
|
|
411
|
+
recompute: body.recompute,
|
|
412
|
+
});
|
|
413
|
+
return c.json(result, 200);
|
|
414
|
+
});
|