@hogsend/engine 0.43.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +10 -9
- package/src/container.ts +229 -2
- package/src/destinations/define-destination.ts +7 -3
- package/src/destinations/presets/posthog.ts +31 -0
- package/src/index.ts +99 -0
- package/src/journeys/execute-journey-run.ts +93 -0
- package/src/journeys/journey-context.ts +4 -0
- package/src/lib/attribution-backfill.ts +285 -0
- package/src/lib/attribution.ts +239 -0
- package/src/lib/contacts.ts +71 -0
- package/src/lib/conversion-dispatch.ts +326 -0
- package/src/lib/conversions.ts +197 -0
- package/src/lib/crm-deals.ts +320 -0
- package/src/lib/crm-ingest.ts +341 -0
- package/src/lib/crm-provider-registry.ts +41 -0
- package/src/lib/crm-registry-singleton.ts +22 -0
- package/src/lib/deal-money-events.ts +85 -0
- package/src/lib/email-service-types.ts +16 -1
- package/src/lib/event-names.ts +1 -1
- package/src/lib/funnel-progress.ts +76 -0
- package/src/lib/funnel-registry.ts +108 -0
- package/src/lib/funnel-transitions.ts +280 -0
- package/src/lib/holdout.ts +62 -0
- package/src/lib/ingestion.ts +218 -9
- package/src/lib/lift-stats.ts +125 -0
- package/src/lib/mailer.ts +1 -0
- package/src/lib/operator.ts +31 -0
- package/src/lib/outbound.ts +51 -0
- package/src/lib/revenue.ts +116 -0
- package/src/lib/sms-service-types.ts +7 -1
- package/src/lib/sms-tracked.ts +35 -0
- package/src/lib/tracked.ts +40 -0
- package/src/lib/tracking-events.ts +26 -0
- package/src/lib/webhook-signing.ts +7 -0
- package/src/routes/admin/attribution.ts +414 -0
- package/src/routes/admin/contacts.ts +71 -11
- package/src/routes/admin/conversions.ts +247 -0
- package/src/routes/admin/deals.ts +472 -0
- package/src/routes/admin/funnels.ts +228 -0
- package/src/routes/admin/index.ts +8 -0
- package/src/routes/admin/journeys.ts +158 -1
- package/src/routes/admin/readiness.ts +97 -2
- package/src/routes/admin/templates.ts +28 -1
- package/src/routes/events/index.ts +12 -0
- package/src/routes/tracking/click-pipeline.ts +26 -18
- package/src/routes/webhooks/crm-provider.ts +117 -0
- package/src/routes/webhooks/index.ts +5 -1
- package/src/sources/define-contact-source.ts +159 -0
- package/src/sources/registry.ts +71 -0
- package/src/sources/webhook-source.ts +109 -0
- package/src/worker.ts +4 -0
- package/src/workflows/crm-reconcile.ts +129 -0
- package/src/workflows/dispatch-conversion.ts +31 -0
- package/src/workflows/send-campaign.ts +5 -0
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
serializePrefs,
|
|
11
11
|
} from "../../lib/contacts.js";
|
|
12
12
|
import { emitOutbound } from "../../lib/outbound.js";
|
|
13
|
+
import { getContactRevenue, revenueExcludedEvents } from "../../lib/revenue.js";
|
|
13
14
|
|
|
14
15
|
const contactSchema = z.object({
|
|
15
16
|
id: z.string(),
|
|
@@ -35,6 +36,19 @@ const preferencesSchema = z
|
|
|
35
36
|
})
|
|
36
37
|
.nullable();
|
|
37
38
|
|
|
39
|
+
// Revenue rollup over the contact's valued events (`user_events.value`),
|
|
40
|
+
// grouped per currency — never summed across currencies.
|
|
41
|
+
const revenueSchema = z.object({
|
|
42
|
+
totals: z.array(
|
|
43
|
+
z.object({
|
|
44
|
+
currency: z.string().nullable(),
|
|
45
|
+
total: z.number(),
|
|
46
|
+
count: z.number(),
|
|
47
|
+
}),
|
|
48
|
+
),
|
|
49
|
+
lastValuedAt: z.string().nullable(),
|
|
50
|
+
});
|
|
51
|
+
|
|
38
52
|
const listRoute = createRoute({
|
|
39
53
|
method: "get",
|
|
40
54
|
path: "/",
|
|
@@ -45,6 +59,11 @@ const listRoute = createRoute({
|
|
|
45
59
|
limit: z.coerce.number().min(1).max(100).default(50),
|
|
46
60
|
offset: z.coerce.number().min(0).default(0),
|
|
47
61
|
search: z.string().optional(),
|
|
62
|
+
// Long-tail value filters (plan §4b.3): the "find my value customers"
|
|
63
|
+
// query surface.
|
|
64
|
+
minRevenue: z.coerce.number().optional(),
|
|
65
|
+
// Plain string: valid stages are the deployment's configured ladder.
|
|
66
|
+
dealStage: z.string().optional(),
|
|
48
67
|
}),
|
|
49
68
|
},
|
|
50
69
|
responses: {
|
|
@@ -79,10 +98,11 @@ const getRoute = createRoute({
|
|
|
79
98
|
schema: z.object({
|
|
80
99
|
contact: contactSchema,
|
|
81
100
|
preferences: preferencesSchema,
|
|
101
|
+
revenue: revenueSchema,
|
|
82
102
|
}),
|
|
83
103
|
},
|
|
84
104
|
},
|
|
85
|
-
description: "Contact with preferences",
|
|
105
|
+
description: "Contact with preferences and revenue rollup",
|
|
86
106
|
},
|
|
87
107
|
404: {
|
|
88
108
|
content: {
|
|
@@ -209,13 +229,45 @@ const serializeContact = (row: typeof contacts.$inferSelect) =>
|
|
|
209
229
|
export const contactsRouter = new OpenAPIHono<AppEnv>()
|
|
210
230
|
.openapi(listRoute, async (c) => {
|
|
211
231
|
const { db } = c.get("container");
|
|
212
|
-
const { limit, offset, search } =
|
|
232
|
+
const { limit, offset, search, minRevenue, dealStage } =
|
|
233
|
+
c.req.valid("query");
|
|
213
234
|
|
|
214
235
|
const searchFilter = search ? contactSearchFilter(search) : undefined;
|
|
215
236
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
237
|
+
// Valued events are keyed by the contact's canonical event key
|
|
238
|
+
// (external_id ?? anonymous_id ?? id) — same precedence ingestEvent
|
|
239
|
+
// resolves. Served by the partial user_events_valued_user_idx.
|
|
240
|
+
// Exclusions come from lib/revenue.ts (static machinery events + funnel
|
|
241
|
+
// milestone triggers + the browser trust gate): one deal's value rides
|
|
242
|
+
// several rows, and pk_-minted values are forgeable.
|
|
243
|
+
const revenueFilter =
|
|
244
|
+
minRevenue !== undefined
|
|
245
|
+
? sql`(
|
|
246
|
+
select coalesce(sum(ue.value), 0)
|
|
247
|
+
from user_events ue
|
|
248
|
+
where ue.user_id = coalesce(${contacts.externalId}, ${contacts.anonymousId}, ${contacts.id}::text)
|
|
249
|
+
and ue.value is not null
|
|
250
|
+
and ue.event not in (${sql.join(
|
|
251
|
+
revenueExcludedEvents().map((e) => sql`${e}`),
|
|
252
|
+
sql`, `,
|
|
253
|
+
)})
|
|
254
|
+
and (ue.source is null or ue.source <> 'inapp')
|
|
255
|
+
) >= ${minRevenue}`
|
|
256
|
+
: undefined;
|
|
257
|
+
const dealStageFilter = dealStage
|
|
258
|
+
? sql`exists (
|
|
259
|
+
select 1 from deals d
|
|
260
|
+
where d.contact_id = ${contacts.id}
|
|
261
|
+
and d.canonical_stage = ${dealStage}
|
|
262
|
+
)`
|
|
263
|
+
: undefined;
|
|
264
|
+
|
|
265
|
+
const where = and(
|
|
266
|
+
isNull(contacts.deletedAt),
|
|
267
|
+
...(searchFilter ? [searchFilter] : []),
|
|
268
|
+
...(revenueFilter ? [revenueFilter] : []),
|
|
269
|
+
...(dealStageFilter ? [dealStageFilter] : []),
|
|
270
|
+
);
|
|
219
271
|
|
|
220
272
|
const [rows, totalRows] = await Promise.all([
|
|
221
273
|
db
|
|
@@ -249,16 +301,24 @@ export const contactsRouter = new OpenAPIHono<AppEnv>()
|
|
|
249
301
|
|
|
250
302
|
// email_preferences.user_id uses external_id when present, else the contact
|
|
251
303
|
// uuid as the deterministic fallback (risk 10 — email-only contacts).
|
|
252
|
-
const prefRows = await
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
304
|
+
const [prefRows, revenue] = await Promise.all([
|
|
305
|
+
db
|
|
306
|
+
.select()
|
|
307
|
+
.from(emailPreferences)
|
|
308
|
+
.where(eq(emailPreferences.userId, contact.externalId ?? contact.id))
|
|
309
|
+
.limit(1),
|
|
310
|
+
// Valued events are keyed by the contact's canonical event key — the
|
|
311
|
+
// same precedence ingestEvent resolves (`external ?? anon ?? id`).
|
|
312
|
+
getContactRevenue({
|
|
313
|
+
db,
|
|
314
|
+
key: contact.externalId ?? contact.anonymousId ?? contact.id,
|
|
315
|
+
}),
|
|
316
|
+
]);
|
|
257
317
|
|
|
258
318
|
const prefs = prefRows[0] ? serializePrefs(prefRows[0]) : null;
|
|
259
319
|
|
|
260
320
|
return c.json(
|
|
261
|
-
{ contact: serializeContact(contact), preferences: prefs },
|
|
321
|
+
{ contact: serializeContact(contact), preferences: prefs, revenue },
|
|
262
322
|
200,
|
|
263
323
|
);
|
|
264
324
|
})
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
import { contacts, conversionDispatches, conversions } from "@hogsend/db";
|
|
2
|
+
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
+
import { and, asc, count, desc, eq, sql } from "drizzle-orm";
|
|
4
|
+
import type { AppEnv } from "../../app.js";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Admin conversions API (docs/revenue-attribution-plan.md §5b.2) — fired
|
|
8
|
+
* conversion points and their per-destination delivery state. Answers the
|
|
9
|
+
* operator question "did the ad platform actually receive the sale?".
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
const dispatchSchema = z.object({
|
|
13
|
+
destinationId: z.string(),
|
|
14
|
+
status: z.string(),
|
|
15
|
+
attempts: z.number(),
|
|
16
|
+
lastError: z.string().nullable(),
|
|
17
|
+
deliveredAt: z.string().nullable(),
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
const conversionSchema = z.object({
|
|
21
|
+
id: z.string(),
|
|
22
|
+
definitionId: z.string(),
|
|
23
|
+
contactId: z.string(),
|
|
24
|
+
contactEmail: z.string().nullable(),
|
|
25
|
+
value: z.number().nullable(),
|
|
26
|
+
currency: z.string().nullable(),
|
|
27
|
+
/** The definition's declared scope (ConversionMeta.scope), if any. */
|
|
28
|
+
scopeJourneyId: z.string().nullable(),
|
|
29
|
+
scopeCampaignId: z.string().nullable(),
|
|
30
|
+
occurredAt: z.string(),
|
|
31
|
+
dispatches: z.array(dispatchSchema),
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
const listRoute = createRoute({
|
|
35
|
+
method: "get",
|
|
36
|
+
path: "/",
|
|
37
|
+
tags: ["Admin"],
|
|
38
|
+
summary: "List fired conversions with delivery state",
|
|
39
|
+
request: {
|
|
40
|
+
query: z.object({
|
|
41
|
+
definitionId: z.string().optional(),
|
|
42
|
+
/** Filter by the definition's declared scope (ConversionMeta.scope). */
|
|
43
|
+
journeyId: z.string().optional(),
|
|
44
|
+
campaignId: z.string().optional(),
|
|
45
|
+
/** Only conversions with at least one dispatch in this status. */
|
|
46
|
+
dispatchStatus: z.enum(["pending", "delivered", "failed"]).optional(),
|
|
47
|
+
sort: z
|
|
48
|
+
.enum(["occurredAt", "value", "definitionId"])
|
|
49
|
+
.default("occurredAt"),
|
|
50
|
+
dir: z.enum(["asc", "desc"]).default("desc"),
|
|
51
|
+
limit: z.coerce.number().min(1).max(200).default(50),
|
|
52
|
+
offset: z.coerce.number().min(0).default(0),
|
|
53
|
+
}),
|
|
54
|
+
},
|
|
55
|
+
responses: {
|
|
56
|
+
200: {
|
|
57
|
+
content: {
|
|
58
|
+
"application/json": {
|
|
59
|
+
schema: z.object({
|
|
60
|
+
conversions: z.array(conversionSchema),
|
|
61
|
+
total: z.number(),
|
|
62
|
+
limit: z.number(),
|
|
63
|
+
offset: z.number(),
|
|
64
|
+
}),
|
|
65
|
+
},
|
|
66
|
+
},
|
|
67
|
+
description: "Paginated fired conversions, newest first",
|
|
68
|
+
},
|
|
69
|
+
},
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
const statsSchema = z.object({
|
|
73
|
+
/** Per-definition firing counts (30d window + lifetime). */
|
|
74
|
+
definitions: z.array(
|
|
75
|
+
z.object({
|
|
76
|
+
definitionId: z.string(),
|
|
77
|
+
count30d: z.number(),
|
|
78
|
+
countLifetime: z.number(),
|
|
79
|
+
lastFiredAt: z.string().nullable(),
|
|
80
|
+
}),
|
|
81
|
+
),
|
|
82
|
+
/** Per-destination delivery health across all dispatches. */
|
|
83
|
+
destinations: z.array(
|
|
84
|
+
z.object({
|
|
85
|
+
destinationId: z.string(),
|
|
86
|
+
pending: z.number(),
|
|
87
|
+
delivered: z.number(),
|
|
88
|
+
failed: z.number(),
|
|
89
|
+
}),
|
|
90
|
+
),
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
const statsRoute = createRoute({
|
|
94
|
+
method: "get",
|
|
95
|
+
path: "/stats",
|
|
96
|
+
tags: ["Admin"],
|
|
97
|
+
summary: "Conversion firing counts + destination delivery health",
|
|
98
|
+
responses: {
|
|
99
|
+
200: {
|
|
100
|
+
content: { "application/json": { schema: statsSchema } },
|
|
101
|
+
description: "Definition and destination summaries",
|
|
102
|
+
},
|
|
103
|
+
},
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
export const adminConversionsRouter = new OpenAPIHono<AppEnv>()
|
|
107
|
+
.openapi(listRoute, async (c) => {
|
|
108
|
+
const { db } = c.get("container");
|
|
109
|
+
const query = c.req.valid("query");
|
|
110
|
+
|
|
111
|
+
const filters = [
|
|
112
|
+
...(query.definitionId
|
|
113
|
+
? [eq(conversions.definitionId, query.definitionId)]
|
|
114
|
+
: []),
|
|
115
|
+
...(query.journeyId
|
|
116
|
+
? [eq(conversions.scopeJourneyId, query.journeyId)]
|
|
117
|
+
: []),
|
|
118
|
+
...(query.campaignId
|
|
119
|
+
? [eq(conversions.scopeCampaignId, query.campaignId)]
|
|
120
|
+
: []),
|
|
121
|
+
...(query.dispatchStatus
|
|
122
|
+
? [
|
|
123
|
+
sql`exists (select 1 from conversion_dispatches d where d.conversion_id = ${conversions.id} and d.status = ${query.dispatchStatus})`,
|
|
124
|
+
]
|
|
125
|
+
: []),
|
|
126
|
+
];
|
|
127
|
+
const where = filters.length > 0 ? and(...filters) : undefined;
|
|
128
|
+
|
|
129
|
+
const sortColumns = {
|
|
130
|
+
occurredAt: conversions.occurredAt,
|
|
131
|
+
value: conversions.value,
|
|
132
|
+
definitionId: conversions.definitionId,
|
|
133
|
+
} as const;
|
|
134
|
+
const sortCol = sortColumns[query.sort];
|
|
135
|
+
|
|
136
|
+
const [rows, totalRows] = await Promise.all([
|
|
137
|
+
db
|
|
138
|
+
.select({ conversion: conversions, contactEmail: contacts.email })
|
|
139
|
+
.from(conversions)
|
|
140
|
+
.leftJoin(contacts, eq(conversions.contactId, contacts.id))
|
|
141
|
+
.where(where)
|
|
142
|
+
.orderBy(
|
|
143
|
+
query.dir === "asc" ? asc(sortCol) : desc(sortCol),
|
|
144
|
+
desc(conversions.occurredAt),
|
|
145
|
+
)
|
|
146
|
+
.limit(query.limit)
|
|
147
|
+
.offset(query.offset),
|
|
148
|
+
db.select({ total: count() }).from(conversions).where(where),
|
|
149
|
+
]);
|
|
150
|
+
|
|
151
|
+
// One IN query for the page's dispatches beats a per-row join explosion.
|
|
152
|
+
const ids = rows.map((r) => r.conversion.id);
|
|
153
|
+
const dispatchRows =
|
|
154
|
+
ids.length > 0
|
|
155
|
+
? await db
|
|
156
|
+
.select()
|
|
157
|
+
.from(conversionDispatches)
|
|
158
|
+
.where(
|
|
159
|
+
sql`${conversionDispatches.conversionId} in (${sql.join(
|
|
160
|
+
ids.map((id) => sql`${id}`),
|
|
161
|
+
sql`, `,
|
|
162
|
+
)})`,
|
|
163
|
+
)
|
|
164
|
+
: [];
|
|
165
|
+
const dispatchesByConversion = new Map<
|
|
166
|
+
string,
|
|
167
|
+
(typeof dispatchRows)[number][]
|
|
168
|
+
>();
|
|
169
|
+
for (const d of dispatchRows) {
|
|
170
|
+
const list = dispatchesByConversion.get(d.conversionId) ?? [];
|
|
171
|
+
list.push(d);
|
|
172
|
+
dispatchesByConversion.set(d.conversionId, list);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
return c.json(
|
|
176
|
+
{
|
|
177
|
+
conversions: rows.map(({ conversion, contactEmail }) => ({
|
|
178
|
+
id: conversion.id,
|
|
179
|
+
definitionId: conversion.definitionId,
|
|
180
|
+
contactId: conversion.contactId,
|
|
181
|
+
contactEmail,
|
|
182
|
+
value: conversion.value,
|
|
183
|
+
currency: conversion.currency,
|
|
184
|
+
scopeJourneyId: conversion.scopeJourneyId,
|
|
185
|
+
scopeCampaignId: conversion.scopeCampaignId,
|
|
186
|
+
occurredAt: conversion.occurredAt.toISOString(),
|
|
187
|
+
dispatches: (dispatchesByConversion.get(conversion.id) ?? []).map(
|
|
188
|
+
(d) => ({
|
|
189
|
+
destinationId: d.destinationId,
|
|
190
|
+
status: d.status,
|
|
191
|
+
attempts: d.attempts,
|
|
192
|
+
lastError: d.lastError,
|
|
193
|
+
deliveredAt: d.deliveredAt?.toISOString() ?? null,
|
|
194
|
+
}),
|
|
195
|
+
),
|
|
196
|
+
})),
|
|
197
|
+
total: totalRows[0]?.total ?? 0,
|
|
198
|
+
limit: query.limit,
|
|
199
|
+
offset: query.offset,
|
|
200
|
+
},
|
|
201
|
+
200,
|
|
202
|
+
);
|
|
203
|
+
})
|
|
204
|
+
.openapi(statsRoute, async (c) => {
|
|
205
|
+
const { db } = c.get("container");
|
|
206
|
+
const thirtyDaysAgo = new Date(
|
|
207
|
+
Date.now() - 30 * 24 * 60 * 60 * 1000,
|
|
208
|
+
).toISOString();
|
|
209
|
+
|
|
210
|
+
const [definitionRows, destinationRows] = await Promise.all([
|
|
211
|
+
db
|
|
212
|
+
.select({
|
|
213
|
+
definitionId: conversions.definitionId,
|
|
214
|
+
count30d: sql<number>`count(*) filter (where ${conversions.occurredAt} >= ${thirtyDaysAgo}::timestamptz)::int`,
|
|
215
|
+
countLifetime: count(),
|
|
216
|
+
lastFiredAt: sql<string | null>`max(${conversions.occurredAt})`,
|
|
217
|
+
})
|
|
218
|
+
.from(conversions)
|
|
219
|
+
.groupBy(conversions.definitionId),
|
|
220
|
+
db
|
|
221
|
+
.select({
|
|
222
|
+
destinationId: conversionDispatches.destinationId,
|
|
223
|
+
pending: sql<number>`count(*) filter (where ${conversionDispatches.status} = 'pending')::int`,
|
|
224
|
+
delivered: sql<number>`count(*) filter (where ${conversionDispatches.status} = 'delivered')::int`,
|
|
225
|
+
failed: sql<number>`count(*) filter (where ${conversionDispatches.status} = 'failed')::int`,
|
|
226
|
+
})
|
|
227
|
+
.from(conversionDispatches)
|
|
228
|
+
.groupBy(conversionDispatches.destinationId),
|
|
229
|
+
]);
|
|
230
|
+
|
|
231
|
+
return c.json(
|
|
232
|
+
{
|
|
233
|
+
definitions: definitionRows
|
|
234
|
+
.map((row) => ({
|
|
235
|
+
definitionId: row.definitionId,
|
|
236
|
+
count30d: row.count30d,
|
|
237
|
+
countLifetime: Number(row.countLifetime),
|
|
238
|
+
lastFiredAt: row.lastFiredAt
|
|
239
|
+
? new Date(row.lastFiredAt).toISOString()
|
|
240
|
+
: null,
|
|
241
|
+
}))
|
|
242
|
+
.sort((a, b) => b.count30d - a.count30d),
|
|
243
|
+
destinations: destinationRows,
|
|
244
|
+
},
|
|
245
|
+
200,
|
|
246
|
+
);
|
|
247
|
+
});
|