@hogsend/engine 0.43.0 → 0.44.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/package.json +10 -9
  2. package/src/container.ts +229 -2
  3. package/src/destinations/define-destination.ts +7 -3
  4. package/src/destinations/presets/posthog.ts +31 -0
  5. package/src/index.ts +99 -0
  6. package/src/journeys/execute-journey-run.ts +93 -0
  7. package/src/journeys/journey-context.ts +4 -0
  8. package/src/lib/attribution-backfill.ts +285 -0
  9. package/src/lib/attribution.ts +239 -0
  10. package/src/lib/contacts.ts +71 -0
  11. package/src/lib/conversion-dispatch.ts +326 -0
  12. package/src/lib/conversions.ts +197 -0
  13. package/src/lib/crm-deals.ts +320 -0
  14. package/src/lib/crm-ingest.ts +341 -0
  15. package/src/lib/crm-provider-registry.ts +41 -0
  16. package/src/lib/crm-registry-singleton.ts +22 -0
  17. package/src/lib/deal-money-events.ts +85 -0
  18. package/src/lib/email-service-types.ts +16 -1
  19. package/src/lib/event-names.ts +1 -1
  20. package/src/lib/funnel-progress.ts +76 -0
  21. package/src/lib/funnel-registry.ts +108 -0
  22. package/src/lib/funnel-transitions.ts +280 -0
  23. package/src/lib/holdout.ts +62 -0
  24. package/src/lib/ingestion.ts +218 -9
  25. package/src/lib/lift-stats.ts +125 -0
  26. package/src/lib/mailer.ts +1 -0
  27. package/src/lib/operator.ts +31 -0
  28. package/src/lib/outbound.ts +51 -0
  29. package/src/lib/revenue.ts +116 -0
  30. package/src/lib/sms-service-types.ts +7 -1
  31. package/src/lib/sms-tracked.ts +35 -0
  32. package/src/lib/tracked.ts +40 -0
  33. package/src/lib/tracking-events.ts +26 -0
  34. package/src/lib/webhook-signing.ts +7 -0
  35. package/src/routes/admin/attribution.ts +414 -0
  36. package/src/routes/admin/contacts.ts +71 -11
  37. package/src/routes/admin/conversions.ts +247 -0
  38. package/src/routes/admin/deals.ts +472 -0
  39. package/src/routes/admin/funnels.ts +228 -0
  40. package/src/routes/admin/index.ts +8 -0
  41. package/src/routes/admin/journeys.ts +158 -1
  42. package/src/routes/admin/readiness.ts +97 -2
  43. package/src/routes/admin/templates.ts +28 -1
  44. package/src/routes/events/index.ts +12 -0
  45. package/src/routes/tracking/click-pipeline.ts +26 -18
  46. package/src/routes/webhooks/crm-provider.ts +117 -0
  47. package/src/routes/webhooks/index.ts +5 -1
  48. package/src/sources/define-contact-source.ts +159 -0
  49. package/src/sources/registry.ts +71 -0
  50. package/src/sources/webhook-source.ts +109 -0
  51. package/src/worker.ts +4 -0
  52. package/src/workflows/crm-reconcile.ts +129 -0
  53. package/src/workflows/dispatch-conversion.ts +31 -0
  54. package/src/workflows/send-campaign.ts +5 -0
@@ -10,6 +10,7 @@ import {
10
10
  serializePrefs,
11
11
  } from "../../lib/contacts.js";
12
12
  import { emitOutbound } from "../../lib/outbound.js";
13
+ import { getContactRevenue, revenueExcludedEvents } from "../../lib/revenue.js";
13
14
 
14
15
  const contactSchema = z.object({
15
16
  id: z.string(),
@@ -35,6 +36,19 @@ const preferencesSchema = z
35
36
  })
36
37
  .nullable();
37
38
 
39
+ // Revenue rollup over the contact's valued events (`user_events.value`),
40
+ // grouped per currency — never summed across currencies.
41
+ const revenueSchema = z.object({
42
+ totals: z.array(
43
+ z.object({
44
+ currency: z.string().nullable(),
45
+ total: z.number(),
46
+ count: z.number(),
47
+ }),
48
+ ),
49
+ lastValuedAt: z.string().nullable(),
50
+ });
51
+
38
52
  const listRoute = createRoute({
39
53
  method: "get",
40
54
  path: "/",
@@ -45,6 +59,11 @@ const listRoute = createRoute({
45
59
  limit: z.coerce.number().min(1).max(100).default(50),
46
60
  offset: z.coerce.number().min(0).default(0),
47
61
  search: z.string().optional(),
62
+ // Long-tail value filters (plan §4b.3): the "find my value customers"
63
+ // query surface.
64
+ minRevenue: z.coerce.number().optional(),
65
+ // Plain string: valid stages are the deployment's configured ladder.
66
+ dealStage: z.string().optional(),
48
67
  }),
49
68
  },
50
69
  responses: {
@@ -79,10 +98,11 @@ const getRoute = createRoute({
79
98
  schema: z.object({
80
99
  contact: contactSchema,
81
100
  preferences: preferencesSchema,
101
+ revenue: revenueSchema,
82
102
  }),
83
103
  },
84
104
  },
85
- description: "Contact with preferences",
105
+ description: "Contact with preferences and revenue rollup",
86
106
  },
87
107
  404: {
88
108
  content: {
@@ -209,13 +229,45 @@ const serializeContact = (row: typeof contacts.$inferSelect) =>
209
229
  export const contactsRouter = new OpenAPIHono<AppEnv>()
210
230
  .openapi(listRoute, async (c) => {
211
231
  const { db } = c.get("container");
212
- const { limit, offset, search } = c.req.valid("query");
232
+ const { limit, offset, search, minRevenue, dealStage } =
233
+ c.req.valid("query");
213
234
 
214
235
  const searchFilter = search ? contactSearchFilter(search) : undefined;
215
236
 
216
- const where = searchFilter
217
- ? and(searchFilter, isNull(contacts.deletedAt))
218
- : isNull(contacts.deletedAt);
237
+ // Valued events are keyed by the contact's canonical event key
238
+ // (external_id ?? anonymous_id ?? id) — same precedence ingestEvent
239
+ // resolves. Served by the partial user_events_valued_user_idx.
240
+ // Exclusions come from lib/revenue.ts (static machinery events + funnel
241
+ // milestone triggers + the browser trust gate): one deal's value rides
242
+ // several rows, and pk_-minted values are forgeable.
243
+ const revenueFilter =
244
+ minRevenue !== undefined
245
+ ? sql`(
246
+ select coalesce(sum(ue.value), 0)
247
+ from user_events ue
248
+ where ue.user_id = coalesce(${contacts.externalId}, ${contacts.anonymousId}, ${contacts.id}::text)
249
+ and ue.value is not null
250
+ and ue.event not in (${sql.join(
251
+ revenueExcludedEvents().map((e) => sql`${e}`),
252
+ sql`, `,
253
+ )})
254
+ and (ue.source is null or ue.source <> 'inapp')
255
+ ) >= ${minRevenue}`
256
+ : undefined;
257
+ const dealStageFilter = dealStage
258
+ ? sql`exists (
259
+ select 1 from deals d
260
+ where d.contact_id = ${contacts.id}
261
+ and d.canonical_stage = ${dealStage}
262
+ )`
263
+ : undefined;
264
+
265
+ const where = and(
266
+ isNull(contacts.deletedAt),
267
+ ...(searchFilter ? [searchFilter] : []),
268
+ ...(revenueFilter ? [revenueFilter] : []),
269
+ ...(dealStageFilter ? [dealStageFilter] : []),
270
+ );
219
271
 
220
272
  const [rows, totalRows] = await Promise.all([
221
273
  db
@@ -249,16 +301,24 @@ export const contactsRouter = new OpenAPIHono<AppEnv>()
249
301
 
250
302
  // email_preferences.user_id uses external_id when present, else the contact
251
303
  // uuid as the deterministic fallback (risk 10 — email-only contacts).
252
- const prefRows = await db
253
- .select()
254
- .from(emailPreferences)
255
- .where(eq(emailPreferences.userId, contact.externalId ?? contact.id))
256
- .limit(1);
304
+ const [prefRows, revenue] = await Promise.all([
305
+ db
306
+ .select()
307
+ .from(emailPreferences)
308
+ .where(eq(emailPreferences.userId, contact.externalId ?? contact.id))
309
+ .limit(1),
310
+ // Valued events are keyed by the contact's canonical event key — the
311
+ // same precedence ingestEvent resolves (`external ?? anon ?? id`).
312
+ getContactRevenue({
313
+ db,
314
+ key: contact.externalId ?? contact.anonymousId ?? contact.id,
315
+ }),
316
+ ]);
257
317
 
258
318
  const prefs = prefRows[0] ? serializePrefs(prefRows[0]) : null;
259
319
 
260
320
  return c.json(
261
- { contact: serializeContact(contact), preferences: prefs },
321
+ { contact: serializeContact(contact), preferences: prefs, revenue },
262
322
  200,
263
323
  );
264
324
  })
@@ -0,0 +1,247 @@
1
+ import { contacts, conversionDispatches, conversions } from "@hogsend/db";
2
+ import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
3
+ import { and, asc, count, desc, eq, sql } from "drizzle-orm";
4
+ import type { AppEnv } from "../../app.js";
5
+
6
+ /**
7
+ * Admin conversions API (docs/revenue-attribution-plan.md §5b.2) — fired
8
+ * conversion points and their per-destination delivery state. Answers the
9
+ * operator question "did the ad platform actually receive the sale?".
10
+ */
11
+
12
+ const dispatchSchema = z.object({
13
+ destinationId: z.string(),
14
+ status: z.string(),
15
+ attempts: z.number(),
16
+ lastError: z.string().nullable(),
17
+ deliveredAt: z.string().nullable(),
18
+ });
19
+
20
+ const conversionSchema = z.object({
21
+ id: z.string(),
22
+ definitionId: z.string(),
23
+ contactId: z.string(),
24
+ contactEmail: z.string().nullable(),
25
+ value: z.number().nullable(),
26
+ currency: z.string().nullable(),
27
+ /** The definition's declared scope (ConversionMeta.scope), if any. */
28
+ scopeJourneyId: z.string().nullable(),
29
+ scopeCampaignId: z.string().nullable(),
30
+ occurredAt: z.string(),
31
+ dispatches: z.array(dispatchSchema),
32
+ });
33
+
34
+ const listRoute = createRoute({
35
+ method: "get",
36
+ path: "/",
37
+ tags: ["Admin"],
38
+ summary: "List fired conversions with delivery state",
39
+ request: {
40
+ query: z.object({
41
+ definitionId: z.string().optional(),
42
+ /** Filter by the definition's declared scope (ConversionMeta.scope). */
43
+ journeyId: z.string().optional(),
44
+ campaignId: z.string().optional(),
45
+ /** Only conversions with at least one dispatch in this status. */
46
+ dispatchStatus: z.enum(["pending", "delivered", "failed"]).optional(),
47
+ sort: z
48
+ .enum(["occurredAt", "value", "definitionId"])
49
+ .default("occurredAt"),
50
+ dir: z.enum(["asc", "desc"]).default("desc"),
51
+ limit: z.coerce.number().min(1).max(200).default(50),
52
+ offset: z.coerce.number().min(0).default(0),
53
+ }),
54
+ },
55
+ responses: {
56
+ 200: {
57
+ content: {
58
+ "application/json": {
59
+ schema: z.object({
60
+ conversions: z.array(conversionSchema),
61
+ total: z.number(),
62
+ limit: z.number(),
63
+ offset: z.number(),
64
+ }),
65
+ },
66
+ },
67
+ description: "Paginated fired conversions, newest first",
68
+ },
69
+ },
70
+ });
71
+
72
+ const statsSchema = z.object({
73
+ /** Per-definition firing counts (30d window + lifetime). */
74
+ definitions: z.array(
75
+ z.object({
76
+ definitionId: z.string(),
77
+ count30d: z.number(),
78
+ countLifetime: z.number(),
79
+ lastFiredAt: z.string().nullable(),
80
+ }),
81
+ ),
82
+ /** Per-destination delivery health across all dispatches. */
83
+ destinations: z.array(
84
+ z.object({
85
+ destinationId: z.string(),
86
+ pending: z.number(),
87
+ delivered: z.number(),
88
+ failed: z.number(),
89
+ }),
90
+ ),
91
+ });
92
+
93
+ const statsRoute = createRoute({
94
+ method: "get",
95
+ path: "/stats",
96
+ tags: ["Admin"],
97
+ summary: "Conversion firing counts + destination delivery health",
98
+ responses: {
99
+ 200: {
100
+ content: { "application/json": { schema: statsSchema } },
101
+ description: "Definition and destination summaries",
102
+ },
103
+ },
104
+ });
105
+
106
+ export const adminConversionsRouter = new OpenAPIHono<AppEnv>()
107
+ .openapi(listRoute, async (c) => {
108
+ const { db } = c.get("container");
109
+ const query = c.req.valid("query");
110
+
111
+ const filters = [
112
+ ...(query.definitionId
113
+ ? [eq(conversions.definitionId, query.definitionId)]
114
+ : []),
115
+ ...(query.journeyId
116
+ ? [eq(conversions.scopeJourneyId, query.journeyId)]
117
+ : []),
118
+ ...(query.campaignId
119
+ ? [eq(conversions.scopeCampaignId, query.campaignId)]
120
+ : []),
121
+ ...(query.dispatchStatus
122
+ ? [
123
+ sql`exists (select 1 from conversion_dispatches d where d.conversion_id = ${conversions.id} and d.status = ${query.dispatchStatus})`,
124
+ ]
125
+ : []),
126
+ ];
127
+ const where = filters.length > 0 ? and(...filters) : undefined;
128
+
129
+ const sortColumns = {
130
+ occurredAt: conversions.occurredAt,
131
+ value: conversions.value,
132
+ definitionId: conversions.definitionId,
133
+ } as const;
134
+ const sortCol = sortColumns[query.sort];
135
+
136
+ const [rows, totalRows] = await Promise.all([
137
+ db
138
+ .select({ conversion: conversions, contactEmail: contacts.email })
139
+ .from(conversions)
140
+ .leftJoin(contacts, eq(conversions.contactId, contacts.id))
141
+ .where(where)
142
+ .orderBy(
143
+ query.dir === "asc" ? asc(sortCol) : desc(sortCol),
144
+ desc(conversions.occurredAt),
145
+ )
146
+ .limit(query.limit)
147
+ .offset(query.offset),
148
+ db.select({ total: count() }).from(conversions).where(where),
149
+ ]);
150
+
151
+ // One IN query for the page's dispatches beats a per-row join explosion.
152
+ const ids = rows.map((r) => r.conversion.id);
153
+ const dispatchRows =
154
+ ids.length > 0
155
+ ? await db
156
+ .select()
157
+ .from(conversionDispatches)
158
+ .where(
159
+ sql`${conversionDispatches.conversionId} in (${sql.join(
160
+ ids.map((id) => sql`${id}`),
161
+ sql`, `,
162
+ )})`,
163
+ )
164
+ : [];
165
+ const dispatchesByConversion = new Map<
166
+ string,
167
+ (typeof dispatchRows)[number][]
168
+ >();
169
+ for (const d of dispatchRows) {
170
+ const list = dispatchesByConversion.get(d.conversionId) ?? [];
171
+ list.push(d);
172
+ dispatchesByConversion.set(d.conversionId, list);
173
+ }
174
+
175
+ return c.json(
176
+ {
177
+ conversions: rows.map(({ conversion, contactEmail }) => ({
178
+ id: conversion.id,
179
+ definitionId: conversion.definitionId,
180
+ contactId: conversion.contactId,
181
+ contactEmail,
182
+ value: conversion.value,
183
+ currency: conversion.currency,
184
+ scopeJourneyId: conversion.scopeJourneyId,
185
+ scopeCampaignId: conversion.scopeCampaignId,
186
+ occurredAt: conversion.occurredAt.toISOString(),
187
+ dispatches: (dispatchesByConversion.get(conversion.id) ?? []).map(
188
+ (d) => ({
189
+ destinationId: d.destinationId,
190
+ status: d.status,
191
+ attempts: d.attempts,
192
+ lastError: d.lastError,
193
+ deliveredAt: d.deliveredAt?.toISOString() ?? null,
194
+ }),
195
+ ),
196
+ })),
197
+ total: totalRows[0]?.total ?? 0,
198
+ limit: query.limit,
199
+ offset: query.offset,
200
+ },
201
+ 200,
202
+ );
203
+ })
204
+ .openapi(statsRoute, async (c) => {
205
+ const { db } = c.get("container");
206
+ const thirtyDaysAgo = new Date(
207
+ Date.now() - 30 * 24 * 60 * 60 * 1000,
208
+ ).toISOString();
209
+
210
+ const [definitionRows, destinationRows] = await Promise.all([
211
+ db
212
+ .select({
213
+ definitionId: conversions.definitionId,
214
+ count30d: sql<number>`count(*) filter (where ${conversions.occurredAt} >= ${thirtyDaysAgo}::timestamptz)::int`,
215
+ countLifetime: count(),
216
+ lastFiredAt: sql<string | null>`max(${conversions.occurredAt})`,
217
+ })
218
+ .from(conversions)
219
+ .groupBy(conversions.definitionId),
220
+ db
221
+ .select({
222
+ destinationId: conversionDispatches.destinationId,
223
+ pending: sql<number>`count(*) filter (where ${conversionDispatches.status} = 'pending')::int`,
224
+ delivered: sql<number>`count(*) filter (where ${conversionDispatches.status} = 'delivered')::int`,
225
+ failed: sql<number>`count(*) filter (where ${conversionDispatches.status} = 'failed')::int`,
226
+ })
227
+ .from(conversionDispatches)
228
+ .groupBy(conversionDispatches.destinationId),
229
+ ]);
230
+
231
+ return c.json(
232
+ {
233
+ definitions: definitionRows
234
+ .map((row) => ({
235
+ definitionId: row.definitionId,
236
+ count30d: row.count30d,
237
+ countLifetime: Number(row.countLifetime),
238
+ lastFiredAt: row.lastFiredAt
239
+ ? new Date(row.lastFiredAt).toISOString()
240
+ : null,
241
+ }))
242
+ .sort((a, b) => b.count30d - a.count30d),
243
+ destinations: destinationRows,
244
+ },
245
+ 200,
246
+ );
247
+ });