@hogsend/engine 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/app.ts +37 -0
- package/src/buckets/check-membership.ts +499 -0
- package/src/buckets/define-bucket.ts +29 -0
- package/src/buckets/registry-singleton.ts +21 -0
- package/src/buckets/registry.ts +62 -0
- package/src/container.ts +50 -2
- package/src/env.ts +10 -0
- package/src/index.ts +40 -1
- package/src/lib/auth.ts +8 -1
- package/src/lib/bucket-emit.ts +107 -0
- package/src/lib/bucket-posthog-sync.ts +63 -0
- package/src/lib/email-service-types.ts +6 -0
- package/src/lib/email.ts +2 -0
- package/src/lib/ingestion.ts +25 -0
- package/src/lib/mailer.ts +9 -1
- package/src/lib/metrics-sql.ts +17 -0
- package/src/lib/studio.ts +105 -0
- package/src/lib/tracked.ts +4 -0
- package/src/middleware/rate-limit.ts +1 -1
- package/src/middleware/require-admin.ts +26 -0
- package/src/routes/admin/buckets.ts +464 -0
- package/src/routes/admin/emails.ts +155 -9
- package/src/routes/admin/index.ts +10 -2
- package/src/routes/admin/metrics.ts +286 -23
- package/src/routes/admin/reporting.ts +369 -0
- package/src/routes/admin/suppressions.ts +99 -0
- package/src/routes/admin/templates.ts +298 -0
- package/src/worker.ts +35 -0
- package/src/workflows/bucket-backfill.ts +556 -0
- package/src/workflows/bucket-reconcile.ts +721 -0
|
@@ -6,7 +6,20 @@ import {
|
|
|
6
6
|
trackedLinks,
|
|
7
7
|
} from "@hogsend/db";
|
|
8
8
|
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
9
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
and,
|
|
11
|
+
asc,
|
|
12
|
+
count,
|
|
13
|
+
desc,
|
|
14
|
+
eq,
|
|
15
|
+
getTableColumns,
|
|
16
|
+
gte,
|
|
17
|
+
inArray,
|
|
18
|
+
isNotNull,
|
|
19
|
+
isNull,
|
|
20
|
+
lte,
|
|
21
|
+
or,
|
|
22
|
+
} from "drizzle-orm";
|
|
10
23
|
import type { AppEnv } from "../../app.js";
|
|
11
24
|
|
|
12
25
|
const emailSchema = z.object({
|
|
@@ -19,6 +32,8 @@ const emailSchema = z.object({
|
|
|
19
32
|
subject: z.string(),
|
|
20
33
|
category: z.string().nullable(),
|
|
21
34
|
status: z.string(),
|
|
35
|
+
userId: z.string().nullable(),
|
|
36
|
+
journeyId: z.string().nullable(),
|
|
22
37
|
sentAt: z.string().nullable(),
|
|
23
38
|
deliveredAt: z.string().nullable(),
|
|
24
39
|
openedAt: z.string().nullable(),
|
|
@@ -29,6 +44,14 @@ const emailSchema = z.object({
|
|
|
29
44
|
updatedAt: z.string(),
|
|
30
45
|
});
|
|
31
46
|
|
|
47
|
+
const eventSchema = z.object({
|
|
48
|
+
type: z.string(),
|
|
49
|
+
timestamp: z.string(),
|
|
50
|
+
url: z.string().optional(),
|
|
51
|
+
ipAddress: z.string().nullable().optional(),
|
|
52
|
+
userAgent: z.string().nullable().optional(),
|
|
53
|
+
});
|
|
54
|
+
|
|
32
55
|
const trackedLinkSchema = z.object({
|
|
33
56
|
id: z.string(),
|
|
34
57
|
originalUrl: z.string(),
|
|
@@ -54,7 +77,13 @@ const journeyContextSchema = z
|
|
|
54
77
|
|
|
55
78
|
import { errorSchema } from "../../lib/schemas.js";
|
|
56
79
|
|
|
57
|
-
function serializeEmail(
|
|
80
|
+
function serializeEmail(
|
|
81
|
+
row: typeof emailSends.$inferSelect,
|
|
82
|
+
identity: { userId: string | null; journeyId: string | null } = {
|
|
83
|
+
userId: null,
|
|
84
|
+
journeyId: null,
|
|
85
|
+
},
|
|
86
|
+
) {
|
|
58
87
|
return {
|
|
59
88
|
id: row.id,
|
|
60
89
|
journeyStateId: row.journeyStateId,
|
|
@@ -65,6 +94,8 @@ function serializeEmail(row: typeof emailSends.$inferSelect) {
|
|
|
65
94
|
subject: row.subject,
|
|
66
95
|
category: row.category,
|
|
67
96
|
status: row.status,
|
|
97
|
+
userId: identity.userId,
|
|
98
|
+
journeyId: identity.journeyId,
|
|
68
99
|
sentAt: row.sentAt?.toISOString() ?? null,
|
|
69
100
|
deliveredAt: row.deliveredAt?.toISOString() ?? null,
|
|
70
101
|
openedAt: row.openedAt?.toISOString() ?? null,
|
|
@@ -100,6 +131,16 @@ const listRoute = createRoute({
|
|
|
100
131
|
"failed",
|
|
101
132
|
])
|
|
102
133
|
.optional(),
|
|
134
|
+
journeyId: z.string().optional(),
|
|
135
|
+
userId: z.string().optional(),
|
|
136
|
+
category: z.string().optional(),
|
|
137
|
+
engagement: z
|
|
138
|
+
.enum(["opened", "clicked", "bounced", "complained"])
|
|
139
|
+
.optional(),
|
|
140
|
+
sort: z
|
|
141
|
+
.enum(["createdAt", "sentAt", "openedAt", "clickedAt"])
|
|
142
|
+
.default("createdAt"),
|
|
143
|
+
order: z.enum(["asc", "desc"]).default("desc"),
|
|
103
144
|
from: z.string().datetime().optional(),
|
|
104
145
|
to: z.string().datetime().optional(),
|
|
105
146
|
}),
|
|
@@ -135,6 +176,7 @@ const getRoute = createRoute({
|
|
|
135
176
|
"application/json": {
|
|
136
177
|
schema: z.object({
|
|
137
178
|
email: emailSchema,
|
|
179
|
+
events: z.array(eventSchema),
|
|
138
180
|
trackedLinks: z.array(trackedLinkSchema),
|
|
139
181
|
journeyContext: journeyContextSchema,
|
|
140
182
|
}),
|
|
@@ -188,32 +230,93 @@ async function fetchTrackedLinksWithClicks(db: Database, emailSendId: string) {
|
|
|
188
230
|
export const emailsRouter = new OpenAPIHono<AppEnv>()
|
|
189
231
|
.openapi(listRoute, async (c) => {
|
|
190
232
|
const { db } = c.get("container");
|
|
191
|
-
const {
|
|
192
|
-
|
|
233
|
+
const {
|
|
234
|
+
limit,
|
|
235
|
+
offset,
|
|
236
|
+
toEmail,
|
|
237
|
+
templateKey,
|
|
238
|
+
status,
|
|
239
|
+
journeyId,
|
|
240
|
+
userId,
|
|
241
|
+
category,
|
|
242
|
+
engagement,
|
|
243
|
+
sort,
|
|
244
|
+
order,
|
|
245
|
+
from,
|
|
246
|
+
to,
|
|
247
|
+
} = c.req.valid("query");
|
|
248
|
+
|
|
249
|
+
const engagementColumn = {
|
|
250
|
+
opened: emailSends.openedAt,
|
|
251
|
+
clicked: emailSends.clickedAt,
|
|
252
|
+
bounced: emailSends.bouncedAt,
|
|
253
|
+
complained: emailSends.complainedAt,
|
|
254
|
+
} as const;
|
|
255
|
+
|
|
256
|
+
const sortColumn = {
|
|
257
|
+
createdAt: emailSends.createdAt,
|
|
258
|
+
sentAt: emailSends.sentAt,
|
|
259
|
+
openedAt: emailSends.openedAt,
|
|
260
|
+
clickedAt: emailSends.clickedAt,
|
|
261
|
+
} as const;
|
|
193
262
|
|
|
194
263
|
const conditions = [];
|
|
195
264
|
if (toEmail) conditions.push(eq(emailSends.toEmail, toEmail));
|
|
196
265
|
if (templateKey) conditions.push(eq(emailSends.templateKey, templateKey));
|
|
197
266
|
if (status) conditions.push(eq(emailSends.status, status));
|
|
267
|
+
if (category) conditions.push(eq(emailSends.category, category));
|
|
268
|
+
if (journeyId) conditions.push(eq(journeyStates.journeyId, journeyId));
|
|
269
|
+
// Match the denormalized identity OR the journey-state join, so journeyless
|
|
270
|
+
// sends (which only carry the denormalized userId) are still filterable.
|
|
271
|
+
if (userId) {
|
|
272
|
+
conditions.push(
|
|
273
|
+
or(eq(emailSends.userId, userId), eq(journeyStates.userId, userId)),
|
|
274
|
+
);
|
|
275
|
+
}
|
|
276
|
+
if (engagement) conditions.push(isNotNull(engagementColumn[engagement]));
|
|
198
277
|
if (from) conditions.push(gte(emailSends.createdAt, new Date(from)));
|
|
199
278
|
if (to) conditions.push(lte(emailSends.createdAt, new Date(to)));
|
|
200
279
|
|
|
201
280
|
const where = conditions.length > 0 ? and(...conditions) : undefined;
|
|
202
281
|
|
|
282
|
+
const orderBy =
|
|
283
|
+
order === "asc" ? asc(sortColumn[sort]) : desc(sortColumn[sort]);
|
|
284
|
+
|
|
285
|
+
const joinCondition = and(
|
|
286
|
+
eq(emailSends.journeyStateId, journeyStates.id),
|
|
287
|
+
isNull(journeyStates.deletedAt),
|
|
288
|
+
);
|
|
289
|
+
|
|
203
290
|
const [rows, totalRows] = await Promise.all([
|
|
204
291
|
db
|
|
205
|
-
.select(
|
|
292
|
+
.select({
|
|
293
|
+
...getTableColumns(emailSends),
|
|
294
|
+
identityUserId: journeyStates.userId,
|
|
295
|
+
identityJourneyId: journeyStates.journeyId,
|
|
296
|
+
})
|
|
206
297
|
.from(emailSends)
|
|
298
|
+
.leftJoin(journeyStates, joinCondition)
|
|
207
299
|
.where(where)
|
|
208
|
-
.orderBy(
|
|
300
|
+
.orderBy(orderBy)
|
|
209
301
|
.limit(limit)
|
|
210
302
|
.offset(offset),
|
|
211
|
-
db
|
|
303
|
+
db
|
|
304
|
+
.select({ count: count() })
|
|
305
|
+
.from(emailSends)
|
|
306
|
+
.leftJoin(journeyStates, joinCondition)
|
|
307
|
+
.where(where),
|
|
212
308
|
]);
|
|
213
309
|
|
|
214
310
|
return c.json(
|
|
215
311
|
{
|
|
216
|
-
emails: rows.map(
|
|
312
|
+
emails: rows.map(({ identityUserId, identityJourneyId, ...row }) =>
|
|
313
|
+
// Prefer the denormalized identity on the send row; fall back to the
|
|
314
|
+
// journey-state join (covers rows written before denormalization).
|
|
315
|
+
serializeEmail(row, {
|
|
316
|
+
userId: row.userId ?? identityUserId,
|
|
317
|
+
journeyId: identityJourneyId,
|
|
318
|
+
}),
|
|
319
|
+
),
|
|
217
320
|
total: totalRows[0]?.count ?? 0,
|
|
218
321
|
limit,
|
|
219
322
|
offset,
|
|
@@ -258,9 +361,52 @@ export const emailsRouter = new OpenAPIHono<AppEnv>()
|
|
|
258
361
|
: Promise.resolve(null),
|
|
259
362
|
]);
|
|
260
363
|
|
|
364
|
+
const events: z.infer<typeof eventSchema>[] = [];
|
|
365
|
+
if (row.createdAt)
|
|
366
|
+
events.push({ type: "queued", timestamp: row.createdAt.toISOString() });
|
|
367
|
+
if (row.sentAt)
|
|
368
|
+
events.push({ type: "sent", timestamp: row.sentAt.toISOString() });
|
|
369
|
+
if (row.deliveredAt)
|
|
370
|
+
events.push({
|
|
371
|
+
type: "delivered",
|
|
372
|
+
timestamp: row.deliveredAt.toISOString(),
|
|
373
|
+
});
|
|
374
|
+
if (row.openedAt)
|
|
375
|
+
events.push({ type: "opened", timestamp: row.openedAt.toISOString() });
|
|
376
|
+
if (row.bouncedAt)
|
|
377
|
+
events.push({ type: "bounced", timestamp: row.bouncedAt.toISOString() });
|
|
378
|
+
if (row.complainedAt)
|
|
379
|
+
events.push({
|
|
380
|
+
type: "complained",
|
|
381
|
+
timestamp: row.complainedAt.toISOString(),
|
|
382
|
+
});
|
|
383
|
+
if (row.status === "failed" && row.updatedAt)
|
|
384
|
+
events.push({ type: "failed", timestamp: row.updatedAt.toISOString() });
|
|
385
|
+
|
|
386
|
+
for (const link of links) {
|
|
387
|
+
for (const click of link.clicks) {
|
|
388
|
+
events.push({
|
|
389
|
+
type: "clicked",
|
|
390
|
+
timestamp: click.clickedAt,
|
|
391
|
+
url: link.originalUrl,
|
|
392
|
+
ipAddress: click.ipAddress,
|
|
393
|
+
userAgent: click.userAgent,
|
|
394
|
+
});
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
events.sort(
|
|
399
|
+
(a, b) =>
|
|
400
|
+
new Date(a.timestamp).getTime() - new Date(b.timestamp).getTime(),
|
|
401
|
+
);
|
|
402
|
+
|
|
261
403
|
return c.json(
|
|
262
404
|
{
|
|
263
|
-
email: serializeEmail(row
|
|
405
|
+
email: serializeEmail(row, {
|
|
406
|
+
userId: row.userId ?? journeyContext?.userId ?? null,
|
|
407
|
+
journeyId: journeyContext?.journeyId ?? null,
|
|
408
|
+
}),
|
|
409
|
+
events,
|
|
264
410
|
trackedLinks: links,
|
|
265
411
|
journeyContext,
|
|
266
412
|
},
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { OpenAPIHono } from "@hono/zod-openapi";
|
|
2
2
|
import type { AppEnv } from "../../app.js";
|
|
3
|
-
import { requireApiKey } from "../../middleware/api-key.js";
|
|
4
3
|
import { auditMiddleware } from "../../middleware/audit.js";
|
|
5
4
|
import { rateLimit } from "../../middleware/rate-limit.js";
|
|
5
|
+
import { requireAdmin } from "../../middleware/require-admin.js";
|
|
6
6
|
import { alertsRouter } from "./alerts.js";
|
|
7
7
|
import { apiKeysRouter } from "./api-keys.js";
|
|
8
8
|
import { auditLogsRouter } from "./audit-logs.js";
|
|
9
|
+
import { bucketsRouter } from "./buckets.js";
|
|
9
10
|
import { bulkRouter } from "./bulk.js";
|
|
10
11
|
import { contactsRouter } from "./contacts.js";
|
|
11
12
|
import { dlqRouter } from "./dlq.js";
|
|
@@ -15,10 +16,13 @@ import { journeyLogsRouter } from "./journey-logs.js";
|
|
|
15
16
|
import { journeysRouter } from "./journeys.js";
|
|
16
17
|
import { metricsRouter } from "./metrics.js";
|
|
17
18
|
import { preferencesRouter } from "./preferences.js";
|
|
19
|
+
import { reportingRouter } from "./reporting.js";
|
|
20
|
+
import { suppressionsRouter } from "./suppressions.js";
|
|
21
|
+
import { templatesRouter } from "./templates.js";
|
|
18
22
|
import { timelineRouter } from "./timeline.js";
|
|
19
23
|
|
|
20
24
|
export const adminRouter = new OpenAPIHono<AppEnv>();
|
|
21
|
-
adminRouter.use("*",
|
|
25
|
+
adminRouter.use("*", requireAdmin);
|
|
22
26
|
adminRouter.use("*", rateLimit);
|
|
23
27
|
adminRouter.use("*", auditMiddleware);
|
|
24
28
|
adminRouter.route("/", bulkRouter);
|
|
@@ -26,10 +30,14 @@ adminRouter.route("/contacts", contactsRouter);
|
|
|
26
30
|
adminRouter.route("/contacts", preferencesRouter);
|
|
27
31
|
adminRouter.route("/contacts", timelineRouter);
|
|
28
32
|
adminRouter.route("/journeys", journeysRouter);
|
|
33
|
+
adminRouter.route("/buckets", bucketsRouter);
|
|
29
34
|
adminRouter.route("/events", eventsRouter);
|
|
30
35
|
adminRouter.route("/emails", emailsRouter);
|
|
31
36
|
adminRouter.route("/journey-logs", journeyLogsRouter);
|
|
32
37
|
adminRouter.route("/metrics", metricsRouter);
|
|
38
|
+
adminRouter.route("/reporting", reportingRouter);
|
|
39
|
+
adminRouter.route("/templates", templatesRouter);
|
|
40
|
+
adminRouter.route("/suppressions", suppressionsRouter);
|
|
33
41
|
adminRouter.route("/api-keys", apiKeysRouter);
|
|
34
42
|
adminRouter.route("/audit-logs", auditLogsRouter);
|
|
35
43
|
adminRouter.route("/alerts", alertsRouter);
|