@hogsend/engine 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/buckets/check-membership.ts +34 -15
- package/src/container.ts +33 -0
- package/src/env.ts +4 -0
- package/src/index.ts +13 -0
- package/src/journeys/journey-context.ts +5 -1
- package/src/lib/boot.ts +1 -1
- package/src/lib/bucket-emit.ts +2 -2
- package/src/lib/contacts.ts +1083 -18
- package/src/lib/email-service-types.ts +8 -0
- package/src/lib/ingestion.ts +63 -33
- package/src/lib/mailer.ts +1 -0
- package/src/lib/preferences.ts +106 -0
- package/src/lib/tracked.ts +159 -34
- package/src/lib/tracking-events.ts +1 -1
- package/src/lists/define-list.ts +81 -0
- package/src/lists/registry-singleton.ts +39 -0
- package/src/lists/registry.ts +95 -0
- package/src/middleware/api-key.ts +33 -7
- package/src/middleware/rate-limit.ts +73 -49
- package/src/routes/_shared.ts +30 -0
- package/src/routes/admin/api-keys.ts +1 -1
- package/src/routes/admin/bulk.ts +7 -3
- package/src/routes/admin/contacts.ts +66 -57
- package/src/routes/admin/events.ts +65 -0
- package/src/routes/admin/journeys.ts +3 -1
- package/src/routes/admin/preferences.ts +2 -2
- package/src/routes/admin/reporting.ts +3 -3
- package/src/routes/admin/timeline.ts +5 -2
- package/src/routes/campaigns/index.ts +252 -0
- package/src/routes/contacts/index.ts +188 -0
- package/src/routes/email/preferences.ts +27 -3
- package/src/routes/email/unsubscribe.ts +7 -49
- package/src/routes/emails/index.ts +133 -0
- package/src/routes/events/index.ts +119 -0
- package/src/routes/index.ts +52 -2
- package/src/routes/lists/index.ts +222 -0
- package/src/worker.ts +6 -0
- package/src/workflows/bucket-backfill.ts +32 -21
- package/src/workflows/bucket-reconcile.ts +20 -5
- package/src/workflows/import-contacts.ts +28 -20
- package/src/workflows/send-campaign.ts +589 -0
- package/src/routes/ingest.ts +0 -71
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { getTemplateNames, type TemplateName } from "@hogsend/email";
|
|
2
|
+
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
+
import type { AppEnv } from "../../app.js";
|
|
4
|
+
import { resolveRecipient } from "../../lib/contacts.js";
|
|
5
|
+
import { errorSchema } from "../../lib/schemas.js";
|
|
6
|
+
import { hasScope } from "../../middleware/api-key.js";
|
|
7
|
+
import { requireIdentity } from "../_shared.js";
|
|
8
|
+
|
|
9
|
+
const emailRequestSchema = z.object({
|
|
10
|
+
to: z.string().email().optional(),
|
|
11
|
+
userId: z.string().min(1).optional(),
|
|
12
|
+
template: z.string().min(1),
|
|
13
|
+
props: z.record(z.string(), z.unknown()).optional(),
|
|
14
|
+
from: z.string().optional(),
|
|
15
|
+
subject: z.string().optional(),
|
|
16
|
+
replyTo: z.union([z.string(), z.array(z.string())]).optional(),
|
|
17
|
+
category: z.string().optional(),
|
|
18
|
+
skipPreferenceCheck: z.boolean().optional(),
|
|
19
|
+
idempotencyKey: z.string().optional(),
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
const emailResponseSchema = z.object({
|
|
23
|
+
emailSendId: z.string(),
|
|
24
|
+
status: z.enum(["queued", "sent", "suppressed", "unsubscribed", "skipped"]),
|
|
25
|
+
reason: z.string().optional(),
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
const sendRoute = createRoute({
|
|
29
|
+
method: "post",
|
|
30
|
+
path: "/",
|
|
31
|
+
tags: ["Emails"],
|
|
32
|
+
summary: "Send a transactional email",
|
|
33
|
+
description:
|
|
34
|
+
"Resolves a recipient by `to` or `userId`, then sends the named template through the engine-owned tracked mailer (journeyless — link-click + open tracking still applies). `skipPreferenceCheck` requires a full-admin key.",
|
|
35
|
+
request: {
|
|
36
|
+
body: {
|
|
37
|
+
content: {
|
|
38
|
+
"application/json": { schema: emailRequestSchema },
|
|
39
|
+
},
|
|
40
|
+
},
|
|
41
|
+
},
|
|
42
|
+
responses: {
|
|
43
|
+
202: {
|
|
44
|
+
content: {
|
|
45
|
+
"application/json": { schema: emailResponseSchema },
|
|
46
|
+
},
|
|
47
|
+
description: "Email send queued / dispatched",
|
|
48
|
+
},
|
|
49
|
+
400: {
|
|
50
|
+
content: { "application/json": { schema: errorSchema } },
|
|
51
|
+
description: "Missing recipient or unknown template",
|
|
52
|
+
},
|
|
53
|
+
403: {
|
|
54
|
+
content: { "application/json": { schema: errorSchema } },
|
|
55
|
+
description: "skipPreferenceCheck requires a full-admin key",
|
|
56
|
+
},
|
|
57
|
+
404: {
|
|
58
|
+
content: { "application/json": { schema: errorSchema } },
|
|
59
|
+
description: "userId has no resolvable email",
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
export const emailsRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
65
|
+
sendRoute,
|
|
66
|
+
async (c) => {
|
|
67
|
+
const { db, emailService, templates } = c.get("container");
|
|
68
|
+
const apiKey = c.get("apiKey");
|
|
69
|
+
const body = c.req.valid("json");
|
|
70
|
+
|
|
71
|
+
const guard = requireIdentity(
|
|
72
|
+
c,
|
|
73
|
+
{ email: body.to, userId: body.userId },
|
|
74
|
+
{ field: "to" },
|
|
75
|
+
);
|
|
76
|
+
if (guard) return guard;
|
|
77
|
+
|
|
78
|
+
// `skipPreferenceCheck` is a privileged bypass — gate it on full-admin
|
|
79
|
+
// (§2.5). The data-plane prefix guard already required the `ingest` scope.
|
|
80
|
+
if (body.skipPreferenceCheck) {
|
|
81
|
+
if (!apiKey || !hasScope(apiKey.scopes, "full-admin")) {
|
|
82
|
+
return c.json(
|
|
83
|
+
{ error: "skipPreferenceCheck requires a full-admin key" },
|
|
84
|
+
403,
|
|
85
|
+
);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
// Validate the template server-side against the wired registry (§2.5).
|
|
90
|
+
if (!getTemplateNames(templates).includes(body.template as TemplateName)) {
|
|
91
|
+
return c.json({ error: `Unknown template: ${body.template}` }, 400);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const recipient = await resolveRecipient({
|
|
95
|
+
db,
|
|
96
|
+
userId: body.userId,
|
|
97
|
+
email: body.to,
|
|
98
|
+
});
|
|
99
|
+
if (!recipient) {
|
|
100
|
+
return c.json({ error: "No resolvable email for recipient" }, 404);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
// The `Idempotency-Key` header wins over the body field (mirrors /v1/events).
|
|
104
|
+
const headerKey = c.req.header("idempotency-key");
|
|
105
|
+
const idempotencyKey = headerKey ?? body.idempotencyKey;
|
|
106
|
+
|
|
107
|
+
// Journeyless send (no journeyStateId) so §5 tracking runs. The
|
|
108
|
+
// denormalized `userId` on the send row is external_id when present, else
|
|
109
|
+
// the contact id fallback (§2.5).
|
|
110
|
+
const result = await emailService.send({
|
|
111
|
+
template: body.template as TemplateName,
|
|
112
|
+
props: (body.props ?? {}) as never,
|
|
113
|
+
to: recipient.email,
|
|
114
|
+
from: body.from,
|
|
115
|
+
subject: body.subject,
|
|
116
|
+
replyTo: body.replyTo,
|
|
117
|
+
category: body.category,
|
|
118
|
+
userId: recipient.externalId ?? recipient.contactId,
|
|
119
|
+
userEmail: recipient.email,
|
|
120
|
+
skipPreferenceCheck: body.skipPreferenceCheck,
|
|
121
|
+
idempotencyKey,
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
return c.json(
|
|
125
|
+
{
|
|
126
|
+
emailSendId: result.emailSendId,
|
|
127
|
+
status: result.status,
|
|
128
|
+
...(result.reason ? { reason: result.reason } : {}),
|
|
129
|
+
},
|
|
130
|
+
202,
|
|
131
|
+
);
|
|
132
|
+
},
|
|
133
|
+
);
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
2
|
+
import type { AppEnv } from "../../app.js";
|
|
3
|
+
import { ingestEvent } from "../../lib/ingestion.js";
|
|
4
|
+
import { applyListMembership } from "../../lib/preferences.js";
|
|
5
|
+
import { errorSchema } from "../../lib/schemas.js";
|
|
6
|
+
import { listMembershipError, requireIdentity } from "../_shared.js";
|
|
7
|
+
|
|
8
|
+
const eventRequestSchema = z.object({
|
|
9
|
+
name: z.string().min(1),
|
|
10
|
+
email: z.string().email().optional(),
|
|
11
|
+
userId: z.string().min(1).optional(),
|
|
12
|
+
eventProperties: z.record(z.string(), z.unknown()).optional(),
|
|
13
|
+
contactProperties: z.record(z.string(), z.unknown()).optional(),
|
|
14
|
+
lists: z.record(z.string(), z.boolean()).optional(),
|
|
15
|
+
idempotencyKey: z.string().optional(),
|
|
16
|
+
timestamp: z.string().datetime().optional(),
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
const eventResponseSchema = z.object({
|
|
20
|
+
stored: z.boolean(),
|
|
21
|
+
exits: z.array(
|
|
22
|
+
z.object({
|
|
23
|
+
journeyId: z.string(),
|
|
24
|
+
stateId: z.string(),
|
|
25
|
+
exited: z.boolean(),
|
|
26
|
+
}),
|
|
27
|
+
),
|
|
28
|
+
// Present only when the event was durably ingested but the (non-atomic,
|
|
29
|
+
// post-ingest) list-membership write failed. The ingest itself succeeded —
|
|
30
|
+
// surfaced as a warning on a 202, not a 400 that conflates "nothing happened"
|
|
31
|
+
// with "event happened, lists failed" (and would tempt a retry double-ingest).
|
|
32
|
+
listsError: z.string().optional(),
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
const eventRoute = createRoute({
|
|
36
|
+
method: "post",
|
|
37
|
+
path: "/",
|
|
38
|
+
tags: ["Events"],
|
|
39
|
+
summary: "Ingest an event",
|
|
40
|
+
description:
|
|
41
|
+
"Stores the event (with eventProperties), merges contactProperties onto the contact, pushes to Hatchet for journey routing, processes exit conditions, and optionally writes list membership. The `Idempotency-Key` header takes precedence over the body field.",
|
|
42
|
+
request: {
|
|
43
|
+
body: {
|
|
44
|
+
content: {
|
|
45
|
+
"application/json": { schema: eventRequestSchema },
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
responses: {
|
|
50
|
+
202: {
|
|
51
|
+
content: {
|
|
52
|
+
"application/json": { schema: eventResponseSchema },
|
|
53
|
+
},
|
|
54
|
+
description: "Event accepted and dispatched",
|
|
55
|
+
},
|
|
56
|
+
400: {
|
|
57
|
+
content: { "application/json": { schema: errorSchema } },
|
|
58
|
+
description: "Missing recipient or unmanageable list membership",
|
|
59
|
+
},
|
|
60
|
+
},
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
export const eventsRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
64
|
+
eventRoute,
|
|
65
|
+
async (c) => {
|
|
66
|
+
const { db, registry, hatchet, logger } = c.get("container");
|
|
67
|
+
const body = c.req.valid("json");
|
|
68
|
+
|
|
69
|
+
const guard = requireIdentity(c, body);
|
|
70
|
+
if (guard) return guard;
|
|
71
|
+
|
|
72
|
+
// The `Idempotency-Key` header wins over the body field (§2.5).
|
|
73
|
+
const headerKey = c.req.header("idempotency-key");
|
|
74
|
+
const idempotencyKey = headerKey ?? body.idempotencyKey;
|
|
75
|
+
|
|
76
|
+
const result = await ingestEvent({
|
|
77
|
+
db,
|
|
78
|
+
registry,
|
|
79
|
+
hatchet,
|
|
80
|
+
logger,
|
|
81
|
+
event: {
|
|
82
|
+
event: body.name,
|
|
83
|
+
userId: body.userId,
|
|
84
|
+
userEmail: body.email,
|
|
85
|
+
eventProperties: body.eventProperties ?? {},
|
|
86
|
+
contactProperties: body.contactProperties,
|
|
87
|
+
idempotencyKey,
|
|
88
|
+
// §2.5: caller-supplied event time (backfill/replay). The validated ISO
|
|
89
|
+
// string is coerced to a Date inside ingestEvent.
|
|
90
|
+
occurredAt: body.timestamp,
|
|
91
|
+
},
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
// Lists applied AFTER ingest so the contact exists (§2.5 lists ordering).
|
|
95
|
+
// `applyListMembership` writes `email_preferences` independently of the
|
|
96
|
+
// contacts row, so it doesn't race the resolve. Requires a resolvable email.
|
|
97
|
+
//
|
|
98
|
+
// The ingest above is already durable (event stored, journeys dispatched,
|
|
99
|
+
// exits processed). A list-write failure here must NOT be reported as a 400
|
|
100
|
+
// — that would (a) hide a successful ingest behind a "nothing happened"
|
|
101
|
+
// status and (b) tempt a client to retry, re-ingesting the event. Surface it
|
|
102
|
+
// as a `listsError` warning on the 202 instead.
|
|
103
|
+
let listsError: string | undefined;
|
|
104
|
+
if (body.lists && Object.keys(body.lists).length > 0) {
|
|
105
|
+
try {
|
|
106
|
+
await applyListMembership({
|
|
107
|
+
db,
|
|
108
|
+
userId: body.userId,
|
|
109
|
+
email: body.email,
|
|
110
|
+
lists: body.lists,
|
|
111
|
+
});
|
|
112
|
+
} catch (err) {
|
|
113
|
+
listsError = listMembershipError(err);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
return c.json({ ...result, ...(listsError ? { listsError } : {}) }, 202);
|
|
118
|
+
},
|
|
119
|
+
);
|
package/src/routes/index.ts
CHANGED
|
@@ -1,10 +1,16 @@
|
|
|
1
1
|
import { OpenAPIHono } from "@hono/zod-openapi";
|
|
2
2
|
import type { AppEnv } from "../app.js";
|
|
3
|
+
import { requireApiKey, requireScope } from "../middleware/api-key.js";
|
|
4
|
+
import { createRateLimit } from "../middleware/rate-limit.js";
|
|
3
5
|
import type { DefinedWebhookSource } from "../webhook-sources/define-webhook-source.js";
|
|
4
6
|
import { adminRouter } from "./admin/index.js";
|
|
7
|
+
import { campaignsRouter } from "./campaigns/index.js";
|
|
8
|
+
import { contactsRouter } from "./contacts/index.js";
|
|
5
9
|
import { emailRouter } from "./email/index.js";
|
|
10
|
+
import { emailsRouter } from "./emails/index.js";
|
|
11
|
+
import { eventsRouter } from "./events/index.js";
|
|
6
12
|
import { healthRouter } from "./health.js";
|
|
7
|
-
import {
|
|
13
|
+
import { listsRouter } from "./lists/index.js";
|
|
8
14
|
import { trackingRouter } from "./tracking/index.js";
|
|
9
15
|
import { registerWebhookRoutes } from "./webhooks/index.js";
|
|
10
16
|
|
|
@@ -12,18 +18,62 @@ export interface RegisterRoutesOptions {
|
|
|
12
18
|
webhookSources: DefinedWebhookSource[];
|
|
13
19
|
}
|
|
14
20
|
|
|
21
|
+
// Conservative per-key email budget. `/v1/emails` MUST use a distinct prefix so
|
|
22
|
+
// transactional sends don't share the sliding-window budget with contact
|
|
23
|
+
// upserts / event ingest (open risk #15). 30/min/key is well under the
|
|
24
|
+
// contact-upsert default (100/min) — an integration loop sending more than that
|
|
25
|
+
// is almost certainly a runaway.
|
|
26
|
+
const EMAIL_RATE_LIMIT_MAX = 30;
|
|
27
|
+
|
|
15
28
|
export function registerRoutes(
|
|
16
29
|
app: OpenAPIHono<AppEnv>,
|
|
17
30
|
opts: RegisterRoutesOptions,
|
|
18
31
|
) {
|
|
19
32
|
const v1 = new OpenAPIHono<AppEnv>();
|
|
20
33
|
|
|
34
|
+
// Open routes: health + tracking pixels/redirects are intentionally
|
|
35
|
+
// unauthenticated (links land in recipient inboxes), and the admin router
|
|
36
|
+
// owns its own `requireAdmin` guard.
|
|
21
37
|
v1.route("/health", healthRouter);
|
|
22
|
-
v1.route("/ingest", ingestRouter);
|
|
23
38
|
v1.route("/email", emailRouter);
|
|
24
39
|
v1.route("/admin", adminRouter);
|
|
25
40
|
v1.route("/t", trackingRouter);
|
|
26
41
|
|
|
42
|
+
// The guarded data plane (D5 / decision #16): `requireApiKey` →
|
|
43
|
+
// `requireScope("ingest")` on `/contacts`, `/events`, `/emails`, `/lists`.
|
|
44
|
+
// Each prefix is guarded EXPLICITLY rather than via a root-mounted catch-all
|
|
45
|
+
// sub-app — a sub-app at "/" with `use("*")` also intercepts sibling paths
|
|
46
|
+
// (e.g. `/v1/webhooks`) and 401s them before they reach their own handlers.
|
|
47
|
+
// Both the bare path and its `/*` subtree are covered (Hono treats them as
|
|
48
|
+
// distinct match patterns). `/emails` layers the per-key email rate-limit on
|
|
49
|
+
// top, in strict order auth → scope → rateLimit.
|
|
50
|
+
const emailRateLimit = createRateLimit({
|
|
51
|
+
prefix: "ratelimit:emails",
|
|
52
|
+
max: EMAIL_RATE_LIMIT_MAX,
|
|
53
|
+
});
|
|
54
|
+
for (const base of [
|
|
55
|
+
"/contacts",
|
|
56
|
+
"/events",
|
|
57
|
+
"/emails",
|
|
58
|
+
"/lists",
|
|
59
|
+
"/campaigns",
|
|
60
|
+
]) {
|
|
61
|
+
v1.use(base, requireApiKey, requireScope("ingest"));
|
|
62
|
+
v1.use(`${base}/*`, requireApiKey, requireScope("ingest"));
|
|
63
|
+
}
|
|
64
|
+
// Register the email rate-limit ONCE. The wildcard pattern `/emails/*` matches
|
|
65
|
+
// BOTH the bare `POST /v1/emails` and any subtree, so a single registration
|
|
66
|
+
// covers the whole emails surface. Registering both bare AND wildcard with the
|
|
67
|
+
// SAME stateful instance double-counts every send (two sliding-window entries
|
|
68
|
+
// per request), halving the effective per-key budget (decision #16 / risk 15).
|
|
69
|
+
v1.use("/emails/*", emailRateLimit);
|
|
70
|
+
|
|
71
|
+
v1.route("/contacts", contactsRouter);
|
|
72
|
+
v1.route("/events", eventsRouter);
|
|
73
|
+
v1.route("/emails", emailsRouter);
|
|
74
|
+
v1.route("/lists", listsRouter);
|
|
75
|
+
v1.route("/campaigns", campaignsRouter);
|
|
76
|
+
|
|
27
77
|
app.route("/v1", v1);
|
|
28
78
|
|
|
29
79
|
// Webhooks (built-in Resend + injected content sources) are registered on the
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import type { Database } from "@hogsend/db";
|
|
2
|
+
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
+
import type { AppEnv } from "../../app.js";
|
|
4
|
+
import { resolveOrCreateContact } from "../../lib/contacts.js";
|
|
5
|
+
import { applyListMembership } from "../../lib/preferences.js";
|
|
6
|
+
import { errorSchema } from "../../lib/schemas.js";
|
|
7
|
+
import { getListRegistry } from "../../lists/registry-singleton.js";
|
|
8
|
+
import { listMembershipError } from "../_shared.js";
|
|
9
|
+
|
|
10
|
+
const listSummarySchema = z.object({
|
|
11
|
+
id: z.string(),
|
|
12
|
+
name: z.string(),
|
|
13
|
+
description: z.string().optional(),
|
|
14
|
+
defaultOptIn: z.boolean(),
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
const bodySchema = z.object({
|
|
18
|
+
email: z.string().optional(),
|
|
19
|
+
userId: z.string().optional(),
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
const listRoute = createRoute({
|
|
23
|
+
method: "get",
|
|
24
|
+
path: "/",
|
|
25
|
+
tags: ["Lists"],
|
|
26
|
+
summary: "List defined email lists",
|
|
27
|
+
description:
|
|
28
|
+
"Returns the enabled, code-defined email lists (D3). Membership lives in `email_preferences.categories`; this only enumerates the catalog.",
|
|
29
|
+
responses: {
|
|
30
|
+
200: {
|
|
31
|
+
content: {
|
|
32
|
+
"application/json": {
|
|
33
|
+
schema: z.object({ lists: z.array(listSummarySchema) }),
|
|
34
|
+
},
|
|
35
|
+
},
|
|
36
|
+
description: "Enabled lists",
|
|
37
|
+
},
|
|
38
|
+
},
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
const subscribeRoute = createRoute({
|
|
42
|
+
method: "post",
|
|
43
|
+
path: "/{id}/subscribe",
|
|
44
|
+
tags: ["Lists"],
|
|
45
|
+
summary: "Subscribe a contact to a list",
|
|
46
|
+
request: {
|
|
47
|
+
params: z.object({ id: z.string() }),
|
|
48
|
+
body: {
|
|
49
|
+
content: { "application/json": { schema: bodySchema } },
|
|
50
|
+
},
|
|
51
|
+
},
|
|
52
|
+
responses: {
|
|
53
|
+
200: {
|
|
54
|
+
content: {
|
|
55
|
+
"application/json": {
|
|
56
|
+
schema: z.object({
|
|
57
|
+
list: z.string(),
|
|
58
|
+
subscribed: z.literal(true),
|
|
59
|
+
}),
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
description: "Contact subscribed",
|
|
63
|
+
},
|
|
64
|
+
400: {
|
|
65
|
+
content: { "application/json": { schema: errorSchema } },
|
|
66
|
+
description: "Missing recipient or no resolvable email",
|
|
67
|
+
},
|
|
68
|
+
404: {
|
|
69
|
+
content: { "application/json": { schema: errorSchema } },
|
|
70
|
+
description: "Unknown list id",
|
|
71
|
+
},
|
|
72
|
+
},
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
const unsubscribeRoute = createRoute({
|
|
76
|
+
method: "post",
|
|
77
|
+
path: "/{id}/unsubscribe",
|
|
78
|
+
tags: ["Lists"],
|
|
79
|
+
summary: "Unsubscribe a contact from a list",
|
|
80
|
+
request: {
|
|
81
|
+
params: z.object({ id: z.string() }),
|
|
82
|
+
body: {
|
|
83
|
+
content: { "application/json": { schema: bodySchema } },
|
|
84
|
+
},
|
|
85
|
+
},
|
|
86
|
+
responses: {
|
|
87
|
+
200: {
|
|
88
|
+
content: {
|
|
89
|
+
"application/json": {
|
|
90
|
+
schema: z.object({
|
|
91
|
+
list: z.string(),
|
|
92
|
+
subscribed: z.literal(false),
|
|
93
|
+
}),
|
|
94
|
+
},
|
|
95
|
+
},
|
|
96
|
+
description: "Contact unsubscribed",
|
|
97
|
+
},
|
|
98
|
+
400: {
|
|
99
|
+
content: { "application/json": { schema: errorSchema } },
|
|
100
|
+
description: "Missing recipient or no resolvable email",
|
|
101
|
+
},
|
|
102
|
+
404: {
|
|
103
|
+
content: { "application/json": { schema: errorSchema } },
|
|
104
|
+
description: "Unknown list id",
|
|
105
|
+
},
|
|
106
|
+
},
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* The shared side-effect of subscribe + unsubscribe (identical apart from the
|
|
111
|
+
* boolean polarity): validate the list id, guard identity, then resolve/create
|
|
112
|
+
* the contact FIRST (mirroring /v1/contacts + /v1/events) so a real row (and
|
|
113
|
+
* uuid id) exists — without it `resolveRecipient` returns the raw email as the
|
|
114
|
+
* contactId fallback and `email_preferences.user_id` is written as the raw email
|
|
115
|
+
* instead of the `external_id ?? contact.id` uuid, breaking risk-10 key
|
|
116
|
+
* consistency.
|
|
117
|
+
*
|
|
118
|
+
* Returns a discriminated result the caller maps to a status: `unknown_list` /
|
|
119
|
+
* `missing_identity` → 404 / 400; `failed` → 400 with the error message; `ok` →
|
|
120
|
+
* the caller's literally-typed success body. The `valid()` reads stay in each
|
|
121
|
+
* typed `.openapi()` handler; only the polarity-invariant work lives here.
|
|
122
|
+
*/
|
|
123
|
+
async function applyListSubscription(opts: {
|
|
124
|
+
db: Database;
|
|
125
|
+
id: string;
|
|
126
|
+
email?: string;
|
|
127
|
+
userId?: string;
|
|
128
|
+
subscribed: boolean;
|
|
129
|
+
}): Promise<
|
|
130
|
+
| { kind: "unknown_list" }
|
|
131
|
+
| { kind: "missing_identity" }
|
|
132
|
+
| { kind: "failed"; message: string }
|
|
133
|
+
| { kind: "ok" }
|
|
134
|
+
> {
|
|
135
|
+
const { db, id, email, userId, subscribed } = opts;
|
|
136
|
+
|
|
137
|
+
if (!getListRegistry().has(id)) {
|
|
138
|
+
return { kind: "unknown_list" };
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
if (!email && !userId) {
|
|
142
|
+
return { kind: "missing_identity" };
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
try {
|
|
146
|
+
await resolveOrCreateContact({ db, userId, email });
|
|
147
|
+
await applyListMembership({
|
|
148
|
+
db,
|
|
149
|
+
userId,
|
|
150
|
+
email,
|
|
151
|
+
lists: { [id]: subscribed },
|
|
152
|
+
});
|
|
153
|
+
} catch (err) {
|
|
154
|
+
return { kind: "failed", message: listMembershipError(err) };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
return { kind: "ok" };
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// The lists router does NOT re-apply auth internally — the data-plane prefix
|
|
161
|
+
// guards in `routes/index.ts` (decision #16) apply `requireApiKey` +
|
|
162
|
+
// `requireScope("ingest")` to `/v1/lists` (bare + `/*`) before requests reach
|
|
163
|
+
// this router. Mounting auth here too would double the middleware.
|
|
164
|
+
export const listsRouter = new OpenAPIHono<AppEnv>()
|
|
165
|
+
.openapi(listRoute, (c) => {
|
|
166
|
+
const lists = getListRegistry()
|
|
167
|
+
.getEnabled()
|
|
168
|
+
.map((l) => ({
|
|
169
|
+
id: l.id,
|
|
170
|
+
name: l.name,
|
|
171
|
+
...(l.description !== undefined ? { description: l.description } : {}),
|
|
172
|
+
defaultOptIn: l.defaultOptIn,
|
|
173
|
+
}));
|
|
174
|
+
|
|
175
|
+
return c.json({ lists }, 200);
|
|
176
|
+
})
|
|
177
|
+
.openapi(subscribeRoute, async (c) => {
|
|
178
|
+
const { db } = c.get("container");
|
|
179
|
+
const { id } = c.req.valid("param");
|
|
180
|
+
const { email, userId } = c.req.valid("json");
|
|
181
|
+
|
|
182
|
+
const result = await applyListSubscription({
|
|
183
|
+
db,
|
|
184
|
+
id,
|
|
185
|
+
email,
|
|
186
|
+
userId,
|
|
187
|
+
subscribed: true,
|
|
188
|
+
});
|
|
189
|
+
if (result.kind === "unknown_list") {
|
|
190
|
+
return c.json({ error: `Unknown list: ${id}` }, 404);
|
|
191
|
+
}
|
|
192
|
+
if (result.kind === "missing_identity") {
|
|
193
|
+
return c.json({ error: "email or userId is required" }, 400);
|
|
194
|
+
}
|
|
195
|
+
if (result.kind === "failed") {
|
|
196
|
+
return c.json({ error: result.message }, 400);
|
|
197
|
+
}
|
|
198
|
+
return c.json({ list: id, subscribed: true as const }, 200);
|
|
199
|
+
})
|
|
200
|
+
.openapi(unsubscribeRoute, async (c) => {
|
|
201
|
+
const { db } = c.get("container");
|
|
202
|
+
const { id } = c.req.valid("param");
|
|
203
|
+
const { email, userId } = c.req.valid("json");
|
|
204
|
+
|
|
205
|
+
const result = await applyListSubscription({
|
|
206
|
+
db,
|
|
207
|
+
id,
|
|
208
|
+
email,
|
|
209
|
+
userId,
|
|
210
|
+
subscribed: false,
|
|
211
|
+
});
|
|
212
|
+
if (result.kind === "unknown_list") {
|
|
213
|
+
return c.json({ error: `Unknown list: ${id}` }, 404);
|
|
214
|
+
}
|
|
215
|
+
if (result.kind === "missing_identity") {
|
|
216
|
+
return c.json({ error: "email or userId is required" }, 400);
|
|
217
|
+
}
|
|
218
|
+
if (result.kind === "failed") {
|
|
219
|
+
return c.json({ error: result.message }, 400);
|
|
220
|
+
}
|
|
221
|
+
return c.json({ list: id, subscribed: false as const }, 200);
|
|
222
|
+
});
|
package/src/worker.ts
CHANGED
|
@@ -17,6 +17,10 @@ import {
|
|
|
17
17
|
import { bucketReconcileTask } from "./workflows/bucket-reconcile.js";
|
|
18
18
|
import { checkAlertsTask } from "./workflows/check-alerts.js";
|
|
19
19
|
import { importContactsTask } from "./workflows/import-contacts.js";
|
|
20
|
+
import {
|
|
21
|
+
reapStuckCampaignsTask,
|
|
22
|
+
sendCampaignTask,
|
|
23
|
+
} from "./workflows/send-campaign.js";
|
|
20
24
|
import { sendEmailTask } from "./workflows/send-email.js";
|
|
21
25
|
|
|
22
26
|
export interface CreateWorkerOptions {
|
|
@@ -62,6 +66,8 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
|
|
|
62
66
|
const baseWorkflows = [
|
|
63
67
|
sendEmailTask,
|
|
64
68
|
importContactsTask,
|
|
69
|
+
sendCampaignTask,
|
|
70
|
+
reapStuckCampaignsTask,
|
|
65
71
|
checkAlertsTask,
|
|
66
72
|
bucketReconcileTask,
|
|
67
73
|
bucketBackfillTask,
|
|
@@ -24,6 +24,7 @@ import {
|
|
|
24
24
|
import { getBucketRegistrySingleton } from "../buckets/registry-singleton.js";
|
|
25
25
|
import { getJourneyRegistrySingleton } from "../journeys/registry-singleton.js";
|
|
26
26
|
import { emitBucketTransition } from "../lib/bucket-emit.js";
|
|
27
|
+
import { contactKeySql } from "../lib/contacts.js";
|
|
27
28
|
import { hatchet } from "../lib/hatchet.js";
|
|
28
29
|
import type { Logger } from "../lib/logger.js";
|
|
29
30
|
import { createLogger } from "../lib/logger.js";
|
|
@@ -231,16 +232,20 @@ async function backfillJoins(opts: {
|
|
|
231
232
|
for (let i = 0; i < matcherIds.length; i += BATCH_SIZE) {
|
|
232
233
|
const chunk = matcherIds.slice(i, i + BATCH_SIZE);
|
|
233
234
|
|
|
234
|
-
// userEmail backfilled from the contacts row where available.
|
|
235
|
+
// userEmail backfilled from the contacts row where available. The chunk
|
|
236
|
+
// holds the RESOLVED key (coalesce(external_id, anonymous_id, id)) — for an
|
|
237
|
+
// email-only / anonymous contact that is the anonymous_id or the uuid id, NOT
|
|
238
|
+
// the (null) external_id. Looking up by `contacts.externalId` would miss
|
|
239
|
+
// those rows and write a NULL userEmail despite the contact having an email,
|
|
240
|
+
// so we key the lookup + the map by the SAME coalesce expression the chunk
|
|
241
|
+
// carries (matches reconcileBucketJoins, which reads userId + email off one
|
|
242
|
+
// contacts row).
|
|
243
|
+
const resolvedKey = contactKeySql();
|
|
235
244
|
const chunkContacts = await db
|
|
236
|
-
.select({
|
|
245
|
+
.select({ userKey: resolvedKey, email: contacts.email })
|
|
237
246
|
.from(contacts)
|
|
238
|
-
.where(
|
|
239
|
-
|
|
240
|
-
);
|
|
241
|
-
const emailByUser = new Map(
|
|
242
|
-
chunkContacts.map((c) => [c.externalId, c.email]),
|
|
243
|
-
);
|
|
247
|
+
.where(and(inArray(resolvedKey, chunk), isNull(contacts.deletedAt)));
|
|
248
|
+
const emailByUser = new Map(chunkContacts.map((c) => [c.userKey, c.email]));
|
|
244
249
|
|
|
245
250
|
// Fix A: entryCount = 1 + prior memberships for each (user, bucket), the
|
|
246
251
|
// same monotonic ordinal the live join computes (check-membership.ts). On a
|
|
@@ -456,7 +461,9 @@ async function selectEventMatchers(
|
|
|
456
461
|
.as("present");
|
|
457
462
|
|
|
458
463
|
const rows = await db
|
|
459
|
-
.select({
|
|
464
|
+
.select({
|
|
465
|
+
userId: contactKeySql(),
|
|
466
|
+
})
|
|
460
467
|
.from(contacts)
|
|
461
468
|
.innerJoin(everFired, eq(everFired.userId, contacts.externalId))
|
|
462
469
|
.leftJoin(present, eq(present.userId, contacts.externalId))
|
|
@@ -496,12 +503,15 @@ async function selectEventMatchers(
|
|
|
496
503
|
* a per-contact `evaluateCondition` loop over live contacts. Property
|
|
497
504
|
* sub-conditions evaluate against the contact's merged properties.
|
|
498
505
|
*
|
|
499
|
-
* KEYSET PAGINATION by `contacts.
|
|
500
|
-
*
|
|
501
|
-
*
|
|
502
|
-
*
|
|
503
|
-
*
|
|
504
|
-
*
|
|
506
|
+
* KEYSET PAGINATION by `contacts.id` in BATCH_SIZE pages: each page selects
|
|
507
|
+
* `WHERE id > :cursor ORDER BY id ASC LIMIT BATCH_SIZE`, evaluates the criteria
|
|
508
|
+
* per contact, then advances the cursor to the last `id` of the page — repeating
|
|
509
|
+
* until a short page ends the scan. The whole contacts table is never held in
|
|
510
|
+
* memory at once. Paging on `id` (the non-null unique PK) — NOT `external_id`,
|
|
511
|
+
* which is nullable (email-only / anonymous contacts) and would drop every
|
|
512
|
+
* null-external_id row and order NULLs unstably. (reconcileBucketJoins is not a
|
|
513
|
+
* keyset scan — it relies on matchers dropping out as they become active
|
|
514
|
+
* members — so this no longer mirrors it.)
|
|
505
515
|
*/
|
|
506
516
|
async function selectCompositeMatchers(
|
|
507
517
|
db: Database,
|
|
@@ -513,17 +523,18 @@ async function selectCompositeMatchers(
|
|
|
513
523
|
for (;;) {
|
|
514
524
|
const page = await db
|
|
515
525
|
.select({
|
|
516
|
-
|
|
526
|
+
id: contacts.id,
|
|
527
|
+
userId: contactKeySql(),
|
|
517
528
|
properties: contacts.properties,
|
|
518
529
|
})
|
|
519
530
|
.from(contacts)
|
|
520
531
|
.where(
|
|
521
532
|
and(
|
|
522
533
|
isNull(contacts.deletedAt),
|
|
523
|
-
cursor != null ? gt(contacts.
|
|
534
|
+
cursor != null ? gt(contacts.id, cursor) : undefined,
|
|
524
535
|
),
|
|
525
536
|
)
|
|
526
|
-
.orderBy(sql`${contacts.
|
|
537
|
+
.orderBy(sql`${contacts.id} asc`)
|
|
527
538
|
.limit(BATCH_SIZE);
|
|
528
539
|
|
|
529
540
|
for (const contact of page) {
|
|
@@ -531,17 +542,17 @@ async function selectCompositeMatchers(
|
|
|
531
542
|
condition: criteria,
|
|
532
543
|
ctx: {
|
|
533
544
|
db,
|
|
534
|
-
userId: contact.
|
|
545
|
+
userId: contact.userId,
|
|
535
546
|
journeyContext:
|
|
536
547
|
(contact.properties as Record<string, unknown> | null) ?? {},
|
|
537
548
|
},
|
|
538
549
|
});
|
|
539
|
-
if (isMember) matchers.push(contact.
|
|
550
|
+
if (isMember) matchers.push(contact.userId);
|
|
540
551
|
}
|
|
541
552
|
|
|
542
553
|
// A short page (fewer than a full batch) means the scan is exhausted.
|
|
543
554
|
if (page.length < BATCH_SIZE) break;
|
|
544
|
-
cursor = page[page.length - 1]?.
|
|
555
|
+
cursor = page[page.length - 1]?.id ?? null;
|
|
545
556
|
if (cursor == null) break;
|
|
546
557
|
}
|
|
547
558
|
|