@hogsend/engine 0.40.0 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +9 -7
- package/src/buckets/check-membership.ts +6 -1
- package/src/campaigns/cohort-sql.ts +198 -0
- package/src/campaigns/define-campaign.ts +207 -28
- package/src/campaigns/reconcile.ts +105 -11
- package/src/campaigns/steps.ts +48 -0
- package/src/connectors/define-action.ts +70 -0
- package/src/container.ts +124 -42
- package/src/destinations/presets/posthog.ts +7 -1
- package/src/index.ts +82 -2
- package/src/journeys/constants.ts +10 -0
- package/src/journeys/define-journey.ts +14 -405
- package/src/journeys/execute-journey-run.ts +608 -0
- package/src/journeys/graph/build-graph.ts +212 -56
- package/src/journeys/journey-boundary.ts +54 -0
- package/src/journeys/journey-context.ts +374 -59
- package/src/journeys/record-once.ts +110 -0
- package/src/lib/blueprints.ts +576 -0
- package/src/lib/campaign-send-key.ts +58 -0
- package/src/lib/connector-actions.ts +203 -19
- package/src/lib/contacts.ts +54 -0
- package/src/lib/email-service-types.ts +4 -1
- package/src/lib/email.ts +13 -2
- package/src/lib/feed.ts +77 -57
- package/src/lib/frequency-cap.ts +45 -15
- package/src/lib/hatchet-duration.ts +22 -0
- package/src/lib/html.ts +1 -0
- package/src/lib/ingestion.ts +145 -4
- package/src/lib/journey-suppress.ts +87 -0
- package/src/lib/links.ts +204 -18
- package/src/lib/outbound.ts +20 -0
- package/src/lib/preferences.ts +80 -1
- package/src/lib/tracked.ts +44 -29
- package/src/lib/tracking-event-names.ts +8 -0
- package/src/lib/webhook-signing.ts +6 -1
- package/src/lists/channels.ts +75 -0
- package/src/lists/define-list.ts +34 -0
- package/src/lists/registry.ts +29 -2
- package/src/mcp/blueprint-tools.ts +460 -0
- package/src/routes/admin/blueprints.ts +711 -0
- package/src/routes/admin/campaigns.ts +179 -3
- package/src/routes/admin/emails.ts +13 -0
- package/src/routes/admin/index.ts +4 -0
- package/src/routes/admin/journeys.ts +4 -2
- package/src/routes/admin/links.ts +303 -36
- package/src/routes/admin/lists.ts +51 -0
- package/src/routes/campaigns/index.ts +44 -6
- package/src/routes/email/preferences.ts +50 -13
- package/src/routes/feed/recipient.ts +2 -43
- package/src/routes/index.ts +6 -0
- package/src/routes/lists/index.ts +138 -18
- package/src/routes/tracking/arrive.ts +260 -0
- package/src/routes/tracking/click-pipeline.ts +329 -0
- package/src/routes/tracking/click.ts +5 -270
- package/src/routes/tracking/index.ts +2 -0
- package/src/routes/tracking/vanity.ts +64 -0
- package/src/worker.ts +5 -0
- package/src/workflows/bucket-backfill.ts +4 -2
- package/src/workflows/bucket-reconcile.ts +11 -4
- package/src/workflows/check-alerts.ts +130 -1
- package/src/workflows/journey-blueprint-interpreter.ts +561 -0
- package/src/workflows/send-campaign.ts +574 -180
|
@@ -1,15 +1,23 @@
|
|
|
1
1
|
import { connectorDeliveries, contacts, type Database } from "@hogsend/db";
|
|
2
|
-
import { and, eq, isNull, or } from "drizzle-orm";
|
|
2
|
+
import { and, eq, isNull, or, sql } from "drizzle-orm";
|
|
3
3
|
import { getConnectorActionRegistry } from "../connectors/action-registry-singleton.js";
|
|
4
|
-
import type {
|
|
4
|
+
import type {
|
|
5
|
+
ConnectorActionSkipped,
|
|
6
|
+
DefinedConnectorAction,
|
|
7
|
+
ResolvedActionContact,
|
|
8
|
+
} from "../connectors/define-action.js";
|
|
5
9
|
import { env } from "../env.js";
|
|
6
10
|
import {
|
|
7
11
|
deriveJourneyKey,
|
|
8
12
|
getJourneyBoundary,
|
|
9
13
|
registerKey,
|
|
10
14
|
} from "../journeys/journey-boundary.js";
|
|
15
|
+
import { getListRegistry } from "../lists/registry-singleton.js";
|
|
11
16
|
import { getDb } from "./db.js";
|
|
12
17
|
import { createLogger } from "./logger.js";
|
|
18
|
+
import { readRecipientPreferences } from "./preferences.js";
|
|
19
|
+
|
|
20
|
+
const logger = createLogger(env.LOG_LEVEL);
|
|
13
21
|
|
|
14
22
|
/**
|
|
15
23
|
* Matches the canonical UUID form of `contacts.id` (8-4-4-4-12 hex). Gates the
|
|
@@ -20,6 +28,13 @@ import { createLogger } from "./logger.js";
|
|
|
20
28
|
const CONTACT_ID_UUID =
|
|
21
29
|
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
22
30
|
|
|
31
|
+
/**
|
|
32
|
+
* Matches a NAMESPACED platform ref (`telegram:<chatId>`, `discord:<id>`, …):
|
|
33
|
+
* a leading `<namespace>:` followed by a non-empty value. Captures the namespace
|
|
34
|
+
* (group 1) and the value (group 2).
|
|
35
|
+
*/
|
|
36
|
+
const NAMESPACED_REF = /^([a-z][a-z0-9_-]*):(.+)$/i;
|
|
37
|
+
|
|
23
38
|
/**
|
|
24
39
|
* Resolve a contact for an outbound action by its canonical key in ANY form:
|
|
25
40
|
* email, external id, a platform id (a Discord snowflake / anonymous id), or —
|
|
@@ -30,12 +45,35 @@ const CONTACT_ID_UUID =
|
|
|
30
45
|
* shape-gated: `contacts.id` is a uuid column and an unguarded `eq(id, ref)`
|
|
31
46
|
* against an email/snowflake text ref throws an invalid-uuid cast. First live
|
|
32
47
|
* match wins.
|
|
48
|
+
*
|
|
49
|
+
* NAMESPACED refs (`telegram:<chatId>`) ADDITIONALLY match the platform metadata
|
|
50
|
+
* under `properties.<namespace>` (the `chat_id`/`id` fields). This is load-bearing
|
|
51
|
+
* for the PRIMARY "reach an existing app user on a chat platform" case: when a
|
|
52
|
+
* platform links onto an ALREADY-identified contact, the engine keeps the row's
|
|
53
|
+
* `external_id = <app id>` and stores the chat id ONLY under
|
|
54
|
+
* `properties.<namespace>` (the DEEP_MERGE_KEYS convention in lib/contacts.ts) —
|
|
55
|
+
* the `telegram:<id>` key is NEVER promoted to `external_id` nor aliased. Without
|
|
56
|
+
* these legs `resolveContact("telegram:<chatId>")` would miss that population and
|
|
57
|
+
* a preference gate would wrongly ALLOW a send to an opted-out user. Both the
|
|
58
|
+
* namespace key and the value are SQL-parameterized (never string-interpolated).
|
|
59
|
+
* PERF: these two legs are un-indexed jsonb reads — acceptable at DM volume; a
|
|
60
|
+
* partial index on `properties -> '<ns>' ->> 'chat_id'` is the follow-up if it
|
|
61
|
+
* ever gets hot.
|
|
33
62
|
*/
|
|
34
63
|
async function resolveContact(
|
|
35
64
|
db: Database,
|
|
36
65
|
ref: string,
|
|
37
66
|
): Promise<ResolvedActionContact | null> {
|
|
38
67
|
if (!ref) return null;
|
|
68
|
+
// A namespaced ref (telegram:<chatId>) additionally matches the platform
|
|
69
|
+
// metadata a link-onto-identified contact stores only in properties.<ns>.
|
|
70
|
+
const nsMatch = NAMESPACED_REF.exec(ref);
|
|
71
|
+
const namespacedLegs = nsMatch
|
|
72
|
+
? [
|
|
73
|
+
sql`${contacts.properties} -> ${nsMatch[1]}::text ->> 'chat_id' = ${nsMatch[2]}`,
|
|
74
|
+
sql`${contacts.properties} -> ${nsMatch[1]}::text ->> 'id' = ${nsMatch[2]}`,
|
|
75
|
+
]
|
|
76
|
+
: [];
|
|
39
77
|
const rows = await db
|
|
40
78
|
.select({
|
|
41
79
|
id: contacts.id,
|
|
@@ -54,6 +92,7 @@ async function resolveContact(
|
|
|
54
92
|
eq(contacts.discordId, ref),
|
|
55
93
|
eq(contacts.anonymousId, ref),
|
|
56
94
|
...(CONTACT_ID_UUID.test(ref) ? [eq(contacts.id, ref)] : []),
|
|
95
|
+
...namespacedLegs,
|
|
57
96
|
),
|
|
58
97
|
),
|
|
59
98
|
)
|
|
@@ -69,6 +108,92 @@ async function resolveContact(
|
|
|
69
108
|
};
|
|
70
109
|
}
|
|
71
110
|
|
|
111
|
+
/**
|
|
112
|
+
* Preference gate for a member-directed outbound action. Returns a
|
|
113
|
+
* {@link ConnectorActionSkipped} verdict when the resolved recipient has opted
|
|
114
|
+
* out, or `null` when the send is allowed.
|
|
115
|
+
*
|
|
116
|
+
* Gating rules:
|
|
117
|
+
* - An action WITHOUT a member audience (`audience?.kind !== "member"`) is
|
|
118
|
+
* ops/channel-directed (grantRole, broadcast, mentions, channel messages) and
|
|
119
|
+
* is NEVER gated — returns null immediately.
|
|
120
|
+
* - The audience extractor is run inside a try/catch: a THROWING extractor
|
|
121
|
+
* FAILS OPEN (`logger.warn` + null) — a bad extractor must never brick a
|
|
122
|
+
* journey that works today.
|
|
123
|
+
* - Candidate refs are tried in order; the FIRST that resolves a live contact
|
|
124
|
+
* wins. A candidate that resolves NO contact (a raw snowflake, a group-chat
|
|
125
|
+
* id, an unlinked ref) means there is NO preference surface to consult — the
|
|
126
|
+
* send is ALLOWED (null). Only once a contact is resolved do preferences gate.
|
|
127
|
+
* - On a resolved contact we read the aggregated {@link readRecipientPreferences}
|
|
128
|
+
* keyed by its email + `externalId ?? id` (the SAME key form preference writes
|
|
129
|
+
* use), then: `unsubscribedAll` → skip `unsubscribed_all`; else channel
|
|
130
|
+
* opt-out (`!getListRegistry().isSubscribed(categories, connectorId)`) → skip
|
|
131
|
+
* `channel_unsubscribed`; else null.
|
|
132
|
+
*
|
|
133
|
+
* Deliberately does NOT consume `prefs.suppressed` — that is an email-transport
|
|
134
|
+
* hard-bounce/complaint signal, irrelevant to a chat channel.
|
|
135
|
+
*/
|
|
136
|
+
async function checkActionAudience(
|
|
137
|
+
db: Database,
|
|
138
|
+
resolveCachedContact: (ref: string) => Promise<ResolvedActionContact | null>,
|
|
139
|
+
action: DefinedConnectorAction,
|
|
140
|
+
connectorId: string,
|
|
141
|
+
actionName: string,
|
|
142
|
+
args: unknown,
|
|
143
|
+
): Promise<ConnectorActionSkipped | null> {
|
|
144
|
+
// Ops/channel-directed actions carry no member audience — never gated.
|
|
145
|
+
if (action.audience?.kind !== "member") return null;
|
|
146
|
+
|
|
147
|
+
// Extract candidate recipient refs. A throwing extractor fails OPEN so a
|
|
148
|
+
// gating bug can never regress a send that works today.
|
|
149
|
+
let candidates: string[];
|
|
150
|
+
try {
|
|
151
|
+
const raw = action.audience.ref(args);
|
|
152
|
+
candidates = (Array.isArray(raw) ? raw : [raw]).filter(
|
|
153
|
+
(r): r is string => typeof r === "string" && r.length > 0,
|
|
154
|
+
);
|
|
155
|
+
} catch (error) {
|
|
156
|
+
logger.warn("connector action audience extractor threw; sending anyway", {
|
|
157
|
+
connectorId,
|
|
158
|
+
action: actionName,
|
|
159
|
+
error: error instanceof Error ? error.message : String(error),
|
|
160
|
+
});
|
|
161
|
+
return null;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
for (const ref of candidates) {
|
|
165
|
+
const contact = await resolveCachedContact(ref);
|
|
166
|
+
// No contact for this candidate → no preference surface → try the next; a
|
|
167
|
+
// fully-unresolved recipient (raw id / group chat) falls through to allow.
|
|
168
|
+
if (!contact) continue;
|
|
169
|
+
|
|
170
|
+
const prefs = await readRecipientPreferences(db, {
|
|
171
|
+
email: contact.email,
|
|
172
|
+
userId: contact.externalId ?? contact.id,
|
|
173
|
+
});
|
|
174
|
+
if (prefs.unsubscribedAll) {
|
|
175
|
+
return {
|
|
176
|
+
skipped: true,
|
|
177
|
+
reason: "unsubscribed_all",
|
|
178
|
+
connectorId,
|
|
179
|
+
action: actionName,
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
if (!getListRegistry().isSubscribed(prefs.categories, connectorId)) {
|
|
183
|
+
return {
|
|
184
|
+
skipped: true,
|
|
185
|
+
reason: "channel_unsubscribed",
|
|
186
|
+
connectorId,
|
|
187
|
+
action: actionName,
|
|
188
|
+
};
|
|
189
|
+
}
|
|
190
|
+
return null;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
// No candidate resolved a contact → allowed (no preference surface).
|
|
194
|
+
return null;
|
|
195
|
+
}
|
|
196
|
+
|
|
72
197
|
export interface SendConnectorActionArgs {
|
|
73
198
|
/** The connector the action belongs to (e.g. "discord"). */
|
|
74
199
|
connectorId: string;
|
|
@@ -100,6 +225,17 @@ export interface SendConnectorActionArgs {
|
|
|
100
225
|
* Independent of any inbound gateway runtime: a deployment with the gateway off
|
|
101
226
|
* (or "Worker Offline") can still send — Discord actions are bot-REST needing
|
|
102
227
|
* only the bot token.
|
|
228
|
+
*
|
|
229
|
+
* MEMBER-DIRECTED actions (those declaring `audience: { kind: "member" }`) are
|
|
230
|
+
* automatically gated on the recipient's channel preferences BEFORE the plugin
|
|
231
|
+
* runs: a resolved contact who unsubscribed globally or opted out of the
|
|
232
|
+
* connector's channel list yields a {@link ConnectorActionSkipped} verdict
|
|
233
|
+
* INSTEAD of a send, and no `connector_deliveries` row is claimed. The gate runs
|
|
234
|
+
* INSIDE the durable memo closure, so its verdict is recorded and replays
|
|
235
|
+
* verbatim (THE LAW: the key derivation, `registerKey`, and the `memoize` call
|
|
236
|
+
* itself stay unconditional). Skips are observable via logs + the returned
|
|
237
|
+
* verdict only — an audit ledger is a future additive migration. Ops/channel-
|
|
238
|
+
* directed actions (no audience) are never gated.
|
|
103
239
|
*/
|
|
104
240
|
export async function sendConnectorAction(
|
|
105
241
|
input: SendConnectorActionArgs,
|
|
@@ -115,17 +251,57 @@ export async function sendConnectorAction(
|
|
|
115
251
|
);
|
|
116
252
|
}
|
|
117
253
|
const db = getDb();
|
|
254
|
+
|
|
255
|
+
// Per-call promise-memoized contact resolver so the preference gate and the
|
|
256
|
+
// plugin share ONE contacts round-trip per ref: `checkActionAudience` and the
|
|
257
|
+
// plugin's `ctx.resolveContact` would otherwise resolve the SAME member ref
|
|
258
|
+
// twice. Scoped to THIS call (never module-level) — a durable replay rebuilds
|
|
259
|
+
// it fresh; it adds/removes no durable calls and lives inside the memoized
|
|
260
|
+
// closure's execution, never moving the gate outside it (THE LAW).
|
|
261
|
+
const contactCache = new Map<string, Promise<ResolvedActionContact | null>>();
|
|
262
|
+
const resolveCached = (ref: string) => {
|
|
263
|
+
let p = contactCache.get(ref);
|
|
264
|
+
if (!p) {
|
|
265
|
+
p = resolveContact(db, ref);
|
|
266
|
+
contactCache.set(ref, p);
|
|
267
|
+
}
|
|
268
|
+
return p;
|
|
269
|
+
};
|
|
270
|
+
|
|
118
271
|
const doRun = () =>
|
|
119
272
|
action.run(input.args, {
|
|
120
273
|
db,
|
|
121
|
-
logger
|
|
122
|
-
resolveContact:
|
|
274
|
+
logger,
|
|
275
|
+
resolveContact: resolveCached,
|
|
123
276
|
});
|
|
124
277
|
|
|
278
|
+
// Preference gate shared by BOTH the no-boundary and memoized paths: consult
|
|
279
|
+
// the recipient's channel preferences and either short-circuit with a skip
|
|
280
|
+
// verdict (memo-recorded so it replays verbatim) or invoke `run`.
|
|
281
|
+
const gate = async (run: () => Promise<unknown>): Promise<unknown> => {
|
|
282
|
+
const skip = await checkActionAudience(
|
|
283
|
+
db,
|
|
284
|
+
resolveCached,
|
|
285
|
+
action,
|
|
286
|
+
input.connectorId,
|
|
287
|
+
input.action,
|
|
288
|
+
input.args,
|
|
289
|
+
);
|
|
290
|
+
if (skip) {
|
|
291
|
+
logger.info("connector action skipped: recipient preferences", {
|
|
292
|
+
connectorId: input.connectorId,
|
|
293
|
+
action: input.action,
|
|
294
|
+
reason: skip.reason,
|
|
295
|
+
});
|
|
296
|
+
return skip;
|
|
297
|
+
}
|
|
298
|
+
return run();
|
|
299
|
+
};
|
|
300
|
+
|
|
125
301
|
// Outside a journey run (an admin/manual send) there is no replay to defend
|
|
126
|
-
// against and no boundary to key from — run the action directly.
|
|
302
|
+
// against and no boundary to key from — gate then run the action directly.
|
|
127
303
|
const boundary = getJourneyBoundary();
|
|
128
|
-
if (!boundary) return doRun
|
|
304
|
+
if (!boundary) return gate(doRun);
|
|
129
305
|
|
|
130
306
|
// The replay-stable, branch-derived key shared by BOTH defense layers. `site`
|
|
131
307
|
// = the explicit `idempotencyLabel` ?? the nearest authored wait/checkpoint
|
|
@@ -145,20 +321,28 @@ export async function sendConnectorAction(
|
|
|
145
321
|
registerKey(boundary, key);
|
|
146
322
|
|
|
147
323
|
// Layer 1 (eviction-gated, FREE) fast path wrapping the Layer-2-backed run.
|
|
148
|
-
// When the engine supports eviction the durable `memo()` skips the
|
|
149
|
-
//
|
|
150
|
-
//
|
|
151
|
-
//
|
|
152
|
-
//
|
|
153
|
-
//
|
|
324
|
+
// When the engine supports eviction the durable `memo()` skips the closure —
|
|
325
|
+
// the preference gate, the action, AND the DB round-trip below — entirely on a
|
|
326
|
+
// replay, replaying whatever the FIRST run recorded (a skip verdict OR the
|
|
327
|
+
// send result). When it does NOT (a pre-eviction / ':latest'-drifted engine),
|
|
328
|
+
// `memoize` falls through to the closure and Layer 2 (the
|
|
329
|
+
// `connector_deliveries` short-circuit) is the version-INDEPENDENT exactly-once
|
|
330
|
+
// guarantee. Belt-and-suspenders, mirroring the email send path (tracked.ts).
|
|
331
|
+
//
|
|
332
|
+
// The gate runs INSIDE the closure so a skip is memo-recorded: a preference
|
|
333
|
+
// flip between run and replay cannot change the recorded verdict. On a skip we
|
|
334
|
+
// never reach `runWithConnectorDelivery`, so NO `connector_deliveries` row is
|
|
335
|
+
// ever claimed for a skipped send (zero migrations; the enum is untouched).
|
|
154
336
|
return boundary.memoize([key], () =>
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
337
|
+
gate(() =>
|
|
338
|
+
runWithConnectorDelivery({
|
|
339
|
+
db,
|
|
340
|
+
connectorId: input.connectorId,
|
|
341
|
+
action: input.action,
|
|
342
|
+
dedupeKey: key,
|
|
343
|
+
run: doRun,
|
|
344
|
+
}),
|
|
345
|
+
),
|
|
162
346
|
);
|
|
163
347
|
}
|
|
164
348
|
|
package/src/lib/contacts.ts
CHANGED
|
@@ -34,6 +34,53 @@ export class PublishableAnonymousMergeError extends Error {
|
|
|
34
34
|
}
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
/**
|
|
38
|
+
* True when `value` is the canonical key of an IDENTIFIED contact — i.e. a
|
|
39
|
+
* live contact's `external_id`, or its `email` when that is its canonical key
|
|
40
|
+
* (no `external_id`). Such a value names an identified person, so a
|
|
41
|
+
* token-less publishable/unauthenticated caller must NOT be allowed to claim
|
|
42
|
+
* it as an "anon id" (the feed-read and arrival-stamp forgery guard).
|
|
43
|
+
*
|
|
44
|
+
* A genuine browser anon id only ever matches a contact via `anonymous_id`
|
|
45
|
+
* whose canonical key is that same anon id (the contact has no `external_id`)
|
|
46
|
+
* — that is the caller's OWN anon contact and is allowed (returns false).
|
|
47
|
+
*
|
|
48
|
+
* Lives here beside `PublishableAnonymousMergeError` because it is the same
|
|
49
|
+
* invariant read-side: consumed by `resolveFeedRecipient` (feed reads) and
|
|
50
|
+
* `POST /v1/t/arrive` (arrival stamps).
|
|
51
|
+
*/
|
|
52
|
+
export async function collidesWithIdentified(
|
|
53
|
+
db: Database,
|
|
54
|
+
value: string,
|
|
55
|
+
): Promise<boolean> {
|
|
56
|
+
const rows = await db
|
|
57
|
+
.select({
|
|
58
|
+
externalId: contacts.externalId,
|
|
59
|
+
email: contacts.email,
|
|
60
|
+
anonymousId: contacts.anonymousId,
|
|
61
|
+
})
|
|
62
|
+
.from(contacts)
|
|
63
|
+
.where(
|
|
64
|
+
and(
|
|
65
|
+
or(
|
|
66
|
+
eq(contacts.externalId, value),
|
|
67
|
+
eq(contacts.email, value),
|
|
68
|
+
eq(contacts.anonymousId, value),
|
|
69
|
+
),
|
|
70
|
+
isNull(contacts.deletedAt),
|
|
71
|
+
),
|
|
72
|
+
);
|
|
73
|
+
for (const row of rows) {
|
|
74
|
+
// The supplied value is this contact's `external_id` → its rows are keyed
|
|
75
|
+
// on it (identified). Reject.
|
|
76
|
+
if (row.externalId === value) return true;
|
|
77
|
+
// The supplied value is this contact's `email` AND that email is its
|
|
78
|
+
// canonical key (no external_id) → identified rows are keyed on it. Reject.
|
|
79
|
+
if (row.email === value && !row.externalId) return true;
|
|
80
|
+
}
|
|
81
|
+
return false;
|
|
82
|
+
}
|
|
83
|
+
|
|
37
84
|
/**
|
|
38
85
|
* Thrown by {@link resolveOrCreateContact}'s engine-internal `contactId` pin when
|
|
39
86
|
* the pinned subject row no longer exists and no merge-alias chain leads to a live
|
|
@@ -150,6 +197,13 @@ export function normalizeEmail(raw: string): string {
|
|
|
150
197
|
return raw.trim().toLowerCase();
|
|
151
198
|
}
|
|
152
199
|
|
|
200
|
+
/** `normalizeEmail` for a maybe-missing address. */
|
|
201
|
+
export function normalizeEmailOrNull(
|
|
202
|
+
email: string | null | undefined,
|
|
203
|
+
): string | null {
|
|
204
|
+
return email ? normalizeEmail(email) : null;
|
|
205
|
+
}
|
|
206
|
+
|
|
153
207
|
// ---------------------------------------------------------------------------
|
|
154
208
|
// Identity resolution
|
|
155
209
|
// ---------------------------------------------------------------------------
|
|
@@ -76,11 +76,14 @@ export interface TrackedSendResult {
|
|
|
76
76
|
/**
|
|
77
77
|
* Present only when `status === "skipped"`:
|
|
78
78
|
* - `"frequency_capped"` — the per-recipient frequency cap was hit.
|
|
79
|
+
* - `"journey_suppressed"` — the enclosing journey's `meta.suppress` min-gap
|
|
80
|
+
* was in force: a non-failed send for this journey to this recipient already
|
|
81
|
+
* went out inside the window, so this one was skipped (no row, no send).
|
|
79
82
|
* - `"test_mode_blocked"` — test mode was active but no redirect address
|
|
80
83
|
* resolved (no `HOGSEND_TEST_EMAIL` / `STUDIO_ADMIN_EMAIL`), so the send was
|
|
81
84
|
* blocked rather than delivered to the real recipient.
|
|
82
85
|
*/
|
|
83
|
-
reason?: "frequency_capped" | "test_mode_blocked";
|
|
86
|
+
reason?: "frequency_capped" | "journey_suppressed" | "test_mode_blocked";
|
|
84
87
|
}
|
|
85
88
|
|
|
86
89
|
/**
|
package/src/lib/email.ts
CHANGED
|
@@ -40,7 +40,13 @@ export interface SendEmailOptions {
|
|
|
40
40
|
* dynamically (e.g. the public `POST /v1/emails`).
|
|
41
41
|
*/
|
|
42
42
|
template: TemplateName;
|
|
43
|
-
|
|
43
|
+
/**
|
|
44
|
+
* The email subject. Optional: when omitted, the tracked mailer falls back
|
|
45
|
+
* to the template registry's `defaultSubject` for {@link template} (the
|
|
46
|
+
* blueprint interpreter's send nodes rely on this — a blueprint send node
|
|
47
|
+
* carries only a template key). Code journeys typically pass it explicitly.
|
|
48
|
+
*/
|
|
49
|
+
subject?: string;
|
|
44
50
|
journeyName?: string;
|
|
45
51
|
journeyStateId?: string;
|
|
46
52
|
props?: Record<string, unknown>;
|
|
@@ -137,7 +143,12 @@ export async function sendEmail(
|
|
|
137
143
|
idempotencyKey: resolvedIdempotencyKey,
|
|
138
144
|
userId: opts.userId,
|
|
139
145
|
userEmail: opts.to,
|
|
140
|
-
category
|
|
146
|
+
// A journey may stamp its own email-preference category (`meta.category`,
|
|
147
|
+
// threaded onto the boundary) — it overrides the template's own category at
|
|
148
|
+
// send time exactly as this hardcoded `"journey"` default did. Outside a
|
|
149
|
+
// journey there is no boundary, so this stays the built-in `journey`
|
|
150
|
+
// category, unchanged.
|
|
151
|
+
category: boundary?.category ?? "journey",
|
|
141
152
|
tags: [
|
|
142
153
|
{ name: "journeyId", value: opts.journeyName ?? opts.template },
|
|
143
154
|
{ name: "templateKey", value: opts.template },
|
package/src/lib/feed.ts
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { and, eq } from "drizzle-orm";
|
|
1
|
+
import { type FeedBlock, feedItems } from "@hogsend/db";
|
|
3
2
|
import {
|
|
4
3
|
deriveJourneyKey,
|
|
5
4
|
getJourneyBoundary,
|
|
6
5
|
registerKey,
|
|
7
6
|
} from "../journeys/journey-boundary.js";
|
|
7
|
+
import { IN_APP_LIST_ID } from "../lists/channels.js";
|
|
8
8
|
import { getListRegistry } from "../lists/registry-singleton.js";
|
|
9
9
|
import { resolveOrCreateContact, resolveRecipient } from "./contacts.js";
|
|
10
10
|
import { getDb } from "./db.js";
|
|
11
11
|
import { createLogger } from "./logger.js";
|
|
12
|
+
import { readRecipientPreferences } from "./preferences.js";
|
|
12
13
|
import { getRedis } from "./redis.js";
|
|
13
14
|
|
|
14
15
|
const logger = createLogger(process.env.LOG_LEVEL);
|
|
@@ -16,9 +17,11 @@ const logger = createLogger(process.env.LOG_LEVEL);
|
|
|
16
17
|
/**
|
|
17
18
|
* Reserved list id governing in-app feed suppression (mirrors the built-in
|
|
18
19
|
* `transactional` / `journey` categories). A recipient unsubscribed from
|
|
19
|
-
* `in_app` (or `unsubscribed_all`) gets no feed items.
|
|
20
|
+
* `in_app` (or `unsubscribed_all`) gets no feed items. Canonically defined in
|
|
21
|
+
* `../lists/channels.js` (where the in-app channel is synthesized); re-exported
|
|
22
|
+
* here so the engine's existing export surface stays stable.
|
|
20
23
|
*/
|
|
21
|
-
export
|
|
24
|
+
export { IN_APP_LIST_ID };
|
|
22
25
|
|
|
23
26
|
export interface SendFeedItemOptions {
|
|
24
27
|
recipient: { userId?: string; email?: string; anonymousId?: string };
|
|
@@ -88,55 +91,23 @@ export async function sendFeedItem(
|
|
|
88
91
|
});
|
|
89
92
|
const recipientKey = resolvedKey;
|
|
90
93
|
|
|
91
|
-
// (2)
|
|
92
|
-
//
|
|
93
|
-
// `(user_id, email)`-keyed; an anon-only recipient has no pref row → categories
|
|
94
|
-
// = {} → opt-in default → not suppressed (you cannot suppress what has no
|
|
95
|
-
// preference surface yet). `unsubscribed_all` on an identified recipient DOES
|
|
96
|
-
// suppress, consistent with the email mailer's `checkEmailPreferences`.
|
|
97
|
-
let categories: Record<string, boolean> = {};
|
|
98
|
-
const recip = await resolveRecipient({
|
|
99
|
-
db,
|
|
100
|
-
userId: recipient.userId,
|
|
101
|
-
email: recipient.email,
|
|
102
|
-
});
|
|
103
|
-
if (recip) {
|
|
104
|
-
const extId = recip.externalId ?? recip.contactId;
|
|
105
|
-
const rows = await db
|
|
106
|
-
.select()
|
|
107
|
-
.from(emailPreferences)
|
|
108
|
-
.where(
|
|
109
|
-
and(
|
|
110
|
-
eq(emailPreferences.userId, extId),
|
|
111
|
-
eq(emailPreferences.email, recip.email),
|
|
112
|
-
),
|
|
113
|
-
)
|
|
114
|
-
.limit(1);
|
|
115
|
-
const row = rows[0];
|
|
116
|
-
if (row) {
|
|
117
|
-
if (row.unsubscribedAll) {
|
|
118
|
-
return {
|
|
119
|
-
feedItemId: null,
|
|
120
|
-
recipientKey,
|
|
121
|
-
suppressed: true,
|
|
122
|
-
createdAt: null,
|
|
123
|
-
};
|
|
124
|
-
}
|
|
125
|
-
categories = (row.categories ?? {}) as Record<string, boolean>;
|
|
126
|
-
}
|
|
127
|
-
}
|
|
128
|
-
if (!getListRegistry().isSubscribed(categories, IN_APP_LIST_ID)) {
|
|
129
|
-
return {
|
|
130
|
-
feedItemId: null,
|
|
131
|
-
recipientKey,
|
|
132
|
-
suppressed: true,
|
|
133
|
-
createdAt: null,
|
|
134
|
-
};
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
// (3) Replay-safe idempotency key (mirrors `sendEmail`). The `feed:`-namespaced
|
|
94
|
+
// (2) Replay-safe idempotency key (mirrors `sendEmail`), derived
|
|
95
|
+
// UNCONDITIONALLY — BEFORE any preference decision. The `feed:`-namespaced
|
|
138
96
|
// discriminant realizes the plan's `feedSend:<runAnchor>:<site>:<type>` shape
|
|
139
97
|
// through the SAME branch-stable key engine (one primitive, not a fork).
|
|
98
|
+
//
|
|
99
|
+
// THE LAW: the Hatchet journal is positional, and `boundary.memoize` (step 4)
|
|
100
|
+
// is a durable call, so its issuance must never be conditional on a live
|
|
101
|
+
// preference read. A recipient's `in_app`/`unsubscribed_all` state can flip
|
|
102
|
+
// between an original run and a replay-from-top; the OLD code early-returned on
|
|
103
|
+
// that read HERE, before the key derivation + `registerKey` + `memoize`, so a
|
|
104
|
+
// flip made the replay conditionally skip (or add) the memoize durable call →
|
|
105
|
+
// positional journal shift → the run is killed with a non-determinism error.
|
|
106
|
+
// The fix (mirroring the connector gate in lib/connector-actions.ts): derive +
|
|
107
|
+
// register the key and issue the memoize UNCONDITIONALLY, and fold the whole
|
|
108
|
+
// preference verdict INSIDE the memo closure (step 3) so the skip/allow verdict
|
|
109
|
+
// is RECORDED by the durable memo and replays verbatim — the live-flipped
|
|
110
|
+
// preference is never re-read on a replay.
|
|
140
111
|
const boundary = getJourneyBoundary();
|
|
141
112
|
let key: string | undefined = opts.idempotencyKey;
|
|
142
113
|
if (!key && boundary) {
|
|
@@ -222,12 +193,61 @@ export async function sendFeedItem(
|
|
|
222
193
|
};
|
|
223
194
|
};
|
|
224
195
|
|
|
225
|
-
//
|
|
226
|
-
//
|
|
227
|
-
//
|
|
228
|
-
//
|
|
196
|
+
// (3) `in_app` suppression gate + insert/publish, folded into ONE closure so
|
|
197
|
+
// the whole verdict (skip OR insert) is recorded by the durable memo (step 4)
|
|
198
|
+
// and replays byte-identically. Suppression is governed by the `in_app`
|
|
199
|
+
// channel list regardless of the item's own category.
|
|
200
|
+
//
|
|
201
|
+
// The preference read is the UNIFIED aggregated `readRecipientPreferences`
|
|
202
|
+
// keyed by BOTH the recipient's `external_id ?? contact_id` AND its email —
|
|
203
|
+
// NOT the old single-row `(extId, email)` lookup. This is a deliberate,
|
|
204
|
+
// suppression-conservative behaviour change: an `unsubscribed_all` (or category
|
|
205
|
+
// opt-out) imported before the contact existed and keyed `(email, email)` now
|
|
206
|
+
// suppresses the feed too, exactly as it already suppresses email. An anon-only
|
|
207
|
+
// recipient has no preference surface (`resolveRecipient` → null), so
|
|
208
|
+
// `unsubscribed_all` is not consulted (you cannot suppress what has no pref row
|
|
209
|
+
// yet); the `in_app` channel check still runs, but empty categories → opt-in
|
|
210
|
+
// default → subscribed, so an anon recipient is never suppressed here.
|
|
211
|
+
const doGatedInsertAndPublish = async (): Promise<SendFeedItemResult> => {
|
|
212
|
+
const suppressedResult: SendFeedItemResult = {
|
|
213
|
+
feedItemId: null,
|
|
214
|
+
recipientKey,
|
|
215
|
+
suppressed: true,
|
|
216
|
+
createdAt: null,
|
|
217
|
+
};
|
|
218
|
+
const recip = await resolveRecipient({
|
|
219
|
+
db,
|
|
220
|
+
userId: recipient.userId,
|
|
221
|
+
email: recipient.email,
|
|
222
|
+
});
|
|
223
|
+
// `external_id ?? contact_id` — the SAME identity key the old single-row read
|
|
224
|
+
// used (and that preference writes key on). Undefined for an anon recipient.
|
|
225
|
+
const extId = recip ? (recip.externalId ?? recip.contactId) : undefined;
|
|
226
|
+
const prefs = await readRecipientPreferences(db, {
|
|
227
|
+
email: recip?.email,
|
|
228
|
+
userId: extId,
|
|
229
|
+
});
|
|
230
|
+
// `unsubscribed_all` on an IDENTIFIED recipient suppresses (consistent with
|
|
231
|
+
// the email mailer's `checkEmailPreferences`); guarded on `recip` so an
|
|
232
|
+
// anon recipient with no preference surface is never blocked.
|
|
233
|
+
if (recip && prefs.unsubscribedAll) {
|
|
234
|
+
return suppressedResult;
|
|
235
|
+
}
|
|
236
|
+
if (!getListRegistry().isSubscribed(prefs.categories, IN_APP_LIST_ID)) {
|
|
237
|
+
return suppressedResult;
|
|
238
|
+
}
|
|
239
|
+
return doInsertAndPublish();
|
|
240
|
+
};
|
|
241
|
+
|
|
242
|
+
// (4) Layer 1 (eviction-gated, FREE) fast path. When inside a journey on an
|
|
243
|
+
// eviction-capable engine, a replay returns the recorded result (a skip verdict
|
|
244
|
+
// OR the insert result) WITHOUT re-reading preferences or re-hitting the DB;
|
|
245
|
+
// Layer 2 (`onConflictDoNothing`) is the version-independent backstop. Outside
|
|
246
|
+
// a journey, run directly. The key derivation, `registerKey`, and this
|
|
247
|
+
// `memoize` call all stay UNCONDITIONAL (THE LAW) — the preference verdict
|
|
248
|
+
// lives inside the closure, never gating the durable call itself.
|
|
229
249
|
if (boundary && key) {
|
|
230
|
-
return boundary.memoize([key],
|
|
250
|
+
return boundary.memoize([key], doGatedInsertAndPublish);
|
|
231
251
|
}
|
|
232
|
-
return
|
|
252
|
+
return doGatedInsertAndPublish();
|
|
233
253
|
}
|
package/src/lib/frequency-cap.ts
CHANGED
|
@@ -6,6 +6,42 @@ import type { FrequencyCapConfig } from "./email-service-types.js";
|
|
|
6
6
|
|
|
7
7
|
const DEFAULT_EXEMPT = ["transactional"];
|
|
8
8
|
|
|
9
|
+
/**
|
|
10
|
+
* The windowed non-failed send count for a recipient — the COUNT behind
|
|
11
|
+
* {@link isFrequencyCapped}, shared with `ctx.throttle` (and later the suppress
|
|
12
|
+
* guard). Served by the `email_sends_freq_cap_idx (to_email, created_at,
|
|
13
|
+
* category)` index. Never-dispatched / failed rows (`status = 'failed'`) are
|
|
14
|
+
* excluded; a `category` narrows the count to that category.
|
|
15
|
+
*
|
|
16
|
+
* NOTE: NO exemption semantics live here — unlike {@link isFrequencyCapped}
|
|
17
|
+
* (which short-circuits an exempt "transactional" category), a `category` passed
|
|
18
|
+
* here genuinely counts that category's sends.
|
|
19
|
+
*/
|
|
20
|
+
export async function countRecentSends(opts: {
|
|
21
|
+
db: Database;
|
|
22
|
+
to: string;
|
|
23
|
+
since: Date;
|
|
24
|
+
category?: string;
|
|
25
|
+
}): Promise<number> {
|
|
26
|
+
const { db, to, since, category } = opts;
|
|
27
|
+
|
|
28
|
+
const conditions = [
|
|
29
|
+
eq(emailSends.toEmail, to),
|
|
30
|
+
gte(emailSends.createdAt, since),
|
|
31
|
+
ne(emailSends.status, "failed"),
|
|
32
|
+
];
|
|
33
|
+
if (category !== undefined) {
|
|
34
|
+
conditions.push(eq(emailSends.category, category));
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const [row] = await db
|
|
38
|
+
.select({ n: count() })
|
|
39
|
+
.from(emailSends)
|
|
40
|
+
.where(and(...conditions));
|
|
41
|
+
|
|
42
|
+
return row?.n ?? 0;
|
|
43
|
+
}
|
|
44
|
+
|
|
9
45
|
/**
|
|
10
46
|
* True if this recipient has hit the configured send cap within the window.
|
|
11
47
|
*
|
|
@@ -35,20 +71,14 @@ export async function isFrequencyCapped(opts: {
|
|
|
35
71
|
|
|
36
72
|
const since = new Date(Date.now() - durationToMs(rule.window));
|
|
37
73
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
const [row] = await db
|
|
49
|
-
.select({ n: count() })
|
|
50
|
-
.from(emailSends)
|
|
51
|
-
.where(and(...conditions));
|
|
74
|
+
// The byCategory branch counts only sends in that category; the global rule
|
|
75
|
+
// counts ALL of the recipient's non-failed sends in the window.
|
|
76
|
+
const n = await countRecentSends({
|
|
77
|
+
db,
|
|
78
|
+
to,
|
|
79
|
+
since,
|
|
80
|
+
...(override && category ? { category } : {}),
|
|
81
|
+
});
|
|
52
82
|
|
|
53
|
-
return
|
|
83
|
+
return n >= rule.count;
|
|
54
84
|
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { type DurationObject, durationToMs } from "@hogsend/core";
|
|
2
|
+
|
|
3
|
+
/** Normalize any duration to a SINGLE-UNIT whole-seconds Go string ("120s")
|
|
4
|
+
* before it reaches a Hatchet durable sleep (`sleepFor` or a `SleepCondition`).
|
|
5
|
+
* The SDK renders ms numbers as multi-unit strings ("1m59s"), which some
|
|
6
|
+
* hatchet-lite versions silently fail to honor as sleep conditions (the wait
|
|
7
|
+
* resolves instantly with no match) — pinned empirically 2026-07-10. Whole
|
|
8
|
+
* seconds match the SDK's own `sleepUntil` normalization. EVERY engine call
|
|
9
|
+
* site that hands Hatchet a sleep duration must route through this — a raw ms
|
|
10
|
+
* number reopens the instant-wake bug.
|
|
11
|
+
*
|
|
12
|
+
* Lives beside (not inside) `lib/hatchet.ts` because tests `vi.mock` that
|
|
13
|
+
* module to stub the client singleton; this helper must survive the mock. */
|
|
14
|
+
export const toSleepDuration = (
|
|
15
|
+
durationOrMs: DurationObject | number,
|
|
16
|
+
): `${number}s` => {
|
|
17
|
+
const ms =
|
|
18
|
+
typeof durationOrMs === "number"
|
|
19
|
+
? durationOrMs
|
|
20
|
+
: durationToMs(durationOrMs);
|
|
21
|
+
return `${Math.max(1, Math.ceil(ms / 1000))}s`;
|
|
22
|
+
};
|