@hogsend/engine 0.40.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -8
- package/src/buckets/check-membership.ts +6 -1
- package/src/campaigns/cohort-sql.ts +198 -0
- package/src/campaigns/define-campaign.ts +207 -28
- package/src/campaigns/reconcile.ts +105 -11
- package/src/campaigns/steps.ts +48 -0
- package/src/connectors/define-action.ts +70 -0
- package/src/container.ts +124 -42
- package/src/destinations/presets/posthog.ts +7 -1
- package/src/index.ts +96 -2
- package/src/journeys/constants.ts +10 -0
- package/src/journeys/define-journey.ts +14 -405
- package/src/journeys/execute-journey-run.ts +641 -0
- package/src/journeys/graph/build-graph.ts +212 -56
- package/src/journeys/journey-boundary.ts +54 -0
- package/src/journeys/journey-context.ts +406 -59
- package/src/journeys/record-once.ts +110 -0
- package/src/lib/blueprint-lock.ts +21 -0
- package/src/lib/blueprints.ts +803 -0
- package/src/lib/campaign-send-key.ts +58 -0
- package/src/lib/connector-actions.ts +203 -19
- package/src/lib/contacts.ts +54 -0
- package/src/lib/email-service-types.ts +4 -1
- package/src/lib/email.ts +13 -2
- package/src/lib/event-names.ts +143 -0
- package/src/lib/feed.ts +77 -57
- package/src/lib/frequency-cap.ts +45 -15
- package/src/lib/hatchet-duration.ts +22 -0
- package/src/lib/html.ts +1 -0
- package/src/lib/ingestion.ts +145 -4
- package/src/lib/journey-suppress.ts +87 -0
- package/src/lib/links.ts +204 -18
- package/src/lib/outbound.ts +20 -0
- package/src/lib/preferences.ts +80 -1
- package/src/lib/tracked.ts +44 -29
- package/src/lib/tracking-event-names.ts +8 -0
- package/src/lib/tracking.ts +1 -5
- package/src/lib/webhook-signing.ts +6 -1
- package/src/lists/channels.ts +75 -0
- package/src/lists/define-list.ts +34 -0
- package/src/lists/registry.ts +29 -2
- package/src/mcp/authoring-guide.ts +160 -0
- package/src/mcp/blueprint-tools.ts +390 -0
- package/src/middleware/require-admin.ts +13 -0
- package/src/routes/admin/api-keys.ts +72 -0
- package/src/routes/admin/blueprints.ts +795 -0
- package/src/routes/admin/campaigns.ts +179 -3
- package/src/routes/admin/emails.ts +13 -0
- package/src/routes/admin/events.ts +55 -0
- package/src/routes/admin/index.ts +4 -0
- package/src/routes/admin/journeys.ts +4 -2
- package/src/routes/admin/links.ts +303 -36
- package/src/routes/admin/lists.ts +51 -0
- package/src/routes/campaigns/index.ts +44 -6
- package/src/routes/email/preferences.ts +50 -13
- package/src/routes/feed/recipient.ts +2 -43
- package/src/routes/index.ts +6 -0
- package/src/routes/lists/index.ts +138 -18
- package/src/routes/tracking/arrive.ts +260 -0
- package/src/routes/tracking/click-pipeline.ts +329 -0
- package/src/routes/tracking/click.ts +5 -270
- package/src/routes/tracking/index.ts +2 -0
- package/src/routes/tracking/vanity.ts +64 -0
- package/src/worker.ts +5 -0
- package/src/workflows/bucket-backfill.ts +4 -2
- package/src/workflows/bucket-reconcile.ts +11 -4
- package/src/workflows/check-alerts.ts +130 -1
- package/src/workflows/journey-blueprint-interpreter.ts +656 -0
- package/src/workflows/send-campaign.ts +574 -180
|
@@ -1,5 +1,12 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type CampaignSendStep,
|
|
3
|
+
type CampaignStep,
|
|
4
|
+
type ConditionEval,
|
|
5
|
+
durationToMs,
|
|
6
|
+
} from "@hogsend/core";
|
|
1
7
|
import {
|
|
2
8
|
bucketMemberships,
|
|
9
|
+
campaignRecipients,
|
|
3
10
|
campaigns,
|
|
4
11
|
contacts,
|
|
5
12
|
type Database,
|
|
@@ -7,6 +14,11 @@ import {
|
|
|
7
14
|
} from "@hogsend/db";
|
|
8
15
|
import type { TemplateName } from "@hogsend/email";
|
|
9
16
|
import { and, eq, gt, inArray, isNull, lt, lte, or, sql } from "drizzle-orm";
|
|
17
|
+
import {
|
|
18
|
+
cohortSuppressionSql,
|
|
19
|
+
waveConditionSql,
|
|
20
|
+
} from "../campaigns/cohort-sql.js";
|
|
21
|
+
import { campaignSendKey } from "../lib/campaign-send-key.js";
|
|
10
22
|
import { normalizeEmail } from "../lib/contacts.js";
|
|
11
23
|
import { getDb } from "../lib/db.js";
|
|
12
24
|
import { getEmailService } from "../lib/email.js";
|
|
@@ -25,44 +37,60 @@ interface CampaignRecipient {
|
|
|
25
37
|
|
|
26
38
|
/**
|
|
27
39
|
* Statuses that are TERMINAL — a duplicate/late enqueue must not re-send.
|
|
28
|
-
* `canceled`/`expired` matter doubly for SCHEDULED campaigns:
|
|
29
|
-
* Hatchet scheduled run cannot be deleted at cancel time (we don't
|
|
30
|
-
* id), so it still fires and must find the row terminal here and
|
|
31
|
-
* `failed` is deliberately NOT terminal — a re-enqueue can revive it.
|
|
40
|
+
* `canceled`/`expired` matter doubly for SCHEDULED (and now WAITING) campaigns:
|
|
41
|
+
* the punctual Hatchet scheduled run cannot be deleted at cancel time (we don't
|
|
42
|
+
* persist its id), so it still fires and must find the row terminal here and
|
|
43
|
+
* no-op. `failed` is deliberately NOT terminal — a re-enqueue can revive it.
|
|
32
44
|
*/
|
|
33
45
|
const TERMINAL_STATUSES = ["sent", "canceled", "expired"] as const;
|
|
34
46
|
|
|
35
47
|
/**
|
|
36
|
-
* How far ahead of `scheduledAt` a run may fire and still
|
|
37
|
-
* clock skew). An earlier fire — the punctual run created for
|
|
38
|
-
* was later moved BACK
|
|
39
|
-
*
|
|
48
|
+
* How far ahead of `scheduledAt` / `nextStepAt` a run may fire and still
|
|
49
|
+
* proceed (absorbs clock skew). An earlier fire — the punctual run created for
|
|
50
|
+
* an instant that was later moved BACK (a reconcile edit for `scheduledAt`; a
|
|
51
|
+
* reap + re-park re-schedule for `nextStepAt`) — skips without sending; the
|
|
52
|
+
* run created for the new instant (or the reaper sweep) delivers it.
|
|
40
53
|
*/
|
|
41
54
|
const EARLY_FIRE_TOLERANCE_MS = 60 * 1000;
|
|
42
55
|
|
|
43
56
|
/**
|
|
44
|
-
* Built-in durable campaign / broadcast task (Loops "campaign" parity).
|
|
45
|
-
*
|
|
46
|
-
* of a bucket).
|
|
57
|
+
* Built-in durable campaign / broadcast task (Loops "campaign" parity). A
|
|
58
|
+
* legacy row (NULL `steps`) sends a single template to every subscribed member
|
|
59
|
+
* of a list (or every active member of a bucket). A multi-step row executes
|
|
60
|
+
* its `steps` blob as WAVES — each send step a SET operation over the
|
|
61
|
+
* campaign's anchored cohort, separated by durable waits (status `waiting`,
|
|
62
|
+
* resumed at `nextStepAt`) — see docs/campaign-steps-spec.md §Wave runtime.
|
|
63
|
+
* The task keeps its name and `{ campaignId }` input either way; the row's
|
|
64
|
+
* `currentStep` is the sole resume cursor.
|
|
47
65
|
*
|
|
48
|
-
* Retry-safety: each send carries an idempotency key
|
|
49
|
-
* `campaign:<
|
|
50
|
-
*
|
|
66
|
+
* Retry-safety: each send carries an idempotency key minted by
|
|
67
|
+
* `campaignSendKey` — legacy `campaign:<id>:<email>` for single-step
|
|
68
|
+
* campaigns, step-scoped `campaign:<id>:<step>:<email>` for EVERY step of a
|
|
69
|
+
* multi-step campaign (email_sends.idempotency_key, migration 0015) — so a
|
|
70
|
+
* Hatchet retry re-runs the wave but every already-dispatched send
|
|
51
71
|
* short-circuits to its prior row instead of dispatching a duplicate provider
|
|
52
|
-
* call. Counts are derived as-you-go from each `send()` result status — which
|
|
53
|
-
* itself idempotency-aware (a retried send returns the prior row's status),
|
|
54
|
-
* the tallies stay consistent across re-attempts.
|
|
55
|
-
* increment) the row, so a retry re-derives them from scratch rather than
|
|
56
|
-
* double-counting.
|
|
72
|
+
* call. Counts are derived as-you-go from each `send()` result status — which
|
|
73
|
+
* is itself idempotency-aware (a retried send returns the prior row's status),
|
|
74
|
+
* so the tallies stay consistent across re-attempts.
|
|
57
75
|
*
|
|
58
|
-
*
|
|
59
|
-
* `
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
76
|
+
* Counts across waves: the row counts are CUMULATIVE across waves.
|
|
77
|
+
* `stepBaseCounts` snapshots the cumulative tallies through the last COMPLETED
|
|
78
|
+
* wave (written with each cursor advance); the current wave always re-derives
|
|
79
|
+
* its own tally from scratch on top of that seed, and every flush stays an
|
|
80
|
+
* absolute overwrite — so a retried wave re-tallies exactly rather than
|
|
81
|
+
* double-counting. Prior-wave counts are deliberately NOT derived from
|
|
82
|
+
* `email_sends`: a suppressed send writes its row WITHOUT the idempotency key
|
|
83
|
+
* and a frequency-capped send writes NO row at all, so the ledger cannot
|
|
84
|
+
* reproduce skipped counts — hence the snapshot column.
|
|
85
|
+
*
|
|
86
|
+
* Resume-on-retry: the terminal guard short-circuits ONLY a terminal campaign —
|
|
87
|
+
* a `failed`/`sending` row is NOT terminal, so a Hatchet retry (or the reaper's
|
|
88
|
+
* re-enqueue) re-enters at `currentStep` and re-runs the current wave.
|
|
89
|
+
* Already-dispatched sends no-op via the idempotency key, so the re-run safely
|
|
90
|
+
* completes the TAIL of a partial wave instead of abandoning it. The catch
|
|
91
|
+
* block therefore does NOT stamp `failed` before re-throwing — that would make
|
|
92
|
+
* the retry short-circuit and silently under-deliver. A run that exhausts its
|
|
93
|
+
* retries is reaped to `failed`/re-enqueued by {@link reapStuckCampaignsTask}.
|
|
66
94
|
*/
|
|
67
95
|
export const sendCampaignTask = hatchet.task({
|
|
68
96
|
name: "send-campaign",
|
|
@@ -89,11 +117,12 @@ export const sendCampaignTask = hatchet.task({
|
|
|
89
117
|
return { status: "failed", reason: "not_found" as const };
|
|
90
118
|
}
|
|
91
119
|
|
|
92
|
-
// Already terminal — a duplicate/late enqueue must not re-send. ONLY
|
|
93
|
-
//
|
|
94
|
-
// Hatchet retry (or a reaper re-enqueue)
|
|
95
|
-
// completes the unsent TAIL of a partial
|
|
96
|
-
// no-op via the per-send idempotency key —
|
|
120
|
+
// Already terminal — a duplicate/late enqueue must not re-send. ONLY
|
|
121
|
+
// sent/canceled/expired are terminal: a `failed`/`sending` row is
|
|
122
|
+
// intentionally re-runnable so a Hatchet retry (or a reaper re-enqueue)
|
|
123
|
+
// re-enters at `currentStep` and completes the unsent TAIL of a partial
|
|
124
|
+
// wave (already-sent recipients no-op via the per-send idempotency key —
|
|
125
|
+
// risk: silent under-delivery).
|
|
97
126
|
if ((TERMINAL_STATUSES as readonly string[]).includes(campaign.status)) {
|
|
98
127
|
return { status: campaign.status, skipped: true };
|
|
99
128
|
}
|
|
@@ -110,20 +139,61 @@ export const sendCampaignTask = hatchet.task({
|
|
|
110
139
|
return { status: "scheduled", skipped: true, reason: "not_due" as const };
|
|
111
140
|
}
|
|
112
141
|
|
|
142
|
+
// Mirror clause for `waiting`: a punctual next-step run that fires while
|
|
143
|
+
// `nextStepAt` is still in the future is stale — a reap + re-park replaced
|
|
144
|
+
// the pending wait after this run was scheduled. The run created for the
|
|
145
|
+
// new instant (or the reaper's waiting-promotion sweep) resumes it.
|
|
146
|
+
if (
|
|
147
|
+
campaign.status === "waiting" &&
|
|
148
|
+
campaign.nextStepAt &&
|
|
149
|
+
campaign.nextStepAt.getTime() > Date.now() + EARLY_FIRE_TOLERANCE_MS
|
|
150
|
+
) {
|
|
151
|
+
return { status: "waiting", skipped: true, reason: "not_due" as const };
|
|
152
|
+
}
|
|
153
|
+
|
|
113
154
|
// Claim via CAS: only a non-terminal row transitions to `sending`, so a
|
|
114
155
|
// cancel landing between the guard read above and this write is honored
|
|
115
|
-
// rather than silently resurrected.
|
|
116
|
-
|
|
156
|
+
// rather than silently resurrected. `startedAt` is claimed ONCE
|
|
157
|
+
// (coalesce): event-condition scoping reads "since campaign startedAt",
|
|
158
|
+
// so a wave-2 claim must NOT reset the anchor wave-1's conditions were
|
|
159
|
+
// already scoped by. The cursor + count seed are re-read FROM the claimed
|
|
160
|
+
// row (not the pre-claim read) so a wave resumed after a crash sees the
|
|
161
|
+
// exact state its predecessor persisted.
|
|
162
|
+
const claimedRows = await db
|
|
117
163
|
.update(campaigns)
|
|
118
|
-
.set({
|
|
164
|
+
.set({
|
|
165
|
+
status: "sending",
|
|
166
|
+
startedAt: sql`coalesce(${campaigns.startedAt}, now())`,
|
|
167
|
+
// A resumed `waiting` row's wait has elapsed — clear the stale instant
|
|
168
|
+
// so serialized rows never show a countdown while `sending`. (Studio
|
|
169
|
+
// keys its countdown on status === "waiting" regardless.)
|
|
170
|
+
nextStepAt: null,
|
|
171
|
+
// `staleSince` deliberately NOT cleared here: claiming is not
|
|
172
|
+
// progress. A poison run claims, crashes, and is re-driven — if the
|
|
173
|
+
// claim reset the marker, its give-up clock would restart every cycle
|
|
174
|
+
// and the reaper could never declare it failed. The first real
|
|
175
|
+
// progress write (chunk flush / park / cursor advance) clears it.
|
|
176
|
+
updatedAt: new Date(),
|
|
177
|
+
})
|
|
119
178
|
.where(
|
|
120
179
|
and(
|
|
121
180
|
eq(campaigns.id, input.campaignId),
|
|
122
|
-
inArray(campaigns.status, [
|
|
181
|
+
inArray(campaigns.status, [
|
|
182
|
+
"queued",
|
|
183
|
+
"scheduled",
|
|
184
|
+
"sending",
|
|
185
|
+
"waiting",
|
|
186
|
+
]),
|
|
123
187
|
),
|
|
124
188
|
)
|
|
125
|
-
.returning({
|
|
126
|
-
|
|
189
|
+
.returning({
|
|
190
|
+
id: campaigns.id,
|
|
191
|
+
currentStep: campaigns.currentStep,
|
|
192
|
+
startedAt: campaigns.startedAt,
|
|
193
|
+
stepBaseCounts: campaigns.stepBaseCounts,
|
|
194
|
+
});
|
|
195
|
+
const claim = claimedRows[0];
|
|
196
|
+
if (!claim) {
|
|
127
197
|
const current = await db
|
|
128
198
|
.select({ status: campaigns.status })
|
|
129
199
|
.from(campaigns)
|
|
@@ -132,12 +202,65 @@ export const sendCampaignTask = hatchet.task({
|
|
|
132
202
|
return { status: current[0]?.status ?? "unknown", skipped: true };
|
|
133
203
|
}
|
|
134
204
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
205
|
+
// The executable step sequence. A NULL blob is a legacy single-send row —
|
|
206
|
+
// one send step synthesized from the row columns, and (per the
|
|
207
|
+
// campaignSendKey contract) the LEGACY per-send key format. A non-NULL
|
|
208
|
+
// blob only ever holds > 1 steps (the reconciler stores single-step
|
|
209
|
+
// definitions as NULL), so `multiStep` doubles as the key-format switch:
|
|
210
|
+
// step-scoped keys for ALL steps including 0.
|
|
211
|
+
const blob = campaign.steps;
|
|
212
|
+
const multiStep = blob != null;
|
|
213
|
+
const executable: CampaignStep[] = blob
|
|
214
|
+
? // db cannot import @hogsend/core, so the blob's elements are opaque
|
|
215
|
+
// Record<string, unknown> there; the engine owns the narrowing.
|
|
216
|
+
(blob.steps as unknown as CampaignStep[])
|
|
217
|
+
: [
|
|
218
|
+
{
|
|
219
|
+
kind: "send",
|
|
220
|
+
template: campaign.templateKey,
|
|
221
|
+
props: campaign.props ?? {},
|
|
222
|
+
...(campaign.subject != null ? { subject: campaign.subject } : {}),
|
|
223
|
+
...(campaign.fromEmail != null ? { from: campaign.fromEmail } : {}),
|
|
224
|
+
},
|
|
225
|
+
];
|
|
226
|
+
|
|
227
|
+
// Immutable row facts captured once (nested fns below can't see the
|
|
228
|
+
// `campaign` narrowing). `startedAt` is non-null post-claim (coalesce);
|
|
229
|
+
// the ?? is a type-level fallback only.
|
|
230
|
+
const audienceKind = campaign.audienceKind;
|
|
231
|
+
const audienceId = campaign.audienceId;
|
|
232
|
+
const startedAt = claim.startedAt ?? new Date();
|
|
233
|
+
|
|
234
|
+
// Seed the cumulative counters from the snapshot through the last
|
|
235
|
+
// COMPLETED wave; the current wave re-derives its own tally on top. Do NOT
|
|
236
|
+
// try to derive the seed from email_sends — suppressed sends write no
|
|
237
|
+
// idempotency key and frequency-capped sends write no row (lib/tracked.ts).
|
|
238
|
+
const base = claim.stepBaseCounts ?? {
|
|
239
|
+
total: 0,
|
|
240
|
+
sent: 0,
|
|
241
|
+
skipped: 0,
|
|
242
|
+
failed: 0,
|
|
243
|
+
};
|
|
244
|
+
let totalRecipients = base.total;
|
|
245
|
+
let sentCount = base.sent;
|
|
246
|
+
let skippedCount = base.skipped;
|
|
247
|
+
let failedCount = base.failed;
|
|
248
|
+
|
|
249
|
+
const snapshotCounts = () => ({
|
|
250
|
+
total: totalRecipients,
|
|
251
|
+
sent: sentCount,
|
|
252
|
+
skipped: skippedCount,
|
|
253
|
+
failed: failedCount,
|
|
254
|
+
});
|
|
139
255
|
|
|
140
|
-
|
|
256
|
+
// Every flush is genuine progress (a dispatched chunk, a cancel/completion
|
|
257
|
+
// race) and clears the reaper's staleness marker — EXCEPT the catch-path
|
|
258
|
+
// flush: a crashed run made no progress, so `staleSince` must survive it
|
|
259
|
+
// or a deterministically-crashing campaign resets its give-up clock on
|
|
260
|
+
// every re-drive and is re-enqueued forever.
|
|
261
|
+
const flushCounts = async (opts?: {
|
|
262
|
+
preserveStaleSince?: boolean;
|
|
263
|
+
}): Promise<void> => {
|
|
141
264
|
await db
|
|
142
265
|
.update(campaigns)
|
|
143
266
|
.set({
|
|
@@ -145,127 +268,160 @@ export const sendCampaignTask = hatchet.task({
|
|
|
145
268
|
sentCount,
|
|
146
269
|
skippedCount,
|
|
147
270
|
failedCount,
|
|
271
|
+
...(opts?.preserveStaleSince ? {} : { staleSince: null }),
|
|
148
272
|
updatedAt: new Date(),
|
|
149
273
|
})
|
|
150
274
|
.where(eq(campaigns.id, input.campaignId));
|
|
151
275
|
};
|
|
152
276
|
|
|
153
277
|
try {
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
278
|
+
// The wave loop: consecutive send steps run sequentially in this same
|
|
279
|
+
// task run; a wait step parks the row (`waiting`) and returns. Resuming
|
|
280
|
+
// with `currentStep` already past the last index (a crash between the
|
|
281
|
+
// final cursor advance and the completion CAS) skips straight to the
|
|
282
|
+
// completion CAS below — exactly right.
|
|
283
|
+
for (let k = claim.currentStep; k < executable.length; k++) {
|
|
284
|
+
const step = executable[k];
|
|
285
|
+
if (step === undefined) break; // unreachable — loop bound
|
|
158
286
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
287
|
+
if (step.kind === "wait") {
|
|
288
|
+
const nextStepAt = new Date(Date.now() + durationToMs(step.duration));
|
|
289
|
+
// Park CAS: only a still-`sending` row goes `waiting`, so a cancel
|
|
290
|
+
// racing the end of the previous wave keeps its `canceled` status.
|
|
291
|
+
// Cursor, wait instant, count snapshot, and counts land in ONE
|
|
292
|
+
// update — a crash can never separate them.
|
|
293
|
+
const parked = await db
|
|
294
|
+
.update(campaigns)
|
|
295
|
+
.set({
|
|
296
|
+
status: "waiting",
|
|
297
|
+
currentStep: k + 1,
|
|
298
|
+
nextStepAt,
|
|
299
|
+
stepBaseCounts: snapshotCounts(),
|
|
300
|
+
totalRecipients,
|
|
301
|
+
sentCount,
|
|
302
|
+
skippedCount,
|
|
303
|
+
failedCount,
|
|
304
|
+
staleSince: null,
|
|
305
|
+
updatedAt: new Date(),
|
|
306
|
+
})
|
|
307
|
+
.where(
|
|
308
|
+
and(
|
|
309
|
+
eq(campaigns.id, input.campaignId),
|
|
310
|
+
eq(campaigns.status, "sending"),
|
|
311
|
+
),
|
|
312
|
+
)
|
|
313
|
+
.returning({ id: campaigns.id });
|
|
314
|
+
if (parked.length === 0) {
|
|
315
|
+
await flushCounts();
|
|
316
|
+
const current = await db
|
|
317
|
+
.select({ status: campaigns.status })
|
|
318
|
+
.from(campaigns)
|
|
319
|
+
.where(eq(campaigns.id, input.campaignId))
|
|
320
|
+
.limit(1);
|
|
321
|
+
return { status: current[0]?.status ?? "unknown", skipped: true };
|
|
322
|
+
}
|
|
163
323
|
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
324
|
+
// Punctual resume run at nextStepAt — best-effort, same split as
|
|
325
|
+
// `scheduledAt`: on failure the row stays `waiting` and the reaper's
|
|
326
|
+
// waiting-promotion sweep resumes it.
|
|
327
|
+
try {
|
|
328
|
+
await sendCampaignTask.schedule(nextStepAt, {
|
|
329
|
+
campaignId: input.campaignId,
|
|
330
|
+
});
|
|
331
|
+
} catch (err) {
|
|
332
|
+
logger.warn(
|
|
333
|
+
"send-campaign: next-step schedule failed (reaper will promote)",
|
|
334
|
+
{
|
|
335
|
+
campaignId: input.campaignId,
|
|
336
|
+
nextStepAt: nextStepAt.toISOString(),
|
|
337
|
+
error: err instanceof Error ? err.message : String(err),
|
|
338
|
+
},
|
|
339
|
+
);
|
|
340
|
+
}
|
|
173
341
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
342
|
+
logger.info("send-campaign: wave complete — waiting", {
|
|
343
|
+
campaignId: input.campaignId,
|
|
344
|
+
currentStep: k + 1,
|
|
345
|
+
nextStepAt: nextStepAt.toISOString(),
|
|
346
|
+
});
|
|
347
|
+
return {
|
|
348
|
+
status: "waiting" as const,
|
|
349
|
+
currentStep: k + 1,
|
|
350
|
+
nextStepAt: nextStepAt.toISOString(),
|
|
351
|
+
totalRecipients,
|
|
352
|
+
sentCount,
|
|
353
|
+
skippedCount,
|
|
354
|
+
failedCount,
|
|
355
|
+
};
|
|
184
356
|
}
|
|
185
357
|
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
emailService.send({
|
|
193
|
-
template: campaign?.templateKey as TemplateName,
|
|
194
|
-
props: (campaign?.props ?? {}) as never,
|
|
195
|
-
to: r.email,
|
|
196
|
-
userId: r.userId,
|
|
197
|
-
userEmail: r.email,
|
|
198
|
-
subject: campaign?.subject ?? undefined,
|
|
199
|
-
from: campaign?.fromEmail ?? undefined,
|
|
200
|
-
// A list's audienceId IS a real subscription category, so pass it
|
|
201
|
-
// through for suppression + the unsubscribe link. A bucket's
|
|
202
|
-
// audienceId is NOT a category — forcing it here would mint an
|
|
203
|
-
// unsubscribe link keyed on the bucket id (`categories[bucketId] =
|
|
204
|
-
// false`) that the bucket resolver never honors (it only checks
|
|
205
|
-
// unsubscribedAll/suppressed), silently no-op'ing the unsubscribe.
|
|
206
|
-
// For a bucket, pass undefined so the template's OWN declared
|
|
207
|
-
// category (e.g. `product-updates`) drives both suppression and a
|
|
208
|
-
// real, honored List-Unsubscribe target.
|
|
209
|
-
category:
|
|
210
|
-
campaign?.audienceKind === "bucket"
|
|
211
|
-
? undefined
|
|
212
|
-
: campaign?.audienceId,
|
|
213
|
-
// The idempotency key dedupes a retried send to its prior row.
|
|
214
|
-
idempotencyKey: `campaign:${input.campaignId}:${r.email}`,
|
|
215
|
-
}),
|
|
216
|
-
),
|
|
217
|
-
);
|
|
218
|
-
|
|
219
|
-
for (const result of results) {
|
|
220
|
-
if (result.status === "rejected") {
|
|
221
|
-
failedCount++;
|
|
222
|
-
continue;
|
|
223
|
-
}
|
|
224
|
-
const status = result.value.status;
|
|
225
|
-
if (status === "sent") {
|
|
226
|
-
sentCount++;
|
|
227
|
-
} else {
|
|
228
|
-
// suppressed | unsubscribed | skipped (frequency-capped) — counted
|
|
229
|
-
// as skipped, not a delivery failure.
|
|
230
|
-
skippedCount++;
|
|
231
|
-
}
|
|
358
|
+
if (step.kind !== "send") {
|
|
359
|
+
// A blob minted by a future engine (steps.v evolution) must fail
|
|
360
|
+
// loudly here, not silently skip a delivery step.
|
|
361
|
+
throw new Error(
|
|
362
|
+
`send-campaign: unsupported step kind "${(step as { kind: string }).kind}" at step ${k} of campaign ${input.campaignId}`,
|
|
363
|
+
);
|
|
232
364
|
}
|
|
233
365
|
|
|
234
|
-
await
|
|
235
|
-
|
|
366
|
+
const canceled = await runWave(k, step);
|
|
367
|
+
if (canceled) {
|
|
368
|
+
// The operator's cancel already stamped status/canceledAt; persist
|
|
369
|
+
// the progress counts so the row shows how far the blast got.
|
|
370
|
+
await flushCounts();
|
|
371
|
+
logger.info("send-campaign: canceled mid-send", {
|
|
372
|
+
campaignId: input.campaignId,
|
|
373
|
+
currentStep: k,
|
|
374
|
+
totalRecipients,
|
|
375
|
+
sentCount,
|
|
376
|
+
skippedCount,
|
|
377
|
+
failedCount,
|
|
378
|
+
});
|
|
379
|
+
return {
|
|
380
|
+
status: "canceled" as const,
|
|
381
|
+
totalRecipients,
|
|
382
|
+
sentCount,
|
|
383
|
+
skippedCount,
|
|
384
|
+
failedCount,
|
|
385
|
+
};
|
|
386
|
+
}
|
|
236
387
|
|
|
237
|
-
|
|
238
|
-
//
|
|
239
|
-
//
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
388
|
+
// Advance the cursor + snapshot the cumulative counters in ONE update
|
|
389
|
+
// — the wave is complete, so a retry entering after this point starts
|
|
390
|
+
// at the NEXT step with these counts as its seed. Plain update (not
|
|
391
|
+
// CAS): a cancel that raced the tail of the wave only bumps cursor
|
|
392
|
+
// metadata on an already-terminal row, which nothing reads.
|
|
393
|
+
await db
|
|
394
|
+
.update(campaigns)
|
|
395
|
+
.set({
|
|
396
|
+
currentStep: k + 1,
|
|
397
|
+
stepBaseCounts: snapshotCounts(),
|
|
398
|
+
totalRecipients,
|
|
399
|
+
sentCount,
|
|
400
|
+
skippedCount,
|
|
401
|
+
failedCount,
|
|
402
|
+
staleSince: null,
|
|
403
|
+
updatedAt: new Date(),
|
|
404
|
+
})
|
|
405
|
+
.where(eq(campaigns.id, input.campaignId));
|
|
255
406
|
}
|
|
256
407
|
|
|
257
408
|
// Completion CAS: only a still-`sending` row is stamped `sent`, so a
|
|
258
409
|
// cancel racing the final chunk (after its last per-chunk check) keeps
|
|
259
|
-
// its `canceled` status instead of being overwritten.
|
|
410
|
+
// its `canceled` status instead of being overwritten. `nextStepAt` +
|
|
411
|
+
// `stepBaseCounts` are nulled — a terminal row carries no pending wait
|
|
412
|
+
// and no resume seed.
|
|
260
413
|
const completed = await db
|
|
261
414
|
.update(campaigns)
|
|
262
415
|
.set({
|
|
263
416
|
status: "sent",
|
|
264
417
|
completedAt: new Date(),
|
|
418
|
+
nextStepAt: null,
|
|
419
|
+
stepBaseCounts: null,
|
|
265
420
|
totalRecipients,
|
|
266
421
|
sentCount,
|
|
267
422
|
skippedCount,
|
|
268
423
|
failedCount,
|
|
424
|
+
staleSince: null,
|
|
269
425
|
updatedAt: new Date(),
|
|
270
426
|
})
|
|
271
427
|
.where(
|
|
@@ -315,21 +471,16 @@ export const sendCampaignTask = hatchet.task({
|
|
|
315
471
|
// Do NOT stamp `failed` here. A `failed` stamp before the re-throw makes
|
|
316
472
|
// the single Hatchet retry hit the terminal guard and short-circuit
|
|
317
473
|
// WITHOUT sending the remaining recipients — silently abandoning the tail
|
|
318
|
-
// of a partial
|
|
319
|
-
// status `sending` (re-runnable
|
|
320
|
-
//
|
|
321
|
-
//
|
|
322
|
-
//
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
skippedCount,
|
|
329
|
-
failedCount,
|
|
330
|
-
updatedAt: new Date(),
|
|
331
|
-
})
|
|
332
|
-
.where(eq(campaigns.id, input.campaignId));
|
|
474
|
+
// of a partial wave. Instead we persist the progress counts, leave the
|
|
475
|
+
// status `sending` (re-runnable — `currentStep` still points at the
|
|
476
|
+
// interrupted wave and `stepBaseCounts` still seeds it), and re-throw so
|
|
477
|
+
// the genuine retry re-enters the wave and finishes the unsent tail
|
|
478
|
+
// (already-sent recipients no-op via their idempotency key). A run that
|
|
479
|
+
// EXHAUSTS its retries is transitioned to `failed` (or re-enqueued) by
|
|
480
|
+
// `reapStuckCampaignsTask`. `staleSince` is preserved — a crashed run is
|
|
481
|
+
// not progress, and clearing it here would reset the give-up clock on
|
|
482
|
+
// every re-drive of a poison campaign.
|
|
483
|
+
await flushCounts({ preserveStaleSince: true });
|
|
333
484
|
|
|
334
485
|
logger.error("send-campaign: errored mid-run (will retry)", {
|
|
335
486
|
campaignId: input.campaignId,
|
|
@@ -337,6 +488,148 @@ export const sendCampaignTask = hatchet.task({
|
|
|
337
488
|
});
|
|
338
489
|
throw error;
|
|
339
490
|
}
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* Execute one per-recipient email wave. Returns `true` when a mid-wave
|
|
494
|
+
* cancel was detected (the caller flushes + returns; nothing after the
|
|
495
|
+
* detecting chunk is dispatched).
|
|
496
|
+
*
|
|
497
|
+
* Qualifier source:
|
|
498
|
+
* - Wave 0 resolves the audience LIVE (list/bucket resolvers, fresh
|
|
499
|
+
* suppression pre-filter) and — for MULTI-step campaigns — ANCHORS the
|
|
500
|
+
* cohort: each chunk is batch-inserted into `campaign_recipients`
|
|
501
|
+
* BEFORE its sends are dispatched — a crash between insert and send is
|
|
502
|
+
* fine (the retry re-resolves and the (campaign_id, email) unique
|
|
503
|
+
* absorbs the replay). The audience is resolved exactly once per
|
|
504
|
+
* campaign: someone added to the list afterwards never receives step 3
|
|
505
|
+
* without step 1. A single-send campaign skips the ledger entirely (no
|
|
506
|
+
* later wave will read it — the legacy write path stays byte-identical).
|
|
507
|
+
* - Waves k > 0 qualify FROM the anchored cohort: `campaign_recipients`
|
|
508
|
+
* ∩ the step's `where` conditions ∩ a fresh suppression re-check
|
|
509
|
+
* (see resolveCohortRecipients — suppression is never snapshotted).
|
|
510
|
+
*/
|
|
511
|
+
async function runWave(
|
|
512
|
+
k: number,
|
|
513
|
+
step: CampaignSendStep,
|
|
514
|
+
): Promise<boolean> {
|
|
515
|
+
const recipients =
|
|
516
|
+
k === 0
|
|
517
|
+
? audienceKind === "bucket"
|
|
518
|
+
? resolveBucketRecipients(db, audienceId)
|
|
519
|
+
: resolveListRecipients(db, audienceId)
|
|
520
|
+
: resolveCohortRecipients(db, {
|
|
521
|
+
campaignId: input.campaignId,
|
|
522
|
+
conditions: step.where,
|
|
523
|
+
startedAt,
|
|
524
|
+
});
|
|
525
|
+
|
|
526
|
+
// Mid-flight cancel: re-checked once per chunk (one cheap SELECT per
|
|
527
|
+
// CHUNK_SIZE sends) so `POST /v1/campaigns/:id/cancel` can stop a blast
|
|
528
|
+
// that is already `sending` — recipients not yet dispatched are spared.
|
|
529
|
+
let cancelDetected = false;
|
|
530
|
+
|
|
531
|
+
let chunk: CampaignRecipient[] = [];
|
|
532
|
+
for await (const recipient of recipients) {
|
|
533
|
+
chunk.push(recipient);
|
|
534
|
+
if (chunk.length < CHUNK_SIZE) continue;
|
|
535
|
+
await sendChunk();
|
|
536
|
+
if (cancelDetected) return true;
|
|
537
|
+
}
|
|
538
|
+
// Final partial chunk.
|
|
539
|
+
if (chunk.length > 0 && !cancelDetected) await sendChunk();
|
|
540
|
+
return cancelDetected;
|
|
541
|
+
|
|
542
|
+
async function sendChunk(): Promise<void> {
|
|
543
|
+
const statusRows = await db
|
|
544
|
+
.select({ status: campaigns.status })
|
|
545
|
+
.from(campaigns)
|
|
546
|
+
.where(eq(campaigns.id, input.campaignId))
|
|
547
|
+
.limit(1);
|
|
548
|
+
if (statusRows[0]?.status === "canceled") {
|
|
549
|
+
cancelDetected = true;
|
|
550
|
+
chunk = [];
|
|
551
|
+
return;
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
const batch = chunk;
|
|
555
|
+
chunk = [];
|
|
556
|
+
|
|
557
|
+
// Anchor the cohort BEFORE dispatching the chunk (wave 0 of a
|
|
558
|
+
// MULTI-step campaign only — a single-send campaign has no later wave
|
|
559
|
+
// to read the ledger, so skipping the insert keeps the legacy blast's
|
|
560
|
+
// write path byte-identical). Emails are already normalized by the
|
|
561
|
+
// resolvers, matching the table's (campaign_id, email) unique
|
|
562
|
+
// keyspace.
|
|
563
|
+
if (k === 0 && multiStep) {
|
|
564
|
+
await db
|
|
565
|
+
.insert(campaignRecipients)
|
|
566
|
+
.values(
|
|
567
|
+
batch.map((r) => ({
|
|
568
|
+
campaignId: input.campaignId,
|
|
569
|
+
userId: r.userId ?? null,
|
|
570
|
+
email: r.email,
|
|
571
|
+
})),
|
|
572
|
+
)
|
|
573
|
+
.onConflictDoNothing({
|
|
574
|
+
target: [campaignRecipients.campaignId, campaignRecipients.email],
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
totalRecipients += batch.length;
|
|
579
|
+
|
|
580
|
+
const results = await Promise.allSettled(
|
|
581
|
+
batch.map((r) =>
|
|
582
|
+
emailService.send({
|
|
583
|
+
template: step.template as TemplateName,
|
|
584
|
+
props: (step.props ?? {}) as never,
|
|
585
|
+
to: r.email,
|
|
586
|
+
userId: r.userId,
|
|
587
|
+
userEmail: r.email,
|
|
588
|
+
subject: step.subject,
|
|
589
|
+
from: step.from,
|
|
590
|
+
// A list's audienceId IS a real subscription category, so pass it
|
|
591
|
+
// through for suppression + the unsubscribe link. A bucket's
|
|
592
|
+
// audienceId is NOT a category — forcing it here would mint an
|
|
593
|
+
// unsubscribe link keyed on the bucket id (`categories[bucketId] =
|
|
594
|
+
// false`) that the bucket resolver never honors (it only checks
|
|
595
|
+
// unsubscribedAll/suppressed), silently no-op'ing the unsubscribe.
|
|
596
|
+
// For a bucket, pass undefined so the template's OWN declared
|
|
597
|
+
// category (e.g. `product-updates`) drives both suppression and a
|
|
598
|
+
// real, honored List-Unsubscribe target. The rule applies to
|
|
599
|
+
// EVERY wave, not just the first.
|
|
600
|
+
category: audienceKind === "bucket" ? undefined : audienceId,
|
|
601
|
+
// The idempotency key dedupes a retried send to its prior row.
|
|
602
|
+
// Legacy format for single-step campaigns; step-scoped for every
|
|
603
|
+
// step of a multi-step campaign (campaignSendKey contract).
|
|
604
|
+
idempotencyKey: campaignSendKey(
|
|
605
|
+
input.campaignId,
|
|
606
|
+
r.email,
|
|
607
|
+
multiStep ? k : undefined,
|
|
608
|
+
),
|
|
609
|
+
}),
|
|
610
|
+
),
|
|
611
|
+
);
|
|
612
|
+
|
|
613
|
+
for (const result of results) {
|
|
614
|
+
if (result.status === "rejected") {
|
|
615
|
+
failedCount++;
|
|
616
|
+
continue;
|
|
617
|
+
}
|
|
618
|
+
const status = result.value.status;
|
|
619
|
+
if (status === "sent") {
|
|
620
|
+
sentCount++;
|
|
621
|
+
} else {
|
|
622
|
+
// suppressed | unsubscribed | skipped (frequency-capped) — counted
|
|
623
|
+
// as skipped, not a delivery failure. The member STAYS in the
|
|
624
|
+
// cohort either way: a provider hiccup or a skipped wave is not an
|
|
625
|
+
// exit, and suppression is re-checked fresh on every later wave.
|
|
626
|
+
skippedCount++;
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
await flushCounts();
|
|
631
|
+
}
|
|
632
|
+
}
|
|
340
633
|
},
|
|
341
634
|
});
|
|
342
635
|
|
|
@@ -352,21 +645,30 @@ const STALE_AFTER_MS = Number(
|
|
|
352
645
|
);
|
|
353
646
|
|
|
354
647
|
/**
|
|
355
|
-
* After a campaign has
|
|
356
|
-
*
|
|
357
|
-
*
|
|
358
|
-
*
|
|
648
|
+
* After a campaign has been CONTINUOUSLY stuck this long with zero progress it
|
|
649
|
+
* is declared `failed` rather than re-enqueued forever — a poison campaign
|
|
650
|
+
* (e.g. a run that always crashes) stops being re-driven and surfaces to
|
|
651
|
+
* operators.
|
|
652
|
+
*
|
|
653
|
+
* For in-flight (`queued`/`sending`) rows the window is measured from
|
|
654
|
+
* `staleSince` — set once by the stale sweep's first re-enqueue, cleared by
|
|
655
|
+
* every genuine progress flush of the send task. NOT from `updatedAt`: the
|
|
656
|
+
* sweep bumps `updatedAt` as its re-pick guard, so a deterministically-
|
|
657
|
+
* crashing row would never age past the window (the give-up was dead code
|
|
658
|
+
* until `stale_since` landed). `scheduled` / `waiting` rows are measured from
|
|
659
|
+
* `scheduledAt` / `nextStepAt` as before.
|
|
359
660
|
*/
|
|
360
661
|
const GIVE_UP_AFTER_MS = Number(
|
|
361
662
|
process.env.CAMPAIGN_GIVE_UP_AFTER_MS ?? 6 * 60 * 60 * 1000,
|
|
362
663
|
);
|
|
363
664
|
|
|
364
665
|
/**
|
|
365
|
-
* How far past `scheduledAt` a `scheduled` campaign
|
|
366
|
-
* promotes it (enqueues the send
|
|
367
|
-
*
|
|
368
|
-
*
|
|
369
|
-
*
|
|
666
|
+
* How far past `scheduledAt` a `scheduled` campaign (or past `nextStepAt` a
|
|
667
|
+
* `waiting` one) may sit before the reaper promotes it (enqueues the send
|
|
668
|
+
* task). The punctual Hatchet scheduled run created at schedule/park time is
|
|
669
|
+
* the primary trigger; this grace keeps the sweep from routinely
|
|
670
|
+
* double-firing right at the boundary. A double fire is harmless anyway
|
|
671
|
+
* (terminal guard + per-send idempotency).
|
|
370
672
|
*/
|
|
371
673
|
const SCHEDULE_PROMOTE_GRACE_MS = Number(
|
|
372
674
|
process.env.CAMPAIGN_PROMOTE_GRACE_MS ?? 2 * 60 * 1000,
|
|
@@ -387,6 +689,10 @@ const SCHEDULE_PROMOTE_GRACE_MS = Number(
|
|
|
387
689
|
* due. A scheduled row stuck past the give-up window (measured from
|
|
388
690
|
* `scheduledAt`, NOT `updatedAt` — a row is legitimately idle between
|
|
389
691
|
* create and send time) is declared `failed`.
|
|
692
|
+
* - A `waiting` campaign whose punctual next-step run was lost — promoted
|
|
693
|
+
* once `nextStepAt` is past the same grace. Give-up is likewise measured
|
|
694
|
+
* from `nextStepAt`, NOT `updatedAt` — a row is legitimately idle mid-wait,
|
|
695
|
+
* exactly like `scheduled`/`scheduledAt`.
|
|
390
696
|
*
|
|
391
697
|
* Recovery is a simple RE-ENQUEUE of `sendCampaignTask` (safe: the per-send
|
|
392
698
|
* idempotency key no-ops already-sent recipients and the re-run completes the
|
|
@@ -410,24 +716,38 @@ export const reapStuckCampaignsTask = hatchet.task({
|
|
|
410
716
|
const now = Date.now();
|
|
411
717
|
const staleBefore = new Date(now - STALE_AFTER_MS);
|
|
412
718
|
const giveUpBefore = new Date(now - GIVE_UP_AFTER_MS);
|
|
719
|
+
const promoteBefore = new Date(now - SCHEDULE_PROMOTE_GRACE_MS);
|
|
413
720
|
|
|
414
721
|
// (1) Declare poison campaigns `failed` first (stuck past the give-up
|
|
415
722
|
// window), so they are not re-enqueued below. In-flight rows are measured
|
|
416
|
-
// from `
|
|
417
|
-
//
|
|
723
|
+
// from `staleSince` (set by the first stale re-enqueue, cleared by every
|
|
724
|
+
// genuine progress flush — a NULL never matches the `lt`, so a row the
|
|
725
|
+
// sweep hasn't touched yet gets its re-drive chance first); `scheduled`
|
|
726
|
+
// rows from `scheduledAt` and `waiting` rows from `nextStepAt` (their
|
|
727
|
+
// `updatedAt` is legitimately old while they wait). `staleSince` is nulled
|
|
728
|
+
// with the stamp so an operator revive starts a fresh give-up window.
|
|
418
729
|
const failedRows = await db
|
|
419
730
|
.update(campaigns)
|
|
420
|
-
.set({
|
|
731
|
+
.set({
|
|
732
|
+
status: "failed",
|
|
733
|
+
completedAt: new Date(),
|
|
734
|
+
staleSince: null,
|
|
735
|
+
updatedAt: new Date(),
|
|
736
|
+
})
|
|
421
737
|
.where(
|
|
422
738
|
or(
|
|
423
739
|
and(
|
|
424
740
|
inArray(campaigns.status, ["queued", "sending"]),
|
|
425
|
-
lt(campaigns.
|
|
741
|
+
lt(campaigns.staleSince, giveUpBefore),
|
|
426
742
|
),
|
|
427
743
|
and(
|
|
428
744
|
eq(campaigns.status, "scheduled"),
|
|
429
745
|
lt(campaigns.scheduledAt, giveUpBefore),
|
|
430
746
|
),
|
|
747
|
+
and(
|
|
748
|
+
eq(campaigns.status, "waiting"),
|
|
749
|
+
lt(campaigns.nextStepAt, giveUpBefore),
|
|
750
|
+
),
|
|
431
751
|
),
|
|
432
752
|
)
|
|
433
753
|
.returning({ id: campaigns.id });
|
|
@@ -436,9 +756,18 @@ export const reapStuckCampaignsTask = hatchet.task({
|
|
|
436
756
|
// `updatedAt` so the same row is not re-picked on the very next tick before
|
|
437
757
|
// the re-driven run makes progress; the per-send idempotency key keeps the
|
|
438
758
|
// re-enqueue safe even if the original run is somehow still alive.
|
|
759
|
+
// `staleSince` is coalesce-set ONCE on the first re-enqueue (later sweeps
|
|
760
|
+
// keep the original instant) — this is the bump-free clock the give-up
|
|
761
|
+
// clause above reads; the send task clears it on genuine progress.
|
|
762
|
+
// `queued`/`sending` ONLY — `waiting` is deliberately excluded: a 2-day
|
|
763
|
+
// wait between waves is not a stuck campaign (its resume path is the
|
|
764
|
+
// promotion sweep below, keyed on `nextStepAt`).
|
|
439
765
|
const staleRows = await db
|
|
440
766
|
.update(campaigns)
|
|
441
|
-
.set({
|
|
767
|
+
.set({
|
|
768
|
+
staleSince: sql`coalesce(${campaigns.staleSince}, now())`,
|
|
769
|
+
updatedAt: new Date(),
|
|
770
|
+
})
|
|
442
771
|
.where(
|
|
443
772
|
and(
|
|
444
773
|
inArray(campaigns.status, ["queued", "sending"]),
|
|
@@ -448,15 +777,22 @@ export const reapStuckCampaignsTask = hatchet.task({
|
|
|
448
777
|
.returning({ id: campaigns.id });
|
|
449
778
|
|
|
450
779
|
// (3) Promote due `scheduled` campaigns whose punctual scheduled run never
|
|
451
|
-
// fired
|
|
780
|
+
// fired, and due `waiting` campaigns whose punctual next-step run was lost.
|
|
781
|
+
// Enqueue-only — the send task owns the scheduled/waiting→sending
|
|
452
782
|
// transition, so a failed enqueue simply retries next sweep.
|
|
453
783
|
const dueRows = await db
|
|
454
784
|
.select({ id: campaigns.id })
|
|
455
785
|
.from(campaigns)
|
|
456
786
|
.where(
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
787
|
+
or(
|
|
788
|
+
and(
|
|
789
|
+
eq(campaigns.status, "scheduled"),
|
|
790
|
+
lte(campaigns.scheduledAt, promoteBefore),
|
|
791
|
+
),
|
|
792
|
+
and(
|
|
793
|
+
eq(campaigns.status, "waiting"),
|
|
794
|
+
lte(campaigns.nextStepAt, promoteBefore),
|
|
795
|
+
),
|
|
460
796
|
),
|
|
461
797
|
);
|
|
462
798
|
|
|
@@ -522,19 +858,76 @@ async function* keysetPaginate<Row>(opts: {
|
|
|
522
858
|
}
|
|
523
859
|
}
|
|
524
860
|
|
|
861
|
+
/**
|
|
862
|
+
* Wave-k (k > 0) qualifier resolver: the campaign's anchored cohort
|
|
863
|
+
* (`campaign_recipients`) ∩ the step's `where` conditions (each compiled to a
|
|
864
|
+
* correlated [NOT] EXISTS — see `campaigns/cohort-sql.ts`) ∩ a fresh
|
|
865
|
+
* suppression/unsubscribe re-check. Membership was anchored at wave 0 and is
|
|
866
|
+
* never re-resolved — a member who left the bucket/list between waves still
|
|
867
|
+
* qualifies (suppression excepted); a member added afterwards never appears.
|
|
868
|
+
* Paged by the keyset cursor on `campaign_recipients.id` (the
|
|
869
|
+
* (campaign_id, id) index), so every page is an indexed join, never a LIKE
|
|
870
|
+
* scan over email_sends.
|
|
871
|
+
*/
|
|
872
|
+
async function* resolveCohortRecipients(
|
|
873
|
+
db: Database,
|
|
874
|
+
opts: {
|
|
875
|
+
campaignId: string;
|
|
876
|
+
conditions: ConditionEval[] | undefined;
|
|
877
|
+
startedAt: Date;
|
|
878
|
+
},
|
|
879
|
+
): AsyncGenerator<CampaignRecipient> {
|
|
880
|
+
// Compiled once per wave — every page reuses the same predicates.
|
|
881
|
+
const qualifiers = (opts.conditions ?? []).map((condition) =>
|
|
882
|
+
waveConditionSql({
|
|
883
|
+
condition,
|
|
884
|
+
campaignId: opts.campaignId,
|
|
885
|
+
startedAt: opts.startedAt,
|
|
886
|
+
}),
|
|
887
|
+
);
|
|
888
|
+
|
|
889
|
+
yield* keysetPaginate({
|
|
890
|
+
page: (cursor) => {
|
|
891
|
+
const conditions = [
|
|
892
|
+
eq(campaignRecipients.campaignId, opts.campaignId),
|
|
893
|
+
cohortSuppressionSql(),
|
|
894
|
+
...qualifiers,
|
|
895
|
+
];
|
|
896
|
+
if (cursor) conditions.push(gt(campaignRecipients.id, cursor));
|
|
897
|
+
|
|
898
|
+
return db
|
|
899
|
+
.select({
|
|
900
|
+
id: campaignRecipients.id,
|
|
901
|
+
userId: campaignRecipients.userId,
|
|
902
|
+
email: campaignRecipients.email,
|
|
903
|
+
})
|
|
904
|
+
.from(campaignRecipients)
|
|
905
|
+
.where(and(...conditions))
|
|
906
|
+
.orderBy(campaignRecipients.id)
|
|
907
|
+
.limit(CHUNK_SIZE);
|
|
908
|
+
},
|
|
909
|
+
cursorOf: (row) => row.id,
|
|
910
|
+
// Cohort emails were normalized at anchor time; userId may be NULL for an
|
|
911
|
+
// email-only member (kept — the mailer falls back the same way wave 0 did).
|
|
912
|
+
map: (row) => ({ email: row.email, userId: row.userId ?? undefined }),
|
|
913
|
+
});
|
|
914
|
+
}
|
|
915
|
+
|
|
525
916
|
/**
|
|
526
917
|
* Active, non-deleted members of a bucket, joined to a live contact for the
|
|
527
918
|
* email — mirrors the bucket-access member query. Paged by the keyset cursor on
|
|
528
919
|
* `bucket_memberships.id`.
|
|
529
920
|
*
|
|
530
|
-
* Compliance: `bucket_memberships.userEmail` is
|
|
531
|
-
*
|
|
532
|
-
*
|
|
533
|
-
*
|
|
534
|
-
* (
|
|
535
|
-
*
|
|
536
|
-
*
|
|
537
|
-
*
|
|
921
|
+
* Compliance: `bucket_memberships.userEmail` is normalized at every write site
|
|
922
|
+
* (and existing rows were backfilled by migration 0043), but this resolver
|
|
923
|
+
* KEEPS its own `normalizeEmail` / `lower(trim(…))` belt-and-braces —
|
|
924
|
+
* TODO(cleanup): safe to drop once `@hogsend/engine` has shipped past 0.40.0
|
|
925
|
+
* (all deployed write sites normalize as of this changeset). An un-normalized
|
|
926
|
+
* membership email (`User@Example.com`) would not
|
|
927
|
+
* case-match its NORMALIZED `email_preferences` row (`user@example.com`) and
|
|
928
|
+
* the mailer's case-sensitive suppression check would MISS the row, leaking a
|
|
929
|
+
* marketing blast to a suppressed/unsubscribed contact (CAN-SPAM/GDPR).
|
|
930
|
+
* Defense-in-depth: this resolver ALSO pre-filters
|
|
538
931
|
* `unsubscribedAll`/`suppressed` at the audience layer (mirroring the list
|
|
539
932
|
* resolver) via a LEFT JOIN to `email_preferences` on the NORMALIZED email, so
|
|
540
933
|
* a globally-unsubscribed / suppressed bucket member is excluded up front
|
|
@@ -546,8 +939,9 @@ async function* resolveBucketRecipients(
|
|
|
546
939
|
db: Database,
|
|
547
940
|
bucketId: string,
|
|
548
941
|
): AsyncGenerator<CampaignRecipient> {
|
|
549
|
-
// The recipient's normalized email —
|
|
550
|
-
// (
|
|
942
|
+
// The recipient's normalized email — writes are normalized now, but the
|
|
943
|
+
// read-side lower(trim(…)) stays as belt-and-braces (TODO(cleanup): see the
|
|
944
|
+
// fn docstring for the drop condition); the contact email is the fallback.
|
|
551
945
|
const recipientEmail = sql<string>`lower(trim(coalesce(${bucketMemberships.userEmail}, ${contacts.email})))`;
|
|
552
946
|
|
|
553
947
|
yield* keysetPaginate({
|