@hogsend/engine 0.39.0 → 0.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +9 -7
- package/src/buckets/check-membership.ts +6 -1
- package/src/campaigns/cohort-sql.ts +198 -0
- package/src/campaigns/define-campaign.ts +301 -0
- package/src/campaigns/reconcile.ts +391 -0
- package/src/campaigns/steps.ts +48 -0
- package/src/connectors/define-action.ts +70 -0
- package/src/container.ts +144 -42
- package/src/destinations/presets/posthog.ts +7 -1
- package/src/env.ts +14 -0
- package/src/index.ts +98 -3
- package/src/journeys/constants.ts +10 -0
- package/src/journeys/define-journey.ts +14 -405
- package/src/journeys/execute-journey-run.ts +608 -0
- package/src/journeys/graph/build-graph.ts +212 -56
- package/src/journeys/journey-boundary.ts +54 -0
- package/src/journeys/journey-context.ts +374 -59
- package/src/journeys/record-once.ts +110 -0
- package/src/lib/auth.ts +33 -1
- package/src/lib/blueprints.ts +576 -0
- package/src/lib/campaign-send-key.ts +58 -0
- package/src/lib/connector-actions.ts +203 -19
- package/src/lib/contacts.ts +54 -0
- package/src/lib/email-service-types.ts +4 -1
- package/src/lib/email.ts +13 -2
- package/src/lib/feed.ts +77 -57
- package/src/lib/frequency-cap.ts +45 -15
- package/src/lib/hatchet-duration.ts +22 -0
- package/src/lib/html.ts +1 -0
- package/src/lib/ingestion.ts +145 -4
- package/src/lib/journey-suppress.ts +87 -0
- package/src/lib/links.ts +204 -18
- package/src/lib/outbound.ts +20 -0
- package/src/lib/preferences.ts +80 -1
- package/src/lib/tracked.ts +44 -29
- package/src/lib/tracking-event-names.ts +8 -0
- package/src/lib/webhook-signing.ts +6 -1
- package/src/lists/channels.ts +75 -0
- package/src/lists/define-list.ts +34 -0
- package/src/lists/registry.ts +29 -2
- package/src/mcp/blueprint-tools.ts +460 -0
- package/src/routes/admin/blueprints.ts +711 -0
- package/src/routes/admin/campaigns.ts +341 -0
- package/src/routes/admin/emails.ts +13 -0
- package/src/routes/admin/index.ts +6 -0
- package/src/routes/admin/journeys.ts +4 -2
- package/src/routes/admin/links.ts +303 -36
- package/src/routes/admin/lists.ts +51 -0
- package/src/routes/campaigns/index.ts +297 -50
- package/src/routes/email/preferences.ts +50 -13
- package/src/routes/feed/recipient.ts +2 -43
- package/src/routes/index.ts +6 -0
- package/src/routes/lists/index.ts +138 -18
- package/src/routes/tracking/arrive.ts +260 -0
- package/src/routes/tracking/click-pipeline.ts +329 -0
- package/src/routes/tracking/click.ts +5 -270
- package/src/routes/tracking/index.ts +2 -0
- package/src/routes/tracking/vanity.ts +64 -0
- package/src/worker.ts +27 -0
- package/src/workflows/bucket-backfill.ts +4 -2
- package/src/workflows/bucket-reconcile.ts +11 -4
- package/src/workflows/check-alerts.ts +130 -1
- package/src/workflows/journey-blueprint-interpreter.ts +561 -0
- package/src/workflows/send-campaign.ts +673 -127
|
@@ -1,12 +1,24 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type CampaignSendStep,
|
|
3
|
+
type CampaignStep,
|
|
4
|
+
type ConditionEval,
|
|
5
|
+
durationToMs,
|
|
6
|
+
} from "@hogsend/core";
|
|
1
7
|
import {
|
|
2
8
|
bucketMemberships,
|
|
9
|
+
campaignRecipients,
|
|
3
10
|
campaigns,
|
|
4
11
|
contacts,
|
|
5
12
|
type Database,
|
|
6
13
|
emailPreferences,
|
|
7
14
|
} from "@hogsend/db";
|
|
8
15
|
import type { TemplateName } from "@hogsend/email";
|
|
9
|
-
import { and, eq, gt, inArray, isNull, lt, sql } from "drizzle-orm";
|
|
16
|
+
import { and, eq, gt, inArray, isNull, lt, lte, or, sql } from "drizzle-orm";
|
|
17
|
+
import {
|
|
18
|
+
cohortSuppressionSql,
|
|
19
|
+
waveConditionSql,
|
|
20
|
+
} from "../campaigns/cohort-sql.js";
|
|
21
|
+
import { campaignSendKey } from "../lib/campaign-send-key.js";
|
|
10
22
|
import { normalizeEmail } from "../lib/contacts.js";
|
|
11
23
|
import { getDb } from "../lib/db.js";
|
|
12
24
|
import { getEmailService } from "../lib/email.js";
|
|
@@ -23,32 +35,62 @@ interface CampaignRecipient {
|
|
|
23
35
|
userId?: string;
|
|
24
36
|
}
|
|
25
37
|
|
|
26
|
-
/**
|
|
27
|
-
|
|
38
|
+
/**
|
|
39
|
+
* Statuses that are TERMINAL — a duplicate/late enqueue must not re-send.
|
|
40
|
+
* `canceled`/`expired` matter doubly for SCHEDULED (and now WAITING) campaigns:
|
|
41
|
+
* the punctual Hatchet scheduled run cannot be deleted at cancel time (we don't
|
|
42
|
+
* persist its id), so it still fires and must find the row terminal here and
|
|
43
|
+
* no-op. `failed` is deliberately NOT terminal — a re-enqueue can revive it.
|
|
44
|
+
*/
|
|
45
|
+
const TERMINAL_STATUSES = ["sent", "canceled", "expired"] as const;
|
|
28
46
|
|
|
29
47
|
/**
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
*
|
|
48
|
+
* How far ahead of `scheduledAt` / `nextStepAt` a run may fire and still
|
|
49
|
+
* proceed (absorbs clock skew). An earlier fire — the punctual run created for
|
|
50
|
+
* an instant that was later moved BACK (a reconcile edit for `scheduledAt`; a
|
|
51
|
+
* reap + re-park re-schedule for `nextStepAt`) — skips without sending; the
|
|
52
|
+
* run created for the new instant (or the reaper sweep) delivers it.
|
|
53
|
+
*/
|
|
54
|
+
const EARLY_FIRE_TOLERANCE_MS = 60 * 1000;
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Built-in durable campaign / broadcast task (Loops "campaign" parity). A
|
|
58
|
+
* legacy row (NULL `steps`) sends a single template to every subscribed member
|
|
59
|
+
* of a list (or every active member of a bucket). A multi-step row executes
|
|
60
|
+
* its `steps` blob as WAVES — each send step a SET operation over the
|
|
61
|
+
* campaign's anchored cohort, separated by durable waits (status `waiting`,
|
|
62
|
+
* resumed at `nextStepAt`) — see docs/campaign-steps-spec.md §Wave runtime.
|
|
63
|
+
* The task keeps its name and `{ campaignId }` input either way; the row's
|
|
64
|
+
* `currentStep` is the sole resume cursor.
|
|
33
65
|
*
|
|
34
|
-
* Retry-safety: each send carries an idempotency key
|
|
35
|
-
* `campaign:<
|
|
36
|
-
*
|
|
66
|
+
* Retry-safety: each send carries an idempotency key minted by
|
|
67
|
+
* `campaignSendKey` — legacy `campaign:<id>:<email>` for single-step
|
|
68
|
+
* campaigns, step-scoped `campaign:<id>:<step>:<email>` for EVERY step of a
|
|
69
|
+
* multi-step campaign (email_sends.idempotency_key, migration 0015) — so a
|
|
70
|
+
* Hatchet retry re-runs the wave but every already-dispatched send
|
|
37
71
|
* short-circuits to its prior row instead of dispatching a duplicate provider
|
|
38
|
-
* call. Counts are derived as-you-go from each `send()` result status — which
|
|
39
|
-
* itself idempotency-aware (a retried send returns the prior row's status),
|
|
40
|
-
* the tallies stay consistent across re-attempts.
|
|
41
|
-
*
|
|
42
|
-
*
|
|
72
|
+
* call. Counts are derived as-you-go from each `send()` result status — which
|
|
73
|
+
* is itself idempotency-aware (a retried send returns the prior row's status),
|
|
74
|
+
* so the tallies stay consistent across re-attempts.
|
|
75
|
+
*
|
|
76
|
+
* Counts across waves: the row counts are CUMULATIVE across waves.
|
|
77
|
+
* `stepBaseCounts` snapshots the cumulative tallies through the last COMPLETED
|
|
78
|
+
* wave (written with each cursor advance); the current wave always re-derives
|
|
79
|
+
* its own tally from scratch on top of that seed, and every flush stays an
|
|
80
|
+
* absolute overwrite — so a retried wave re-tallies exactly rather than
|
|
81
|
+
* double-counting. Prior-wave counts are deliberately NOT derived from
|
|
82
|
+
* `email_sends`: a suppressed send writes its row WITHOUT the idempotency key
|
|
83
|
+
* and a frequency-capped send writes NO row at all, so the ledger cannot
|
|
84
|
+
* reproduce skipped counts — hence the snapshot column.
|
|
43
85
|
*
|
|
44
|
-
* Resume-on-retry: the terminal guard short-circuits ONLY a
|
|
45
|
-
* `failed`/`sending` row is NOT terminal, so a Hatchet retry (or the reaper's
|
|
46
|
-
* re-enqueue) re-
|
|
47
|
-
* no-op via the idempotency key, so the re-run safely
|
|
48
|
-
* partial
|
|
49
|
-
* stamp `failed` before re-throwing — that would make
|
|
50
|
-
* and silently under-deliver. A run that exhausts its
|
|
51
|
-
* `failed`/re-enqueued by {@link reapStuckCampaignsTask}.
|
|
86
|
+
* Resume-on-retry: the terminal guard short-circuits ONLY a terminal campaign —
|
|
87
|
+
* a `failed`/`sending` row is NOT terminal, so a Hatchet retry (or the reaper's
|
|
88
|
+
* re-enqueue) re-enters at `currentStep` and re-runs the current wave.
|
|
89
|
+
* Already-dispatched sends no-op via the idempotency key, so the re-run safely
|
|
90
|
+
* completes the TAIL of a partial wave instead of abandoning it. The catch
|
|
91
|
+
* block therefore does NOT stamp `failed` before re-throwing — that would make
|
|
92
|
+
* the retry short-circuit and silently under-deliver. A run that exhausts its
|
|
93
|
+
* retries is reaped to `failed`/re-enqueued by {@link reapStuckCampaignsTask}.
|
|
52
94
|
*/
|
|
53
95
|
export const sendCampaignTask = hatchet.task({
|
|
54
96
|
name: "send-campaign",
|
|
@@ -75,26 +117,150 @@ export const sendCampaignTask = hatchet.task({
|
|
|
75
117
|
return { status: "failed", reason: "not_found" as const };
|
|
76
118
|
}
|
|
77
119
|
|
|
78
|
-
// Already terminal — a duplicate/late enqueue must not re-send. ONLY
|
|
79
|
-
//
|
|
80
|
-
// Hatchet retry (or a reaper re-enqueue)
|
|
81
|
-
// completes the unsent TAIL of a partial
|
|
82
|
-
// no-op via the per-send idempotency key —
|
|
120
|
+
// Already terminal — a duplicate/late enqueue must not re-send. ONLY
|
|
121
|
+
// sent/canceled/expired are terminal: a `failed`/`sending` row is
|
|
122
|
+
// intentionally re-runnable so a Hatchet retry (or a reaper re-enqueue)
|
|
123
|
+
// re-enters at `currentStep` and completes the unsent TAIL of a partial
|
|
124
|
+
// wave (already-sent recipients no-op via the per-send idempotency key —
|
|
125
|
+
// risk: silent under-delivery).
|
|
83
126
|
if ((TERMINAL_STATUSES as readonly string[]).includes(campaign.status)) {
|
|
84
127
|
return { status: campaign.status, skipped: true };
|
|
85
128
|
}
|
|
86
129
|
|
|
87
|
-
|
|
130
|
+
// Early-fire guard: a `scheduled` campaign whose instant is still in the
|
|
131
|
+
// future must not send. This is a stale punctual run — the `sendAt` was
|
|
132
|
+
// moved later after this run was scheduled. The run for the new instant
|
|
133
|
+
// (or the reaper's due-scheduled sweep) delivers it.
|
|
134
|
+
if (
|
|
135
|
+
campaign.status === "scheduled" &&
|
|
136
|
+
campaign.scheduledAt &&
|
|
137
|
+
campaign.scheduledAt.getTime() > Date.now() + EARLY_FIRE_TOLERANCE_MS
|
|
138
|
+
) {
|
|
139
|
+
return { status: "scheduled", skipped: true, reason: "not_due" as const };
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
// Mirror clause for `waiting`: a punctual next-step run that fires while
|
|
143
|
+
// `nextStepAt` is still in the future is stale — a reap + re-park replaced
|
|
144
|
+
// the pending wait after this run was scheduled. The run created for the
|
|
145
|
+
// new instant (or the reaper's waiting-promotion sweep) resumes it.
|
|
146
|
+
if (
|
|
147
|
+
campaign.status === "waiting" &&
|
|
148
|
+
campaign.nextStepAt &&
|
|
149
|
+
campaign.nextStepAt.getTime() > Date.now() + EARLY_FIRE_TOLERANCE_MS
|
|
150
|
+
) {
|
|
151
|
+
return { status: "waiting", skipped: true, reason: "not_due" as const };
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// Claim via CAS: only a non-terminal row transitions to `sending`, so a
|
|
155
|
+
// cancel landing between the guard read above and this write is honored
|
|
156
|
+
// rather than silently resurrected. `startedAt` is claimed ONCE
|
|
157
|
+
// (coalesce): event-condition scoping reads "since campaign startedAt",
|
|
158
|
+
// so a wave-2 claim must NOT reset the anchor wave-1's conditions were
|
|
159
|
+
// already scoped by. The cursor + count seed are re-read FROM the claimed
|
|
160
|
+
// row (not the pre-claim read) so a wave resumed after a crash sees the
|
|
161
|
+
// exact state its predecessor persisted.
|
|
162
|
+
const claimedRows = await db
|
|
88
163
|
.update(campaigns)
|
|
89
|
-
.set({
|
|
90
|
-
|
|
164
|
+
.set({
|
|
165
|
+
status: "sending",
|
|
166
|
+
startedAt: sql`coalesce(${campaigns.startedAt}, now())`,
|
|
167
|
+
// A resumed `waiting` row's wait has elapsed — clear the stale instant
|
|
168
|
+
// so serialized rows never show a countdown while `sending`. (Studio
|
|
169
|
+
// keys its countdown on status === "waiting" regardless.)
|
|
170
|
+
nextStepAt: null,
|
|
171
|
+
// `staleSince` deliberately NOT cleared here: claiming is not
|
|
172
|
+
// progress. A poison run claims, crashes, and is re-driven — if the
|
|
173
|
+
// claim reset the marker, its give-up clock would restart every cycle
|
|
174
|
+
// and the reaper could never declare it failed. The first real
|
|
175
|
+
// progress write (chunk flush / park / cursor advance) clears it.
|
|
176
|
+
updatedAt: new Date(),
|
|
177
|
+
})
|
|
178
|
+
.where(
|
|
179
|
+
and(
|
|
180
|
+
eq(campaigns.id, input.campaignId),
|
|
181
|
+
inArray(campaigns.status, [
|
|
182
|
+
"queued",
|
|
183
|
+
"scheduled",
|
|
184
|
+
"sending",
|
|
185
|
+
"waiting",
|
|
186
|
+
]),
|
|
187
|
+
),
|
|
188
|
+
)
|
|
189
|
+
.returning({
|
|
190
|
+
id: campaigns.id,
|
|
191
|
+
currentStep: campaigns.currentStep,
|
|
192
|
+
startedAt: campaigns.startedAt,
|
|
193
|
+
stepBaseCounts: campaigns.stepBaseCounts,
|
|
194
|
+
});
|
|
195
|
+
const claim = claimedRows[0];
|
|
196
|
+
if (!claim) {
|
|
197
|
+
const current = await db
|
|
198
|
+
.select({ status: campaigns.status })
|
|
199
|
+
.from(campaigns)
|
|
200
|
+
.where(eq(campaigns.id, input.campaignId))
|
|
201
|
+
.limit(1);
|
|
202
|
+
return { status: current[0]?.status ?? "unknown", skipped: true };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// The executable step sequence. A NULL blob is a legacy single-send row —
|
|
206
|
+
// one send step synthesized from the row columns, and (per the
|
|
207
|
+
// campaignSendKey contract) the LEGACY per-send key format. A non-NULL
|
|
208
|
+
// blob only ever holds > 1 steps (the reconciler stores single-step
|
|
209
|
+
// definitions as NULL), so `multiStep` doubles as the key-format switch:
|
|
210
|
+
// step-scoped keys for ALL steps including 0.
|
|
211
|
+
const blob = campaign.steps;
|
|
212
|
+
const multiStep = blob != null;
|
|
213
|
+
const executable: CampaignStep[] = blob
|
|
214
|
+
? // db cannot import @hogsend/core, so the blob's elements are opaque
|
|
215
|
+
// Record<string, unknown> there; the engine owns the narrowing.
|
|
216
|
+
(blob.steps as unknown as CampaignStep[])
|
|
217
|
+
: [
|
|
218
|
+
{
|
|
219
|
+
kind: "send",
|
|
220
|
+
template: campaign.templateKey,
|
|
221
|
+
props: campaign.props ?? {},
|
|
222
|
+
...(campaign.subject != null ? { subject: campaign.subject } : {}),
|
|
223
|
+
...(campaign.fromEmail != null ? { from: campaign.fromEmail } : {}),
|
|
224
|
+
},
|
|
225
|
+
];
|
|
226
|
+
|
|
227
|
+
// Immutable row facts captured once (nested fns below can't see the
|
|
228
|
+
// `campaign` narrowing). `startedAt` is non-null post-claim (coalesce);
|
|
229
|
+
// the ?? is a type-level fallback only.
|
|
230
|
+
const audienceKind = campaign.audienceKind;
|
|
231
|
+
const audienceId = campaign.audienceId;
|
|
232
|
+
const startedAt = claim.startedAt ?? new Date();
|
|
233
|
+
|
|
234
|
+
// Seed the cumulative counters from the snapshot through the last
|
|
235
|
+
// COMPLETED wave; the current wave re-derives its own tally on top. Do NOT
|
|
236
|
+
// try to derive the seed from email_sends — suppressed sends write no
|
|
237
|
+
// idempotency key and frequency-capped sends write no row (lib/tracked.ts).
|
|
238
|
+
const base = claim.stepBaseCounts ?? {
|
|
239
|
+
total: 0,
|
|
240
|
+
sent: 0,
|
|
241
|
+
skipped: 0,
|
|
242
|
+
failed: 0,
|
|
243
|
+
};
|
|
244
|
+
let totalRecipients = base.total;
|
|
245
|
+
let sentCount = base.sent;
|
|
246
|
+
let skippedCount = base.skipped;
|
|
247
|
+
let failedCount = base.failed;
|
|
91
248
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
249
|
+
const snapshotCounts = () => ({
|
|
250
|
+
total: totalRecipients,
|
|
251
|
+
sent: sentCount,
|
|
252
|
+
skipped: skippedCount,
|
|
253
|
+
failed: failedCount,
|
|
254
|
+
});
|
|
96
255
|
|
|
97
|
-
|
|
256
|
+
// Every flush is genuine progress (a dispatched chunk, a cancel/completion
|
|
257
|
+
// race) and clears the reaper's staleness marker — EXCEPT the catch-path
|
|
258
|
+
// flush: a crashed run made no progress, so `staleSince` must survive it
|
|
259
|
+
// or a deterministically-crashing campaign resets its give-up clock on
|
|
260
|
+
// every re-drive and is re-enqueued forever.
|
|
261
|
+
const flushCounts = async (opts?: {
|
|
262
|
+
preserveStaleSince?: boolean;
|
|
263
|
+
}): Promise<void> => {
|
|
98
264
|
await db
|
|
99
265
|
.update(campaigns)
|
|
100
266
|
.set({
|
|
@@ -102,41 +268,325 @@ export const sendCampaignTask = hatchet.task({
|
|
|
102
268
|
sentCount,
|
|
103
269
|
skippedCount,
|
|
104
270
|
failedCount,
|
|
271
|
+
...(opts?.preserveStaleSince ? {} : { staleSince: null }),
|
|
105
272
|
updatedAt: new Date(),
|
|
106
273
|
})
|
|
107
274
|
.where(eq(campaigns.id, input.campaignId));
|
|
108
275
|
};
|
|
109
276
|
|
|
110
277
|
try {
|
|
278
|
+
// The wave loop: consecutive send steps run sequentially in this same
|
|
279
|
+
// task run; a wait step parks the row (`waiting`) and returns. Resuming
|
|
280
|
+
// with `currentStep` already past the last index (a crash between the
|
|
281
|
+
// final cursor advance and the completion CAS) skips straight to the
|
|
282
|
+
// completion CAS below — exactly right.
|
|
283
|
+
for (let k = claim.currentStep; k < executable.length; k++) {
|
|
284
|
+
const step = executable[k];
|
|
285
|
+
if (step === undefined) break; // unreachable — loop bound
|
|
286
|
+
|
|
287
|
+
if (step.kind === "wait") {
|
|
288
|
+
const nextStepAt = new Date(Date.now() + durationToMs(step.duration));
|
|
289
|
+
// Park CAS: only a still-`sending` row goes `waiting`, so a cancel
|
|
290
|
+
// racing the end of the previous wave keeps its `canceled` status.
|
|
291
|
+
// Cursor, wait instant, count snapshot, and counts land in ONE
|
|
292
|
+
// update — a crash can never separate them.
|
|
293
|
+
const parked = await db
|
|
294
|
+
.update(campaigns)
|
|
295
|
+
.set({
|
|
296
|
+
status: "waiting",
|
|
297
|
+
currentStep: k + 1,
|
|
298
|
+
nextStepAt,
|
|
299
|
+
stepBaseCounts: snapshotCounts(),
|
|
300
|
+
totalRecipients,
|
|
301
|
+
sentCount,
|
|
302
|
+
skippedCount,
|
|
303
|
+
failedCount,
|
|
304
|
+
staleSince: null,
|
|
305
|
+
updatedAt: new Date(),
|
|
306
|
+
})
|
|
307
|
+
.where(
|
|
308
|
+
and(
|
|
309
|
+
eq(campaigns.id, input.campaignId),
|
|
310
|
+
eq(campaigns.status, "sending"),
|
|
311
|
+
),
|
|
312
|
+
)
|
|
313
|
+
.returning({ id: campaigns.id });
|
|
314
|
+
if (parked.length === 0) {
|
|
315
|
+
await flushCounts();
|
|
316
|
+
const current = await db
|
|
317
|
+
.select({ status: campaigns.status })
|
|
318
|
+
.from(campaigns)
|
|
319
|
+
.where(eq(campaigns.id, input.campaignId))
|
|
320
|
+
.limit(1);
|
|
321
|
+
return { status: current[0]?.status ?? "unknown", skipped: true };
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
// Punctual resume run at nextStepAt — best-effort, same split as
|
|
325
|
+
// `scheduledAt`: on failure the row stays `waiting` and the reaper's
|
|
326
|
+
// waiting-promotion sweep resumes it.
|
|
327
|
+
try {
|
|
328
|
+
await sendCampaignTask.schedule(nextStepAt, {
|
|
329
|
+
campaignId: input.campaignId,
|
|
330
|
+
});
|
|
331
|
+
} catch (err) {
|
|
332
|
+
logger.warn(
|
|
333
|
+
"send-campaign: next-step schedule failed (reaper will promote)",
|
|
334
|
+
{
|
|
335
|
+
campaignId: input.campaignId,
|
|
336
|
+
nextStepAt: nextStepAt.toISOString(),
|
|
337
|
+
error: err instanceof Error ? err.message : String(err),
|
|
338
|
+
},
|
|
339
|
+
);
|
|
340
|
+
}
|
|
341
|
+
|
|
342
|
+
logger.info("send-campaign: wave complete — waiting", {
|
|
343
|
+
campaignId: input.campaignId,
|
|
344
|
+
currentStep: k + 1,
|
|
345
|
+
nextStepAt: nextStepAt.toISOString(),
|
|
346
|
+
});
|
|
347
|
+
return {
|
|
348
|
+
status: "waiting" as const,
|
|
349
|
+
currentStep: k + 1,
|
|
350
|
+
nextStepAt: nextStepAt.toISOString(),
|
|
351
|
+
totalRecipients,
|
|
352
|
+
sentCount,
|
|
353
|
+
skippedCount,
|
|
354
|
+
failedCount,
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
if (step.kind !== "send") {
|
|
359
|
+
// A blob minted by a future engine (steps.v evolution) must fail
|
|
360
|
+
// loudly here, not silently skip a delivery step.
|
|
361
|
+
throw new Error(
|
|
362
|
+
`send-campaign: unsupported step kind "${(step as { kind: string }).kind}" at step ${k} of campaign ${input.campaignId}`,
|
|
363
|
+
);
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
const canceled = await runWave(k, step);
|
|
367
|
+
if (canceled) {
|
|
368
|
+
// The operator's cancel already stamped status/canceledAt; persist
|
|
369
|
+
// the progress counts so the row shows how far the blast got.
|
|
370
|
+
await flushCounts();
|
|
371
|
+
logger.info("send-campaign: canceled mid-send", {
|
|
372
|
+
campaignId: input.campaignId,
|
|
373
|
+
currentStep: k,
|
|
374
|
+
totalRecipients,
|
|
375
|
+
sentCount,
|
|
376
|
+
skippedCount,
|
|
377
|
+
failedCount,
|
|
378
|
+
});
|
|
379
|
+
return {
|
|
380
|
+
status: "canceled" as const,
|
|
381
|
+
totalRecipients,
|
|
382
|
+
sentCount,
|
|
383
|
+
skippedCount,
|
|
384
|
+
failedCount,
|
|
385
|
+
};
|
|
386
|
+
}
|
|
387
|
+
|
|
388
|
+
// Advance the cursor + snapshot the cumulative counters in ONE update
|
|
389
|
+
// — the wave is complete, so a retry entering after this point starts
|
|
390
|
+
// at the NEXT step with these counts as its seed. Plain update (not
|
|
391
|
+
// CAS): a cancel that raced the tail of the wave only bumps cursor
|
|
392
|
+
// metadata on an already-terminal row, which nothing reads.
|
|
393
|
+
await db
|
|
394
|
+
.update(campaigns)
|
|
395
|
+
.set({
|
|
396
|
+
currentStep: k + 1,
|
|
397
|
+
stepBaseCounts: snapshotCounts(),
|
|
398
|
+
totalRecipients,
|
|
399
|
+
sentCount,
|
|
400
|
+
skippedCount,
|
|
401
|
+
failedCount,
|
|
402
|
+
staleSince: null,
|
|
403
|
+
updatedAt: new Date(),
|
|
404
|
+
})
|
|
405
|
+
.where(eq(campaigns.id, input.campaignId));
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
// Completion CAS: only a still-`sending` row is stamped `sent`, so a
|
|
409
|
+
// cancel racing the final chunk (after its last per-chunk check) keeps
|
|
410
|
+
// its `canceled` status instead of being overwritten. `nextStepAt` +
|
|
411
|
+
// `stepBaseCounts` are nulled — a terminal row carries no pending wait
|
|
412
|
+
// and no resume seed.
|
|
413
|
+
const completed = await db
|
|
414
|
+
.update(campaigns)
|
|
415
|
+
.set({
|
|
416
|
+
status: "sent",
|
|
417
|
+
completedAt: new Date(),
|
|
418
|
+
nextStepAt: null,
|
|
419
|
+
stepBaseCounts: null,
|
|
420
|
+
totalRecipients,
|
|
421
|
+
sentCount,
|
|
422
|
+
skippedCount,
|
|
423
|
+
failedCount,
|
|
424
|
+
staleSince: null,
|
|
425
|
+
updatedAt: new Date(),
|
|
426
|
+
})
|
|
427
|
+
.where(
|
|
428
|
+
and(
|
|
429
|
+
eq(campaigns.id, input.campaignId),
|
|
430
|
+
eq(campaigns.status, "sending"),
|
|
431
|
+
),
|
|
432
|
+
)
|
|
433
|
+
.returning({ id: campaigns.id });
|
|
434
|
+
if (completed.length === 0) {
|
|
435
|
+
await flushCounts();
|
|
436
|
+
const current = await db
|
|
437
|
+
.select({ status: campaigns.status })
|
|
438
|
+
.from(campaigns)
|
|
439
|
+
.where(eq(campaigns.id, input.campaignId))
|
|
440
|
+
.limit(1);
|
|
441
|
+
const status = current[0]?.status ?? "unknown";
|
|
442
|
+
logger.info("send-campaign: finished but row left `sending` first", {
|
|
443
|
+
campaignId: input.campaignId,
|
|
444
|
+
status,
|
|
445
|
+
});
|
|
446
|
+
return {
|
|
447
|
+
status,
|
|
448
|
+
totalRecipients,
|
|
449
|
+
sentCount,
|
|
450
|
+
skippedCount,
|
|
451
|
+
failedCount,
|
|
452
|
+
};
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
logger.info("send-campaign: complete", {
|
|
456
|
+
campaignId: input.campaignId,
|
|
457
|
+
totalRecipients,
|
|
458
|
+
sentCount,
|
|
459
|
+
skippedCount,
|
|
460
|
+
failedCount,
|
|
461
|
+
});
|
|
462
|
+
|
|
463
|
+
return {
|
|
464
|
+
status: "sent" as const,
|
|
465
|
+
totalRecipients,
|
|
466
|
+
sentCount,
|
|
467
|
+
skippedCount,
|
|
468
|
+
failedCount,
|
|
469
|
+
};
|
|
470
|
+
} catch (error) {
|
|
471
|
+
// Do NOT stamp `failed` here. A `failed` stamp before the re-throw makes
|
|
472
|
+
// the single Hatchet retry hit the terminal guard and short-circuit
|
|
473
|
+
// WITHOUT sending the remaining recipients — silently abandoning the tail
|
|
474
|
+
// of a partial wave. Instead we persist the progress counts, leave the
|
|
475
|
+
// status `sending` (re-runnable — `currentStep` still points at the
|
|
476
|
+
// interrupted wave and `stepBaseCounts` still seeds it), and re-throw so
|
|
477
|
+
// the genuine retry re-enters the wave and finishes the unsent tail
|
|
478
|
+
// (already-sent recipients no-op via their idempotency key). A run that
|
|
479
|
+
// EXHAUSTS its retries is transitioned to `failed` (or re-enqueued) by
|
|
480
|
+
// `reapStuckCampaignsTask`. `staleSince` is preserved — a crashed run is
|
|
481
|
+
// not progress, and clearing it here would reset the give-up clock on
|
|
482
|
+
// every re-drive of a poison campaign.
|
|
483
|
+
await flushCounts({ preserveStaleSince: true });
|
|
484
|
+
|
|
485
|
+
logger.error("send-campaign: errored mid-run (will retry)", {
|
|
486
|
+
campaignId: input.campaignId,
|
|
487
|
+
error: error instanceof Error ? error.message : String(error),
|
|
488
|
+
});
|
|
489
|
+
throw error;
|
|
490
|
+
}
|
|
491
|
+
|
|
492
|
+
/**
|
|
493
|
+
* Execute one per-recipient email wave. Returns `true` when a mid-wave
|
|
494
|
+
* cancel was detected (the caller flushes + returns; nothing after the
|
|
495
|
+
* detecting chunk is dispatched).
|
|
496
|
+
*
|
|
497
|
+
* Qualifier source:
|
|
498
|
+
* - Wave 0 resolves the audience LIVE (list/bucket resolvers, fresh
|
|
499
|
+
* suppression pre-filter) and — for MULTI-step campaigns — ANCHORS the
|
|
500
|
+
* cohort: each chunk is batch-inserted into `campaign_recipients`
|
|
501
|
+
* BEFORE its sends are dispatched — a crash between insert and send is
|
|
502
|
+
* fine (the retry re-resolves and the (campaign_id, email) unique
|
|
503
|
+
* absorbs the replay). The audience is resolved exactly once per
|
|
504
|
+
* campaign: someone added to the list afterwards never receives step 3
|
|
505
|
+
* without step 1. A single-send campaign skips the ledger entirely (no
|
|
506
|
+
* later wave will read it — the legacy write path stays byte-identical).
|
|
507
|
+
* - Waves k > 0 qualify FROM the anchored cohort: `campaign_recipients`
|
|
508
|
+
* ∩ the step's `where` conditions ∩ a fresh suppression re-check
|
|
509
|
+
* (see resolveCohortRecipients — suppression is never snapshotted).
|
|
510
|
+
*/
|
|
511
|
+
async function runWave(
|
|
512
|
+
k: number,
|
|
513
|
+
step: CampaignSendStep,
|
|
514
|
+
): Promise<boolean> {
|
|
111
515
|
const recipients =
|
|
112
|
-
|
|
113
|
-
?
|
|
114
|
-
|
|
516
|
+
k === 0
|
|
517
|
+
? audienceKind === "bucket"
|
|
518
|
+
? resolveBucketRecipients(db, audienceId)
|
|
519
|
+
: resolveListRecipients(db, audienceId)
|
|
520
|
+
: resolveCohortRecipients(db, {
|
|
521
|
+
campaignId: input.campaignId,
|
|
522
|
+
conditions: step.where,
|
|
523
|
+
startedAt,
|
|
524
|
+
});
|
|
525
|
+
|
|
526
|
+
// Mid-flight cancel: re-checked once per chunk (one cheap SELECT per
|
|
527
|
+
// CHUNK_SIZE sends) so `POST /v1/campaigns/:id/cancel` can stop a blast
|
|
528
|
+
// that is already `sending` — recipients not yet dispatched are spared.
|
|
529
|
+
let cancelDetected = false;
|
|
115
530
|
|
|
116
531
|
let chunk: CampaignRecipient[] = [];
|
|
117
532
|
for await (const recipient of recipients) {
|
|
118
533
|
chunk.push(recipient);
|
|
119
534
|
if (chunk.length < CHUNK_SIZE) continue;
|
|
120
535
|
await sendChunk();
|
|
536
|
+
if (cancelDetected) return true;
|
|
121
537
|
}
|
|
122
538
|
// Final partial chunk.
|
|
123
|
-
if (chunk.length > 0) await sendChunk();
|
|
539
|
+
if (chunk.length > 0 && !cancelDetected) await sendChunk();
|
|
540
|
+
return cancelDetected;
|
|
124
541
|
|
|
125
542
|
async function sendChunk(): Promise<void> {
|
|
543
|
+
const statusRows = await db
|
|
544
|
+
.select({ status: campaigns.status })
|
|
545
|
+
.from(campaigns)
|
|
546
|
+
.where(eq(campaigns.id, input.campaignId))
|
|
547
|
+
.limit(1);
|
|
548
|
+
if (statusRows[0]?.status === "canceled") {
|
|
549
|
+
cancelDetected = true;
|
|
550
|
+
chunk = [];
|
|
551
|
+
return;
|
|
552
|
+
}
|
|
553
|
+
|
|
126
554
|
const batch = chunk;
|
|
127
555
|
chunk = [];
|
|
556
|
+
|
|
557
|
+
// Anchor the cohort BEFORE dispatching the chunk (wave 0 of a
|
|
558
|
+
// MULTI-step campaign only — a single-send campaign has no later wave
|
|
559
|
+
// to read the ledger, so skipping the insert keeps the legacy blast's
|
|
560
|
+
// write path byte-identical). Emails are already normalized by the
|
|
561
|
+
// resolvers, matching the table's (campaign_id, email) unique
|
|
562
|
+
// keyspace.
|
|
563
|
+
if (k === 0 && multiStep) {
|
|
564
|
+
await db
|
|
565
|
+
.insert(campaignRecipients)
|
|
566
|
+
.values(
|
|
567
|
+
batch.map((r) => ({
|
|
568
|
+
campaignId: input.campaignId,
|
|
569
|
+
userId: r.userId ?? null,
|
|
570
|
+
email: r.email,
|
|
571
|
+
})),
|
|
572
|
+
)
|
|
573
|
+
.onConflictDoNothing({
|
|
574
|
+
target: [campaignRecipients.campaignId, campaignRecipients.email],
|
|
575
|
+
});
|
|
576
|
+
}
|
|
577
|
+
|
|
128
578
|
totalRecipients += batch.length;
|
|
129
579
|
|
|
130
580
|
const results = await Promise.allSettled(
|
|
131
581
|
batch.map((r) =>
|
|
132
582
|
emailService.send({
|
|
133
|
-
template:
|
|
134
|
-
props: (
|
|
583
|
+
template: step.template as TemplateName,
|
|
584
|
+
props: (step.props ?? {}) as never,
|
|
135
585
|
to: r.email,
|
|
136
586
|
userId: r.userId,
|
|
137
587
|
userEmail: r.email,
|
|
138
|
-
subject:
|
|
139
|
-
from:
|
|
588
|
+
subject: step.subject,
|
|
589
|
+
from: step.from,
|
|
140
590
|
// A list's audienceId IS a real subscription category, so pass it
|
|
141
591
|
// through for suppression + the unsubscribe link. A bucket's
|
|
142
592
|
// audienceId is NOT a category — forcing it here would mint an
|
|
@@ -145,13 +595,17 @@ export const sendCampaignTask = hatchet.task({
|
|
|
145
595
|
// unsubscribedAll/suppressed), silently no-op'ing the unsubscribe.
|
|
146
596
|
// For a bucket, pass undefined so the template's OWN declared
|
|
147
597
|
// category (e.g. `product-updates`) drives both suppression and a
|
|
148
|
-
// real, honored List-Unsubscribe target.
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
? undefined
|
|
152
|
-
: campaign?.audienceId,
|
|
598
|
+
// real, honored List-Unsubscribe target. The rule applies to
|
|
599
|
+
// EVERY wave, not just the first.
|
|
600
|
+
category: audienceKind === "bucket" ? undefined : audienceId,
|
|
153
601
|
// The idempotency key dedupes a retried send to its prior row.
|
|
154
|
-
|
|
602
|
+
// Legacy format for single-step campaigns; step-scoped for every
|
|
603
|
+
// step of a multi-step campaign (campaignSendKey contract).
|
|
604
|
+
idempotencyKey: campaignSendKey(
|
|
605
|
+
input.campaignId,
|
|
606
|
+
r.email,
|
|
607
|
+
multiStep ? k : undefined,
|
|
608
|
+
),
|
|
155
609
|
}),
|
|
156
610
|
),
|
|
157
611
|
);
|
|
@@ -166,67 +620,15 @@ export const sendCampaignTask = hatchet.task({
|
|
|
166
620
|
sentCount++;
|
|
167
621
|
} else {
|
|
168
622
|
// suppressed | unsubscribed | skipped (frequency-capped) — counted
|
|
169
|
-
// as skipped, not a delivery failure.
|
|
623
|
+
// as skipped, not a delivery failure. The member STAYS in the
|
|
624
|
+
// cohort either way: a provider hiccup or a skipped wave is not an
|
|
625
|
+
// exit, and suppression is re-checked fresh on every later wave.
|
|
170
626
|
skippedCount++;
|
|
171
627
|
}
|
|
172
628
|
}
|
|
173
629
|
|
|
174
630
|
await flushCounts();
|
|
175
631
|
}
|
|
176
|
-
|
|
177
|
-
await db
|
|
178
|
-
.update(campaigns)
|
|
179
|
-
.set({
|
|
180
|
-
status: "sent",
|
|
181
|
-
completedAt: new Date(),
|
|
182
|
-
totalRecipients,
|
|
183
|
-
sentCount,
|
|
184
|
-
skippedCount,
|
|
185
|
-
failedCount,
|
|
186
|
-
updatedAt: new Date(),
|
|
187
|
-
})
|
|
188
|
-
.where(eq(campaigns.id, input.campaignId));
|
|
189
|
-
|
|
190
|
-
logger.info("send-campaign: complete", {
|
|
191
|
-
campaignId: input.campaignId,
|
|
192
|
-
totalRecipients,
|
|
193
|
-
sentCount,
|
|
194
|
-
skippedCount,
|
|
195
|
-
failedCount,
|
|
196
|
-
});
|
|
197
|
-
|
|
198
|
-
return {
|
|
199
|
-
status: "sent" as const,
|
|
200
|
-
totalRecipients,
|
|
201
|
-
sentCount,
|
|
202
|
-
skippedCount,
|
|
203
|
-
failedCount,
|
|
204
|
-
};
|
|
205
|
-
} catch (error) {
|
|
206
|
-
// Do NOT stamp `failed` here. A `failed` stamp before the re-throw makes
|
|
207
|
-
// the single Hatchet retry hit the terminal guard and short-circuit
|
|
208
|
-
// WITHOUT sending the remaining recipients — silently abandoning the tail
|
|
209
|
-
// of a partial send. Instead we persist the progress counts, leave the
|
|
210
|
-
// status `sending` (re-runnable), and re-throw so the genuine retry
|
|
211
|
-
// re-enters the loop and finishes the unsent tail (already-sent recipients
|
|
212
|
-
// no-op via their idempotency key). A run that EXHAUSTS its retries is
|
|
213
|
-
// transitioned to `failed` (or re-enqueued) by `reapStuckCampaignsTask`.
|
|
214
|
-
await db
|
|
215
|
-
.update(campaigns)
|
|
216
|
-
.set({
|
|
217
|
-
totalRecipients,
|
|
218
|
-
sentCount,
|
|
219
|
-
skippedCount,
|
|
220
|
-
failedCount,
|
|
221
|
-
updatedAt: new Date(),
|
|
222
|
-
})
|
|
223
|
-
.where(eq(campaigns.id, input.campaignId));
|
|
224
|
-
|
|
225
|
-
logger.error("send-campaign: errored mid-run (will retry)", {
|
|
226
|
-
campaignId: input.campaignId,
|
|
227
|
-
error: error instanceof Error ? error.message : String(error),
|
|
228
|
-
});
|
|
229
|
-
throw error;
|
|
230
632
|
}
|
|
231
633
|
},
|
|
232
634
|
});
|
|
@@ -243,15 +645,35 @@ const STALE_AFTER_MS = Number(
|
|
|
243
645
|
);
|
|
244
646
|
|
|
245
647
|
/**
|
|
246
|
-
* After a campaign has
|
|
247
|
-
*
|
|
248
|
-
*
|
|
249
|
-
*
|
|
648
|
+
* After a campaign has been CONTINUOUSLY stuck this long with zero progress it
|
|
649
|
+
* is declared `failed` rather than re-enqueued forever — a poison campaign
|
|
650
|
+
* (e.g. a run that always crashes) stops being re-driven and surfaces to
|
|
651
|
+
* operators.
|
|
652
|
+
*
|
|
653
|
+
* For in-flight (`queued`/`sending`) rows the window is measured from
|
|
654
|
+
* `staleSince` — set once by the stale sweep's first re-enqueue, cleared by
|
|
655
|
+
* every genuine progress flush of the send task. NOT from `updatedAt`: the
|
|
656
|
+
* sweep bumps `updatedAt` as its re-pick guard, so a deterministically-
|
|
657
|
+
* crashing row would never age past the window (the give-up was dead code
|
|
658
|
+
* until `stale_since` landed). `scheduled` / `waiting` rows are measured from
|
|
659
|
+
* `scheduledAt` / `nextStepAt` as before.
|
|
250
660
|
*/
|
|
251
661
|
const GIVE_UP_AFTER_MS = Number(
|
|
252
662
|
process.env.CAMPAIGN_GIVE_UP_AFTER_MS ?? 6 * 60 * 60 * 1000,
|
|
253
663
|
);
|
|
254
664
|
|
|
665
|
+
/**
|
|
666
|
+
* How far past `scheduledAt` a `scheduled` campaign (or past `nextStepAt` a
|
|
667
|
+
* `waiting` one) may sit before the reaper promotes it (enqueues the send
|
|
668
|
+
* task). The punctual Hatchet scheduled run created at schedule/park time is
|
|
669
|
+
* the primary trigger; this grace keeps the sweep from routinely
|
|
670
|
+
* double-firing right at the boundary. A double fire is harmless anyway
|
|
671
|
+
* (terminal guard + per-send idempotency).
|
|
672
|
+
*/
|
|
673
|
+
const SCHEDULE_PROMOTE_GRACE_MS = Number(
|
|
674
|
+
process.env.CAMPAIGN_PROMOTE_GRACE_MS ?? 2 * 60 * 1000,
|
|
675
|
+
);
|
|
676
|
+
|
|
255
677
|
/**
|
|
256
678
|
* Engine-owned reaper cron for campaigns left in a non-terminal in-flight state
|
|
257
679
|
* with no live run to finish them (closes the "stuck forever" gap):
|
|
@@ -261,6 +683,16 @@ const GIVE_UP_AFTER_MS = Number(
|
|
|
261
683
|
* catch never ran, so the row is stuck `sending` with no live run.
|
|
262
684
|
* - A `queued` campaign whose enqueue threw at create time (broker down /
|
|
263
685
|
* network) — the row was committed but no run was ever created (orphan).
|
|
686
|
+
* - A `scheduled` campaign whose punctual Hatchet scheduled run never fired
|
|
687
|
+
* (schedule-create failed at POST/reconcile time, or the run was lost) —
|
|
688
|
+
* promoted once `scheduledAt` is {@link SCHEDULE_PROMOTE_GRACE_MS} past
|
|
689
|
+
* due. A scheduled row stuck past the give-up window (measured from
|
|
690
|
+
* `scheduledAt`, NOT `updatedAt` — a row is legitimately idle between
|
|
691
|
+
* create and send time) is declared `failed`.
|
|
692
|
+
* - A `waiting` campaign whose punctual next-step run was lost — promoted
|
|
693
|
+
* once `nextStepAt` is past the same grace. Give-up is likewise measured
|
|
694
|
+
* from `nextStepAt`, NOT `updatedAt` — a row is legitimately idle mid-wait,
|
|
695
|
+
* exactly like `scheduled`/`scheduledAt`.
|
|
264
696
|
*
|
|
265
697
|
* Recovery is a simple RE-ENQUEUE of `sendCampaignTask` (safe: the per-send
|
|
266
698
|
* idempotency key no-ops already-sent recipients and the re-run completes the
|
|
@@ -284,16 +716,38 @@ export const reapStuckCampaignsTask = hatchet.task({
|
|
|
284
716
|
const now = Date.now();
|
|
285
717
|
const staleBefore = new Date(now - STALE_AFTER_MS);
|
|
286
718
|
const giveUpBefore = new Date(now - GIVE_UP_AFTER_MS);
|
|
719
|
+
const promoteBefore = new Date(now - SCHEDULE_PROMOTE_GRACE_MS);
|
|
287
720
|
|
|
288
721
|
// (1) Declare poison campaigns `failed` first (stuck past the give-up
|
|
289
|
-
// window), so they are not re-enqueued below.
|
|
722
|
+
// window), so they are not re-enqueued below. In-flight rows are measured
|
|
723
|
+
// from `staleSince` (set by the first stale re-enqueue, cleared by every
|
|
724
|
+
// genuine progress flush — a NULL never matches the `lt`, so a row the
|
|
725
|
+
// sweep hasn't touched yet gets its re-drive chance first); `scheduled`
|
|
726
|
+
// rows from `scheduledAt` and `waiting` rows from `nextStepAt` (their
|
|
727
|
+
// `updatedAt` is legitimately old while they wait). `staleSince` is nulled
|
|
728
|
+
// with the stamp so an operator revive starts a fresh give-up window.
|
|
290
729
|
const failedRows = await db
|
|
291
730
|
.update(campaigns)
|
|
292
|
-
.set({
|
|
731
|
+
.set({
|
|
732
|
+
status: "failed",
|
|
733
|
+
completedAt: new Date(),
|
|
734
|
+
staleSince: null,
|
|
735
|
+
updatedAt: new Date(),
|
|
736
|
+
})
|
|
293
737
|
.where(
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
738
|
+
or(
|
|
739
|
+
and(
|
|
740
|
+
inArray(campaigns.status, ["queued", "sending"]),
|
|
741
|
+
lt(campaigns.staleSince, giveUpBefore),
|
|
742
|
+
),
|
|
743
|
+
and(
|
|
744
|
+
eq(campaigns.status, "scheduled"),
|
|
745
|
+
lt(campaigns.scheduledAt, giveUpBefore),
|
|
746
|
+
),
|
|
747
|
+
and(
|
|
748
|
+
eq(campaigns.status, "waiting"),
|
|
749
|
+
lt(campaigns.nextStepAt, giveUpBefore),
|
|
750
|
+
),
|
|
297
751
|
),
|
|
298
752
|
)
|
|
299
753
|
.returning({ id: campaigns.id });
|
|
@@ -302,9 +756,18 @@ export const reapStuckCampaignsTask = hatchet.task({
|
|
|
302
756
|
// `updatedAt` so the same row is not re-picked on the very next tick before
|
|
303
757
|
// the re-driven run makes progress; the per-send idempotency key keeps the
|
|
304
758
|
// re-enqueue safe even if the original run is somehow still alive.
|
|
759
|
+
// `staleSince` is coalesce-set ONCE on the first re-enqueue (later sweeps
|
|
760
|
+
// keep the original instant) — this is the bump-free clock the give-up
|
|
761
|
+
// clause above reads; the send task clears it on genuine progress.
|
|
762
|
+
// `queued`/`sending` ONLY — `waiting` is deliberately excluded: a 2-day
|
|
763
|
+
// wait between waves is not a stuck campaign (its resume path is the
|
|
764
|
+
// promotion sweep below, keyed on `nextStepAt`).
|
|
305
765
|
const staleRows = await db
|
|
306
766
|
.update(campaigns)
|
|
307
|
-
.set({
|
|
767
|
+
.set({
|
|
768
|
+
staleSince: sql`coalesce(${campaigns.staleSince}, now())`,
|
|
769
|
+
updatedAt: new Date(),
|
|
770
|
+
})
|
|
308
771
|
.where(
|
|
309
772
|
and(
|
|
310
773
|
inArray(campaigns.status, ["queued", "sending"]),
|
|
@@ -313,9 +776,32 @@ export const reapStuckCampaignsTask = hatchet.task({
|
|
|
313
776
|
)
|
|
314
777
|
.returning({ id: campaigns.id });
|
|
315
778
|
|
|
316
|
-
|
|
779
|
+
// (3) Promote due `scheduled` campaigns whose punctual scheduled run never
|
|
780
|
+
// fired, and due `waiting` campaigns whose punctual next-step run was lost.
|
|
781
|
+
// Enqueue-only — the send task owns the scheduled/waiting→sending
|
|
782
|
+
// transition, so a failed enqueue simply retries next sweep.
|
|
783
|
+
const dueRows = await db
|
|
784
|
+
.select({ id: campaigns.id })
|
|
785
|
+
.from(campaigns)
|
|
786
|
+
.where(
|
|
787
|
+
or(
|
|
788
|
+
and(
|
|
789
|
+
eq(campaigns.status, "scheduled"),
|
|
790
|
+
lte(campaigns.scheduledAt, promoteBefore),
|
|
791
|
+
),
|
|
792
|
+
and(
|
|
793
|
+
eq(campaigns.status, "waiting"),
|
|
794
|
+
lte(campaigns.nextStepAt, promoteBefore),
|
|
795
|
+
),
|
|
796
|
+
),
|
|
797
|
+
);
|
|
798
|
+
|
|
799
|
+
// Enqueue WITHOUT waiting for results — `.run()` blocks until the whole
|
|
800
|
+
// blast completes, which would serialize entire sends inside this cron's
|
|
801
|
+
// 120s executionTimeout.
|
|
802
|
+
for (const row of [...staleRows, ...dueRows]) {
|
|
317
803
|
try {
|
|
318
|
-
await sendCampaignTask.
|
|
804
|
+
await sendCampaignTask.runNoWait({ campaignId: row.id });
|
|
319
805
|
} catch (err) {
|
|
320
806
|
logger.warn("reap-stuck-campaigns: re-enqueue failed", {
|
|
321
807
|
campaignId: row.id,
|
|
@@ -324,16 +810,18 @@ export const reapStuckCampaignsTask = hatchet.task({
|
|
|
324
810
|
}
|
|
325
811
|
}
|
|
326
812
|
|
|
327
|
-
if (failedRows.length > 0 || staleRows.length > 0) {
|
|
813
|
+
if (failedRows.length > 0 || staleRows.length > 0 || dueRows.length > 0) {
|
|
328
814
|
logger.info("reap-stuck-campaigns: swept", {
|
|
329
815
|
failed: failedRows.length,
|
|
330
816
|
reEnqueued: staleRows.length,
|
|
817
|
+
promoted: dueRows.length,
|
|
331
818
|
});
|
|
332
819
|
}
|
|
333
820
|
|
|
334
821
|
return {
|
|
335
822
|
failed: failedRows.length,
|
|
336
823
|
reEnqueued: staleRows.length,
|
|
824
|
+
promoted: dueRows.length,
|
|
337
825
|
};
|
|
338
826
|
},
|
|
339
827
|
});
|
|
@@ -370,19 +858,76 @@ async function* keysetPaginate<Row>(opts: {
|
|
|
370
858
|
}
|
|
371
859
|
}
|
|
372
860
|
|
|
861
|
+
/**
|
|
862
|
+
* Wave-k (k > 0) qualifier resolver: the campaign's anchored cohort
|
|
863
|
+
* (`campaign_recipients`) ∩ the step's `where` conditions (each compiled to a
|
|
864
|
+
* correlated [NOT] EXISTS — see `campaigns/cohort-sql.ts`) ∩ a fresh
|
|
865
|
+
* suppression/unsubscribe re-check. Membership was anchored at wave 0 and is
|
|
866
|
+
* never re-resolved — a member who left the bucket/list between waves still
|
|
867
|
+
* qualifies (suppression excepted); a member added afterwards never appears.
|
|
868
|
+
* Paged by the keyset cursor on `campaign_recipients.id` (the
|
|
869
|
+
* (campaign_id, id) index), so every page is an indexed join, never a LIKE
|
|
870
|
+
* scan over email_sends.
|
|
871
|
+
*/
|
|
872
|
+
async function* resolveCohortRecipients(
|
|
873
|
+
db: Database,
|
|
874
|
+
opts: {
|
|
875
|
+
campaignId: string;
|
|
876
|
+
conditions: ConditionEval[] | undefined;
|
|
877
|
+
startedAt: Date;
|
|
878
|
+
},
|
|
879
|
+
): AsyncGenerator<CampaignRecipient> {
|
|
880
|
+
// Compiled once per wave — every page reuses the same predicates.
|
|
881
|
+
const qualifiers = (opts.conditions ?? []).map((condition) =>
|
|
882
|
+
waveConditionSql({
|
|
883
|
+
condition,
|
|
884
|
+
campaignId: opts.campaignId,
|
|
885
|
+
startedAt: opts.startedAt,
|
|
886
|
+
}),
|
|
887
|
+
);
|
|
888
|
+
|
|
889
|
+
yield* keysetPaginate({
|
|
890
|
+
page: (cursor) => {
|
|
891
|
+
const conditions = [
|
|
892
|
+
eq(campaignRecipients.campaignId, opts.campaignId),
|
|
893
|
+
cohortSuppressionSql(),
|
|
894
|
+
...qualifiers,
|
|
895
|
+
];
|
|
896
|
+
if (cursor) conditions.push(gt(campaignRecipients.id, cursor));
|
|
897
|
+
|
|
898
|
+
return db
|
|
899
|
+
.select({
|
|
900
|
+
id: campaignRecipients.id,
|
|
901
|
+
userId: campaignRecipients.userId,
|
|
902
|
+
email: campaignRecipients.email,
|
|
903
|
+
})
|
|
904
|
+
.from(campaignRecipients)
|
|
905
|
+
.where(and(...conditions))
|
|
906
|
+
.orderBy(campaignRecipients.id)
|
|
907
|
+
.limit(CHUNK_SIZE);
|
|
908
|
+
},
|
|
909
|
+
cursorOf: (row) => row.id,
|
|
910
|
+
// Cohort emails were normalized at anchor time; userId may be NULL for an
|
|
911
|
+
// email-only member (kept — the mailer falls back the same way wave 0 did).
|
|
912
|
+
map: (row) => ({ email: row.email, userId: row.userId ?? undefined }),
|
|
913
|
+
});
|
|
914
|
+
}
|
|
915
|
+
|
|
373
916
|
/**
|
|
374
917
|
* Active, non-deleted members of a bucket, joined to a live contact for the
|
|
375
918
|
* email — mirrors the bucket-access member query. Paged by the keyset cursor on
|
|
376
919
|
* `bucket_memberships.id`.
|
|
377
920
|
*
|
|
378
|
-
* Compliance: `bucket_memberships.userEmail` is
|
|
379
|
-
*
|
|
380
|
-
*
|
|
381
|
-
*
|
|
382
|
-
* (
|
|
383
|
-
*
|
|
384
|
-
*
|
|
385
|
-
*
|
|
921
|
+
* Compliance: `bucket_memberships.userEmail` is normalized at every write site
|
|
922
|
+
* (and existing rows were backfilled by migration 0043), but this resolver
|
|
923
|
+
* KEEPS its own `normalizeEmail` / `lower(trim(…))` belt-and-braces —
|
|
924
|
+
* TODO(cleanup): safe to drop once `@hogsend/engine` has shipped past 0.40.0
|
|
925
|
+
* (all deployed write sites normalize as of this changeset). An un-normalized
|
|
926
|
+
* membership email (`User@Example.com`) would not
|
|
927
|
+
* case-match its NORMALIZED `email_preferences` row (`user@example.com`) and
|
|
928
|
+
* the mailer's case-sensitive suppression check would MISS the row, leaking a
|
|
929
|
+
* marketing blast to a suppressed/unsubscribed contact (CAN-SPAM/GDPR).
|
|
930
|
+
* Defense-in-depth: this resolver ALSO pre-filters
|
|
386
931
|
* `unsubscribedAll`/`suppressed` at the audience layer (mirroring the list
|
|
387
932
|
* resolver) via a LEFT JOIN to `email_preferences` on the NORMALIZED email, so
|
|
388
933
|
* a globally-unsubscribed / suppressed bucket member is excluded up front
|
|
@@ -394,8 +939,9 @@ async function* resolveBucketRecipients(
|
|
|
394
939
|
db: Database,
|
|
395
940
|
bucketId: string,
|
|
396
941
|
): AsyncGenerator<CampaignRecipient> {
|
|
397
|
-
// The recipient's normalized email —
|
|
398
|
-
// (
|
|
942
|
+
// The recipient's normalized email — writes are normalized now, but the
|
|
943
|
+
// read-side lower(trim(…)) stays as belt-and-braces (TODO(cleanup): see the
|
|
944
|
+
// fn docstring for the drop condition); the contact email is the fallback.
|
|
399
945
|
const recipientEmail = sql<string>`lower(trim(coalesce(${bucketMemberships.userEmail}, ${contacts.email})))`;
|
|
400
946
|
|
|
401
947
|
yield* keysetPaginate({
|