@hogsend/engine 0.40.0 → 0.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/package.json +11 -8
  2. package/src/buckets/check-membership.ts +6 -1
  3. package/src/campaigns/cohort-sql.ts +198 -0
  4. package/src/campaigns/define-campaign.ts +207 -28
  5. package/src/campaigns/reconcile.ts +105 -11
  6. package/src/campaigns/steps.ts +48 -0
  7. package/src/connectors/define-action.ts +70 -0
  8. package/src/container.ts +124 -42
  9. package/src/destinations/presets/posthog.ts +7 -1
  10. package/src/index.ts +96 -2
  11. package/src/journeys/constants.ts +10 -0
  12. package/src/journeys/define-journey.ts +14 -405
  13. package/src/journeys/execute-journey-run.ts +641 -0
  14. package/src/journeys/graph/build-graph.ts +212 -56
  15. package/src/journeys/journey-boundary.ts +54 -0
  16. package/src/journeys/journey-context.ts +406 -59
  17. package/src/journeys/record-once.ts +110 -0
  18. package/src/lib/blueprint-lock.ts +21 -0
  19. package/src/lib/blueprints.ts +803 -0
  20. package/src/lib/campaign-send-key.ts +58 -0
  21. package/src/lib/connector-actions.ts +203 -19
  22. package/src/lib/contacts.ts +54 -0
  23. package/src/lib/email-service-types.ts +4 -1
  24. package/src/lib/email.ts +13 -2
  25. package/src/lib/event-names.ts +143 -0
  26. package/src/lib/feed.ts +77 -57
  27. package/src/lib/frequency-cap.ts +45 -15
  28. package/src/lib/hatchet-duration.ts +22 -0
  29. package/src/lib/html.ts +1 -0
  30. package/src/lib/ingestion.ts +145 -4
  31. package/src/lib/journey-suppress.ts +87 -0
  32. package/src/lib/links.ts +204 -18
  33. package/src/lib/outbound.ts +20 -0
  34. package/src/lib/preferences.ts +80 -1
  35. package/src/lib/tracked.ts +44 -29
  36. package/src/lib/tracking-event-names.ts +8 -0
  37. package/src/lib/tracking.ts +1 -5
  38. package/src/lib/webhook-signing.ts +6 -1
  39. package/src/lists/channels.ts +75 -0
  40. package/src/lists/define-list.ts +34 -0
  41. package/src/lists/registry.ts +29 -2
  42. package/src/mcp/authoring-guide.ts +160 -0
  43. package/src/mcp/blueprint-tools.ts +390 -0
  44. package/src/middleware/require-admin.ts +13 -0
  45. package/src/routes/admin/api-keys.ts +72 -0
  46. package/src/routes/admin/blueprints.ts +795 -0
  47. package/src/routes/admin/campaigns.ts +179 -3
  48. package/src/routes/admin/emails.ts +13 -0
  49. package/src/routes/admin/events.ts +55 -0
  50. package/src/routes/admin/index.ts +4 -0
  51. package/src/routes/admin/journeys.ts +4 -2
  52. package/src/routes/admin/links.ts +303 -36
  53. package/src/routes/admin/lists.ts +51 -0
  54. package/src/routes/campaigns/index.ts +44 -6
  55. package/src/routes/email/preferences.ts +50 -13
  56. package/src/routes/feed/recipient.ts +2 -43
  57. package/src/routes/index.ts +6 -0
  58. package/src/routes/lists/index.ts +138 -18
  59. package/src/routes/tracking/arrive.ts +260 -0
  60. package/src/routes/tracking/click-pipeline.ts +329 -0
  61. package/src/routes/tracking/click.ts +5 -270
  62. package/src/routes/tracking/index.ts +2 -0
  63. package/src/routes/tracking/vanity.ts +64 -0
  64. package/src/worker.ts +5 -0
  65. package/src/workflows/bucket-backfill.ts +4 -2
  66. package/src/workflows/bucket-reconcile.ts +11 -4
  67. package/src/workflows/check-alerts.ts +130 -1
  68. package/src/workflows/journey-blueprint-interpreter.ts +656 -0
  69. package/src/workflows/send-campaign.ts +574 -180
@@ -1,5 +1,12 @@
1
+ import {
2
+ type CampaignSendStep,
3
+ type CampaignStep,
4
+ type ConditionEval,
5
+ durationToMs,
6
+ } from "@hogsend/core";
1
7
  import {
2
8
  bucketMemberships,
9
+ campaignRecipients,
3
10
  campaigns,
4
11
  contacts,
5
12
  type Database,
@@ -7,6 +14,11 @@ import {
7
14
  } from "@hogsend/db";
8
15
  import type { TemplateName } from "@hogsend/email";
9
16
  import { and, eq, gt, inArray, isNull, lt, lte, or, sql } from "drizzle-orm";
17
+ import {
18
+ cohortSuppressionSql,
19
+ waveConditionSql,
20
+ } from "../campaigns/cohort-sql.js";
21
+ import { campaignSendKey } from "../lib/campaign-send-key.js";
10
22
  import { normalizeEmail } from "../lib/contacts.js";
11
23
  import { getDb } from "../lib/db.js";
12
24
  import { getEmailService } from "../lib/email.js";
@@ -25,44 +37,60 @@ interface CampaignRecipient {
25
37
 
26
38
  /**
27
39
  * Statuses that are TERMINAL — a duplicate/late enqueue must not re-send.
28
- * `canceled`/`expired` matter doubly for SCHEDULED campaigns: the punctual
29
- * Hatchet scheduled run cannot be deleted at cancel time (we don't persist its
30
- * id), so it still fires and must find the row terminal here and no-op.
31
- * `failed` is deliberately NOT terminal — a re-enqueue can revive it.
40
+ * `canceled`/`expired` matter doubly for SCHEDULED (and now WAITING) campaigns:
41
+ * the punctual Hatchet scheduled run cannot be deleted at cancel time (we don't
42
+ * persist its id), so it still fires and must find the row terminal here and
43
+ * no-op. `failed` is deliberately NOT terminal — a re-enqueue can revive it.
32
44
  */
33
45
  const TERMINAL_STATUSES = ["sent", "canceled", "expired"] as const;
34
46
 
35
47
  /**
36
- * How far ahead of `scheduledAt` a run may fire and still proceed (absorbs
37
- * clock skew). An earlier fire — the punctual run created for a `sendAt` that
38
- * was later moved BACK by a reconcile/edit — skips without sending; the run
39
- * created for the new instant (or the reaper sweep) delivers it.
48
+ * How far ahead of `scheduledAt` / `nextStepAt` a run may fire and still
49
+ * proceed (absorbs clock skew). An earlier fire — the punctual run created for
50
+ * an instant that was later moved BACK (a reconcile edit for `scheduledAt`; a
51
+ * reap + re-park re-schedule for `nextStepAt`) — skips without sending; the
52
+ * run created for the new instant (or the reaper sweep) delivers it.
40
53
  */
41
54
  const EARLY_FIRE_TOLERANCE_MS = 60 * 1000;
42
55
 
43
56
  /**
44
- * Built-in durable campaign / broadcast task (Loops "campaign" parity). Sends a
45
- * single template to every subscribed member of a list (or every active member
46
- * of a bucket).
57
+ * Built-in durable campaign / broadcast task (Loops "campaign" parity). A
58
+ * legacy row (NULL `steps`) sends a single template to every subscribed member
59
+ * of a list (or every active member of a bucket). A multi-step row executes
60
+ * its `steps` blob as WAVES — each send step a SET operation over the
61
+ * campaign's anchored cohort, separated by durable waits (status `waiting`,
62
+ * resumed at `nextStepAt`) — see docs/campaign-steps-spec.md §Wave runtime.
63
+ * The task keeps its name and `{ campaignId }` input either way; the row's
64
+ * `currentStep` is the sole resume cursor.
47
65
  *
48
- * Retry-safety: each send carries an idempotency key
49
- * `campaign:<campaignId>:<email>` (email_sends.idempotency_key, migration 0015),
50
- * so a Hatchet retry re-runs the whole loop but every already-dispatched send
66
+ * Retry-safety: each send carries an idempotency key minted by
67
+ * `campaignSendKey` — legacy `campaign:<id>:<email>` for single-step
68
+ * campaigns, step-scoped `campaign:<id>:<step>:<email>` for EVERY step of a
69
+ * multi-step campaign (email_sends.idempotency_key, migration 0015) — so a
70
+ * Hatchet retry re-runs the wave but every already-dispatched send
51
71
  * short-circuits to its prior row instead of dispatching a duplicate provider
52
- * call. Counts are derived as-you-go from each `send()` result status — which is
53
- * itself idempotency-aware (a retried send returns the prior row's status), so
54
- * the tallies stay consistent across re-attempts. Final counts overwrite (not
55
- * increment) the row, so a retry re-derives them from scratch rather than
56
- * double-counting.
72
+ * call. Counts are derived as-you-go from each `send()` result status — which
73
+ * is itself idempotency-aware (a retried send returns the prior row's status),
74
+ * so the tallies stay consistent across re-attempts.
57
75
  *
58
- * Resume-on-retry: the terminal guard short-circuits ONLY a `sent` campaign — a
59
- * `failed`/`sending` row is NOT terminal, so a Hatchet retry (or the reaper's
60
- * re-enqueue) re-resolves the audience and re-loops. Already-dispatched sends
61
- * no-op via the idempotency key, so the re-run safely completes the TAIL of a
62
- * partial send instead of abandoning it. The catch block therefore does NOT
63
- * stamp `failed` before re-throwing — that would make the retry short-circuit
64
- * and silently under-deliver. A run that exhausts its retries is reaped to
65
- * `failed`/re-enqueued by {@link reapStuckCampaignsTask}.
76
+ * Counts across waves: the row counts are CUMULATIVE across waves.
77
+ * `stepBaseCounts` snapshots the cumulative tallies through the last COMPLETED
78
+ * wave (written with each cursor advance); the current wave always re-derives
79
+ * its own tally from scratch on top of that seed, and every flush stays an
80
+ * absolute overwrite — so a retried wave re-tallies exactly rather than
81
+ * double-counting. Prior-wave counts are deliberately NOT derived from
82
+ * `email_sends`: a suppressed send writes its row WITHOUT the idempotency key
83
+ * and a frequency-capped send writes NO row at all, so the ledger cannot
84
+ * reproduce skipped counts — hence the snapshot column.
85
+ *
86
+ * Resume-on-retry: the terminal guard short-circuits ONLY a terminal campaign —
87
+ * a `failed`/`sending` row is NOT terminal, so a Hatchet retry (or the reaper's
88
+ * re-enqueue) re-enters at `currentStep` and re-runs the current wave.
89
+ * Already-dispatched sends no-op via the idempotency key, so the re-run safely
90
+ * completes the TAIL of a partial wave instead of abandoning it. The catch
91
+ * block therefore does NOT stamp `failed` before re-throwing — that would make
92
+ * the retry short-circuit and silently under-deliver. A run that exhausts its
93
+ * retries is reaped to `failed`/re-enqueued by {@link reapStuckCampaignsTask}.
66
94
  */
67
95
  export const sendCampaignTask = hatchet.task({
68
96
  name: "send-campaign",
@@ -89,11 +117,12 @@ export const sendCampaignTask = hatchet.task({
89
117
  return { status: "failed", reason: "not_found" as const };
90
118
  }
91
119
 
92
- // Already terminal — a duplicate/late enqueue must not re-send. ONLY `sent`
93
- // is terminal: a `failed`/`sending` row is intentionally re-runnable so a
94
- // Hatchet retry (or a reaper re-enqueue) re-resolves the audience and
95
- // completes the unsent TAIL of a partial send (already-sent recipients
96
- // no-op via the per-send idempotency key — risk: silent under-delivery).
120
+ // Already terminal — a duplicate/late enqueue must not re-send. ONLY
121
+ // sent/canceled/expired are terminal: a `failed`/`sending` row is
122
+ // intentionally re-runnable so a Hatchet retry (or a reaper re-enqueue)
123
+ // re-enters at `currentStep` and completes the unsent TAIL of a partial
124
+ // wave (already-sent recipients no-op via the per-send idempotency key —
125
+ // risk: silent under-delivery).
97
126
  if ((TERMINAL_STATUSES as readonly string[]).includes(campaign.status)) {
98
127
  return { status: campaign.status, skipped: true };
99
128
  }
@@ -110,20 +139,61 @@ export const sendCampaignTask = hatchet.task({
110
139
  return { status: "scheduled", skipped: true, reason: "not_due" as const };
111
140
  }
112
141
 
142
+ // Mirror clause for `waiting`: a punctual next-step run that fires while
143
+ // `nextStepAt` is still in the future is stale — a reap + re-park replaced
144
+ // the pending wait after this run was scheduled. The run created for the
145
+ // new instant (or the reaper's waiting-promotion sweep) resumes it.
146
+ if (
147
+ campaign.status === "waiting" &&
148
+ campaign.nextStepAt &&
149
+ campaign.nextStepAt.getTime() > Date.now() + EARLY_FIRE_TOLERANCE_MS
150
+ ) {
151
+ return { status: "waiting", skipped: true, reason: "not_due" as const };
152
+ }
153
+
113
154
  // Claim via CAS: only a non-terminal row transitions to `sending`, so a
114
155
  // cancel landing between the guard read above and this write is honored
115
- // rather than silently resurrected.
116
- const claimed = await db
156
+ // rather than silently resurrected. `startedAt` is claimed ONCE
157
+ // (coalesce): event-condition scoping reads "since campaign startedAt",
158
+ // so a wave-2 claim must NOT reset the anchor wave-1's conditions were
159
+ // already scoped by. The cursor + count seed are re-read FROM the claimed
160
+ // row (not the pre-claim read) so a wave resumed after a crash sees the
161
+ // exact state its predecessor persisted.
162
+ const claimedRows = await db
117
163
  .update(campaigns)
118
- .set({ status: "sending", startedAt: new Date(), updatedAt: new Date() })
164
+ .set({
165
+ status: "sending",
166
+ startedAt: sql`coalesce(${campaigns.startedAt}, now())`,
167
+ // A resumed `waiting` row's wait has elapsed — clear the stale instant
168
+ // so serialized rows never show a countdown while `sending`. (Studio
169
+ // keys its countdown on status === "waiting" regardless.)
170
+ nextStepAt: null,
171
+ // `staleSince` deliberately NOT cleared here: claiming is not
172
+ // progress. A poison run claims, crashes, and is re-driven — if the
173
+ // claim reset the marker, its give-up clock would restart every cycle
174
+ // and the reaper could never declare it failed. The first real
175
+ // progress write (chunk flush / park / cursor advance) clears it.
176
+ updatedAt: new Date(),
177
+ })
119
178
  .where(
120
179
  and(
121
180
  eq(campaigns.id, input.campaignId),
122
- inArray(campaigns.status, ["queued", "scheduled", "sending"]),
181
+ inArray(campaigns.status, [
182
+ "queued",
183
+ "scheduled",
184
+ "sending",
185
+ "waiting",
186
+ ]),
123
187
  ),
124
188
  )
125
- .returning({ id: campaigns.id });
126
- if (claimed.length === 0) {
189
+ .returning({
190
+ id: campaigns.id,
191
+ currentStep: campaigns.currentStep,
192
+ startedAt: campaigns.startedAt,
193
+ stepBaseCounts: campaigns.stepBaseCounts,
194
+ });
195
+ const claim = claimedRows[0];
196
+ if (!claim) {
127
197
  const current = await db
128
198
  .select({ status: campaigns.status })
129
199
  .from(campaigns)
@@ -132,12 +202,65 @@ export const sendCampaignTask = hatchet.task({
132
202
  return { status: current[0]?.status ?? "unknown", skipped: true };
133
203
  }
134
204
 
135
- let sentCount = 0;
136
- let skippedCount = 0;
137
- let failedCount = 0;
138
- let totalRecipients = 0;
205
+ // The executable step sequence. A NULL blob is a legacy single-send row —
206
+ // one send step synthesized from the row columns, and (per the
207
+ // campaignSendKey contract) the LEGACY per-send key format. A non-NULL
208
+ // blob only ever holds > 1 steps (the reconciler stores single-step
209
+ // definitions as NULL), so `multiStep` doubles as the key-format switch:
210
+ // step-scoped keys for ALL steps including 0.
211
+ const blob = campaign.steps;
212
+ const multiStep = blob != null;
213
+ const executable: CampaignStep[] = blob
214
+ ? // db cannot import @hogsend/core, so the blob's elements are opaque
215
+ // Record<string, unknown> there; the engine owns the narrowing.
216
+ (blob.steps as unknown as CampaignStep[])
217
+ : [
218
+ {
219
+ kind: "send",
220
+ template: campaign.templateKey,
221
+ props: campaign.props ?? {},
222
+ ...(campaign.subject != null ? { subject: campaign.subject } : {}),
223
+ ...(campaign.fromEmail != null ? { from: campaign.fromEmail } : {}),
224
+ },
225
+ ];
226
+
227
+ // Immutable row facts captured once (nested fns below can't see the
228
+ // `campaign` narrowing). `startedAt` is non-null post-claim (coalesce);
229
+ // the ?? is a type-level fallback only.
230
+ const audienceKind = campaign.audienceKind;
231
+ const audienceId = campaign.audienceId;
232
+ const startedAt = claim.startedAt ?? new Date();
233
+
234
+ // Seed the cumulative counters from the snapshot through the last
235
+ // COMPLETED wave; the current wave re-derives its own tally on top. Do NOT
236
+ // try to derive the seed from email_sends — suppressed sends write no
237
+ // idempotency key and frequency-capped sends write no row (lib/tracked.ts).
238
+ const base = claim.stepBaseCounts ?? {
239
+ total: 0,
240
+ sent: 0,
241
+ skipped: 0,
242
+ failed: 0,
243
+ };
244
+ let totalRecipients = base.total;
245
+ let sentCount = base.sent;
246
+ let skippedCount = base.skipped;
247
+ let failedCount = base.failed;
248
+
249
+ const snapshotCounts = () => ({
250
+ total: totalRecipients,
251
+ sent: sentCount,
252
+ skipped: skippedCount,
253
+ failed: failedCount,
254
+ });
139
255
 
140
- const flushCounts = async (): Promise<void> => {
256
+ // Every flush is genuine progress (a dispatched chunk, a cancel/completion
257
+ // race) and clears the reaper's staleness marker — EXCEPT the catch-path
258
+ // flush: a crashed run made no progress, so `staleSince` must survive it
259
+ // or a deterministically-crashing campaign resets its give-up clock on
260
+ // every re-drive and is re-enqueued forever.
261
+ const flushCounts = async (opts?: {
262
+ preserveStaleSince?: boolean;
263
+ }): Promise<void> => {
141
264
  await db
142
265
  .update(campaigns)
143
266
  .set({
@@ -145,127 +268,160 @@ export const sendCampaignTask = hatchet.task({
145
268
  sentCount,
146
269
  skippedCount,
147
270
  failedCount,
271
+ ...(opts?.preserveStaleSince ? {} : { staleSince: null }),
148
272
  updatedAt: new Date(),
149
273
  })
150
274
  .where(eq(campaigns.id, input.campaignId));
151
275
  };
152
276
 
153
277
  try {
154
- const recipients =
155
- campaign.audienceKind === "bucket"
156
- ? resolveBucketRecipients(db, campaign.audienceId)
157
- : resolveListRecipients(db, campaign.audienceId);
278
+ // The wave loop: consecutive send steps run sequentially in this same
279
+ // task run; a wait step parks the row (`waiting`) and returns. Resuming
280
+ // with `currentStep` already past the last index (a crash between the
281
+ // final cursor advance and the completion CAS) skips straight to the
282
+ // completion CAS below — exactly right.
283
+ for (let k = claim.currentStep; k < executable.length; k++) {
284
+ const step = executable[k];
285
+ if (step === undefined) break; // unreachable — loop bound
158
286
 
159
- // Mid-flight cancel: re-checked once per chunk (one cheap SELECT per
160
- // CHUNK_SIZE sends) so `POST /v1/campaigns/:id/cancel` can stop a blast
161
- // that is already `sending` — recipients not yet dispatched are spared.
162
- let cancelDetected = false;
287
+ if (step.kind === "wait") {
288
+ const nextStepAt = new Date(Date.now() + durationToMs(step.duration));
289
+ // Park CAS: only a still-`sending` row goes `waiting`, so a cancel
290
+ // racing the end of the previous wave keeps its `canceled` status.
291
+ // Cursor, wait instant, count snapshot, and counts land in ONE
292
+ // update — a crash can never separate them.
293
+ const parked = await db
294
+ .update(campaigns)
295
+ .set({
296
+ status: "waiting",
297
+ currentStep: k + 1,
298
+ nextStepAt,
299
+ stepBaseCounts: snapshotCounts(),
300
+ totalRecipients,
301
+ sentCount,
302
+ skippedCount,
303
+ failedCount,
304
+ staleSince: null,
305
+ updatedAt: new Date(),
306
+ })
307
+ .where(
308
+ and(
309
+ eq(campaigns.id, input.campaignId),
310
+ eq(campaigns.status, "sending"),
311
+ ),
312
+ )
313
+ .returning({ id: campaigns.id });
314
+ if (parked.length === 0) {
315
+ await flushCounts();
316
+ const current = await db
317
+ .select({ status: campaigns.status })
318
+ .from(campaigns)
319
+ .where(eq(campaigns.id, input.campaignId))
320
+ .limit(1);
321
+ return { status: current[0]?.status ?? "unknown", skipped: true };
322
+ }
163
323
 
164
- let chunk: CampaignRecipient[] = [];
165
- for await (const recipient of recipients) {
166
- chunk.push(recipient);
167
- if (chunk.length < CHUNK_SIZE) continue;
168
- await sendChunk();
169
- if (cancelDetected) break;
170
- }
171
- // Final partial chunk.
172
- if (chunk.length > 0 && !cancelDetected) await sendChunk();
324
+ // Punctual resume run at nextStepAt — best-effort, same split as
325
+ // `scheduledAt`: on failure the row stays `waiting` and the reaper's
326
+ // waiting-promotion sweep resumes it.
327
+ try {
328
+ await sendCampaignTask.schedule(nextStepAt, {
329
+ campaignId: input.campaignId,
330
+ });
331
+ } catch (err) {
332
+ logger.warn(
333
+ "send-campaign: next-step schedule failed (reaper will promote)",
334
+ {
335
+ campaignId: input.campaignId,
336
+ nextStepAt: nextStepAt.toISOString(),
337
+ error: err instanceof Error ? err.message : String(err),
338
+ },
339
+ );
340
+ }
173
341
 
174
- async function sendChunk(): Promise<void> {
175
- const statusRows = await db
176
- .select({ status: campaigns.status })
177
- .from(campaigns)
178
- .where(eq(campaigns.id, input.campaignId))
179
- .limit(1);
180
- if (statusRows[0]?.status === "canceled") {
181
- cancelDetected = true;
182
- chunk = [];
183
- return;
342
+ logger.info("send-campaign: wave complete — waiting", {
343
+ campaignId: input.campaignId,
344
+ currentStep: k + 1,
345
+ nextStepAt: nextStepAt.toISOString(),
346
+ });
347
+ return {
348
+ status: "waiting" as const,
349
+ currentStep: k + 1,
350
+ nextStepAt: nextStepAt.toISOString(),
351
+ totalRecipients,
352
+ sentCount,
353
+ skippedCount,
354
+ failedCount,
355
+ };
184
356
  }
185
357
 
186
- const batch = chunk;
187
- chunk = [];
188
- totalRecipients += batch.length;
189
-
190
- const results = await Promise.allSettled(
191
- batch.map((r) =>
192
- emailService.send({
193
- template: campaign?.templateKey as TemplateName,
194
- props: (campaign?.props ?? {}) as never,
195
- to: r.email,
196
- userId: r.userId,
197
- userEmail: r.email,
198
- subject: campaign?.subject ?? undefined,
199
- from: campaign?.fromEmail ?? undefined,
200
- // A list's audienceId IS a real subscription category, so pass it
201
- // through for suppression + the unsubscribe link. A bucket's
202
- // audienceId is NOT a category — forcing it here would mint an
203
- // unsubscribe link keyed on the bucket id (`categories[bucketId] =
204
- // false`) that the bucket resolver never honors (it only checks
205
- // unsubscribedAll/suppressed), silently no-op'ing the unsubscribe.
206
- // For a bucket, pass undefined so the template's OWN declared
207
- // category (e.g. `product-updates`) drives both suppression and a
208
- // real, honored List-Unsubscribe target.
209
- category:
210
- campaign?.audienceKind === "bucket"
211
- ? undefined
212
- : campaign?.audienceId,
213
- // The idempotency key dedupes a retried send to its prior row.
214
- idempotencyKey: `campaign:${input.campaignId}:${r.email}`,
215
- }),
216
- ),
217
- );
218
-
219
- for (const result of results) {
220
- if (result.status === "rejected") {
221
- failedCount++;
222
- continue;
223
- }
224
- const status = result.value.status;
225
- if (status === "sent") {
226
- sentCount++;
227
- } else {
228
- // suppressed | unsubscribed | skipped (frequency-capped) — counted
229
- // as skipped, not a delivery failure.
230
- skippedCount++;
231
- }
358
+ if (step.kind !== "send") {
359
+ // A blob minted by a future engine (steps.v evolution) must fail
360
+ // loudly here, not silently skip a delivery step.
361
+ throw new Error(
362
+ `send-campaign: unsupported step kind "${(step as { kind: string }).kind}" at step ${k} of campaign ${input.campaignId}`,
363
+ );
232
364
  }
233
365
 
234
- await flushCounts();
235
- }
366
+ const canceled = await runWave(k, step);
367
+ if (canceled) {
368
+ // The operator's cancel already stamped status/canceledAt; persist
369
+ // the progress counts so the row shows how far the blast got.
370
+ await flushCounts();
371
+ logger.info("send-campaign: canceled mid-send", {
372
+ campaignId: input.campaignId,
373
+ currentStep: k,
374
+ totalRecipients,
375
+ sentCount,
376
+ skippedCount,
377
+ failedCount,
378
+ });
379
+ return {
380
+ status: "canceled" as const,
381
+ totalRecipients,
382
+ sentCount,
383
+ skippedCount,
384
+ failedCount,
385
+ };
386
+ }
236
387
 
237
- if (cancelDetected) {
238
- // The operator's cancel already stamped status/canceledAt; persist the
239
- // progress counts so the row shows how far the blast got.
240
- await flushCounts();
241
- logger.info("send-campaign: canceled mid-send", {
242
- campaignId: input.campaignId,
243
- totalRecipients,
244
- sentCount,
245
- skippedCount,
246
- failedCount,
247
- });
248
- return {
249
- status: "canceled" as const,
250
- totalRecipients,
251
- sentCount,
252
- skippedCount,
253
- failedCount,
254
- };
388
+ // Advance the cursor + snapshot the cumulative counters in ONE update
389
+ // — the wave is complete, so a retry entering after this point starts
390
+ // at the NEXT step with these counts as its seed. Plain update (not
391
+ // CAS): a cancel that raced the tail of the wave only bumps cursor
392
+ // metadata on an already-terminal row, which nothing reads.
393
+ await db
394
+ .update(campaigns)
395
+ .set({
396
+ currentStep: k + 1,
397
+ stepBaseCounts: snapshotCounts(),
398
+ totalRecipients,
399
+ sentCount,
400
+ skippedCount,
401
+ failedCount,
402
+ staleSince: null,
403
+ updatedAt: new Date(),
404
+ })
405
+ .where(eq(campaigns.id, input.campaignId));
255
406
  }
256
407
 
257
408
  // Completion CAS: only a still-`sending` row is stamped `sent`, so a
258
409
  // cancel racing the final chunk (after its last per-chunk check) keeps
259
- // its `canceled` status instead of being overwritten.
410
+ // its `canceled` status instead of being overwritten. `nextStepAt` +
411
+ // `stepBaseCounts` are nulled — a terminal row carries no pending wait
412
+ // and no resume seed.
260
413
  const completed = await db
261
414
  .update(campaigns)
262
415
  .set({
263
416
  status: "sent",
264
417
  completedAt: new Date(),
418
+ nextStepAt: null,
419
+ stepBaseCounts: null,
265
420
  totalRecipients,
266
421
  sentCount,
267
422
  skippedCount,
268
423
  failedCount,
424
+ staleSince: null,
269
425
  updatedAt: new Date(),
270
426
  })
271
427
  .where(
@@ -315,21 +471,16 @@ export const sendCampaignTask = hatchet.task({
315
471
  // Do NOT stamp `failed` here. A `failed` stamp before the re-throw makes
316
472
  // the single Hatchet retry hit the terminal guard and short-circuit
317
473
  // WITHOUT sending the remaining recipients — silently abandoning the tail
318
- // of a partial send. Instead we persist the progress counts, leave the
319
- // status `sending` (re-runnable), and re-throw so the genuine retry
320
- // re-enters the loop and finishes the unsent tail (already-sent recipients
321
- // no-op via their idempotency key). A run that EXHAUSTS its retries is
322
- // transitioned to `failed` (or re-enqueued) by `reapStuckCampaignsTask`.
323
- await db
324
- .update(campaigns)
325
- .set({
326
- totalRecipients,
327
- sentCount,
328
- skippedCount,
329
- failedCount,
330
- updatedAt: new Date(),
331
- })
332
- .where(eq(campaigns.id, input.campaignId));
474
+ // of a partial wave. Instead we persist the progress counts, leave the
475
+ // status `sending` (re-runnable — `currentStep` still points at the
476
+ // interrupted wave and `stepBaseCounts` still seeds it), and re-throw so
477
+ // the genuine retry re-enters the wave and finishes the unsent tail
478
+ // (already-sent recipients no-op via their idempotency key). A run that
479
+ // EXHAUSTS its retries is transitioned to `failed` (or re-enqueued) by
480
+ // `reapStuckCampaignsTask`. `staleSince` is preserved — a crashed run is
481
+ // not progress, and clearing it here would reset the give-up clock on
482
+ // every re-drive of a poison campaign.
483
+ await flushCounts({ preserveStaleSince: true });
333
484
 
334
485
  logger.error("send-campaign: errored mid-run (will retry)", {
335
486
  campaignId: input.campaignId,
@@ -337,6 +488,148 @@ export const sendCampaignTask = hatchet.task({
337
488
  });
338
489
  throw error;
339
490
  }
491
+
492
+ /**
493
+ * Execute one per-recipient email wave. Returns `true` when a mid-wave
494
+ * cancel was detected (the caller flushes + returns; nothing after the
495
+ * detecting chunk is dispatched).
496
+ *
497
+ * Qualifier source:
498
+ * - Wave 0 resolves the audience LIVE (list/bucket resolvers, fresh
499
+ * suppression pre-filter) and — for MULTI-step campaigns — ANCHORS the
500
+ * cohort: each chunk is batch-inserted into `campaign_recipients`
501
+ * BEFORE its sends are dispatched — a crash between insert and send is
502
+ * fine (the retry re-resolves and the (campaign_id, email) unique
503
+ * absorbs the replay). The audience is resolved exactly once per
504
+ * campaign: someone added to the list afterwards never receives step 3
505
+ * without step 1. A single-send campaign skips the ledger entirely (no
506
+ * later wave will read it — the legacy write path stays byte-identical).
507
+ * - Waves k > 0 qualify FROM the anchored cohort: `campaign_recipients`
508
+ * ∩ the step's `where` conditions ∩ a fresh suppression re-check
509
+ * (see resolveCohortRecipients — suppression is never snapshotted).
510
+ */
511
+ async function runWave(
512
+ k: number,
513
+ step: CampaignSendStep,
514
+ ): Promise<boolean> {
515
+ const recipients =
516
+ k === 0
517
+ ? audienceKind === "bucket"
518
+ ? resolveBucketRecipients(db, audienceId)
519
+ : resolveListRecipients(db, audienceId)
520
+ : resolveCohortRecipients(db, {
521
+ campaignId: input.campaignId,
522
+ conditions: step.where,
523
+ startedAt,
524
+ });
525
+
526
+ // Mid-flight cancel: re-checked once per chunk (one cheap SELECT per
527
+ // CHUNK_SIZE sends) so `POST /v1/campaigns/:id/cancel` can stop a blast
528
+ // that is already `sending` — recipients not yet dispatched are spared.
529
+ let cancelDetected = false;
530
+
531
+ let chunk: CampaignRecipient[] = [];
532
+ for await (const recipient of recipients) {
533
+ chunk.push(recipient);
534
+ if (chunk.length < CHUNK_SIZE) continue;
535
+ await sendChunk();
536
+ if (cancelDetected) return true;
537
+ }
538
+ // Final partial chunk.
539
+ if (chunk.length > 0 && !cancelDetected) await sendChunk();
540
+ return cancelDetected;
541
+
542
+ async function sendChunk(): Promise<void> {
543
+ const statusRows = await db
544
+ .select({ status: campaigns.status })
545
+ .from(campaigns)
546
+ .where(eq(campaigns.id, input.campaignId))
547
+ .limit(1);
548
+ if (statusRows[0]?.status === "canceled") {
549
+ cancelDetected = true;
550
+ chunk = [];
551
+ return;
552
+ }
553
+
554
+ const batch = chunk;
555
+ chunk = [];
556
+
557
+ // Anchor the cohort BEFORE dispatching the chunk (wave 0 of a
558
+ // MULTI-step campaign only — a single-send campaign has no later wave
559
+ // to read the ledger, so skipping the insert keeps the legacy blast's
560
+ // write path byte-identical). Emails are already normalized by the
561
+ // resolvers, matching the table's (campaign_id, email) unique
562
+ // keyspace.
563
+ if (k === 0 && multiStep) {
564
+ await db
565
+ .insert(campaignRecipients)
566
+ .values(
567
+ batch.map((r) => ({
568
+ campaignId: input.campaignId,
569
+ userId: r.userId ?? null,
570
+ email: r.email,
571
+ })),
572
+ )
573
+ .onConflictDoNothing({
574
+ target: [campaignRecipients.campaignId, campaignRecipients.email],
575
+ });
576
+ }
577
+
578
+ totalRecipients += batch.length;
579
+
580
+ const results = await Promise.allSettled(
581
+ batch.map((r) =>
582
+ emailService.send({
583
+ template: step.template as TemplateName,
584
+ props: (step.props ?? {}) as never,
585
+ to: r.email,
586
+ userId: r.userId,
587
+ userEmail: r.email,
588
+ subject: step.subject,
589
+ from: step.from,
590
+ // A list's audienceId IS a real subscription category, so pass it
591
+ // through for suppression + the unsubscribe link. A bucket's
592
+ // audienceId is NOT a category — forcing it here would mint an
593
+ // unsubscribe link keyed on the bucket id (`categories[bucketId] =
594
+ // false`) that the bucket resolver never honors (it only checks
595
+ // unsubscribedAll/suppressed), silently no-op'ing the unsubscribe.
596
+ // For a bucket, pass undefined so the template's OWN declared
597
+ // category (e.g. `product-updates`) drives both suppression and a
598
+ // real, honored List-Unsubscribe target. The rule applies to
599
+ // EVERY wave, not just the first.
600
+ category: audienceKind === "bucket" ? undefined : audienceId,
601
+ // The idempotency key dedupes a retried send to its prior row.
602
+ // Legacy format for single-step campaigns; step-scoped for every
603
+ // step of a multi-step campaign (campaignSendKey contract).
604
+ idempotencyKey: campaignSendKey(
605
+ input.campaignId,
606
+ r.email,
607
+ multiStep ? k : undefined,
608
+ ),
609
+ }),
610
+ ),
611
+ );
612
+
613
+ for (const result of results) {
614
+ if (result.status === "rejected") {
615
+ failedCount++;
616
+ continue;
617
+ }
618
+ const status = result.value.status;
619
+ if (status === "sent") {
620
+ sentCount++;
621
+ } else {
622
+ // suppressed | unsubscribed | skipped (frequency-capped) — counted
623
+ // as skipped, not a delivery failure. The member STAYS in the
624
+ // cohort either way: a provider hiccup or a skipped wave is not an
625
+ // exit, and suppression is re-checked fresh on every later wave.
626
+ skippedCount++;
627
+ }
628
+ }
629
+
630
+ await flushCounts();
631
+ }
632
+ }
340
633
  },
341
634
  });
342
635
 
@@ -352,21 +645,30 @@ const STALE_AFTER_MS = Number(
352
645
  );
353
646
 
354
647
  /**
355
- * After a campaign has sat in a non-terminal in-flight state this long (measured
356
- * from `updatedAt`, which the send task bumps on every progress flush) it is
357
- * declared `failed` rather than re-enqueued forever — a poison campaign (e.g. a
358
- * template that always throws) stops being re-driven and surfaces to operators.
648
+ * After a campaign has been CONTINUOUSLY stuck this long with zero progress it
649
+ * is declared `failed` rather than re-enqueued forever — a poison campaign
650
+ * (e.g. a run that always crashes) stops being re-driven and surfaces to
651
+ * operators.
652
+ *
653
+ * For in-flight (`queued`/`sending`) rows the window is measured from
654
+ * `staleSince` — set once by the stale sweep's first re-enqueue, cleared by
655
+ * every genuine progress flush of the send task. NOT from `updatedAt`: the
656
+ * sweep bumps `updatedAt` as its re-pick guard, so a deterministically-
657
+ * crashing row would never age past the window (the give-up was dead code
658
+ * until `stale_since` landed). `scheduled` / `waiting` rows are measured from
659
+ * `scheduledAt` / `nextStepAt` as before.
359
660
  */
360
661
  const GIVE_UP_AFTER_MS = Number(
361
662
  process.env.CAMPAIGN_GIVE_UP_AFTER_MS ?? 6 * 60 * 60 * 1000,
362
663
  );
363
664
 
364
665
  /**
365
- * How far past `scheduledAt` a `scheduled` campaign may sit before the reaper
366
- * promotes it (enqueues the send task). The punctual Hatchet scheduled run
367
- * created at schedule time is the primary trigger; this grace keeps the sweep
368
- * from routinely double-firing right at the boundary. A double fire is
369
- * harmless anyway (terminal guard + per-send idempotency).
666
+ * How far past `scheduledAt` a `scheduled` campaign (or past `nextStepAt` a
667
+ * `waiting` one) may sit before the reaper promotes it (enqueues the send
668
+ * task). The punctual Hatchet scheduled run created at schedule/park time is
669
+ * the primary trigger; this grace keeps the sweep from routinely
670
+ * double-firing right at the boundary. A double fire is harmless anyway
671
+ * (terminal guard + per-send idempotency).
370
672
  */
371
673
  const SCHEDULE_PROMOTE_GRACE_MS = Number(
372
674
  process.env.CAMPAIGN_PROMOTE_GRACE_MS ?? 2 * 60 * 1000,
@@ -387,6 +689,10 @@ const SCHEDULE_PROMOTE_GRACE_MS = Number(
387
689
  * due. A scheduled row stuck past the give-up window (measured from
388
690
  * `scheduledAt`, NOT `updatedAt` — a row is legitimately idle between
389
691
  * create and send time) is declared `failed`.
692
+ * - A `waiting` campaign whose punctual next-step run was lost — promoted
693
+ * once `nextStepAt` is past the same grace. Give-up is likewise measured
694
+ * from `nextStepAt`, NOT `updatedAt` — a row is legitimately idle mid-wait,
695
+ * exactly like `scheduled`/`scheduledAt`.
390
696
  *
391
697
  * Recovery is a simple RE-ENQUEUE of `sendCampaignTask` (safe: the per-send
392
698
  * idempotency key no-ops already-sent recipients and the re-run completes the
@@ -410,24 +716,38 @@ export const reapStuckCampaignsTask = hatchet.task({
410
716
  const now = Date.now();
411
717
  const staleBefore = new Date(now - STALE_AFTER_MS);
412
718
  const giveUpBefore = new Date(now - GIVE_UP_AFTER_MS);
719
+ const promoteBefore = new Date(now - SCHEDULE_PROMOTE_GRACE_MS);
413
720
 
414
721
  // (1) Declare poison campaigns `failed` first (stuck past the give-up
415
722
  // window), so they are not re-enqueued below. In-flight rows are measured
416
- // from `updatedAt` (bumped on every progress flush); `scheduled` rows from
417
- // `scheduledAt` (their `updatedAt` is legitimately old while they wait).
723
+ // from `staleSince` (set by the first stale re-enqueue, cleared by every
724
+ // genuine progress flush — a NULL never matches the `lt`, so a row the
725
+ // sweep hasn't touched yet gets its re-drive chance first); `scheduled`
726
+ // rows from `scheduledAt` and `waiting` rows from `nextStepAt` (their
727
+ // `updatedAt` is legitimately old while they wait). `staleSince` is nulled
728
+ // with the stamp so an operator revive starts a fresh give-up window.
418
729
  const failedRows = await db
419
730
  .update(campaigns)
420
- .set({ status: "failed", completedAt: new Date(), updatedAt: new Date() })
731
+ .set({
732
+ status: "failed",
733
+ completedAt: new Date(),
734
+ staleSince: null,
735
+ updatedAt: new Date(),
736
+ })
421
737
  .where(
422
738
  or(
423
739
  and(
424
740
  inArray(campaigns.status, ["queued", "sending"]),
425
- lt(campaigns.updatedAt, giveUpBefore),
741
+ lt(campaigns.staleSince, giveUpBefore),
426
742
  ),
427
743
  and(
428
744
  eq(campaigns.status, "scheduled"),
429
745
  lt(campaigns.scheduledAt, giveUpBefore),
430
746
  ),
747
+ and(
748
+ eq(campaigns.status, "waiting"),
749
+ lt(campaigns.nextStepAt, giveUpBefore),
750
+ ),
431
751
  ),
432
752
  )
433
753
  .returning({ id: campaigns.id });
@@ -436,9 +756,18 @@ export const reapStuckCampaignsTask = hatchet.task({
436
756
  // `updatedAt` so the same row is not re-picked on the very next tick before
437
757
  // the re-driven run makes progress; the per-send idempotency key keeps the
438
758
  // re-enqueue safe even if the original run is somehow still alive.
759
+ // `staleSince` is coalesce-set ONCE on the first re-enqueue (later sweeps
760
+ // keep the original instant) — this is the bump-free clock the give-up
761
+ // clause above reads; the send task clears it on genuine progress.
762
+ // `queued`/`sending` ONLY — `waiting` is deliberately excluded: a 2-day
763
+ // wait between waves is not a stuck campaign (its resume path is the
764
+ // promotion sweep below, keyed on `nextStepAt`).
439
765
  const staleRows = await db
440
766
  .update(campaigns)
441
- .set({ updatedAt: new Date() })
767
+ .set({
768
+ staleSince: sql`coalesce(${campaigns.staleSince}, now())`,
769
+ updatedAt: new Date(),
770
+ })
442
771
  .where(
443
772
  and(
444
773
  inArray(campaigns.status, ["queued", "sending"]),
@@ -448,15 +777,22 @@ export const reapStuckCampaignsTask = hatchet.task({
448
777
  .returning({ id: campaigns.id });
449
778
 
450
779
  // (3) Promote due `scheduled` campaigns whose punctual scheduled run never
451
- // fired. Enqueue-only — the send task owns the scheduled→sending
780
+ // fired, and due `waiting` campaigns whose punctual next-step run was lost.
781
+ // Enqueue-only — the send task owns the scheduled/waiting→sending
452
782
  // transition, so a failed enqueue simply retries next sweep.
453
783
  const dueRows = await db
454
784
  .select({ id: campaigns.id })
455
785
  .from(campaigns)
456
786
  .where(
457
- and(
458
- eq(campaigns.status, "scheduled"),
459
- lte(campaigns.scheduledAt, new Date(now - SCHEDULE_PROMOTE_GRACE_MS)),
787
+ or(
788
+ and(
789
+ eq(campaigns.status, "scheduled"),
790
+ lte(campaigns.scheduledAt, promoteBefore),
791
+ ),
792
+ and(
793
+ eq(campaigns.status, "waiting"),
794
+ lte(campaigns.nextStepAt, promoteBefore),
795
+ ),
460
796
  ),
461
797
  );
462
798
 
@@ -522,19 +858,76 @@ async function* keysetPaginate<Row>(opts: {
522
858
  }
523
859
  }
524
860
 
861
+ /**
862
+ * Wave-k (k > 0) qualifier resolver: the campaign's anchored cohort
863
+ * (`campaign_recipients`) ∩ the step's `where` conditions (each compiled to a
864
+ * correlated [NOT] EXISTS — see `campaigns/cohort-sql.ts`) ∩ a fresh
865
+ * suppression/unsubscribe re-check. Membership was anchored at wave 0 and is
866
+ * never re-resolved — a member who left the bucket/list between waves still
867
+ * qualifies (suppression excepted); a member added afterwards never appears.
868
+ * Paged by the keyset cursor on `campaign_recipients.id` (the
869
+ * (campaign_id, id) index), so every page is an indexed join, never a LIKE
870
+ * scan over email_sends.
871
+ */
872
+ async function* resolveCohortRecipients(
873
+ db: Database,
874
+ opts: {
875
+ campaignId: string;
876
+ conditions: ConditionEval[] | undefined;
877
+ startedAt: Date;
878
+ },
879
+ ): AsyncGenerator<CampaignRecipient> {
880
+ // Compiled once per wave — every page reuses the same predicates.
881
+ const qualifiers = (opts.conditions ?? []).map((condition) =>
882
+ waveConditionSql({
883
+ condition,
884
+ campaignId: opts.campaignId,
885
+ startedAt: opts.startedAt,
886
+ }),
887
+ );
888
+
889
+ yield* keysetPaginate({
890
+ page: (cursor) => {
891
+ const conditions = [
892
+ eq(campaignRecipients.campaignId, opts.campaignId),
893
+ cohortSuppressionSql(),
894
+ ...qualifiers,
895
+ ];
896
+ if (cursor) conditions.push(gt(campaignRecipients.id, cursor));
897
+
898
+ return db
899
+ .select({
900
+ id: campaignRecipients.id,
901
+ userId: campaignRecipients.userId,
902
+ email: campaignRecipients.email,
903
+ })
904
+ .from(campaignRecipients)
905
+ .where(and(...conditions))
906
+ .orderBy(campaignRecipients.id)
907
+ .limit(CHUNK_SIZE);
908
+ },
909
+ cursorOf: (row) => row.id,
910
+ // Cohort emails were normalized at anchor time; userId may be NULL for an
911
+ // email-only member (kept — the mailer falls back the same way wave 0 did).
912
+ map: (row) => ({ email: row.email, userId: row.userId ?? undefined }),
913
+ });
914
+ }
915
+
525
916
  /**
526
917
  * Active, non-deleted members of a bucket, joined to a live contact for the
527
918
  * email — mirrors the bucket-access member query. Paged by the keyset cursor on
528
919
  * `bucket_memberships.id`.
529
920
  *
530
- * Compliance: `bucket_memberships.userEmail` is written verbatim from the RAW
531
- * event payload on the realtime join path (un-normalized, unlike
532
- * `contacts.email`), so the recipient email is NORMALIZED (`normalizeEmail`)
533
- * before it is yielded — otherwise a mixed-case membership email
534
- * (`User@Example.com`) would not case-match its NORMALIZED `email_preferences`
535
- * row (`user@example.com`) and the mailer's case-sensitive suppression check
536
- * would MISS the row, leaking a marketing blast to a suppressed/unsubscribed
537
- * contact (CAN-SPAM/GDPR). Defense-in-depth: this resolver ALSO pre-filters
921
+ * Compliance: `bucket_memberships.userEmail` is normalized at every write site
922
+ * (and existing rows were backfilled by migration 0043), but this resolver
923
+ * KEEPS its own `normalizeEmail` / `lower(trim(…))` belt-and-braces —
924
+ * TODO(cleanup): safe to drop once `@hogsend/engine` has shipped past 0.40.0
925
+ * (all deployed write sites normalize as of this changeset). An un-normalized
926
+ * membership email (`User@Example.com`) would not
927
+ * case-match its NORMALIZED `email_preferences` row (`user@example.com`) and
928
+ * the mailer's case-sensitive suppression check would MISS the row, leaking a
929
+ * marketing blast to a suppressed/unsubscribed contact (CAN-SPAM/GDPR).
930
+ * Defense-in-depth: this resolver ALSO pre-filters
538
931
  * `unsubscribedAll`/`suppressed` at the audience layer (mirroring the list
539
932
  * resolver) via a LEFT JOIN to `email_preferences` on the NORMALIZED email, so
540
933
  * a globally-unsubscribed / suppressed bucket member is excluded up front
@@ -546,8 +939,9 @@ async function* resolveBucketRecipients(
546
939
  db: Database,
547
940
  bucketId: string,
548
941
  ): AsyncGenerator<CampaignRecipient> {
549
- // The recipient's normalized email — the membership email may be mixed-case
550
- // (written verbatim from the raw event), the contact email is the fallback.
942
+ // The recipient's normalized email — writes are normalized now, but the
943
+ // read-side lower(trim(…)) stays as belt-and-braces (TODO(cleanup): see the
944
+ // fn docstring for the drop condition); the contact email is the fallback.
551
945
  const recipientEmail = sql<string>`lower(trim(coalesce(${bucketMemberships.userEmail}, ${contacts.email})))`;
552
946
 
553
947
  yield* keysetPaginate({