@hogsend/engine 0.39.0 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/package.json +9 -7
  2. package/src/buckets/check-membership.ts +6 -1
  3. package/src/campaigns/cohort-sql.ts +198 -0
  4. package/src/campaigns/define-campaign.ts +301 -0
  5. package/src/campaigns/reconcile.ts +391 -0
  6. package/src/campaigns/steps.ts +48 -0
  7. package/src/connectors/define-action.ts +70 -0
  8. package/src/container.ts +144 -42
  9. package/src/destinations/presets/posthog.ts +7 -1
  10. package/src/env.ts +14 -0
  11. package/src/index.ts +98 -3
  12. package/src/journeys/constants.ts +10 -0
  13. package/src/journeys/define-journey.ts +14 -405
  14. package/src/journeys/execute-journey-run.ts +608 -0
  15. package/src/journeys/graph/build-graph.ts +212 -56
  16. package/src/journeys/journey-boundary.ts +54 -0
  17. package/src/journeys/journey-context.ts +374 -59
  18. package/src/journeys/record-once.ts +110 -0
  19. package/src/lib/auth.ts +33 -1
  20. package/src/lib/blueprints.ts +576 -0
  21. package/src/lib/campaign-send-key.ts +58 -0
  22. package/src/lib/connector-actions.ts +203 -19
  23. package/src/lib/contacts.ts +54 -0
  24. package/src/lib/email-service-types.ts +4 -1
  25. package/src/lib/email.ts +13 -2
  26. package/src/lib/feed.ts +77 -57
  27. package/src/lib/frequency-cap.ts +45 -15
  28. package/src/lib/hatchet-duration.ts +22 -0
  29. package/src/lib/html.ts +1 -0
  30. package/src/lib/ingestion.ts +145 -4
  31. package/src/lib/journey-suppress.ts +87 -0
  32. package/src/lib/links.ts +204 -18
  33. package/src/lib/outbound.ts +20 -0
  34. package/src/lib/preferences.ts +80 -1
  35. package/src/lib/tracked.ts +44 -29
  36. package/src/lib/tracking-event-names.ts +8 -0
  37. package/src/lib/webhook-signing.ts +6 -1
  38. package/src/lists/channels.ts +75 -0
  39. package/src/lists/define-list.ts +34 -0
  40. package/src/lists/registry.ts +29 -2
  41. package/src/mcp/blueprint-tools.ts +460 -0
  42. package/src/routes/admin/blueprints.ts +711 -0
  43. package/src/routes/admin/campaigns.ts +341 -0
  44. package/src/routes/admin/emails.ts +13 -0
  45. package/src/routes/admin/index.ts +6 -0
  46. package/src/routes/admin/journeys.ts +4 -2
  47. package/src/routes/admin/links.ts +303 -36
  48. package/src/routes/admin/lists.ts +51 -0
  49. package/src/routes/campaigns/index.ts +297 -50
  50. package/src/routes/email/preferences.ts +50 -13
  51. package/src/routes/feed/recipient.ts +2 -43
  52. package/src/routes/index.ts +6 -0
  53. package/src/routes/lists/index.ts +138 -18
  54. package/src/routes/tracking/arrive.ts +260 -0
  55. package/src/routes/tracking/click-pipeline.ts +329 -0
  56. package/src/routes/tracking/click.ts +5 -270
  57. package/src/routes/tracking/index.ts +2 -0
  58. package/src/routes/tracking/vanity.ts +64 -0
  59. package/src/worker.ts +27 -0
  60. package/src/workflows/bucket-backfill.ts +4 -2
  61. package/src/workflows/bucket-reconcile.ts +11 -4
  62. package/src/workflows/check-alerts.ts +130 -1
  63. package/src/workflows/journey-blueprint-interpreter.ts +561 -0
  64. package/src/workflows/send-campaign.ts +673 -127
@@ -1,12 +1,24 @@
1
+ import {
2
+ type CampaignSendStep,
3
+ type CampaignStep,
4
+ type ConditionEval,
5
+ durationToMs,
6
+ } from "@hogsend/core";
1
7
  import {
2
8
  bucketMemberships,
9
+ campaignRecipients,
3
10
  campaigns,
4
11
  contacts,
5
12
  type Database,
6
13
  emailPreferences,
7
14
  } from "@hogsend/db";
8
15
  import type { TemplateName } from "@hogsend/email";
9
- import { and, eq, gt, inArray, isNull, lt, sql } from "drizzle-orm";
16
+ import { and, eq, gt, inArray, isNull, lt, lte, or, sql } from "drizzle-orm";
17
+ import {
18
+ cohortSuppressionSql,
19
+ waveConditionSql,
20
+ } from "../campaigns/cohort-sql.js";
21
+ import { campaignSendKey } from "../lib/campaign-send-key.js";
10
22
  import { normalizeEmail } from "../lib/contacts.js";
11
23
  import { getDb } from "../lib/db.js";
12
24
  import { getEmailService } from "../lib/email.js";
@@ -23,32 +35,62 @@ interface CampaignRecipient {
23
35
  userId?: string;
24
36
  }
25
37
 
26
- /** Statuses that are TERMINAL — a duplicate/late enqueue must not re-send. */
27
- const TERMINAL_STATUSES = ["sent"] as const;
38
+ /**
39
+ * Statuses that are TERMINAL — a duplicate/late enqueue must not re-send.
40
+ * `canceled`/`expired` matter doubly for SCHEDULED (and now WAITING) campaigns:
41
+ * the punctual Hatchet scheduled run cannot be deleted at cancel time (we don't
42
+ * persist its id), so it still fires and must find the row terminal here and
43
+ * no-op. `failed` is deliberately NOT terminal — a re-enqueue can revive it.
44
+ */
45
+ const TERMINAL_STATUSES = ["sent", "canceled", "expired"] as const;
28
46
 
29
47
  /**
30
- * Built-in durable campaign / broadcast task (Loops "campaign" parity). Sends a
31
- * single template to every subscribed member of a list (or every active member
32
- * of a bucket).
48
+ * How far ahead of `scheduledAt` / `nextStepAt` a run may fire and still
49
+ * proceed (absorbs clock skew). An earlier fire — the punctual run created for
50
+ * an instant that was later moved BACK (a reconcile edit for `scheduledAt`; a
51
+ * reap + re-park re-schedule for `nextStepAt`) — skips without sending; the
52
+ * run created for the new instant (or the reaper sweep) delivers it.
53
+ */
54
+ const EARLY_FIRE_TOLERANCE_MS = 60 * 1000;
55
+
56
+ /**
57
+ * Built-in durable campaign / broadcast task (Loops "campaign" parity). A
58
+ * legacy row (NULL `steps`) sends a single template to every subscribed member
59
+ * of a list (or every active member of a bucket). A multi-step row executes
60
+ * its `steps` blob as WAVES — each send step a SET operation over the
61
+ * campaign's anchored cohort, separated by durable waits (status `waiting`,
62
+ * resumed at `nextStepAt`) — see docs/campaign-steps-spec.md §Wave runtime.
63
+ * The task keeps its name and `{ campaignId }` input either way; the row's
64
+ * `currentStep` is the sole resume cursor.
33
65
  *
34
- * Retry-safety: each send carries an idempotency key
35
- * `campaign:<campaignId>:<email>` (email_sends.idempotency_key, migration 0015),
36
- * so a Hatchet retry re-runs the whole loop but every already-dispatched send
66
+ * Retry-safety: each send carries an idempotency key minted by
67
+ * `campaignSendKey` — legacy `campaign:<id>:<email>` for single-step
68
+ * campaigns, step-scoped `campaign:<id>:<step>:<email>` for EVERY step of a
69
+ * multi-step campaign (email_sends.idempotency_key, migration 0015) — so a
70
+ * Hatchet retry re-runs the wave but every already-dispatched send
37
71
  * short-circuits to its prior row instead of dispatching a duplicate provider
38
- * call. Counts are derived as-you-go from each `send()` result status — which is
39
- * itself idempotency-aware (a retried send returns the prior row's status), so
40
- * the tallies stay consistent across re-attempts. Final counts overwrite (not
41
- * increment) the row, so a retry re-derives them from scratch rather than
42
- * double-counting.
72
+ * call. Counts are derived as-you-go from each `send()` result status — which
73
+ * is itself idempotency-aware (a retried send returns the prior row's status),
74
+ * so the tallies stay consistent across re-attempts.
75
+ *
76
+ * Counts across waves: the row counts are CUMULATIVE across waves.
77
+ * `stepBaseCounts` snapshots the cumulative tallies through the last COMPLETED
78
+ * wave (written with each cursor advance); the current wave always re-derives
79
+ * its own tally from scratch on top of that seed, and every flush stays an
80
+ * absolute overwrite — so a retried wave re-tallies exactly rather than
81
+ * double-counting. Prior-wave counts are deliberately NOT derived from
82
+ * `email_sends`: a suppressed send writes its row WITHOUT the idempotency key
83
+ * and a frequency-capped send writes NO row at all, so the ledger cannot
84
+ * reproduce skipped counts — hence the snapshot column.
43
85
  *
44
- * Resume-on-retry: the terminal guard short-circuits ONLY a `sent` campaign — a
45
- * `failed`/`sending` row is NOT terminal, so a Hatchet retry (or the reaper's
46
- * re-enqueue) re-resolves the audience and re-loops. Already-dispatched sends
47
- * no-op via the idempotency key, so the re-run safely completes the TAIL of a
48
- * partial send instead of abandoning it. The catch block therefore does NOT
49
- * stamp `failed` before re-throwing — that would make the retry short-circuit
50
- * and silently under-deliver. A run that exhausts its retries is reaped to
51
- * `failed`/re-enqueued by {@link reapStuckCampaignsTask}.
86
+ * Resume-on-retry: the terminal guard short-circuits ONLY a terminal campaign —
87
+ * a `failed`/`sending` row is NOT terminal, so a Hatchet retry (or the reaper's
88
+ * re-enqueue) re-enters at `currentStep` and re-runs the current wave.
89
+ * Already-dispatched sends no-op via the idempotency key, so the re-run safely
90
+ * completes the TAIL of a partial wave instead of abandoning it. The catch
91
+ * block therefore does NOT stamp `failed` before re-throwing — that would make
92
+ * the retry short-circuit and silently under-deliver. A run that exhausts its
93
+ * retries is reaped to `failed`/re-enqueued by {@link reapStuckCampaignsTask}.
52
94
  */
53
95
  export const sendCampaignTask = hatchet.task({
54
96
  name: "send-campaign",
@@ -75,26 +117,150 @@ export const sendCampaignTask = hatchet.task({
75
117
  return { status: "failed", reason: "not_found" as const };
76
118
  }
77
119
 
78
- // Already terminal — a duplicate/late enqueue must not re-send. ONLY `sent`
79
- // is terminal: a `failed`/`sending` row is intentionally re-runnable so a
80
- // Hatchet retry (or a reaper re-enqueue) re-resolves the audience and
81
- // completes the unsent TAIL of a partial send (already-sent recipients
82
- // no-op via the per-send idempotency key — risk: silent under-delivery).
120
+ // Already terminal — a duplicate/late enqueue must not re-send. ONLY
121
+ // sent/canceled/expired are terminal: a `failed`/`sending` row is
122
+ // intentionally re-runnable so a Hatchet retry (or a reaper re-enqueue)
123
+ // re-enters at `currentStep` and completes the unsent TAIL of a partial
124
+ // wave (already-sent recipients no-op via the per-send idempotency key —
125
+ // risk: silent under-delivery).
83
126
  if ((TERMINAL_STATUSES as readonly string[]).includes(campaign.status)) {
84
127
  return { status: campaign.status, skipped: true };
85
128
  }
86
129
 
87
- await db
130
+ // Early-fire guard: a `scheduled` campaign whose instant is still in the
131
+ // future must not send. This is a stale punctual run — the `sendAt` was
132
+ // moved later after this run was scheduled. The run for the new instant
133
+ // (or the reaper's due-scheduled sweep) delivers it.
134
+ if (
135
+ campaign.status === "scheduled" &&
136
+ campaign.scheduledAt &&
137
+ campaign.scheduledAt.getTime() > Date.now() + EARLY_FIRE_TOLERANCE_MS
138
+ ) {
139
+ return { status: "scheduled", skipped: true, reason: "not_due" as const };
140
+ }
141
+
142
+ // Mirror clause for `waiting`: a punctual next-step run that fires while
143
+ // `nextStepAt` is still in the future is stale — a reap + re-park replaced
144
+ // the pending wait after this run was scheduled. The run created for the
145
+ // new instant (or the reaper's waiting-promotion sweep) resumes it.
146
+ if (
147
+ campaign.status === "waiting" &&
148
+ campaign.nextStepAt &&
149
+ campaign.nextStepAt.getTime() > Date.now() + EARLY_FIRE_TOLERANCE_MS
150
+ ) {
151
+ return { status: "waiting", skipped: true, reason: "not_due" as const };
152
+ }
153
+
154
+ // Claim via CAS: only a non-terminal row transitions to `sending`, so a
155
+ // cancel landing between the guard read above and this write is honored
156
+ // rather than silently resurrected. `startedAt` is claimed ONCE
157
+ // (coalesce): event-condition scoping reads "since campaign startedAt",
158
+ // so a wave-2 claim must NOT reset the anchor wave-1's conditions were
159
+ // already scoped by. The cursor + count seed are re-read FROM the claimed
160
+ // row (not the pre-claim read) so a wave resumed after a crash sees the
161
+ // exact state its predecessor persisted.
162
+ const claimedRows = await db
88
163
  .update(campaigns)
89
- .set({ status: "sending", startedAt: new Date(), updatedAt: new Date() })
90
- .where(eq(campaigns.id, input.campaignId));
164
+ .set({
165
+ status: "sending",
166
+ startedAt: sql`coalesce(${campaigns.startedAt}, now())`,
167
+ // A resumed `waiting` row's wait has elapsed — clear the stale instant
168
+ // so serialized rows never show a countdown while `sending`. (Studio
169
+ // keys its countdown on status === "waiting" regardless.)
170
+ nextStepAt: null,
171
+ // `staleSince` deliberately NOT cleared here: claiming is not
172
+ // progress. A poison run claims, crashes, and is re-driven — if the
173
+ // claim reset the marker, its give-up clock would restart every cycle
174
+ // and the reaper could never declare it failed. The first real
175
+ // progress write (chunk flush / park / cursor advance) clears it.
176
+ updatedAt: new Date(),
177
+ })
178
+ .where(
179
+ and(
180
+ eq(campaigns.id, input.campaignId),
181
+ inArray(campaigns.status, [
182
+ "queued",
183
+ "scheduled",
184
+ "sending",
185
+ "waiting",
186
+ ]),
187
+ ),
188
+ )
189
+ .returning({
190
+ id: campaigns.id,
191
+ currentStep: campaigns.currentStep,
192
+ startedAt: campaigns.startedAt,
193
+ stepBaseCounts: campaigns.stepBaseCounts,
194
+ });
195
+ const claim = claimedRows[0];
196
+ if (!claim) {
197
+ const current = await db
198
+ .select({ status: campaigns.status })
199
+ .from(campaigns)
200
+ .where(eq(campaigns.id, input.campaignId))
201
+ .limit(1);
202
+ return { status: current[0]?.status ?? "unknown", skipped: true };
203
+ }
204
+
205
+ // The executable step sequence. A NULL blob is a legacy single-send row —
206
+ // one send step synthesized from the row columns, and (per the
207
+ // campaignSendKey contract) the LEGACY per-send key format. A non-NULL
208
+ // blob only ever holds > 1 steps (the reconciler stores single-step
209
+ // definitions as NULL), so `multiStep` doubles as the key-format switch:
210
+ // step-scoped keys for ALL steps including 0.
211
+ const blob = campaign.steps;
212
+ const multiStep = blob != null;
213
+ const executable: CampaignStep[] = blob
214
+ ? // db cannot import @hogsend/core, so the blob's elements are opaque
215
+ // Record<string, unknown> there; the engine owns the narrowing.
216
+ (blob.steps as unknown as CampaignStep[])
217
+ : [
218
+ {
219
+ kind: "send",
220
+ template: campaign.templateKey,
221
+ props: campaign.props ?? {},
222
+ ...(campaign.subject != null ? { subject: campaign.subject } : {}),
223
+ ...(campaign.fromEmail != null ? { from: campaign.fromEmail } : {}),
224
+ },
225
+ ];
226
+
227
+ // Immutable row facts captured once (nested fns below can't see the
228
+ // `campaign` narrowing). `startedAt` is non-null post-claim (coalesce);
229
+ // the ?? is a type-level fallback only.
230
+ const audienceKind = campaign.audienceKind;
231
+ const audienceId = campaign.audienceId;
232
+ const startedAt = claim.startedAt ?? new Date();
233
+
234
+ // Seed the cumulative counters from the snapshot through the last
235
+ // COMPLETED wave; the current wave re-derives its own tally on top. Do NOT
236
+ // try to derive the seed from email_sends — suppressed sends write no
237
+ // idempotency key and frequency-capped sends write no row (lib/tracked.ts).
238
+ const base = claim.stepBaseCounts ?? {
239
+ total: 0,
240
+ sent: 0,
241
+ skipped: 0,
242
+ failed: 0,
243
+ };
244
+ let totalRecipients = base.total;
245
+ let sentCount = base.sent;
246
+ let skippedCount = base.skipped;
247
+ let failedCount = base.failed;
91
248
 
92
- let sentCount = 0;
93
- let skippedCount = 0;
94
- let failedCount = 0;
95
- let totalRecipients = 0;
249
+ const snapshotCounts = () => ({
250
+ total: totalRecipients,
251
+ sent: sentCount,
252
+ skipped: skippedCount,
253
+ failed: failedCount,
254
+ });
96
255
 
97
- const flushCounts = async (): Promise<void> => {
256
+ // Every flush is genuine progress (a dispatched chunk, a cancel/completion
257
+ // race) and clears the reaper's staleness marker — EXCEPT the catch-path
258
+ // flush: a crashed run made no progress, so `staleSince` must survive it
259
+ // or a deterministically-crashing campaign resets its give-up clock on
260
+ // every re-drive and is re-enqueued forever.
261
+ const flushCounts = async (opts?: {
262
+ preserveStaleSince?: boolean;
263
+ }): Promise<void> => {
98
264
  await db
99
265
  .update(campaigns)
100
266
  .set({
@@ -102,41 +268,325 @@ export const sendCampaignTask = hatchet.task({
102
268
  sentCount,
103
269
  skippedCount,
104
270
  failedCount,
271
+ ...(opts?.preserveStaleSince ? {} : { staleSince: null }),
105
272
  updatedAt: new Date(),
106
273
  })
107
274
  .where(eq(campaigns.id, input.campaignId));
108
275
  };
109
276
 
110
277
  try {
278
+ // The wave loop: consecutive send steps run sequentially in this same
279
+ // task run; a wait step parks the row (`waiting`) and returns. Resuming
280
+ // with `currentStep` already past the last index (a crash between the
281
+ // final cursor advance and the completion CAS) skips straight to the
282
+ // completion CAS below — exactly right.
283
+ for (let k = claim.currentStep; k < executable.length; k++) {
284
+ const step = executable[k];
285
+ if (step === undefined) break; // unreachable — loop bound
286
+
287
+ if (step.kind === "wait") {
288
+ const nextStepAt = new Date(Date.now() + durationToMs(step.duration));
289
+ // Park CAS: only a still-`sending` row goes `waiting`, so a cancel
290
+ // racing the end of the previous wave keeps its `canceled` status.
291
+ // Cursor, wait instant, count snapshot, and counts land in ONE
292
+ // update — a crash can never separate them.
293
+ const parked = await db
294
+ .update(campaigns)
295
+ .set({
296
+ status: "waiting",
297
+ currentStep: k + 1,
298
+ nextStepAt,
299
+ stepBaseCounts: snapshotCounts(),
300
+ totalRecipients,
301
+ sentCount,
302
+ skippedCount,
303
+ failedCount,
304
+ staleSince: null,
305
+ updatedAt: new Date(),
306
+ })
307
+ .where(
308
+ and(
309
+ eq(campaigns.id, input.campaignId),
310
+ eq(campaigns.status, "sending"),
311
+ ),
312
+ )
313
+ .returning({ id: campaigns.id });
314
+ if (parked.length === 0) {
315
+ await flushCounts();
316
+ const current = await db
317
+ .select({ status: campaigns.status })
318
+ .from(campaigns)
319
+ .where(eq(campaigns.id, input.campaignId))
320
+ .limit(1);
321
+ return { status: current[0]?.status ?? "unknown", skipped: true };
322
+ }
323
+
324
+ // Punctual resume run at nextStepAt — best-effort, same split as
325
+ // `scheduledAt`: on failure the row stays `waiting` and the reaper's
326
+ // waiting-promotion sweep resumes it.
327
+ try {
328
+ await sendCampaignTask.schedule(nextStepAt, {
329
+ campaignId: input.campaignId,
330
+ });
331
+ } catch (err) {
332
+ logger.warn(
333
+ "send-campaign: next-step schedule failed (reaper will promote)",
334
+ {
335
+ campaignId: input.campaignId,
336
+ nextStepAt: nextStepAt.toISOString(),
337
+ error: err instanceof Error ? err.message : String(err),
338
+ },
339
+ );
340
+ }
341
+
342
+ logger.info("send-campaign: wave complete — waiting", {
343
+ campaignId: input.campaignId,
344
+ currentStep: k + 1,
345
+ nextStepAt: nextStepAt.toISOString(),
346
+ });
347
+ return {
348
+ status: "waiting" as const,
349
+ currentStep: k + 1,
350
+ nextStepAt: nextStepAt.toISOString(),
351
+ totalRecipients,
352
+ sentCount,
353
+ skippedCount,
354
+ failedCount,
355
+ };
356
+ }
357
+
358
+ if (step.kind !== "send") {
359
+ // A blob minted by a future engine (steps.v evolution) must fail
360
+ // loudly here, not silently skip a delivery step.
361
+ throw new Error(
362
+ `send-campaign: unsupported step kind "${(step as { kind: string }).kind}" at step ${k} of campaign ${input.campaignId}`,
363
+ );
364
+ }
365
+
366
+ const canceled = await runWave(k, step);
367
+ if (canceled) {
368
+ // The operator's cancel already stamped status/canceledAt; persist
369
+ // the progress counts so the row shows how far the blast got.
370
+ await flushCounts();
371
+ logger.info("send-campaign: canceled mid-send", {
372
+ campaignId: input.campaignId,
373
+ currentStep: k,
374
+ totalRecipients,
375
+ sentCount,
376
+ skippedCount,
377
+ failedCount,
378
+ });
379
+ return {
380
+ status: "canceled" as const,
381
+ totalRecipients,
382
+ sentCount,
383
+ skippedCount,
384
+ failedCount,
385
+ };
386
+ }
387
+
388
+ // Advance the cursor + snapshot the cumulative counters in ONE update
389
+ // — the wave is complete, so a retry entering after this point starts
390
+ // at the NEXT step with these counts as its seed. Plain update (not
391
+ // CAS): a cancel that raced the tail of the wave only bumps cursor
392
+ // metadata on an already-terminal row, which nothing reads.
393
+ await db
394
+ .update(campaigns)
395
+ .set({
396
+ currentStep: k + 1,
397
+ stepBaseCounts: snapshotCounts(),
398
+ totalRecipients,
399
+ sentCount,
400
+ skippedCount,
401
+ failedCount,
402
+ staleSince: null,
403
+ updatedAt: new Date(),
404
+ })
405
+ .where(eq(campaigns.id, input.campaignId));
406
+ }
407
+
408
+ // Completion CAS: only a still-`sending` row is stamped `sent`, so a
409
+ // cancel racing the final chunk (after its last per-chunk check) keeps
410
+ // its `canceled` status instead of being overwritten. `nextStepAt` +
411
+ // `stepBaseCounts` are nulled — a terminal row carries no pending wait
412
+ // and no resume seed.
413
+ const completed = await db
414
+ .update(campaigns)
415
+ .set({
416
+ status: "sent",
417
+ completedAt: new Date(),
418
+ nextStepAt: null,
419
+ stepBaseCounts: null,
420
+ totalRecipients,
421
+ sentCount,
422
+ skippedCount,
423
+ failedCount,
424
+ staleSince: null,
425
+ updatedAt: new Date(),
426
+ })
427
+ .where(
428
+ and(
429
+ eq(campaigns.id, input.campaignId),
430
+ eq(campaigns.status, "sending"),
431
+ ),
432
+ )
433
+ .returning({ id: campaigns.id });
434
+ if (completed.length === 0) {
435
+ await flushCounts();
436
+ const current = await db
437
+ .select({ status: campaigns.status })
438
+ .from(campaigns)
439
+ .where(eq(campaigns.id, input.campaignId))
440
+ .limit(1);
441
+ const status = current[0]?.status ?? "unknown";
442
+ logger.info("send-campaign: finished but row left `sending` first", {
443
+ campaignId: input.campaignId,
444
+ status,
445
+ });
446
+ return {
447
+ status,
448
+ totalRecipients,
449
+ sentCount,
450
+ skippedCount,
451
+ failedCount,
452
+ };
453
+ }
454
+
455
+ logger.info("send-campaign: complete", {
456
+ campaignId: input.campaignId,
457
+ totalRecipients,
458
+ sentCount,
459
+ skippedCount,
460
+ failedCount,
461
+ });
462
+
463
+ return {
464
+ status: "sent" as const,
465
+ totalRecipients,
466
+ sentCount,
467
+ skippedCount,
468
+ failedCount,
469
+ };
470
+ } catch (error) {
471
+ // Do NOT stamp `failed` here. A `failed` stamp before the re-throw makes
472
+ // the single Hatchet retry hit the terminal guard and short-circuit
473
+ // WITHOUT sending the remaining recipients — silently abandoning the tail
474
+ // of a partial wave. Instead we persist the progress counts, leave the
475
+ // status `sending` (re-runnable — `currentStep` still points at the
476
+ // interrupted wave and `stepBaseCounts` still seeds it), and re-throw so
477
+ // the genuine retry re-enters the wave and finishes the unsent tail
478
+ // (already-sent recipients no-op via their idempotency key). A run that
479
+ // EXHAUSTS its retries is transitioned to `failed` (or re-enqueued) by
480
+ // `reapStuckCampaignsTask`. `staleSince` is preserved — a crashed run is
481
+ // not progress, and clearing it here would reset the give-up clock on
482
+ // every re-drive of a poison campaign.
483
+ await flushCounts({ preserveStaleSince: true });
484
+
485
+ logger.error("send-campaign: errored mid-run (will retry)", {
486
+ campaignId: input.campaignId,
487
+ error: error instanceof Error ? error.message : String(error),
488
+ });
489
+ throw error;
490
+ }
491
+
492
+ /**
493
+ * Execute one per-recipient email wave. Returns `true` when a mid-wave
494
+ * cancel was detected (the caller flushes + returns; nothing after the
495
+ * detecting chunk is dispatched).
496
+ *
497
+ * Qualifier source:
498
+ * - Wave 0 resolves the audience LIVE (list/bucket resolvers, fresh
499
+ * suppression pre-filter) and — for MULTI-step campaigns — ANCHORS the
500
+ * cohort: each chunk is batch-inserted into `campaign_recipients`
501
+ * BEFORE its sends are dispatched — a crash between insert and send is
502
+ * fine (the retry re-resolves and the (campaign_id, email) unique
503
+ * absorbs the replay). The audience is resolved exactly once per
504
+ * campaign: someone added to the list afterwards never receives step 3
505
+ * without step 1. A single-send campaign skips the ledger entirely (no
506
+ * later wave will read it — the legacy write path stays byte-identical).
507
+ * - Waves k > 0 qualify FROM the anchored cohort: `campaign_recipients`
508
+ * ∩ the step's `where` conditions ∩ a fresh suppression re-check
509
+ * (see resolveCohortRecipients — suppression is never snapshotted).
510
+ */
511
+ async function runWave(
512
+ k: number,
513
+ step: CampaignSendStep,
514
+ ): Promise<boolean> {
111
515
  const recipients =
112
- campaign.audienceKind === "bucket"
113
- ? resolveBucketRecipients(db, campaign.audienceId)
114
- : resolveListRecipients(db, campaign.audienceId);
516
+ k === 0
517
+ ? audienceKind === "bucket"
518
+ ? resolveBucketRecipients(db, audienceId)
519
+ : resolveListRecipients(db, audienceId)
520
+ : resolveCohortRecipients(db, {
521
+ campaignId: input.campaignId,
522
+ conditions: step.where,
523
+ startedAt,
524
+ });
525
+
526
+ // Mid-flight cancel: re-checked once per chunk (one cheap SELECT per
527
+ // CHUNK_SIZE sends) so `POST /v1/campaigns/:id/cancel` can stop a blast
528
+ // that is already `sending` — recipients not yet dispatched are spared.
529
+ let cancelDetected = false;
115
530
 
116
531
  let chunk: CampaignRecipient[] = [];
117
532
  for await (const recipient of recipients) {
118
533
  chunk.push(recipient);
119
534
  if (chunk.length < CHUNK_SIZE) continue;
120
535
  await sendChunk();
536
+ if (cancelDetected) return true;
121
537
  }
122
538
  // Final partial chunk.
123
- if (chunk.length > 0) await sendChunk();
539
+ if (chunk.length > 0 && !cancelDetected) await sendChunk();
540
+ return cancelDetected;
124
541
 
125
542
  async function sendChunk(): Promise<void> {
543
+ const statusRows = await db
544
+ .select({ status: campaigns.status })
545
+ .from(campaigns)
546
+ .where(eq(campaigns.id, input.campaignId))
547
+ .limit(1);
548
+ if (statusRows[0]?.status === "canceled") {
549
+ cancelDetected = true;
550
+ chunk = [];
551
+ return;
552
+ }
553
+
126
554
  const batch = chunk;
127
555
  chunk = [];
556
+
557
+ // Anchor the cohort BEFORE dispatching the chunk (wave 0 of a
558
+ // MULTI-step campaign only — a single-send campaign has no later wave
559
+ // to read the ledger, so skipping the insert keeps the legacy blast's
560
+ // write path byte-identical). Emails are already normalized by the
561
+ // resolvers, matching the table's (campaign_id, email) unique
562
+ // keyspace.
563
+ if (k === 0 && multiStep) {
564
+ await db
565
+ .insert(campaignRecipients)
566
+ .values(
567
+ batch.map((r) => ({
568
+ campaignId: input.campaignId,
569
+ userId: r.userId ?? null,
570
+ email: r.email,
571
+ })),
572
+ )
573
+ .onConflictDoNothing({
574
+ target: [campaignRecipients.campaignId, campaignRecipients.email],
575
+ });
576
+ }
577
+
128
578
  totalRecipients += batch.length;
129
579
 
130
580
  const results = await Promise.allSettled(
131
581
  batch.map((r) =>
132
582
  emailService.send({
133
- template: campaign?.templateKey as TemplateName,
134
- props: (campaign?.props ?? {}) as never,
583
+ template: step.template as TemplateName,
584
+ props: (step.props ?? {}) as never,
135
585
  to: r.email,
136
586
  userId: r.userId,
137
587
  userEmail: r.email,
138
- subject: campaign?.subject ?? undefined,
139
- from: campaign?.fromEmail ?? undefined,
588
+ subject: step.subject,
589
+ from: step.from,
140
590
  // A list's audienceId IS a real subscription category, so pass it
141
591
  // through for suppression + the unsubscribe link. A bucket's
142
592
  // audienceId is NOT a category — forcing it here would mint an
@@ -145,13 +595,17 @@ export const sendCampaignTask = hatchet.task({
145
595
  // unsubscribedAll/suppressed), silently no-op'ing the unsubscribe.
146
596
  // For a bucket, pass undefined so the template's OWN declared
147
597
  // category (e.g. `product-updates`) drives both suppression and a
148
- // real, honored List-Unsubscribe target.
149
- category:
150
- campaign?.audienceKind === "bucket"
151
- ? undefined
152
- : campaign?.audienceId,
598
+ // real, honored List-Unsubscribe target. The rule applies to
599
+ // EVERY wave, not just the first.
600
+ category: audienceKind === "bucket" ? undefined : audienceId,
153
601
  // The idempotency key dedupes a retried send to its prior row.
154
- idempotencyKey: `campaign:${input.campaignId}:${r.email}`,
602
+ // Legacy format for single-step campaigns; step-scoped for every
603
+ // step of a multi-step campaign (campaignSendKey contract).
604
+ idempotencyKey: campaignSendKey(
605
+ input.campaignId,
606
+ r.email,
607
+ multiStep ? k : undefined,
608
+ ),
155
609
  }),
156
610
  ),
157
611
  );
@@ -166,67 +620,15 @@ export const sendCampaignTask = hatchet.task({
166
620
  sentCount++;
167
621
  } else {
168
622
  // suppressed | unsubscribed | skipped (frequency-capped) — counted
169
- // as skipped, not a delivery failure.
623
+ // as skipped, not a delivery failure. The member STAYS in the
624
+ // cohort either way: a provider hiccup or a skipped wave is not an
625
+ // exit, and suppression is re-checked fresh on every later wave.
170
626
  skippedCount++;
171
627
  }
172
628
  }
173
629
 
174
630
  await flushCounts();
175
631
  }
176
-
177
- await db
178
- .update(campaigns)
179
- .set({
180
- status: "sent",
181
- completedAt: new Date(),
182
- totalRecipients,
183
- sentCount,
184
- skippedCount,
185
- failedCount,
186
- updatedAt: new Date(),
187
- })
188
- .where(eq(campaigns.id, input.campaignId));
189
-
190
- logger.info("send-campaign: complete", {
191
- campaignId: input.campaignId,
192
- totalRecipients,
193
- sentCount,
194
- skippedCount,
195
- failedCount,
196
- });
197
-
198
- return {
199
- status: "sent" as const,
200
- totalRecipients,
201
- sentCount,
202
- skippedCount,
203
- failedCount,
204
- };
205
- } catch (error) {
206
- // Do NOT stamp `failed` here. A `failed` stamp before the re-throw makes
207
- // the single Hatchet retry hit the terminal guard and short-circuit
208
- // WITHOUT sending the remaining recipients — silently abandoning the tail
209
- // of a partial send. Instead we persist the progress counts, leave the
210
- // status `sending` (re-runnable), and re-throw so the genuine retry
211
- // re-enters the loop and finishes the unsent tail (already-sent recipients
212
- // no-op via their idempotency key). A run that EXHAUSTS its retries is
213
- // transitioned to `failed` (or re-enqueued) by `reapStuckCampaignsTask`.
214
- await db
215
- .update(campaigns)
216
- .set({
217
- totalRecipients,
218
- sentCount,
219
- skippedCount,
220
- failedCount,
221
- updatedAt: new Date(),
222
- })
223
- .where(eq(campaigns.id, input.campaignId));
224
-
225
- logger.error("send-campaign: errored mid-run (will retry)", {
226
- campaignId: input.campaignId,
227
- error: error instanceof Error ? error.message : String(error),
228
- });
229
- throw error;
230
632
  }
231
633
  },
232
634
  });
@@ -243,15 +645,35 @@ const STALE_AFTER_MS = Number(
243
645
  );
244
646
 
245
647
  /**
246
- * After a campaign has sat in a non-terminal in-flight state this long (measured
247
- * from `updatedAt`, which the send task bumps on every progress flush) it is
248
- * declared `failed` rather than re-enqueued forever — a poison campaign (e.g. a
249
- * template that always throws) stops being re-driven and surfaces to operators.
648
+ * After a campaign has been CONTINUOUSLY stuck this long with zero progress it
649
+ * is declared `failed` rather than re-enqueued forever — a poison campaign
650
+ * (e.g. a run that always crashes) stops being re-driven and surfaces to
651
+ * operators.
652
+ *
653
+ * For in-flight (`queued`/`sending`) rows the window is measured from
654
+ * `staleSince` — set once by the stale sweep's first re-enqueue, cleared by
655
+ * every genuine progress flush of the send task. NOT from `updatedAt`: the
656
+ * sweep bumps `updatedAt` as its re-pick guard, so a deterministically-
657
+ * crashing row would never age past the window (the give-up was dead code
658
+ * until `stale_since` landed). `scheduled` / `waiting` rows are measured from
659
+ * `scheduledAt` / `nextStepAt` as before.
250
660
  */
251
661
  const GIVE_UP_AFTER_MS = Number(
252
662
  process.env.CAMPAIGN_GIVE_UP_AFTER_MS ?? 6 * 60 * 60 * 1000,
253
663
  );
254
664
 
665
+ /**
666
+ * How far past `scheduledAt` a `scheduled` campaign (or past `nextStepAt` a
667
+ * `waiting` one) may sit before the reaper promotes it (enqueues the send
668
+ * task). The punctual Hatchet scheduled run created at schedule/park time is
669
+ * the primary trigger; this grace keeps the sweep from routinely
670
+ * double-firing right at the boundary. A double fire is harmless anyway
671
+ * (terminal guard + per-send idempotency).
672
+ */
673
+ const SCHEDULE_PROMOTE_GRACE_MS = Number(
674
+ process.env.CAMPAIGN_PROMOTE_GRACE_MS ?? 2 * 60 * 1000,
675
+ );
676
+
255
677
  /**
256
678
  * Engine-owned reaper cron for campaigns left in a non-terminal in-flight state
257
679
  * with no live run to finish them (closes the "stuck forever" gap):
@@ -261,6 +683,16 @@ const GIVE_UP_AFTER_MS = Number(
261
683
  * catch never ran, so the row is stuck `sending` with no live run.
262
684
  * - A `queued` campaign whose enqueue threw at create time (broker down /
263
685
  * network) — the row was committed but no run was ever created (orphan).
686
+ * - A `scheduled` campaign whose punctual Hatchet scheduled run never fired
687
+ * (schedule-create failed at POST/reconcile time, or the run was lost) —
688
+ * promoted once `scheduledAt` is {@link SCHEDULE_PROMOTE_GRACE_MS} past
689
+ * due. A scheduled row stuck past the give-up window (measured from
690
+ * `scheduledAt`, NOT `updatedAt` — a row is legitimately idle between
691
+ * create and send time) is declared `failed`.
692
+ * - A `waiting` campaign whose punctual next-step run was lost — promoted
693
+ * once `nextStepAt` is past the same grace. Give-up is likewise measured
694
+ * from `nextStepAt`, NOT `updatedAt` — a row is legitimately idle mid-wait,
695
+ * exactly like `scheduled`/`scheduledAt`.
264
696
  *
265
697
  * Recovery is a simple RE-ENQUEUE of `sendCampaignTask` (safe: the per-send
266
698
  * idempotency key no-ops already-sent recipients and the re-run completes the
@@ -284,16 +716,38 @@ export const reapStuckCampaignsTask = hatchet.task({
284
716
  const now = Date.now();
285
717
  const staleBefore = new Date(now - STALE_AFTER_MS);
286
718
  const giveUpBefore = new Date(now - GIVE_UP_AFTER_MS);
719
+ const promoteBefore = new Date(now - SCHEDULE_PROMOTE_GRACE_MS);
287
720
 
288
721
  // (1) Declare poison campaigns `failed` first (stuck past the give-up
289
- // window), so they are not re-enqueued below.
722
+ // window), so they are not re-enqueued below. In-flight rows are measured
723
+ // from `staleSince` (set by the first stale re-enqueue, cleared by every
724
+ // genuine progress flush — a NULL never matches the `lt`, so a row the
725
+ // sweep hasn't touched yet gets its re-drive chance first); `scheduled`
726
+ // rows from `scheduledAt` and `waiting` rows from `nextStepAt` (their
727
+ // `updatedAt` is legitimately old while they wait). `staleSince` is nulled
728
+ // with the stamp so an operator revive starts a fresh give-up window.
290
729
  const failedRows = await db
291
730
  .update(campaigns)
292
- .set({ status: "failed", completedAt: new Date(), updatedAt: new Date() })
731
+ .set({
732
+ status: "failed",
733
+ completedAt: new Date(),
734
+ staleSince: null,
735
+ updatedAt: new Date(),
736
+ })
293
737
  .where(
294
- and(
295
- inArray(campaigns.status, ["queued", "sending"]),
296
- lt(campaigns.updatedAt, giveUpBefore),
738
+ or(
739
+ and(
740
+ inArray(campaigns.status, ["queued", "sending"]),
741
+ lt(campaigns.staleSince, giveUpBefore),
742
+ ),
743
+ and(
744
+ eq(campaigns.status, "scheduled"),
745
+ lt(campaigns.scheduledAt, giveUpBefore),
746
+ ),
747
+ and(
748
+ eq(campaigns.status, "waiting"),
749
+ lt(campaigns.nextStepAt, giveUpBefore),
750
+ ),
297
751
  ),
298
752
  )
299
753
  .returning({ id: campaigns.id });
@@ -302,9 +756,18 @@ export const reapStuckCampaignsTask = hatchet.task({
302
756
  // `updatedAt` so the same row is not re-picked on the very next tick before
303
757
  // the re-driven run makes progress; the per-send idempotency key keeps the
304
758
  // re-enqueue safe even if the original run is somehow still alive.
759
+ // `staleSince` is coalesce-set ONCE on the first re-enqueue (later sweeps
760
+ // keep the original instant) — this is the bump-free clock the give-up
761
+ // clause above reads; the send task clears it on genuine progress.
762
+ // `queued`/`sending` ONLY — `waiting` is deliberately excluded: a 2-day
763
+ // wait between waves is not a stuck campaign (its resume path is the
764
+ // promotion sweep below, keyed on `nextStepAt`).
305
765
  const staleRows = await db
306
766
  .update(campaigns)
307
- .set({ updatedAt: new Date() })
767
+ .set({
768
+ staleSince: sql`coalesce(${campaigns.staleSince}, now())`,
769
+ updatedAt: new Date(),
770
+ })
308
771
  .where(
309
772
  and(
310
773
  inArray(campaigns.status, ["queued", "sending"]),
@@ -313,9 +776,32 @@ export const reapStuckCampaignsTask = hatchet.task({
313
776
  )
314
777
  .returning({ id: campaigns.id });
315
778
 
316
- for (const row of staleRows) {
779
+ // (3) Promote due `scheduled` campaigns whose punctual scheduled run never
780
+ // fired, and due `waiting` campaigns whose punctual next-step run was lost.
781
+ // Enqueue-only — the send task owns the scheduled/waiting→sending
782
+ // transition, so a failed enqueue simply retries next sweep.
783
+ const dueRows = await db
784
+ .select({ id: campaigns.id })
785
+ .from(campaigns)
786
+ .where(
787
+ or(
788
+ and(
789
+ eq(campaigns.status, "scheduled"),
790
+ lte(campaigns.scheduledAt, promoteBefore),
791
+ ),
792
+ and(
793
+ eq(campaigns.status, "waiting"),
794
+ lte(campaigns.nextStepAt, promoteBefore),
795
+ ),
796
+ ),
797
+ );
798
+
799
+ // Enqueue WITHOUT waiting for results — `.run()` blocks until the whole
800
+ // blast completes, which would serialize entire sends inside this cron's
801
+ // 120s executionTimeout.
802
+ for (const row of [...staleRows, ...dueRows]) {
317
803
  try {
318
- await sendCampaignTask.run({ campaignId: row.id });
804
+ await sendCampaignTask.runNoWait({ campaignId: row.id });
319
805
  } catch (err) {
320
806
  logger.warn("reap-stuck-campaigns: re-enqueue failed", {
321
807
  campaignId: row.id,
@@ -324,16 +810,18 @@ export const reapStuckCampaignsTask = hatchet.task({
324
810
  }
325
811
  }
326
812
 
327
- if (failedRows.length > 0 || staleRows.length > 0) {
813
+ if (failedRows.length > 0 || staleRows.length > 0 || dueRows.length > 0) {
328
814
  logger.info("reap-stuck-campaigns: swept", {
329
815
  failed: failedRows.length,
330
816
  reEnqueued: staleRows.length,
817
+ promoted: dueRows.length,
331
818
  });
332
819
  }
333
820
 
334
821
  return {
335
822
  failed: failedRows.length,
336
823
  reEnqueued: staleRows.length,
824
+ promoted: dueRows.length,
337
825
  };
338
826
  },
339
827
  });
@@ -370,19 +858,76 @@ async function* keysetPaginate<Row>(opts: {
370
858
  }
371
859
  }
372
860
 
861
+ /**
862
+ * Wave-k (k > 0) qualifier resolver: the campaign's anchored cohort
863
+ * (`campaign_recipients`) ∩ the step's `where` conditions (each compiled to a
864
+ * correlated [NOT] EXISTS — see `campaigns/cohort-sql.ts`) ∩ a fresh
865
+ * suppression/unsubscribe re-check. Membership was anchored at wave 0 and is
866
+ * never re-resolved — a member who left the bucket/list between waves still
867
+ * qualifies (suppression excepted); a member added afterwards never appears.
868
+ * Paged by the keyset cursor on `campaign_recipients.id` (the
869
+ * (campaign_id, id) index), so every page is an indexed join, never a LIKE
870
+ * scan over email_sends.
871
+ */
872
+ async function* resolveCohortRecipients(
873
+ db: Database,
874
+ opts: {
875
+ campaignId: string;
876
+ conditions: ConditionEval[] | undefined;
877
+ startedAt: Date;
878
+ },
879
+ ): AsyncGenerator<CampaignRecipient> {
880
+ // Compiled once per wave — every page reuses the same predicates.
881
+ const qualifiers = (opts.conditions ?? []).map((condition) =>
882
+ waveConditionSql({
883
+ condition,
884
+ campaignId: opts.campaignId,
885
+ startedAt: opts.startedAt,
886
+ }),
887
+ );
888
+
889
+ yield* keysetPaginate({
890
+ page: (cursor) => {
891
+ const conditions = [
892
+ eq(campaignRecipients.campaignId, opts.campaignId),
893
+ cohortSuppressionSql(),
894
+ ...qualifiers,
895
+ ];
896
+ if (cursor) conditions.push(gt(campaignRecipients.id, cursor));
897
+
898
+ return db
899
+ .select({
900
+ id: campaignRecipients.id,
901
+ userId: campaignRecipients.userId,
902
+ email: campaignRecipients.email,
903
+ })
904
+ .from(campaignRecipients)
905
+ .where(and(...conditions))
906
+ .orderBy(campaignRecipients.id)
907
+ .limit(CHUNK_SIZE);
908
+ },
909
+ cursorOf: (row) => row.id,
910
+ // Cohort emails were normalized at anchor time; userId may be NULL for an
911
+ // email-only member (kept — the mailer falls back the same way wave 0 did).
912
+ map: (row) => ({ email: row.email, userId: row.userId ?? undefined }),
913
+ });
914
+ }
915
+
373
916
  /**
374
917
  * Active, non-deleted members of a bucket, joined to a live contact for the
375
918
  * email — mirrors the bucket-access member query. Paged by the keyset cursor on
376
919
  * `bucket_memberships.id`.
377
920
  *
378
- * Compliance: `bucket_memberships.userEmail` is written verbatim from the RAW
379
- * event payload on the realtime join path (un-normalized, unlike
380
- * `contacts.email`), so the recipient email is NORMALIZED (`normalizeEmail`)
381
- * before it is yielded — otherwise a mixed-case membership email
382
- * (`User@Example.com`) would not case-match its NORMALIZED `email_preferences`
383
- * row (`user@example.com`) and the mailer's case-sensitive suppression check
384
- * would MISS the row, leaking a marketing blast to a suppressed/unsubscribed
385
- * contact (CAN-SPAM/GDPR). Defense-in-depth: this resolver ALSO pre-filters
921
+ * Compliance: `bucket_memberships.userEmail` is normalized at every write site
922
+ * (and existing rows were backfilled by migration 0043), but this resolver
923
+ * KEEPS its own `normalizeEmail` / `lower(trim(…))` belt-and-braces —
924
+ * TODO(cleanup): safe to drop once `@hogsend/engine` has shipped past 0.40.0
925
+ * (all deployed write sites normalize as of this changeset). An un-normalized
926
+ * membership email (`User@Example.com`) would not
927
+ * case-match its NORMALIZED `email_preferences` row (`user@example.com`) and
928
+ * the mailer's case-sensitive suppression check would MISS the row, leaking a
929
+ * marketing blast to a suppressed/unsubscribed contact (CAN-SPAM/GDPR).
930
+ * Defense-in-depth: this resolver ALSO pre-filters
386
931
  * `unsubscribedAll`/`suppressed` at the audience layer (mirroring the list
387
932
  * resolver) via a LEFT JOIN to `email_preferences` on the NORMALIZED email, so
388
933
  * a globally-unsubscribed / suppressed bucket member is excluded up front
@@ -394,8 +939,9 @@ async function* resolveBucketRecipients(
394
939
  db: Database,
395
940
  bucketId: string,
396
941
  ): AsyncGenerator<CampaignRecipient> {
397
- // The recipient's normalized email — the membership email may be mixed-case
398
- // (written verbatim from the raw event), the contact email is the fallback.
942
+ // The recipient's normalized email — writes are normalized now, but the
943
+ // read-side lower(trim(…)) stays as belt-and-braces (TODO(cleanup): see the
944
+ // fn docstring for the drop condition); the contact email is the fallback.
399
945
  const recipientEmail = sql<string>`lower(trim(coalesce(${bucketMemberships.userEmail}, ${contacts.email})))`;
400
946
 
401
947
  yield* keysetPaginate({