@hogsend/engine 0.38.2 → 0.40.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,7 +6,7 @@ import {
6
6
  emailPreferences,
7
7
  } from "@hogsend/db";
8
8
  import type { TemplateName } from "@hogsend/email";
9
- import { and, eq, gt, inArray, isNull, lt, sql } from "drizzle-orm";
9
+ import { and, eq, gt, inArray, isNull, lt, lte, or, sql } from "drizzle-orm";
10
10
  import { normalizeEmail } from "../lib/contacts.js";
11
11
  import { getDb } from "../lib/db.js";
12
12
  import { getEmailService } from "../lib/email.js";
@@ -23,8 +23,22 @@ interface CampaignRecipient {
23
23
  userId?: string;
24
24
  }
25
25
 
26
- /** Statuses that are TERMINAL — a duplicate/late enqueue must not re-send. */
27
- const TERMINAL_STATUSES = ["sent"] as const;
26
+ /**
27
+ * Statuses that are TERMINAL — a duplicate/late enqueue must not re-send.
28
+ * `canceled`/`expired` matter doubly for SCHEDULED campaigns: the punctual
29
+ * Hatchet scheduled run cannot be deleted at cancel time (we don't persist its
30
+ * id), so it still fires and must find the row terminal here and no-op.
31
+ * `failed` is deliberately NOT terminal — a re-enqueue can revive it.
32
+ */
33
+ const TERMINAL_STATUSES = ["sent", "canceled", "expired"] as const;
34
+
35
+ /**
36
+ * How far ahead of `scheduledAt` a run may fire and still proceed (absorbs
37
+ * clock skew). An earlier fire — the punctual run created for a `sendAt` that
38
+ * was later moved BACK by a reconcile/edit — skips without sending; the run
39
+ * created for the new instant (or the reaper sweep) delivers it.
40
+ */
41
+ const EARLY_FIRE_TOLERANCE_MS = 60 * 1000;
28
42
 
29
43
  /**
30
44
  * Built-in durable campaign / broadcast task (Loops "campaign" parity). Sends a
@@ -84,10 +98,39 @@ export const sendCampaignTask = hatchet.task({
84
98
  return { status: campaign.status, skipped: true };
85
99
  }
86
100
 
87
- await db
101
+ // Early-fire guard: a `scheduled` campaign whose instant is still in the
102
+ // future must not send. This is a stale punctual run — the `sendAt` was
103
+ // moved later after this run was scheduled. The run for the new instant
104
+ // (or the reaper's due-scheduled sweep) delivers it.
105
+ if (
106
+ campaign.status === "scheduled" &&
107
+ campaign.scheduledAt &&
108
+ campaign.scheduledAt.getTime() > Date.now() + EARLY_FIRE_TOLERANCE_MS
109
+ ) {
110
+ return { status: "scheduled", skipped: true, reason: "not_due" as const };
111
+ }
112
+
113
+ // Claim via CAS: only a non-terminal row transitions to `sending`, so a
114
+ // cancel landing between the guard read above and this write is honored
115
+ // rather than silently resurrected.
116
+ const claimed = await db
88
117
  .update(campaigns)
89
118
  .set({ status: "sending", startedAt: new Date(), updatedAt: new Date() })
90
- .where(eq(campaigns.id, input.campaignId));
119
+ .where(
120
+ and(
121
+ eq(campaigns.id, input.campaignId),
122
+ inArray(campaigns.status, ["queued", "scheduled", "sending"]),
123
+ ),
124
+ )
125
+ .returning({ id: campaigns.id });
126
+ if (claimed.length === 0) {
127
+ const current = await db
128
+ .select({ status: campaigns.status })
129
+ .from(campaigns)
130
+ .where(eq(campaigns.id, input.campaignId))
131
+ .limit(1);
132
+ return { status: current[0]?.status ?? "unknown", skipped: true };
133
+ }
91
134
 
92
135
  let sentCount = 0;
93
136
  let skippedCount = 0;
@@ -113,16 +156,33 @@ export const sendCampaignTask = hatchet.task({
113
156
  ? resolveBucketRecipients(db, campaign.audienceId)
114
157
  : resolveListRecipients(db, campaign.audienceId);
115
158
 
159
+ // Mid-flight cancel: re-checked once per chunk (one cheap SELECT per
160
+ // CHUNK_SIZE sends) so `POST /v1/campaigns/:id/cancel` can stop a blast
161
+ // that is already `sending` — recipients not yet dispatched are spared.
162
+ let cancelDetected = false;
163
+
116
164
  let chunk: CampaignRecipient[] = [];
117
165
  for await (const recipient of recipients) {
118
166
  chunk.push(recipient);
119
167
  if (chunk.length < CHUNK_SIZE) continue;
120
168
  await sendChunk();
169
+ if (cancelDetected) break;
121
170
  }
122
171
  // Final partial chunk.
123
- if (chunk.length > 0) await sendChunk();
172
+ if (chunk.length > 0 && !cancelDetected) await sendChunk();
124
173
 
125
174
  async function sendChunk(): Promise<void> {
175
+ const statusRows = await db
176
+ .select({ status: campaigns.status })
177
+ .from(campaigns)
178
+ .where(eq(campaigns.id, input.campaignId))
179
+ .limit(1);
180
+ if (statusRows[0]?.status === "canceled") {
181
+ cancelDetected = true;
182
+ chunk = [];
183
+ return;
184
+ }
185
+
126
186
  const batch = chunk;
127
187
  chunk = [];
128
188
  totalRecipients += batch.length;
@@ -174,7 +234,30 @@ export const sendCampaignTask = hatchet.task({
174
234
  await flushCounts();
175
235
  }
176
236
 
177
- await db
237
+ if (cancelDetected) {
238
+ // The operator's cancel already stamped status/canceledAt; persist the
239
+ // progress counts so the row shows how far the blast got.
240
+ await flushCounts();
241
+ logger.info("send-campaign: canceled mid-send", {
242
+ campaignId: input.campaignId,
243
+ totalRecipients,
244
+ sentCount,
245
+ skippedCount,
246
+ failedCount,
247
+ });
248
+ return {
249
+ status: "canceled" as const,
250
+ totalRecipients,
251
+ sentCount,
252
+ skippedCount,
253
+ failedCount,
254
+ };
255
+ }
256
+
257
+ // Completion CAS: only a still-`sending` row is stamped `sent`, so a
258
+ // cancel racing the final chunk (after its last per-chunk check) keeps
259
+ // its `canceled` status instead of being overwritten.
260
+ const completed = await db
178
261
  .update(campaigns)
179
262
  .set({
180
263
  status: "sent",
@@ -185,7 +268,33 @@ export const sendCampaignTask = hatchet.task({
185
268
  failedCount,
186
269
  updatedAt: new Date(),
187
270
  })
188
- .where(eq(campaigns.id, input.campaignId));
271
+ .where(
272
+ and(
273
+ eq(campaigns.id, input.campaignId),
274
+ eq(campaigns.status, "sending"),
275
+ ),
276
+ )
277
+ .returning({ id: campaigns.id });
278
+ if (completed.length === 0) {
279
+ await flushCounts();
280
+ const current = await db
281
+ .select({ status: campaigns.status })
282
+ .from(campaigns)
283
+ .where(eq(campaigns.id, input.campaignId))
284
+ .limit(1);
285
+ const status = current[0]?.status ?? "unknown";
286
+ logger.info("send-campaign: finished but row left `sending` first", {
287
+ campaignId: input.campaignId,
288
+ status,
289
+ });
290
+ return {
291
+ status,
292
+ totalRecipients,
293
+ sentCount,
294
+ skippedCount,
295
+ failedCount,
296
+ };
297
+ }
189
298
 
190
299
  logger.info("send-campaign: complete", {
191
300
  campaignId: input.campaignId,
@@ -252,6 +361,17 @@ const GIVE_UP_AFTER_MS = Number(
252
361
  process.env.CAMPAIGN_GIVE_UP_AFTER_MS ?? 6 * 60 * 60 * 1000,
253
362
  );
254
363
 
364
+ /**
365
+ * How far past `scheduledAt` a `scheduled` campaign may sit before the reaper
366
+ * promotes it (enqueues the send task). The punctual Hatchet scheduled run
367
+ * created at schedule time is the primary trigger; this grace keeps the sweep
368
+ * from routinely double-firing right at the boundary. A double fire is
369
+ * harmless anyway (terminal guard + per-send idempotency).
370
+ */
371
+ const SCHEDULE_PROMOTE_GRACE_MS = Number(
372
+ process.env.CAMPAIGN_PROMOTE_GRACE_MS ?? 2 * 60 * 1000,
373
+ );
374
+
255
375
  /**
256
376
  * Engine-owned reaper cron for campaigns left in a non-terminal in-flight state
257
377
  * with no live run to finish them (closes the "stuck forever" gap):
@@ -261,6 +381,12 @@ const GIVE_UP_AFTER_MS = Number(
261
381
  * catch never ran, so the row is stuck `sending` with no live run.
262
382
  * - A `queued` campaign whose enqueue threw at create time (broker down /
263
383
  * network) — the row was committed but no run was ever created (orphan).
384
+ * - A `scheduled` campaign whose punctual Hatchet scheduled run never fired
385
+ * (schedule-create failed at POST/reconcile time, or the run was lost) —
386
+ * promoted once `scheduledAt` is {@link SCHEDULE_PROMOTE_GRACE_MS} past
387
+ * due. A scheduled row stuck past the give-up window (measured from
388
+ * `scheduledAt`, NOT `updatedAt` — a row is legitimately idle between
389
+ * create and send time) is declared `failed`.
264
390
  *
265
391
  * Recovery is a simple RE-ENQUEUE of `sendCampaignTask` (safe: the per-send
266
392
  * idempotency key no-ops already-sent recipients and the re-run completes the
@@ -286,14 +412,22 @@ export const reapStuckCampaignsTask = hatchet.task({
286
412
  const giveUpBefore = new Date(now - GIVE_UP_AFTER_MS);
287
413
 
288
414
  // (1) Declare poison campaigns `failed` first (stuck past the give-up
289
- // window), so they are not re-enqueued below.
415
+ // window), so they are not re-enqueued below. In-flight rows are measured
416
+ // from `updatedAt` (bumped on every progress flush); `scheduled` rows from
417
+ // `scheduledAt` (their `updatedAt` is legitimately old while they wait).
290
418
  const failedRows = await db
291
419
  .update(campaigns)
292
420
  .set({ status: "failed", completedAt: new Date(), updatedAt: new Date() })
293
421
  .where(
294
- and(
295
- inArray(campaigns.status, ["queued", "sending"]),
296
- lt(campaigns.updatedAt, giveUpBefore),
422
+ or(
423
+ and(
424
+ inArray(campaigns.status, ["queued", "sending"]),
425
+ lt(campaigns.updatedAt, giveUpBefore),
426
+ ),
427
+ and(
428
+ eq(campaigns.status, "scheduled"),
429
+ lt(campaigns.scheduledAt, giveUpBefore),
430
+ ),
297
431
  ),
298
432
  )
299
433
  .returning({ id: campaigns.id });
@@ -313,9 +447,25 @@ export const reapStuckCampaignsTask = hatchet.task({
313
447
  )
314
448
  .returning({ id: campaigns.id });
315
449
 
316
- for (const row of staleRows) {
450
+ // (3) Promote due `scheduled` campaigns whose punctual scheduled run never
451
+ // fired. Enqueue-only — the send task owns the scheduled→sending
452
+ // transition, so a failed enqueue simply retries next sweep.
453
+ const dueRows = await db
454
+ .select({ id: campaigns.id })
455
+ .from(campaigns)
456
+ .where(
457
+ and(
458
+ eq(campaigns.status, "scheduled"),
459
+ lte(campaigns.scheduledAt, new Date(now - SCHEDULE_PROMOTE_GRACE_MS)),
460
+ ),
461
+ );
462
+
463
+ // Enqueue WITHOUT waiting for results — `.run()` blocks until the whole
464
+ // blast completes, which would serialize entire sends inside this cron's
465
+ // 120s executionTimeout.
466
+ for (const row of [...staleRows, ...dueRows]) {
317
467
  try {
318
- await sendCampaignTask.run({ campaignId: row.id });
468
+ await sendCampaignTask.runNoWait({ campaignId: row.id });
319
469
  } catch (err) {
320
470
  logger.warn("reap-stuck-campaigns: re-enqueue failed", {
321
471
  campaignId: row.id,
@@ -324,16 +474,18 @@ export const reapStuckCampaignsTask = hatchet.task({
324
474
  }
325
475
  }
326
476
 
327
- if (failedRows.length > 0 || staleRows.length > 0) {
477
+ if (failedRows.length > 0 || staleRows.length > 0 || dueRows.length > 0) {
328
478
  logger.info("reap-stuck-campaigns: swept", {
329
479
  failed: failedRows.length,
330
480
  reEnqueued: staleRows.length,
481
+ promoted: dueRows.length,
331
482
  });
332
483
  }
333
484
 
334
485
  return {
335
486
  failed: failedRows.length,
336
487
  reEnqueued: staleRows.length,
488
+ promoted: dueRows.length,
337
489
  };
338
490
  },
339
491
  });