@hogsend/engine 0.38.2 → 0.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +10 -7
- package/src/campaigns/define-campaign.ts +122 -0
- package/src/campaigns/reconcile.ts +297 -0
- package/src/container.ts +153 -10
- package/src/env.ts +14 -0
- package/src/index.ts +39 -1
- package/src/journeys/define-journey.ts +142 -2
- package/src/journeys/graph/build-graph.ts +1159 -0
- package/src/journeys/journey-boundary.ts +33 -0
- package/src/journeys/journey-context.ts +58 -8
- package/src/journeys/journey-log.ts +64 -0
- package/src/journeys/journey-source-locations-singleton.ts +19 -0
- package/src/journeys/journey-sources-singleton.ts +14 -0
- package/src/journeys/registry.ts +148 -5
- package/src/lib/auth.ts +33 -1
- package/src/lib/email.ts +17 -6
- package/src/lib/ingestion.ts +13 -0
- package/src/lib/tracked.ts +32 -0
- package/src/lists/define-list.ts +19 -3
- package/src/routes/admin/campaigns.ts +165 -0
- package/src/routes/admin/index.ts +4 -0
- package/src/routes/admin/journeys.ts +278 -0
- package/src/routes/admin/open-editor.ts +106 -0
- package/src/routes/campaigns/index.ts +259 -50
- package/src/routes/emails/index.ts +25 -1
- package/src/worker.ts +37 -2
- package/src/workflows/send-campaign.ts +167 -15
|
@@ -6,7 +6,7 @@ import {
|
|
|
6
6
|
emailPreferences,
|
|
7
7
|
} from "@hogsend/db";
|
|
8
8
|
import type { TemplateName } from "@hogsend/email";
|
|
9
|
-
import { and, eq, gt, inArray, isNull, lt, sql } from "drizzle-orm";
|
|
9
|
+
import { and, eq, gt, inArray, isNull, lt, lte, or, sql } from "drizzle-orm";
|
|
10
10
|
import { normalizeEmail } from "../lib/contacts.js";
|
|
11
11
|
import { getDb } from "../lib/db.js";
|
|
12
12
|
import { getEmailService } from "../lib/email.js";
|
|
@@ -23,8 +23,22 @@ interface CampaignRecipient {
|
|
|
23
23
|
userId?: string;
|
|
24
24
|
}
|
|
25
25
|
|
|
26
|
-
/**
|
|
27
|
-
|
|
26
|
+
/**
|
|
27
|
+
* Statuses that are TERMINAL — a duplicate/late enqueue must not re-send.
|
|
28
|
+
* `canceled`/`expired` matter doubly for SCHEDULED campaigns: the punctual
|
|
29
|
+
* Hatchet scheduled run cannot be deleted at cancel time (we don't persist its
|
|
30
|
+
* id), so it still fires and must find the row terminal here and no-op.
|
|
31
|
+
* `failed` is deliberately NOT terminal — a re-enqueue can revive it.
|
|
32
|
+
*/
|
|
33
|
+
const TERMINAL_STATUSES = ["sent", "canceled", "expired"] as const;
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* How far ahead of `scheduledAt` a run may fire and still proceed (absorbs
|
|
37
|
+
* clock skew). An earlier fire — the punctual run created for a `sendAt` that
|
|
38
|
+
* was later moved BACK by a reconcile/edit — skips without sending; the run
|
|
39
|
+
* created for the new instant (or the reaper sweep) delivers it.
|
|
40
|
+
*/
|
|
41
|
+
const EARLY_FIRE_TOLERANCE_MS = 60 * 1000;
|
|
28
42
|
|
|
29
43
|
/**
|
|
30
44
|
* Built-in durable campaign / broadcast task (Loops "campaign" parity). Sends a
|
|
@@ -84,10 +98,39 @@ export const sendCampaignTask = hatchet.task({
|
|
|
84
98
|
return { status: campaign.status, skipped: true };
|
|
85
99
|
}
|
|
86
100
|
|
|
87
|
-
|
|
101
|
+
// Early-fire guard: a `scheduled` campaign whose instant is still in the
|
|
102
|
+
// future must not send. This is a stale punctual run — the `sendAt` was
|
|
103
|
+
// moved later after this run was scheduled. The run for the new instant
|
|
104
|
+
// (or the reaper's due-scheduled sweep) delivers it.
|
|
105
|
+
if (
|
|
106
|
+
campaign.status === "scheduled" &&
|
|
107
|
+
campaign.scheduledAt &&
|
|
108
|
+
campaign.scheduledAt.getTime() > Date.now() + EARLY_FIRE_TOLERANCE_MS
|
|
109
|
+
) {
|
|
110
|
+
return { status: "scheduled", skipped: true, reason: "not_due" as const };
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// Claim via CAS: only a non-terminal row transitions to `sending`, so a
|
|
114
|
+
// cancel landing between the guard read above and this write is honored
|
|
115
|
+
// rather than silently resurrected.
|
|
116
|
+
const claimed = await db
|
|
88
117
|
.update(campaigns)
|
|
89
118
|
.set({ status: "sending", startedAt: new Date(), updatedAt: new Date() })
|
|
90
|
-
.where(
|
|
119
|
+
.where(
|
|
120
|
+
and(
|
|
121
|
+
eq(campaigns.id, input.campaignId),
|
|
122
|
+
inArray(campaigns.status, ["queued", "scheduled", "sending"]),
|
|
123
|
+
),
|
|
124
|
+
)
|
|
125
|
+
.returning({ id: campaigns.id });
|
|
126
|
+
if (claimed.length === 0) {
|
|
127
|
+
const current = await db
|
|
128
|
+
.select({ status: campaigns.status })
|
|
129
|
+
.from(campaigns)
|
|
130
|
+
.where(eq(campaigns.id, input.campaignId))
|
|
131
|
+
.limit(1);
|
|
132
|
+
return { status: current[0]?.status ?? "unknown", skipped: true };
|
|
133
|
+
}
|
|
91
134
|
|
|
92
135
|
let sentCount = 0;
|
|
93
136
|
let skippedCount = 0;
|
|
@@ -113,16 +156,33 @@ export const sendCampaignTask = hatchet.task({
|
|
|
113
156
|
? resolveBucketRecipients(db, campaign.audienceId)
|
|
114
157
|
: resolveListRecipients(db, campaign.audienceId);
|
|
115
158
|
|
|
159
|
+
// Mid-flight cancel: re-checked once per chunk (one cheap SELECT per
|
|
160
|
+
// CHUNK_SIZE sends) so `POST /v1/campaigns/:id/cancel` can stop a blast
|
|
161
|
+
// that is already `sending` — recipients not yet dispatched are spared.
|
|
162
|
+
let cancelDetected = false;
|
|
163
|
+
|
|
116
164
|
let chunk: CampaignRecipient[] = [];
|
|
117
165
|
for await (const recipient of recipients) {
|
|
118
166
|
chunk.push(recipient);
|
|
119
167
|
if (chunk.length < CHUNK_SIZE) continue;
|
|
120
168
|
await sendChunk();
|
|
169
|
+
if (cancelDetected) break;
|
|
121
170
|
}
|
|
122
171
|
// Final partial chunk.
|
|
123
|
-
if (chunk.length > 0) await sendChunk();
|
|
172
|
+
if (chunk.length > 0 && !cancelDetected) await sendChunk();
|
|
124
173
|
|
|
125
174
|
async function sendChunk(): Promise<void> {
|
|
175
|
+
const statusRows = await db
|
|
176
|
+
.select({ status: campaigns.status })
|
|
177
|
+
.from(campaigns)
|
|
178
|
+
.where(eq(campaigns.id, input.campaignId))
|
|
179
|
+
.limit(1);
|
|
180
|
+
if (statusRows[0]?.status === "canceled") {
|
|
181
|
+
cancelDetected = true;
|
|
182
|
+
chunk = [];
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
|
|
126
186
|
const batch = chunk;
|
|
127
187
|
chunk = [];
|
|
128
188
|
totalRecipients += batch.length;
|
|
@@ -174,7 +234,30 @@ export const sendCampaignTask = hatchet.task({
|
|
|
174
234
|
await flushCounts();
|
|
175
235
|
}
|
|
176
236
|
|
|
177
|
-
|
|
237
|
+
if (cancelDetected) {
|
|
238
|
+
// The operator's cancel already stamped status/canceledAt; persist the
|
|
239
|
+
// progress counts so the row shows how far the blast got.
|
|
240
|
+
await flushCounts();
|
|
241
|
+
logger.info("send-campaign: canceled mid-send", {
|
|
242
|
+
campaignId: input.campaignId,
|
|
243
|
+
totalRecipients,
|
|
244
|
+
sentCount,
|
|
245
|
+
skippedCount,
|
|
246
|
+
failedCount,
|
|
247
|
+
});
|
|
248
|
+
return {
|
|
249
|
+
status: "canceled" as const,
|
|
250
|
+
totalRecipients,
|
|
251
|
+
sentCount,
|
|
252
|
+
skippedCount,
|
|
253
|
+
failedCount,
|
|
254
|
+
};
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
// Completion CAS: only a still-`sending` row is stamped `sent`, so a
|
|
258
|
+
// cancel racing the final chunk (after its last per-chunk check) keeps
|
|
259
|
+
// its `canceled` status instead of being overwritten.
|
|
260
|
+
const completed = await db
|
|
178
261
|
.update(campaigns)
|
|
179
262
|
.set({
|
|
180
263
|
status: "sent",
|
|
@@ -185,7 +268,33 @@ export const sendCampaignTask = hatchet.task({
|
|
|
185
268
|
failedCount,
|
|
186
269
|
updatedAt: new Date(),
|
|
187
270
|
})
|
|
188
|
-
.where(
|
|
271
|
+
.where(
|
|
272
|
+
and(
|
|
273
|
+
eq(campaigns.id, input.campaignId),
|
|
274
|
+
eq(campaigns.status, "sending"),
|
|
275
|
+
),
|
|
276
|
+
)
|
|
277
|
+
.returning({ id: campaigns.id });
|
|
278
|
+
if (completed.length === 0) {
|
|
279
|
+
await flushCounts();
|
|
280
|
+
const current = await db
|
|
281
|
+
.select({ status: campaigns.status })
|
|
282
|
+
.from(campaigns)
|
|
283
|
+
.where(eq(campaigns.id, input.campaignId))
|
|
284
|
+
.limit(1);
|
|
285
|
+
const status = current[0]?.status ?? "unknown";
|
|
286
|
+
logger.info("send-campaign: finished but row left `sending` first", {
|
|
287
|
+
campaignId: input.campaignId,
|
|
288
|
+
status,
|
|
289
|
+
});
|
|
290
|
+
return {
|
|
291
|
+
status,
|
|
292
|
+
totalRecipients,
|
|
293
|
+
sentCount,
|
|
294
|
+
skippedCount,
|
|
295
|
+
failedCount,
|
|
296
|
+
};
|
|
297
|
+
}
|
|
189
298
|
|
|
190
299
|
logger.info("send-campaign: complete", {
|
|
191
300
|
campaignId: input.campaignId,
|
|
@@ -252,6 +361,17 @@ const GIVE_UP_AFTER_MS = Number(
|
|
|
252
361
|
process.env.CAMPAIGN_GIVE_UP_AFTER_MS ?? 6 * 60 * 60 * 1000,
|
|
253
362
|
);
|
|
254
363
|
|
|
364
|
+
/**
|
|
365
|
+
* How far past `scheduledAt` a `scheduled` campaign may sit before the reaper
|
|
366
|
+
* promotes it (enqueues the send task). The punctual Hatchet scheduled run
|
|
367
|
+
* created at schedule time is the primary trigger; this grace keeps the sweep
|
|
368
|
+
* from routinely double-firing right at the boundary. A double fire is
|
|
369
|
+
* harmless anyway (terminal guard + per-send idempotency).
|
|
370
|
+
*/
|
|
371
|
+
const SCHEDULE_PROMOTE_GRACE_MS = Number(
|
|
372
|
+
process.env.CAMPAIGN_PROMOTE_GRACE_MS ?? 2 * 60 * 1000,
|
|
373
|
+
);
|
|
374
|
+
|
|
255
375
|
/**
|
|
256
376
|
* Engine-owned reaper cron for campaigns left in a non-terminal in-flight state
|
|
257
377
|
* with no live run to finish them (closes the "stuck forever" gap):
|
|
@@ -261,6 +381,12 @@ const GIVE_UP_AFTER_MS = Number(
|
|
|
261
381
|
* catch never ran, so the row is stuck `sending` with no live run.
|
|
262
382
|
* - A `queued` campaign whose enqueue threw at create time (broker down /
|
|
263
383
|
* network) — the row was committed but no run was ever created (orphan).
|
|
384
|
+
* - A `scheduled` campaign whose punctual Hatchet scheduled run never fired
|
|
385
|
+
* (schedule-create failed at POST/reconcile time, or the run was lost) —
|
|
386
|
+
* promoted once `scheduledAt` is {@link SCHEDULE_PROMOTE_GRACE_MS} past
|
|
387
|
+
* due. A scheduled row stuck past the give-up window (measured from
|
|
388
|
+
* `scheduledAt`, NOT `updatedAt` — a row is legitimately idle between
|
|
389
|
+
* create and send time) is declared `failed`.
|
|
264
390
|
*
|
|
265
391
|
* Recovery is a simple RE-ENQUEUE of `sendCampaignTask` (safe: the per-send
|
|
266
392
|
* idempotency key no-ops already-sent recipients and the re-run completes the
|
|
@@ -286,14 +412,22 @@ export const reapStuckCampaignsTask = hatchet.task({
|
|
|
286
412
|
const giveUpBefore = new Date(now - GIVE_UP_AFTER_MS);
|
|
287
413
|
|
|
288
414
|
// (1) Declare poison campaigns `failed` first (stuck past the give-up
|
|
289
|
-
// window), so they are not re-enqueued below.
|
|
415
|
+
// window), so they are not re-enqueued below. In-flight rows are measured
|
|
416
|
+
// from `updatedAt` (bumped on every progress flush); `scheduled` rows from
|
|
417
|
+
// `scheduledAt` (their `updatedAt` is legitimately old while they wait).
|
|
290
418
|
const failedRows = await db
|
|
291
419
|
.update(campaigns)
|
|
292
420
|
.set({ status: "failed", completedAt: new Date(), updatedAt: new Date() })
|
|
293
421
|
.where(
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
422
|
+
or(
|
|
423
|
+
and(
|
|
424
|
+
inArray(campaigns.status, ["queued", "sending"]),
|
|
425
|
+
lt(campaigns.updatedAt, giveUpBefore),
|
|
426
|
+
),
|
|
427
|
+
and(
|
|
428
|
+
eq(campaigns.status, "scheduled"),
|
|
429
|
+
lt(campaigns.scheduledAt, giveUpBefore),
|
|
430
|
+
),
|
|
297
431
|
),
|
|
298
432
|
)
|
|
299
433
|
.returning({ id: campaigns.id });
|
|
@@ -313,9 +447,25 @@ export const reapStuckCampaignsTask = hatchet.task({
|
|
|
313
447
|
)
|
|
314
448
|
.returning({ id: campaigns.id });
|
|
315
449
|
|
|
316
|
-
|
|
450
|
+
// (3) Promote due `scheduled` campaigns whose punctual scheduled run never
|
|
451
|
+
// fired. Enqueue-only — the send task owns the scheduled→sending
|
|
452
|
+
// transition, so a failed enqueue simply retries next sweep.
|
|
453
|
+
const dueRows = await db
|
|
454
|
+
.select({ id: campaigns.id })
|
|
455
|
+
.from(campaigns)
|
|
456
|
+
.where(
|
|
457
|
+
and(
|
|
458
|
+
eq(campaigns.status, "scheduled"),
|
|
459
|
+
lte(campaigns.scheduledAt, new Date(now - SCHEDULE_PROMOTE_GRACE_MS)),
|
|
460
|
+
),
|
|
461
|
+
);
|
|
462
|
+
|
|
463
|
+
// Enqueue WITHOUT waiting for results — `.run()` blocks until the whole
|
|
464
|
+
// blast completes, which would serialize entire sends inside this cron's
|
|
465
|
+
// 120s executionTimeout.
|
|
466
|
+
for (const row of [...staleRows, ...dueRows]) {
|
|
317
467
|
try {
|
|
318
|
-
await sendCampaignTask.
|
|
468
|
+
await sendCampaignTask.runNoWait({ campaignId: row.id });
|
|
319
469
|
} catch (err) {
|
|
320
470
|
logger.warn("reap-stuck-campaigns: re-enqueue failed", {
|
|
321
471
|
campaignId: row.id,
|
|
@@ -324,16 +474,18 @@ export const reapStuckCampaignsTask = hatchet.task({
|
|
|
324
474
|
}
|
|
325
475
|
}
|
|
326
476
|
|
|
327
|
-
if (failedRows.length > 0 || staleRows.length > 0) {
|
|
477
|
+
if (failedRows.length > 0 || staleRows.length > 0 || dueRows.length > 0) {
|
|
328
478
|
logger.info("reap-stuck-campaigns: swept", {
|
|
329
479
|
failed: failedRows.length,
|
|
330
480
|
reEnqueued: staleRows.length,
|
|
481
|
+
promoted: dueRows.length,
|
|
331
482
|
});
|
|
332
483
|
}
|
|
333
484
|
|
|
334
485
|
return {
|
|
335
486
|
failed: failedRows.length,
|
|
336
487
|
reEnqueued: staleRows.length,
|
|
488
|
+
promoted: dueRows.length,
|
|
337
489
|
};
|
|
338
490
|
},
|
|
339
491
|
});
|