@hogsend/engine 0.39.0 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/package.json +9 -7
  2. package/src/buckets/check-membership.ts +6 -1
  3. package/src/campaigns/cohort-sql.ts +198 -0
  4. package/src/campaigns/define-campaign.ts +301 -0
  5. package/src/campaigns/reconcile.ts +391 -0
  6. package/src/campaigns/steps.ts +48 -0
  7. package/src/connectors/define-action.ts +70 -0
  8. package/src/container.ts +144 -42
  9. package/src/destinations/presets/posthog.ts +7 -1
  10. package/src/env.ts +14 -0
  11. package/src/index.ts +98 -3
  12. package/src/journeys/constants.ts +10 -0
  13. package/src/journeys/define-journey.ts +14 -405
  14. package/src/journeys/execute-journey-run.ts +608 -0
  15. package/src/journeys/graph/build-graph.ts +212 -56
  16. package/src/journeys/journey-boundary.ts +54 -0
  17. package/src/journeys/journey-context.ts +374 -59
  18. package/src/journeys/record-once.ts +110 -0
  19. package/src/lib/auth.ts +33 -1
  20. package/src/lib/blueprints.ts +576 -0
  21. package/src/lib/campaign-send-key.ts +58 -0
  22. package/src/lib/connector-actions.ts +203 -19
  23. package/src/lib/contacts.ts +54 -0
  24. package/src/lib/email-service-types.ts +4 -1
  25. package/src/lib/email.ts +13 -2
  26. package/src/lib/feed.ts +77 -57
  27. package/src/lib/frequency-cap.ts +45 -15
  28. package/src/lib/hatchet-duration.ts +22 -0
  29. package/src/lib/html.ts +1 -0
  30. package/src/lib/ingestion.ts +145 -4
  31. package/src/lib/journey-suppress.ts +87 -0
  32. package/src/lib/links.ts +204 -18
  33. package/src/lib/outbound.ts +20 -0
  34. package/src/lib/preferences.ts +80 -1
  35. package/src/lib/tracked.ts +44 -29
  36. package/src/lib/tracking-event-names.ts +8 -0
  37. package/src/lib/webhook-signing.ts +6 -1
  38. package/src/lists/channels.ts +75 -0
  39. package/src/lists/define-list.ts +34 -0
  40. package/src/lists/registry.ts +29 -2
  41. package/src/mcp/blueprint-tools.ts +460 -0
  42. package/src/routes/admin/blueprints.ts +711 -0
  43. package/src/routes/admin/campaigns.ts +341 -0
  44. package/src/routes/admin/emails.ts +13 -0
  45. package/src/routes/admin/index.ts +6 -0
  46. package/src/routes/admin/journeys.ts +4 -2
  47. package/src/routes/admin/links.ts +303 -36
  48. package/src/routes/admin/lists.ts +51 -0
  49. package/src/routes/campaigns/index.ts +297 -50
  50. package/src/routes/email/preferences.ts +50 -13
  51. package/src/routes/feed/recipient.ts +2 -43
  52. package/src/routes/index.ts +6 -0
  53. package/src/routes/lists/index.ts +138 -18
  54. package/src/routes/tracking/arrive.ts +260 -0
  55. package/src/routes/tracking/click-pipeline.ts +329 -0
  56. package/src/routes/tracking/click.ts +5 -270
  57. package/src/routes/tracking/index.ts +2 -0
  58. package/src/routes/tracking/vanity.ts +64 -0
  59. package/src/worker.ts +27 -0
  60. package/src/workflows/bucket-backfill.ts +4 -2
  61. package/src/workflows/bucket-reconcile.ts +11 -4
  62. package/src/workflows/check-alerts.ts +130 -1
  63. package/src/workflows/journey-blueprint-interpreter.ts +561 -0
  64. package/src/workflows/send-campaign.ts +673 -127
@@ -1,72 +1,20 @@
1
1
  import { fileURLToPath } from "node:url";
2
- import type { JsonValue } from "@hatchet-dev/typescript-sdk/v1/types.js";
3
- import {
4
- evaluatePropertyConditions,
5
- type JourneySourceLocation,
6
- normalizeWhere,
7
- } from "@hogsend/core";
2
+ import type { JourneySourceLocation } from "@hogsend/core";
3
+ import { normalizeWhere } from "@hogsend/core";
8
4
  import type {
9
5
  JourneyMeta,
10
6
  JourneyMetaInput,
11
7
  JourneyRunFn,
12
- JourneyUser,
13
8
  } from "@hogsend/core/types";
14
- import { contacts, journeyConfigs, journeyStates } from "@hogsend/db";
15
- import { and, eq, inArray, notInArray } from "drizzle-orm";
16
- import { getAnalytics } from "../lib/analytics-singleton.js";
17
- import { getDb } from "../lib/db.js";
18
- import {
19
- checkEmailPreferences,
20
- checkEntryLimit,
21
- } from "../lib/enrollment-guards.js";
22
9
  import { hatchet } from "../lib/hatchet.js";
23
- import { createLogger } from "../lib/logger.js";
24
- import { emitOutbound } from "../lib/outbound.js";
25
- import { resolveTimezoneWithSource } from "../lib/timezone.js";
26
- import { getClientScheduleDefaults } from "./client-defaults-singleton.js";
27
10
  import {
28
11
  JOURNEY_EXECUTION_TIMEOUT,
29
12
  JOURNEY_SCHEDULE_TIMEOUT,
30
13
  } from "./constants.js";
31
- import { JourneyExitedError } from "./errors.js";
32
14
  import {
33
- createMemoize,
34
- type JourneyBoundary,
35
- runWithJourneyBoundary,
36
- supportsEviction,
37
- } from "./journey-boundary.js";
38
- import { createJourneyContext, TERMINAL_STATUSES } from "./journey-context.js";
39
- import { logTransition } from "./journey-log.js";
40
- import { getJourneyRegistrySingleton } from "./registry-singleton.js";
41
-
42
- const logger = createLogger(process.env.LOG_LEVEL);
43
-
44
- /**
45
- * Log whether Hatchet's durable `memo` (Layer-1 fast path) is actually durable
46
- * on this engine — ONCE per worker process. `supportsEviction === false`
47
- * (hatchet-lite < v0.80.0) means `memo` silently no-ops, so Layer 2 (the
48
- * Postgres `email_sends`/`user_events` unique-index dedup) is the sole live
49
- * guarantee. Surfaced at boot so the team knows which layer is carrying the load.
50
- */
51
- let evictionSupportLogged = false;
52
- function logEvictionSupportOnce(hatchetCtx: unknown): void {
53
- if (evictionSupportLogged) return;
54
- evictionSupportLogged = true;
55
- const live = supportsEviction(hatchetCtx);
56
- logger.info(
57
- live
58
- ? "journey replay-safety: Layer-1 memo fast path is LIVE (engine supports eviction)"
59
- : "journey replay-safety: Layer-1 memo is INERT (engine < v0.80.0); " +
60
- "exactly-once relies on the Layer-2 Postgres unique-index dedup",
61
- );
62
- }
63
-
64
- interface EventPayloadInput {
65
- userId: JsonValue;
66
- userEmail: JsonValue;
67
- properties: JsonValue;
68
- [key: string]: JsonValue;
69
- }
15
+ type EventPayloadInput,
16
+ executeJourneyRun,
17
+ } from "./execute-journey-run.js";
70
18
 
71
19
  export interface DefinedJourney {
72
20
  meta: JourneyMeta;
@@ -215,355 +163,16 @@ export function defineJourney(options: {
215
163
  // a slot instead of being cancelled (which strands the enrollment). Unlike
216
164
  // retries this adds NO replay — it is pure head-room, so it is safe on its own.
217
165
  scheduleTimeout: JOURNEY_SCHEDULE_TIMEOUT,
218
- fn: async (input: EventPayloadInput, hatchetCtx) => {
219
- const db = getDb();
220
- const userId = input.userId as string;
221
- const userEmail = input.userEmail as string;
222
- const properties = (input.properties ?? {}) as Record<
223
- string,
224
- string | number | boolean | null
225
- >;
226
-
227
- // The replay-stable anchor for this enrollment: the Hatchet run id is
228
- // preserved across replays of the SAME logical durable run (crash / OOM /
229
- // eviction / redeploy mid-run), whereas a freshly-minted journeyStates.id
230
- // is not. Both dedup-key derivation AND the enrollment-recovery below key
231
- // on it so a replay re-derives the SAME stateId and the SAME keys.
232
- const workflowRunId = hatchetCtx.workflowRunId();
233
-
234
- // REPLAY RECOVERY — must run BEFORE the enrollment guards below. An
235
- // eviction-capable Hatchet engine (hatchet-lite >= v0.80.0) replays `fn`
236
- // from the top on every durable-wait resume, so every statement above the
237
- // first durable primitive re-runs on each resume. The enrollment guards
238
- // (entryLimit / preferences / trigger conditions / enabled / active-state)
239
- // are ENTRY gates: re-running them on a resume is wrong. In particular a
240
- // `once` journey's own entry-limit guard would find the row it created on
241
- // first entry and skip EVERY resume, stranding the journey in `waiting` and
242
- // silently dropping all sends after the first. Recovering by the run-stable
243
- // id first lets a resume reuse its enrollment and bypass the entry gates (a
244
- // resume is not an entry). It also keeps the original purpose: a journey
245
- // whose prior enrollment reached a TERMINAL status (unlimited /
246
- // once_per_period) recovers the SAME stateId so derived keys still collide.
247
- const recovered = workflowRunId
248
- ? await db.query.journeyStates.findFirst({
249
- where: and(
250
- eq(journeyStates.hatchetRunId, workflowRunId),
251
- eq(journeyStates.journeyId, meta.id),
252
- ),
253
- })
254
- : undefined;
255
-
256
- let state = recovered;
257
-
258
- // FIRST ENTRY ONLY — the enrollment guards gate ENTRY, not resume. A
259
- // recovered enrollment already passed them on first entry, so it skips
260
- // straight to resuming the run. (Sends inside `run` still re-check
261
- // subscription via `ctx.guard.isSubscribed()` after every wait, so
262
- // bypassing the entry-time preference gate never emails an unsubscriber.)
263
- if (!state) {
264
- if (!meta.enabled) {
265
- return { status: "skipped", reason: "journey_disabled" };
266
- }
267
-
268
- const configOverride = await db.query.journeyConfigs.findFirst({
269
- where: eq(journeyConfigs.journeyId, meta.id),
270
- });
271
- if (configOverride && !configOverride.enabled) {
272
- return { status: "skipped", reason: "journey_disabled_by_admin" };
273
- }
274
-
275
- if (meta.trigger.where?.length) {
276
- if (
277
- !evaluatePropertyConditions({
278
- conditions: meta.trigger.where,
279
- properties,
280
- })
281
- ) {
282
- return { status: "skipped", reason: "trigger_conditions_not_met" };
283
- }
284
- }
285
-
286
- const entryAllowed = await checkEntryLimit({
287
- db,
288
- journey: meta,
289
- userId,
290
- });
291
- if (!entryAllowed.allowed) {
292
- return { status: "skipped", reason: entryAllowed.reason };
293
- }
294
-
295
- const prefs = await checkEmailPreferences({ db, userId });
296
- if (prefs.unsubscribed) {
297
- return { status: "skipped", reason: "user_unsubscribed" };
298
- }
299
-
300
- const activeState = await db.query.journeyStates.findFirst({
301
- where: and(
302
- eq(journeyStates.userId, userId),
303
- eq(journeyStates.journeyId, meta.id),
304
- inArray(journeyStates.status, ["active", "waiting"]),
305
- ),
306
- });
307
- if (activeState) {
308
- return { status: "skipped", reason: "already_active" };
309
- }
310
-
311
- [state] = await db
312
- .insert(journeyStates)
313
- .values({
314
- userId,
315
- userEmail,
316
- journeyId: meta.id,
317
- currentNodeId: "start",
318
- status: "active",
319
- context: properties,
320
- hatchetRunId: workflowRunId,
321
- })
322
- .returning();
323
- }
324
-
325
- if (!state) {
326
- return { status: "skipped", reason: "state_creation_failed" };
327
- }
328
-
329
- const stateId = state.id;
330
-
331
- // Fire-and-forget enrollment transition (FRESH entry only — a replay
332
- // recovers `state` by run id above and must NOT re-log an entry). Writes
333
- // `journey_logs` best-effort; never throws into / alters execution.
334
- if (!recovered) {
335
- logTransition({
336
- db,
337
- journeyStateId: stateId,
338
- from: null,
339
- to: "start",
340
- action: "entered",
341
- });
342
- }
343
-
344
- // The replay-stable key anchor: prefer the run id (constant across replays
345
- // of this run), fall back to stateId when the engine has no run id.
346
- const runAnchor = workflowRunId || stateId;
347
-
348
- const user: JourneyUser = {
349
- id: userId,
350
- email: userEmail,
351
- properties,
352
- stateId,
353
- journeyId: meta.id,
354
- journeyName: meta.name,
355
- };
356
-
357
- // The injected analytics instance (set by createHogsendClient). Same
358
- // object as container.analytics; undefined when POSTHOG_API_KEY is unset.
359
- const posthog = getAnalytics();
360
- const scheduleDefaults = getClientScheduleDefaults();
361
-
362
- // Resolve the user's timezone via the precedence chain (explicit is N/A
363
- // at enrollment; PostHog person props → contacts row → client default →
364
- // UTC). Best-effort: failures fall through to the client default tz.
365
- // Independent I/O — fetch the contact row and PostHog person props
366
- // concurrently. PostHog failures fall through to undefined.
367
- const [contact, posthogProperties] = await Promise.all([
368
- // Both legs are best-effort tz inputs: a failure here must fall through
369
- // to the client-default tz, never reject the `Promise.all` and throw out
370
- // of `fn` BEFORE the try/catch below (which would strand the row —
371
- // already inserted "active" — outside the failure handling). The PostHog
372
- // leg already `.catch`es; mirror it on the contact read.
373
- db.query.contacts
374
- .findFirst({ where: eq(contacts.externalId, userId) })
375
- .catch(() => undefined),
376
- posthog?.getPersonProperties(userId).catch(() => undefined),
377
- ]);
378
-
379
- const tz = resolveTimezoneWithSource({
380
- posthogProperties,
381
- contactTimezone: contact?.timezone ?? null,
382
- contactProperties: contact?.properties ?? null,
383
- defaultTimezone: scheduleDefaults.timezone,
384
- logger,
385
- });
386
-
387
- // Opportunistic cache write: when the tz came from a PostHog source and
388
- // the contacts.timezone column is empty, persist it (fire-and-forget;
389
- // PostHog/JSONB remain authoritative so nothing blocks on the column).
390
- if (
391
- (tz.source === "posthog_timezone" || tz.source === "posthog_geoip") &&
392
- contact &&
393
- !contact.timezone
394
- ) {
395
- db.update(contacts)
396
- .set({ timezone: tz.timezone, updatedAt: new Date() })
397
- .where(eq(contacts.id, contact.id))
398
- .catch(() => {
399
- // best-effort cache write; never block the journey
400
- });
401
- }
402
-
403
- const ctx = createJourneyContext({
404
- db,
405
- hatchet,
406
- // The real DurableContext structurally satisfies the (widened) config —
407
- // it exposes `sleepFor`/`waitFor` AND a replay-memoized `now()` (SDK
408
- // 1.22.3). Forwarding it whole lets ctx clock reads (ctx.when,
409
- // sleepUntil delta, lookback, ctx.now) read the memoized clock on an
410
- // eviction-capable engine, falling back to the live clock pre-eviction.
166
+ // The full enrollment + run lifecycle lives in `executeJourneyRun`, shared
167
+ // verbatim with the blueprint interpreter task (spec §6) — code journeys
168
+ // and blueprints execute through the IDENTICAL machinery.
169
+ fn: async (input: EventPayloadInput, hatchetCtx) =>
170
+ executeJourneyRun({
171
+ meta,
172
+ run: options.run,
173
+ input,
411
174
  hatchetCtx,
412
- registry: getJourneyRegistrySingleton(),
413
- logger,
414
- stateId,
415
- userId,
416
- userEmail,
417
- journeyContext: { ...properties },
418
- resolvedTimezone: tz.timezone,
419
- defaultSendWindow: scheduleDefaults.sendWindow,
420
- });
421
-
422
- // The journey boundary makes journey side effects (sendEmail, ctx.trigger)
423
- // EXACTLY-ONCE across a durable replay WITHOUT any change to journey
424
- // authoring. It is established once around `run()`: a replay-from-top
425
- // re-enters this scope from the top. Its `runAnchor` is the replay-stable
426
- // Hatchet run id (recovered above so the SAME stateId is reused on replay),
427
- // so the derived keys collide across a replay of the same run even if the
428
- // enrollment row had to be recovered rather than freshly inserted.
429
- // `memoize` is the Layer-1 fast path (durable only when the engine supports
430
- // eviction); the auto-derived `email_sends`/`user_events` idempotency keys
431
- // (anchored on `runAnchor`) are the version-independent Layer-2 guarantee.
432
- logEvictionSupportOnce(hatchetCtx);
433
- const boundary: JourneyBoundary = {
434
- stateId,
435
- runAnchor,
436
- currentLabel: undefined,
437
- seenKeys: new Set<string>(),
438
- memoize: createMemoize(hatchetCtx),
439
- };
440
-
441
- // Seed the context's memoized-clock snapshot ONCE before run() so a
442
- // `ctx.when` chain used BEFORE the first durable step reads a replay-stable
443
- // instant (on an eviction engine) instead of the construction-time
444
- // `new Date()` seed. Best-effort: on a pre-eviction engine this reads the
445
- // live clock, same as before.
446
- await ctx.now();
447
-
448
- try {
449
- await runWithJourneyBoundary(boundary, () => options.run(user, ctx));
450
-
451
- const completedAt = new Date();
452
- await db
453
- .update(journeyStates)
454
- .set({
455
- status: "completed",
456
- completedAt,
457
- updatedAt: completedAt,
458
- })
459
- .where(eq(journeyStates.id, stateId));
460
-
461
- // Fire-and-forget completion transition (best-effort; never throws).
462
- logTransition({
463
- db,
464
- journeyStateId: stateId,
465
- from: boundary.currentLabel ?? null,
466
- to: "end-completed",
467
- action: "completed",
468
- });
469
-
470
- await hatchet.events.push("journey:completed", {
471
- journeyId: meta.id,
472
- stateId,
473
- userId,
474
- });
475
-
476
- // OUTBOUND `journey.completed` — fired alongside the internal
477
- // `journey:completed` push. Runs in the WORKER (this durable task), so it
478
- // uses the engine `db`/`hatchet`/`logger` singletons. `dedupeKey` =
479
- // `journey.completed:<stateId>`: a Hatchet re-execution recomputes the
480
- // identical key and the unique `(endpointId, dedupeKey)` index absorbs the
481
- // duplicate (risk 3). `journey:failed` is NOT in the catalog → no emit.
482
- void emitOutbound({
483
- db,
484
- hatchet,
485
- logger,
486
- event: "journey.completed",
487
- dedupeKey: `journey.completed:${stateId}`,
488
- payload: {
489
- journeyId: meta.id,
490
- journeyName: meta.name,
491
- stateId,
492
- userId,
493
- userEmail,
494
- completedAt: completedAt.toISOString(),
495
- },
496
- }).catch(logger.warn);
497
-
498
- return { stateId, status: "completed" };
499
- } catch (err) {
500
- // The journey reached a terminal state (exitOn / cancel) while suspended
501
- // in a durable wait. The state row is already terminal — stop gracefully
502
- // without marking it "failed" or re-pushing a journey:failed event.
503
- if (err instanceof JourneyExitedError) {
504
- return { stateId, status: "exited" };
505
- }
506
-
507
- const message =
508
- err instanceof Error ? err.message : "Unknown error during journey";
509
-
510
- // Mark "failed" ONLY if the row isn't already terminal. A run cancelled
511
- // by exitOn (ingestEvent sets "exited" then `runs.cancel`) or by the
512
- // admin route surfaces here as a Hatchet AbortError thrown from the
513
- // suspended waitFor/sleepFor — NOT a JourneyExitedError. Guarding on a
514
- // non-terminal status prevents clobbering that "exited" row to "failed"
515
- // and emitting a spurious journey:failed event.
516
- const [failed] = await db
517
- .update(journeyStates)
518
- .set({
519
- status: "failed",
520
- errorMessage: message,
521
- updatedAt: new Date(),
522
- })
523
- .where(
524
- and(
525
- eq(journeyStates.id, stateId),
526
- notInArray(journeyStates.status, [...TERMINAL_STATUSES]),
527
- ),
528
- )
529
- .returning({ id: journeyStates.id });
530
-
531
- if (!failed) {
532
- // Already terminal (cancelled after exit) — swallow the cancellation
533
- // so the run doesn't double-report as failed.
534
- return { stateId, status: "exited" };
535
- }
536
-
537
- logger.error("Journey run failed", {
538
- journeyId: meta.id,
539
- journeyName: meta.name,
540
- stateId,
541
- userId,
542
- error: message,
543
- });
544
-
545
- // Fire-and-forget failure transition — only reached when a row was
546
- // actually flipped to "failed" (the guard above). `to` is the best-effort
547
- // last durable node (approximate — see plan Guardrails). Never throws.
548
- logTransition({
549
- db,
550
- journeyStateId: stateId,
551
- from: null,
552
- to: boundary.currentLabel ?? state.currentNodeId ?? null,
553
- action: "failed",
554
- detail: { error: message },
555
- });
556
-
557
- await hatchet.events.push("journey:failed", {
558
- journeyId: meta.id,
559
- stateId,
560
- userId,
561
- error: message,
562
- });
563
-
564
- throw err;
565
- }
566
- },
175
+ }),
567
176
  });
568
177
 
569
178
  return { meta, task, runSource, source };