@hogsend/engine 0.40.0 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/package.json +9 -7
  2. package/src/buckets/check-membership.ts +6 -1
  3. package/src/campaigns/cohort-sql.ts +198 -0
  4. package/src/campaigns/define-campaign.ts +207 -28
  5. package/src/campaigns/reconcile.ts +105 -11
  6. package/src/campaigns/steps.ts +48 -0
  7. package/src/connectors/define-action.ts +70 -0
  8. package/src/container.ts +124 -42
  9. package/src/destinations/presets/posthog.ts +7 -1
  10. package/src/index.ts +82 -2
  11. package/src/journeys/constants.ts +10 -0
  12. package/src/journeys/define-journey.ts +14 -405
  13. package/src/journeys/execute-journey-run.ts +608 -0
  14. package/src/journeys/graph/build-graph.ts +212 -56
  15. package/src/journeys/journey-boundary.ts +54 -0
  16. package/src/journeys/journey-context.ts +374 -59
  17. package/src/journeys/record-once.ts +110 -0
  18. package/src/lib/blueprints.ts +576 -0
  19. package/src/lib/campaign-send-key.ts +58 -0
  20. package/src/lib/connector-actions.ts +203 -19
  21. package/src/lib/contacts.ts +54 -0
  22. package/src/lib/email-service-types.ts +4 -1
  23. package/src/lib/email.ts +13 -2
  24. package/src/lib/feed.ts +77 -57
  25. package/src/lib/frequency-cap.ts +45 -15
  26. package/src/lib/hatchet-duration.ts +22 -0
  27. package/src/lib/html.ts +1 -0
  28. package/src/lib/ingestion.ts +145 -4
  29. package/src/lib/journey-suppress.ts +87 -0
  30. package/src/lib/links.ts +204 -18
  31. package/src/lib/outbound.ts +20 -0
  32. package/src/lib/preferences.ts +80 -1
  33. package/src/lib/tracked.ts +44 -29
  34. package/src/lib/tracking-event-names.ts +8 -0
  35. package/src/lib/webhook-signing.ts +6 -1
  36. package/src/lists/channels.ts +75 -0
  37. package/src/lists/define-list.ts +34 -0
  38. package/src/lists/registry.ts +29 -2
  39. package/src/mcp/blueprint-tools.ts +460 -0
  40. package/src/routes/admin/blueprints.ts +711 -0
  41. package/src/routes/admin/campaigns.ts +179 -3
  42. package/src/routes/admin/emails.ts +13 -0
  43. package/src/routes/admin/index.ts +4 -0
  44. package/src/routes/admin/journeys.ts +4 -2
  45. package/src/routes/admin/links.ts +303 -36
  46. package/src/routes/admin/lists.ts +51 -0
  47. package/src/routes/campaigns/index.ts +44 -6
  48. package/src/routes/email/preferences.ts +50 -13
  49. package/src/routes/feed/recipient.ts +2 -43
  50. package/src/routes/index.ts +6 -0
  51. package/src/routes/lists/index.ts +138 -18
  52. package/src/routes/tracking/arrive.ts +260 -0
  53. package/src/routes/tracking/click-pipeline.ts +329 -0
  54. package/src/routes/tracking/click.ts +5 -270
  55. package/src/routes/tracking/index.ts +2 -0
  56. package/src/routes/tracking/vanity.ts +64 -0
  57. package/src/worker.ts +5 -0
  58. package/src/workflows/bucket-backfill.ts +4 -2
  59. package/src/workflows/bucket-reconcile.ts +11 -4
  60. package/src/workflows/check-alerts.ts +130 -1
  61. package/src/workflows/journey-blueprint-interpreter.ts +561 -0
  62. package/src/workflows/send-campaign.ts +574 -180
@@ -0,0 +1,608 @@
1
+ import type { Conditions } from "@hatchet-dev/typescript-sdk/v1/index.js";
2
+ import type { JsonValue } from "@hatchet-dev/typescript-sdk/v1/types.js";
3
+ import {
4
+ type DurationObject,
5
+ durationToMs,
6
+ evaluatePropertyConditions,
7
+ } from "@hogsend/core";
8
+ import type {
9
+ JourneyMeta,
10
+ JourneyRunFn,
11
+ JourneyUser,
12
+ } from "@hogsend/core/types";
13
+ import {
14
+ contacts,
15
+ type Database,
16
+ journeyConfigs,
17
+ journeyStates,
18
+ } from "@hogsend/db";
19
+ import { and, eq, inArray, notInArray, sql } from "drizzle-orm";
20
+ import { getAnalytics } from "../lib/analytics-singleton.js";
21
+ import { getDb } from "../lib/db.js";
22
+ import {
23
+ checkEmailPreferences,
24
+ checkEntryLimit,
25
+ } from "../lib/enrollment-guards.js";
26
+ import { hatchet } from "../lib/hatchet.js";
27
+ import { createLogger } from "../lib/logger.js";
28
+ import { emitOutbound } from "../lib/outbound.js";
29
+ import { resolveTimezoneWithSource } from "../lib/timezone.js";
30
+ import { getClientScheduleDefaults } from "./client-defaults-singleton.js";
31
+ import { JourneyExitedError } from "./errors.js";
32
+ import {
33
+ createMemoize,
34
+ type JourneyBoundary,
35
+ runWithJourneyBoundary,
36
+ supportsEviction,
37
+ } from "./journey-boundary.js";
38
+ import { createJourneyContext, TERMINAL_STATUSES } from "./journey-context.js";
39
+ import { logTransition } from "./journey-log.js";
40
+ import { getJourneyRegistrySingleton } from "./registry-singleton.js";
41
+
42
+ const logger = createLogger(process.env.LOG_LEVEL);
43
+
44
+ /**
45
+ * Log whether Hatchet's durable `memo` (Layer-1 fast path) is actually durable
46
+ * on this engine — ONCE per worker process. `supportsEviction === false`
47
+ * (hatchet-lite < v0.80.0) means `memo` silently no-ops, so Layer 2 (the
48
+ * Postgres `email_sends`/`user_events` unique-index dedup) is the sole live
49
+ * guarantee. Surfaced at boot so the team knows which layer is carrying the load.
50
+ */
51
+ let evictionSupportLogged = false;
52
+ function logEvictionSupportOnce(hatchetCtx: unknown): void {
53
+ if (evictionSupportLogged) return;
54
+ evictionSupportLogged = true;
55
+ const live = supportsEviction(hatchetCtx);
56
+ logger.info(
57
+ live
58
+ ? "journey replay-safety: Layer-1 memo fast path is LIVE (engine supports eviction)"
59
+ : "journey replay-safety: Layer-1 memo is INERT (engine < v0.80.0); " +
60
+ "exactly-once relies on the Layer-2 Postgres unique-index dedup",
61
+ );
62
+ }
63
+
64
+ /** The event payload a journey durable task receives from `hatchet.events.push`. */
65
+ export interface EventPayloadInput {
66
+ userId: JsonValue;
67
+ userEmail: JsonValue;
68
+ properties: JsonValue;
69
+ [key: string]: JsonValue;
70
+ }
71
+
72
+ /**
73
+ * The structural slice of Hatchet's `DurableContext` the journey run lifecycle
74
+ * needs. The real SDK context satisfies it; tests pass a stub. Mirrors the
75
+ * shape `createJourneyContext` documents (sleepFor/waitFor normalization,
76
+ * memoized `now`), plus the run-id anchor and the abort signal the
77
+ * shutdown-release path reads.
78
+ */
79
+ export interface JourneyDurableCtx {
80
+ /** Replay-stable id of THIS logical durable run (see runAnchor below). */
81
+ workflowRunId: () => string;
82
+ sleepFor: (
83
+ duration: DurationObject | number | `${number}s`,
84
+ ) => Promise<unknown>;
85
+ waitFor: (
86
+ conditions: Conditions | Conditions[],
87
+ ) => Promise<Record<string, unknown>>;
88
+ /** Replay-memoized clock (SDK >= 1.22.3); optional so stubs fall back live. */
89
+ now?: () => Promise<Date>;
90
+ /** Aborted when the worker releases the run (graceful shutdown). */
91
+ abortController?: { signal?: { aborted?: boolean } };
92
+ }
93
+
94
+ /** The `journey_states` row an enrollment insert returns. */
95
+ export type JourneyStateRow = typeof journeyStates.$inferSelect;
96
+
97
+ /**
98
+ * Insert the enrollment row, tolerating the partial-unique-index race.
99
+ *
100
+ * The active-state read guard in the run lifecycle (a `findFirst` over the live
101
+ * statuses) is NOT atomic with this insert: two near-simultaneous FIRST events
102
+ * for the same (user, journey) — a burst, which is the digest primitive's whole
103
+ * target workload — can BOTH clear that guard and race here. The loser would
104
+ * otherwise hit the `uq_user_journey_active` partial unique index with a raw
105
+ * 23505 that escapes the task fn (durable tasks run with `retries: 0`) and
106
+ * surfaces as a FAILED Hatchet run. `onConflictDoNothing` against that index
107
+ * absorbs the loser, returning `undefined` (0 rows) — the SAME outcome the read
108
+ * guard produces, so the caller maps it to the `already_active` skip.
109
+ *
110
+ * DRIZZLE GOTCHA (from prior prod debugging in this repo — mirrors
111
+ * campaigns/reconcile.ts): for a PARTIAL unique index the arbiter predicate goes
112
+ * in `where`, NOT `targetWhere`. A `targetWhere` is silently ignored and
113
+ * Postgres then throws 42P10 ("no unique or exclusion constraint matching the ON
114
+ * CONFLICT specification") at runtime. The `where` reproduces the index
115
+ * predicate (`status IN ('active','waiting')`) EXACTLY — see
116
+ * journey-states.ts:uq_user_journey_active.
117
+ */
118
+ export async function insertEnrollment(opts: {
119
+ db: Database;
120
+ userId: string;
121
+ userEmail: string;
122
+ journeyId: string;
123
+ context: Record<string, unknown>;
124
+ hatchetRunId?: string;
125
+ }): Promise<JourneyStateRow | undefined> {
126
+ const [row] = await opts.db
127
+ .insert(journeyStates)
128
+ .values({
129
+ userId: opts.userId,
130
+ userEmail: opts.userEmail,
131
+ journeyId: opts.journeyId,
132
+ currentNodeId: "start",
133
+ status: "active",
134
+ context: opts.context,
135
+ hatchetRunId: opts.hatchetRunId,
136
+ })
137
+ .onConflictDoNothing({
138
+ target: [journeyStates.userId, journeyStates.journeyId],
139
+ where: sql`status IN ('active', 'waiting')`,
140
+ })
141
+ .returning();
142
+ return row;
143
+ }
144
+
145
+ export interface ExecuteJourneyRunOptions {
146
+ /**
147
+ * The journey's resolved meta. For a code journey this is `defineJourney`'s
148
+ * normalized `JourneyMeta`; for a blueprint it is built from the
149
+ * `journey_blueprints` row (same shape — that is the whole point: the
150
+ * enrollment guards below receive it verbatim, spec §6).
151
+ */
152
+ meta: JourneyMeta;
153
+ /** The run body — a code journey's `run` or the blueprint tree-walk. */
154
+ run: JourneyRunFn;
155
+ /** The Hatchet event payload the durable task received. */
156
+ input: EventPayloadInput;
157
+ /** The Hatchet `DurableContext` (or a structural stub in tests). */
158
+ hatchetCtx: JourneyDurableCtx;
159
+ /**
160
+ * Extra keys merged into the enrollment row's `context` jsonb on FIRST entry
161
+ * (a replay recovers the existing row and never re-writes it). The blueprint
162
+ * interpreter pins `__blueprintVersion` here (spec §12).
163
+ */
164
+ extraContext?: Record<string, unknown>;
165
+ /**
166
+ * Serialize a run failure into `journeyStates.errorMessage`. Defaults to the
167
+ * error's message; the blueprint interpreter writes the structured
168
+ * `{ blueprintId, nodeId, message }` JSON instead (spec §6.1).
169
+ */
170
+ serializeError?: (err: unknown) => string;
171
+ }
172
+
173
+ export type ExecuteJourneyRunResult =
174
+ | { status: "skipped"; reason: string }
175
+ | { stateId: string; status: "completed" | "exited" };
176
+
177
+ /**
178
+ * The full journey durable-run lifecycle, shared VERBATIM by `defineJourney`
179
+ * tasks and the blueprint interpreter: replay recovery by run id → enrollment
180
+ * guards (entry only) → enrollment insert → timezone resolve → journey context
181
+ * + replay-safety boundary → `run()` → terminal transitions (completed /
182
+ * exited / released / failed). Extracted from `defineJourney` so the blueprint
183
+ * interpreter executes through the IDENTICAL machinery instead of a parallel
184
+ * reimplementation (spec §6) — behavior for code journeys is unchanged.
185
+ */
186
+ export async function executeJourneyRun(
187
+ options: ExecuteJourneyRunOptions,
188
+ ): Promise<ExecuteJourneyRunResult> {
189
+ const { meta, input, hatchetCtx } = options;
190
+ const db = getDb();
191
+ const userId = input.userId as string;
192
+ const userEmail = input.userEmail as string;
193
+ const properties = (input.properties ?? {}) as Record<
194
+ string,
195
+ string | number | boolean | null
196
+ >;
197
+
198
+ // The replay-stable anchor for this enrollment: the Hatchet run id is
199
+ // preserved across replays of the SAME logical durable run (crash / OOM /
200
+ // eviction / redeploy mid-run), whereas a freshly-minted journeyStates.id
201
+ // is not. Both dedup-key derivation AND the enrollment-recovery below key
202
+ // on it so a replay re-derives the SAME stateId and the SAME keys.
203
+ const workflowRunId = hatchetCtx.workflowRunId();
204
+
205
+ // REPLAY RECOVERY — must run BEFORE the enrollment guards below. An
206
+ // eviction-capable Hatchet engine (hatchet-lite >= v0.80.0) replays the fn
207
+ // from the top on every durable-wait resume, so every statement above the
208
+ // first durable primitive re-runs on each resume. The enrollment guards
209
+ // (entryLimit / preferences / trigger conditions / enabled / active-state)
210
+ // are ENTRY gates: re-running them on a resume is wrong. In particular a
211
+ // `once` journey's own entry-limit guard would find the row it created on
212
+ // first entry and skip EVERY resume, stranding the journey in `waiting` and
213
+ // silently dropping all sends after the first. Recovering by the run-stable
214
+ // id first lets a resume reuse its enrollment and bypass the entry gates (a
215
+ // resume is not an entry). It also keeps the original purpose: a journey
216
+ // whose prior enrollment reached a TERMINAL status (unlimited /
217
+ // once_per_period) recovers the SAME stateId so derived keys still collide.
218
+ const recovered = workflowRunId
219
+ ? await db.query.journeyStates.findFirst({
220
+ where: and(
221
+ eq(journeyStates.hatchetRunId, workflowRunId),
222
+ eq(journeyStates.journeyId, meta.id),
223
+ ),
224
+ })
225
+ : undefined;
226
+
227
+ let state = recovered;
228
+
229
+ // FIRST ENTRY ONLY — the enrollment guards gate ENTRY, not resume. A
230
+ // recovered enrollment already passed them on first entry, so it skips
231
+ // straight to resuming the run. (Sends inside `run` still re-check
232
+ // subscription via `ctx.guard.isSubscribed()` after every wait, so
233
+ // bypassing the entry-time preference gate never emails an unsubscriber.)
234
+ if (!state) {
235
+ if (!meta.enabled) {
236
+ return { status: "skipped", reason: "journey_disabled" };
237
+ }
238
+
239
+ const configOverride = await db.query.journeyConfigs.findFirst({
240
+ where: eq(journeyConfigs.journeyId, meta.id),
241
+ });
242
+ if (configOverride && !configOverride.enabled) {
243
+ return { status: "skipped", reason: "journey_disabled_by_admin" };
244
+ }
245
+
246
+ if (meta.trigger.where?.length) {
247
+ if (
248
+ !evaluatePropertyConditions({
249
+ conditions: meta.trigger.where,
250
+ properties,
251
+ })
252
+ ) {
253
+ return { status: "skipped", reason: "trigger_conditions_not_met" };
254
+ }
255
+ }
256
+
257
+ const entryAllowed = await checkEntryLimit({
258
+ db,
259
+ journey: meta,
260
+ userId,
261
+ });
262
+ if (!entryAllowed.allowed) {
263
+ return {
264
+ status: "skipped",
265
+ reason: entryAllowed.reason ?? "entry_limit",
266
+ };
267
+ }
268
+
269
+ const prefs = await checkEmailPreferences({ db, userId });
270
+ if (prefs.unsubscribed) {
271
+ return { status: "skipped", reason: "user_unsubscribed" };
272
+ }
273
+
274
+ const activeState = await db.query.journeyStates.findFirst({
275
+ where: and(
276
+ eq(journeyStates.userId, userId),
277
+ eq(journeyStates.journeyId, meta.id),
278
+ inArray(journeyStates.status, ["active", "waiting"]),
279
+ ),
280
+ });
281
+ if (activeState) {
282
+ return { status: "skipped", reason: "already_active" };
283
+ }
284
+
285
+ // A successful insert always returns exactly one row, so undefined here
286
+ // can ONLY mean a burst-race conflict (see insertEnrollment's doc for
287
+ // the race + arbiter mechanics) — the same outcome as the read guard,
288
+ // mapped to the identical skip.
289
+ state = await insertEnrollment({
290
+ db,
291
+ userId,
292
+ userEmail,
293
+ journeyId: meta.id,
294
+ context: options.extraContext
295
+ ? { ...properties, ...options.extraContext }
296
+ : properties,
297
+ hatchetRunId: workflowRunId,
298
+ });
299
+ if (!state) {
300
+ return { status: "skipped", reason: "already_active" };
301
+ }
302
+ }
303
+
304
+ const stateId = state.id;
305
+
306
+ // Fire-and-forget enrollment transition (FRESH entry only — a replay
307
+ // recovers `state` by run id above and must NOT re-log an entry). Writes
308
+ // `journey_logs` best-effort; never throws into / alters execution.
309
+ if (!recovered) {
310
+ logTransition({
311
+ db,
312
+ journeyStateId: stateId,
313
+ from: null,
314
+ to: "start",
315
+ action: "entered",
316
+ });
317
+ }
318
+
319
+ // The replay-stable key anchor: prefer the run id (constant across replays
320
+ // of this run), fall back to stateId when the engine has no run id.
321
+ const runAnchor = workflowRunId || stateId;
322
+
323
+ const user: JourneyUser = {
324
+ id: userId,
325
+ email: userEmail,
326
+ properties,
327
+ stateId,
328
+ journeyId: meta.id,
329
+ journeyName: meta.name,
330
+ };
331
+
332
+ // The injected analytics instance (set by createHogsendClient). Same
333
+ // object as container.analytics; undefined when POSTHOG_API_KEY is unset.
334
+ const posthog = getAnalytics();
335
+ const scheduleDefaults = getClientScheduleDefaults();
336
+
337
+ // Resolve the user's timezone via the precedence chain (explicit is N/A
338
+ // at enrollment; PostHog person props → contacts row → client default →
339
+ // UTC). Best-effort: failures fall through to the client default tz.
340
+ // Independent I/O — fetch the contact row and PostHog person props
341
+ // concurrently. PostHog failures fall through to undefined.
342
+ const [contact, posthogProperties] = await Promise.all([
343
+ // Both legs are best-effort tz inputs: a failure here must fall through
344
+ // to the client-default tz, never reject the `Promise.all` and throw out
345
+ // of the fn BEFORE the try/catch below (which would strand the row —
346
+ // already inserted "active" — outside the failure handling). The PostHog
347
+ // leg already `.catch`es; mirror it on the contact read.
348
+ db.query.contacts
349
+ .findFirst({ where: eq(contacts.externalId, userId) })
350
+ .catch(() => undefined),
351
+ posthog?.getPersonProperties(userId).catch(() => undefined),
352
+ ]);
353
+
354
+ const tz = resolveTimezoneWithSource({
355
+ posthogProperties,
356
+ contactTimezone: contact?.timezone ?? null,
357
+ contactProperties: contact?.properties ?? null,
358
+ defaultTimezone: scheduleDefaults.timezone,
359
+ logger,
360
+ });
361
+
362
+ // Opportunistic cache write: when the tz came from a PostHog source and
363
+ // the contacts.timezone column is empty, persist it (fire-and-forget;
364
+ // PostHog/JSONB remain authoritative so nothing blocks on the column).
365
+ if (
366
+ (tz.source === "posthog_timezone" || tz.source === "posthog_geoip") &&
367
+ contact &&
368
+ !contact.timezone
369
+ ) {
370
+ db.update(contacts)
371
+ .set({ timezone: tz.timezone, updatedAt: new Date() })
372
+ .where(eq(contacts.id, contact.id))
373
+ .catch(() => {
374
+ // best-effort cache write; never block the journey
375
+ });
376
+ }
377
+
378
+ const ctx = createJourneyContext({
379
+ db,
380
+ hatchet,
381
+ // The real DurableContext structurally satisfies the (widened) config —
382
+ // it exposes `sleepFor`/`waitFor` AND a replay-memoized `now()` (SDK
383
+ // 1.22.3). Forwarding it whole lets ctx clock reads (ctx.when,
384
+ // sleepUntil delta, lookback, ctx.now) read the memoized clock on an
385
+ // eviction-capable engine, falling back to the live clock pre-eviction.
386
+ hatchetCtx,
387
+ registry: getJourneyRegistrySingleton(),
388
+ logger,
389
+ stateId,
390
+ userId,
391
+ userEmail,
392
+ journeyContext: { ...properties },
393
+ resolvedTimezone: tz.timezone,
394
+ defaultSendWindow: scheduleDefaults.sendWindow,
395
+ // Digest defaults: the trigger event + its already-normalized `where`
396
+ // (so a digest of the journey's own trigger honors the trigger contract
397
+ // without restating it), plus the enrollment shape for the digest's
398
+ // definition-interplay warnings.
399
+ triggerEvent: meta.trigger.event,
400
+ ...(meta.trigger.where ? { triggerWhere: meta.trigger.where } : {}),
401
+ journeyId: meta.id,
402
+ entryLimit: meta.entryLimit,
403
+ ...(meta.entryPeriod ? { entryPeriod: meta.entryPeriod } : {}),
404
+ });
405
+
406
+ // The journey boundary makes journey side effects (sendEmail, ctx.trigger)
407
+ // EXACTLY-ONCE across a durable replay WITHOUT any change to journey
408
+ // authoring. It is established once around `run()`: a replay-from-top
409
+ // re-enters this scope from the top. Its `runAnchor` is the replay-stable
410
+ // Hatchet run id (recovered above so the SAME stateId is reused on replay),
411
+ // so the derived keys collide across a replay of the same run even if the
412
+ // enrollment row had to be recovered rather than freshly inserted.
413
+ // `memoize` is the Layer-1 fast path (durable only when the engine supports
414
+ // eviction); the auto-derived `email_sends`/`user_events` idempotency keys
415
+ // (anchored on `runAnchor`) are the version-independent Layer-2 guarantee.
416
+ logEvictionSupportOnce(hatchetCtx);
417
+ const boundary: JourneyBoundary = {
418
+ stateId,
419
+ runAnchor,
420
+ currentLabel: undefined,
421
+ seenKeys: new Set<string>(),
422
+ seenRecordLabels: new Set<string>(),
423
+ memoize: createMemoize(hatchetCtx),
424
+ journeyId: meta.id,
425
+ // `meta.suppress` is a required DurationObject, but a `{}` / zero
426
+ // duration must yield 0 (disabled); `durationToMs` maps both to 0.
427
+ // Guard against a runtime-absent value so an undefined never reaches
428
+ // durationToMs (which would throw dereferencing `.hours`).
429
+ suppressMs: meta.suppress ? durationToMs(meta.suppress) : 0,
430
+ // The journey-level email-preference category (validated fail-closed at
431
+ // boot). The standalone `sendEmail` reads it off the boundary and stamps
432
+ // it on the send, overriding the template's own category. Undefined for
433
+ // journeys without a `meta.category` (incl. bucket reactions) ⇒ the send
434
+ // keeps the built-in `journey` default.
435
+ category: meta.category,
436
+ };
437
+
438
+ // Seed the context's memoized-clock snapshot ONCE before run() so a
439
+ // `ctx.when` chain used BEFORE the first durable step reads a replay-stable
440
+ // instant (on an eviction engine) instead of the construction-time
441
+ // `new Date()` seed. Best-effort: on a pre-eviction engine this reads the
442
+ // live clock, same as before.
443
+ await ctx.now();
444
+
445
+ try {
446
+ await runWithJourneyBoundary(boundary, () => options.run(user, ctx));
447
+
448
+ const completedAt = new Date();
449
+ await db
450
+ .update(journeyStates)
451
+ .set({
452
+ status: "completed",
453
+ completedAt,
454
+ updatedAt: completedAt,
455
+ })
456
+ .where(eq(journeyStates.id, stateId));
457
+
458
+ // Fire-and-forget completion transition (best-effort; never throws).
459
+ logTransition({
460
+ db,
461
+ journeyStateId: stateId,
462
+ from: boundary.currentLabel ?? null,
463
+ to: "end-completed",
464
+ action: "completed",
465
+ });
466
+
467
+ await hatchet.events.push("journey:completed", {
468
+ journeyId: meta.id,
469
+ stateId,
470
+ userId,
471
+ });
472
+
473
+ // OUTBOUND `journey.completed` — fired alongside the internal
474
+ // `journey:completed` push. Runs in the WORKER (this durable task), so it
475
+ // uses the engine `db`/`hatchet`/`logger` singletons. `dedupeKey` =
476
+ // `journey.completed:<stateId>`: a Hatchet re-execution recomputes the
477
+ // identical key and the unique `(endpointId, dedupeKey)` index absorbs the
478
+ // duplicate (risk 3). `journey:failed` is NOT in the catalog → no emit.
479
+ void emitOutbound({
480
+ db,
481
+ hatchet,
482
+ logger,
483
+ event: "journey.completed",
484
+ dedupeKey: `journey.completed:${stateId}`,
485
+ payload: {
486
+ journeyId: meta.id,
487
+ journeyName: meta.name,
488
+ stateId,
489
+ userId,
490
+ userEmail,
491
+ completedAt: completedAt.toISOString(),
492
+ },
493
+ }).catch(logger.warn);
494
+
495
+ return { stateId, status: "completed" };
496
+ } catch (err) {
497
+ // The journey reached a terminal state (exitOn / cancel) while suspended
498
+ // in a durable wait. The state row is already terminal — stop gracefully
499
+ // without marking it "failed" or re-pushing a journey:failed event.
500
+ if (err instanceof JourneyExitedError) {
501
+ return { stateId, status: "exited" };
502
+ }
503
+
504
+ // Graceful worker shutdown (SIGTERM → worker.stop()) ABORTS in-flight
505
+ // durable runs so Hatchet can REASSIGN them: the suspended sleepFor/
506
+ // waitFor rejects with the SDK's AbortError. That is a RELEASE, not a
507
+ // failure — writing "failed" + pushing journey:failed here permanently
508
+ // POISONS the enrollment (recovery-first later finds a terminal row and
509
+ // never resumes), turning every graceful redeploy mid-wait into enrollment
510
+ // death. Detect the abort by the SDK's error CONTRACT (name/code — the
511
+ // message text varies, so never match on it), with the DurableContext's
512
+ // aborted signal as a belt-and-braces fallback.
513
+ const isAbort =
514
+ (err instanceof Error &&
515
+ (err.name === "AbortError" ||
516
+ (err as { code?: string }).code === "ABORT_ERR")) ||
517
+ hatchetCtx.abortController?.signal?.aborted === true;
518
+
519
+ if (isAbort) {
520
+ // Read the CURRENT status: an exitOn/admin cancel may have flipped the
521
+ // row terminal BEFORE the abort surfaced — keep today's "exited"
522
+ // outcome there. Otherwise the row is still waiting/active (the
523
+ // graceful-shutdown release) — leave it EXACTLY as-is so recovery-first
524
+ // (by hatchetRunId) resumes the recorded window on re-dispatch. The old
525
+ // "failed" write was converting a redeploy into permanent enrollment
526
+ // death; if Hatchet ever fails to re-dispatch, the stranded-waiting
527
+ // alert flags the row — strictly better than a false "failed".
528
+ const current = await db.query.journeyStates.findFirst({
529
+ where: eq(journeyStates.id, stateId),
530
+ columns: { status: true },
531
+ });
532
+ const status = current?.status;
533
+ if (status && (TERMINAL_STATUSES as readonly string[]).includes(status)) {
534
+ return { stateId, status: "exited" };
535
+ }
536
+ logger.info(
537
+ "journey run released mid-wait (worker shutdown/reassignment); " +
538
+ "enrollment left intact for re-dispatch",
539
+ { journeyId: meta.id, stateId, status },
540
+ );
541
+ // Rethrow so the SDK completes its cancellation flow. NO row write, NO
542
+ // journey:failed push, NO "failed" transition.
543
+ throw err;
544
+ }
545
+
546
+ const message = options.serializeError
547
+ ? options.serializeError(err)
548
+ : err instanceof Error
549
+ ? err.message
550
+ : "Unknown error during journey";
551
+
552
+ // Mark "failed" ONLY if the row isn't already terminal. A run cancelled
553
+ // by exitOn (ingestEvent sets "exited" then `runs.cancel`) or by the
554
+ // admin route surfaces here as a Hatchet AbortError thrown from the
555
+ // suspended waitFor/sleepFor — NOT a JourneyExitedError. Guarding on a
556
+ // non-terminal status prevents clobbering that "exited" row to "failed"
557
+ // and emitting a spurious journey:failed event.
558
+ const [failed] = await db
559
+ .update(journeyStates)
560
+ .set({
561
+ status: "failed",
562
+ errorMessage: message,
563
+ updatedAt: new Date(),
564
+ })
565
+ .where(
566
+ and(
567
+ eq(journeyStates.id, stateId),
568
+ notInArray(journeyStates.status, [...TERMINAL_STATUSES]),
569
+ ),
570
+ )
571
+ .returning({ id: journeyStates.id });
572
+
573
+ if (!failed) {
574
+ // Already terminal (cancelled after exit) — swallow the cancellation
575
+ // so the run doesn't double-report as failed.
576
+ return { stateId, status: "exited" };
577
+ }
578
+
579
+ logger.error("Journey run failed", {
580
+ journeyId: meta.id,
581
+ journeyName: meta.name,
582
+ stateId,
583
+ userId,
584
+ error: message,
585
+ });
586
+
587
+ // Fire-and-forget failure transition — only reached when a row was
588
+ // actually flipped to "failed" (the guard above). `to` is the best-effort
589
+ // last durable node (approximate — see plan Guardrails). Never throws.
590
+ logTransition({
591
+ db,
592
+ journeyStateId: stateId,
593
+ from: null,
594
+ to: boundary.currentLabel ?? state.currentNodeId ?? null,
595
+ action: "failed",
596
+ detail: { error: message },
597
+ });
598
+
599
+ await hatchet.events.push("journey:failed", {
600
+ journeyId: meta.id,
601
+ stateId,
602
+ userId,
603
+ error: message,
604
+ });
605
+
606
+ throw err;
607
+ }
608
+ }