@hogsend/engine 0.31.1 → 0.32.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@hogsend/engine",
3
- "version": "0.31.1",
3
+ "version": "0.32.0",
4
4
  "type": "module",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -40,14 +40,14 @@
40
40
  "svix": "^1.95.1",
41
41
  "winston": "^3.19.0",
42
42
  "zod": "^4.4.3",
43
- "@hogsend/core": "^0.31.1",
44
- "@hogsend/db": "^0.31.1",
45
- "@hogsend/email": "^0.31.1",
46
- "@hogsend/plugin-posthog": "^0.31.1",
47
- "@hogsend/plugin-resend": "^0.31.1"
43
+ "@hogsend/core": "^0.32.0",
44
+ "@hogsend/email": "^0.32.0",
45
+ "@hogsend/db": "^0.32.0",
46
+ "@hogsend/plugin-resend": "^0.32.0",
47
+ "@hogsend/plugin-posthog": "^0.32.0"
48
48
  },
49
49
  "optionalDependencies": {
50
- "@hogsend/plugin-postmark": "^0.31.1"
50
+ "@hogsend/plugin-postmark": "^0.32.0"
51
51
  },
52
52
  "devDependencies": {
53
53
  "@types/node": "^22.15.3",
@@ -212,6 +212,7 @@ export function createColdConnect<S = Record<string, unknown>>(
212
212
  userEmail: binding.email,
213
213
  eventProperties,
214
214
  contactProperties,
215
+ source: "connector",
215
216
  idempotencyKey: `cc:confirm:${connectorId}:${binding.platformUserId}:${tok}`,
216
217
  },
217
218
  });
@@ -178,8 +178,18 @@ export interface DefinedConnector<T = unknown> {
178
178
  credential?: StoredCredentialRef;
179
179
  /** Optional Zod schema validating the payload BEFORE transform. */
180
180
  schema?: z.ZodSchema<T>;
181
- /** The transport-invariant heart: raw platform payload → IngestEvent | null. */
182
- transform(payload: T, ctx: ConnectorCtx): Promise<IngestEvent | null>;
181
+ /**
182
+ * The transport-invariant heart: raw platform payload →
183
+ * `IngestEvent | IngestEvent[] | null`. Returning an ARRAY fans one inbound
184
+ * dispatch into multiple subject-keyed events (e.g. a Discord reaction →
185
+ * reactor-keyed `reaction_added` + author-keyed `reaction_received`); each
186
+ * element ingests independently with its own idempotencyKey (see
187
+ * `ingestTransformResult`).
188
+ */
189
+ transform(
190
+ payload: T,
191
+ ctx: ConnectorCtx,
192
+ ): Promise<IngestEvent | IngestEvent[] | null>;
183
193
  /** OPTIONAL OAuth + interactions hooks for the generic dispatch routes. */
184
194
  handlers?: ConnectorHandlers;
185
195
  }
@@ -3,7 +3,7 @@ import {
3
3
  type ConnectorHeartbeatHandle,
4
4
  startConnectorHeartbeat,
5
5
  } from "../lib/connector-heartbeat.js";
6
- import { ingestEvent } from "../lib/ingestion.js";
6
+ import { ingestTransformResult } from "../lib/ingestion.js";
7
7
  import {
8
8
  acquireLeaderLease,
9
9
  newLeaseToken,
@@ -87,21 +87,23 @@ function makeIngest(client: HogsendClient, connector: DefinedConnector) {
87
87
  // (`{ __t, d }`) so the connector transform is byte-identical whether the
88
88
  // dispatch arrived over HTTP or in-process.
89
89
  const payload = { __t: dispatchType, d: data };
90
- const event = await connector.transform(payload, {
90
+ const result = await connector.transform(payload, {
91
91
  db: client.db,
92
92
  logger: client.logger,
93
93
  transport: "gateway",
94
94
  });
95
- if (!event) return { ok: true, status: 200 };
96
- await ingestEvent({
95
+ // A transform may return a single event, an ARRAY (dual-side fan-out), or
96
+ // null — ingestTransformResult normalizes + per-element-isolates. Pass the
97
+ // active analytics provider so a Discord-keyed contact merge stitches the
98
+ // analytics person too (the HTTP route omits this; in-proc holds the
99
+ // container).
100
+ await ingestTransformResult({
101
+ result,
97
102
  db: client.db,
98
103
  registry: client.registry,
99
104
  hatchet: client.hatchet,
100
105
  logger: client.logger,
101
- event,
102
- // Pass the active analytics provider so a Discord-keyed contact merge
103
- // stitches the analytics person too (the HTTP route omits this; in-proc
104
- // can do better since it already holds the container).
106
+ source: "connector",
105
107
  analytics: client.analytics,
106
108
  });
107
109
  return { ok: true, status: 200 };
package/src/index.ts CHANGED
@@ -186,6 +186,14 @@ export {
186
186
  defineJourney,
187
187
  } from "./journeys/define-journey.js";
188
188
  export { JourneyExitedError } from "./journeys/errors.js";
189
+ export {
190
+ createMemoize,
191
+ deriveJourneyKey,
192
+ getJourneyBoundary,
193
+ type JourneyBoundary,
194
+ registerKey,
195
+ runWithJourneyBoundary,
196
+ } from "./journeys/journey-boundary.js";
189
197
  export { createJourneyContext } from "./journeys/journey-context.js";
190
198
  export {
191
199
  buildJourneyRegistry,
@@ -1,12 +1,10 @@
1
1
  import type { JsonValue } from "@hatchet-dev/typescript-sdk/v1/types.js";
2
- import { criteriaBuilder, evaluatePropertyConditions } from "@hogsend/core";
2
+ import { evaluatePropertyConditions, normalizeWhere } from "@hogsend/core";
3
3
  import type {
4
4
  JourneyMeta,
5
5
  JourneyMetaInput,
6
6
  JourneyRunFn,
7
7
  JourneyUser,
8
- JourneyWhere,
9
- PropertyCondition,
10
8
  } from "@hogsend/core/types";
11
9
  import { contacts, journeyConfigs, journeyStates } from "@hogsend/db";
12
10
  import { and, eq, inArray, notInArray } from "drizzle-orm";
@@ -23,11 +21,37 @@ import { resolveTimezoneWithSource } from "../lib/timezone.js";
23
21
  import { getClientScheduleDefaults } from "./client-defaults-singleton.js";
24
22
  import { JOURNEY_EXECUTION_TIMEOUT } from "./constants.js";
25
23
  import { JourneyExitedError } from "./errors.js";
24
+ import {
25
+ createMemoize,
26
+ type JourneyBoundary,
27
+ runWithJourneyBoundary,
28
+ supportsEviction,
29
+ } from "./journey-boundary.js";
26
30
  import { createJourneyContext, TERMINAL_STATUSES } from "./journey-context.js";
27
31
  import { getJourneyRegistrySingleton } from "./registry-singleton.js";
28
32
 
29
33
  const logger = createLogger(process.env.LOG_LEVEL);
30
34
 
35
+ /**
36
+ * Log whether Hatchet's durable `memo` (Layer-1 fast path) is actually durable
37
+ * on this engine — ONCE per worker process. `supportsEviction === false`
38
+ * (hatchet-lite < v0.80.0) means `memo` silently no-ops, so Layer 2 (the
39
+ * Postgres `email_sends`/`user_events` unique-index dedup) is the sole live
40
+ * guarantee. Surfaced at boot so the team knows which layer is carrying the load.
41
+ */
42
+ let evictionSupportLogged = false;
43
+ function logEvictionSupportOnce(hatchetCtx: unknown): void {
44
+ if (evictionSupportLogged) return;
45
+ evictionSupportLogged = true;
46
+ const live = supportsEviction(hatchetCtx);
47
+ logger.info(
48
+ live
49
+ ? "journey replay-safety: Layer-1 memo fast path is LIVE (engine supports eviction)"
50
+ : "journey replay-safety: Layer-1 memo is INERT (engine < v0.80.0); " +
51
+ "exactly-once relies on the Layer-2 Postgres unique-index dedup",
52
+ );
53
+ }
54
+
31
55
  interface EventPayloadInput {
32
56
  userId: JsonValue;
33
57
  userEmail: JsonValue;
@@ -40,21 +64,6 @@ export interface DefinedJourney {
40
64
  task: ReturnType<typeof hatchet.durableTask>;
41
65
  }
42
66
 
43
- /**
44
- * Resolve a builder-function `where` to its `PropertyCondition[]` — a
45
- * one-shot, definition-time call mirroring `defineBucket`'s criteria
46
- * normalization. A declarative array passes straight through, so existing
47
- * journeys are unaffected; downstream (registry parse, checkExits, admin
48
- * routes, Studio) only ever sees plain data.
49
- */
50
- function normalizeWhere(
51
- where: JourneyWhere | undefined,
52
- ): PropertyCondition[] | undefined {
53
- if (typeof where !== "function") return where;
54
- const resolved = where(criteriaBuilder);
55
- return Array.isArray(resolved) ? resolved : [resolved];
56
- }
57
-
58
67
  export function defineJourney(options: {
59
68
  meta: JourneyMetaInput;
60
69
  run: JourneyRunFn;
@@ -130,35 +139,65 @@ export function defineJourney(options: {
130
139
  return { status: "skipped", reason: "user_unsubscribed" };
131
140
  }
132
141
 
133
- const activeState = await db.query.journeyStates.findFirst({
134
- where: and(
135
- eq(journeyStates.userId, userId),
136
- eq(journeyStates.journeyId, meta.id),
137
- inArray(journeyStates.status, ["active", "waiting"]),
138
- ),
139
- });
140
- if (activeState) {
141
- return { status: "skipped", reason: "already_active" };
142
- }
142
+ // The replay-stable anchor for this enrollment: the Hatchet run id is
143
+ // preserved across replays of the SAME logical durable run (crash / OOM /
144
+ // eviction / redeploy mid-run), whereas a freshly-minted journeyStates.id
145
+ // is not. Both dedup-key derivation AND the enrollment-recovery below key
146
+ // on it so a replay re-derives the SAME stateId and the SAME keys.
147
+ const workflowRunId = hatchetCtx.workflowRunId();
148
+
149
+ // REPLAY RECOVERY (must run before the active-state guard / insert).
150
+ // A replay-from-top of the same run must reuse the enrollment it already
151
+ // created — otherwise, for a journey whose prior enrollment reached a
152
+ // TERMINAL status (the normal end state for unlimited/once_per_period), the
153
+ // active-state guard would miss it, a fresh stateId would be minted, and the
154
+ // derived keys would no longer collide → a duplicate send. Recovering by the
155
+ // run-stable id makes the replay deterministic regardless of the prior row's
156
+ // status.
157
+ const recovered = workflowRunId
158
+ ? await db.query.journeyStates.findFirst({
159
+ where: and(
160
+ eq(journeyStates.hatchetRunId, workflowRunId),
161
+ eq(journeyStates.journeyId, meta.id),
162
+ ),
163
+ })
164
+ : undefined;
143
165
 
144
- const [state] = await db
145
- .insert(journeyStates)
146
- .values({
147
- userId,
148
- userEmail,
149
- journeyId: meta.id,
150
- currentNodeId: "start",
151
- status: "active",
152
- context: properties,
153
- hatchetRunId: hatchetCtx.workflowRunId(),
154
- })
155
- .returning();
166
+ let state = recovered;
167
+ if (!state) {
168
+ const activeState = await db.query.journeyStates.findFirst({
169
+ where: and(
170
+ eq(journeyStates.userId, userId),
171
+ eq(journeyStates.journeyId, meta.id),
172
+ inArray(journeyStates.status, ["active", "waiting"]),
173
+ ),
174
+ });
175
+ if (activeState) {
176
+ return { status: "skipped", reason: "already_active" };
177
+ }
178
+
179
+ [state] = await db
180
+ .insert(journeyStates)
181
+ .values({
182
+ userId,
183
+ userEmail,
184
+ journeyId: meta.id,
185
+ currentNodeId: "start",
186
+ status: "active",
187
+ context: properties,
188
+ hatchetRunId: workflowRunId,
189
+ })
190
+ .returning();
191
+ }
156
192
 
157
193
  if (!state) {
158
194
  return { status: "skipped", reason: "state_creation_failed" };
159
195
  }
160
196
 
161
197
  const stateId = state.id;
198
+ // The replay-stable key anchor: prefer the run id (constant across replays
199
+ // of this run), fall back to stateId when the engine has no run id.
200
+ const runAnchor = workflowRunId || stateId;
162
201
 
163
202
  const user: JourneyUser = {
164
203
  id: userId,
@@ -213,6 +252,11 @@ export function defineJourney(options: {
213
252
  const ctx = createJourneyContext({
214
253
  db,
215
254
  hatchet,
255
+ // The real DurableContext structurally satisfies the (widened) config —
256
+ // it exposes `sleepFor`/`waitFor` AND a replay-memoized `now()` (SDK
257
+ // 1.22.3). Forwarding it whole lets ctx clock reads (ctx.when,
258
+ // sleepUntil delta, lookback, ctx.now) read the memoized clock on an
259
+ // eviction-capable engine, falling back to the live clock pre-eviction.
216
260
  hatchetCtx,
217
261
  registry: getJourneyRegistrySingleton(),
218
262
  logger,
@@ -224,8 +268,34 @@ export function defineJourney(options: {
224
268
  defaultSendWindow: scheduleDefaults.sendWindow,
225
269
  });
226
270
 
271
+ // The journey boundary makes journey side effects (sendEmail, ctx.trigger)
272
+ // EXACTLY-ONCE across a durable replay WITHOUT any change to journey
273
+ // authoring. It is established once around `run()`: a replay-from-top
274
+ // re-enters this scope from the top. Its `runAnchor` is the replay-stable
275
+ // Hatchet run id (recovered above so the SAME stateId is reused on replay),
276
+ // so the derived keys collide across a replay of the same run even if the
277
+ // enrollment row had to be recovered rather than freshly inserted.
278
+ // `memoize` is the Layer-1 fast path (durable only when the engine supports
279
+ // eviction); the auto-derived `email_sends`/`user_events` idempotency keys
280
+ // (anchored on `runAnchor`) are the version-independent Layer-2 guarantee.
281
+ logEvictionSupportOnce(hatchetCtx);
282
+ const boundary: JourneyBoundary = {
283
+ stateId,
284
+ runAnchor,
285
+ currentLabel: undefined,
286
+ seenKeys: new Set<string>(),
287
+ memoize: createMemoize(hatchetCtx),
288
+ };
289
+
290
+ // Seed the context's memoized-clock snapshot ONCE before run() so a
291
+ // `ctx.when` chain used BEFORE the first durable step reads a replay-stable
292
+ // instant (on an eviction engine) instead of the construction-time
293
+ // `new Date()` seed. Best-effort: on a pre-eviction engine this reads the
294
+ // live clock, same as before.
295
+ await ctx.now();
296
+
227
297
  try {
228
- await options.run(user, ctx);
298
+ await runWithJourneyBoundary(boundary, () => options.run(user, ctx));
229
299
 
230
300
  const completedAt = new Date();
231
301
  await db
@@ -0,0 +1,166 @@
1
+ import { AsyncLocalStorage } from "node:async_hooks";
2
+
3
+ /**
4
+ * The minimal slice of Hatchet's `DurableContext` the boundary needs for its
5
+ * Layer-1 fast path. The real context structurally satisfies this; tests pass a
6
+ * stub. `memo(fn, deps)` is TS-`private` on the SDK class (1.22.3,
7
+ * context.d.ts:457) so it can only be reached via an `as any` cast at the call
8
+ * site — we model it here as optional so a probe (`typeof … === "function"`)
9
+ * gates it. `supportsEviction` (engineVersion >= v0.80.0) is the public getter
10
+ * that decides whether `memo` is actually durable: when false, the SDK's `memo`
11
+ * returns a bare `fn()` with ZERO durability, so we must NOT rely on it.
12
+ */
13
+ export interface HatchetMemoCtx {
14
+ memo?: <T>(fn: () => Promise<T> | T, deps: unknown[]) => Promise<T>;
15
+ supportsEviction?: boolean;
16
+ }
17
+
18
+ /**
19
+ * Per-enrollment, per-run state threaded through `AsyncLocalStorage` so the
20
+ * standalone `sendEmail` / `ctx.trigger` side-effect helpers can be made
21
+ * EXACTLY-ONCE across a durable replay WITHOUT any change to journey-authoring
22
+ * signatures. The boundary is established once in `define-journey` around the
23
+ * `run()` call; a replay-from-top re-enters that scope from the top, so the
24
+ * boundary is re-derived deterministically each time.
25
+ */
26
+ export interface JourneyBoundary {
27
+ /** The `journeyStates.id` — used for FK/observability and as the key anchor
28
+ * fallback when no replay-stable run id is available (tests, outside a durable
29
+ * run). NOT the primary key anchor — see {@link runAnchor}. */
30
+ stateId: string;
31
+ /**
32
+ * The replay-stable per-enrollment anchor every derived key uses: the Hatchet
33
+ * `workflowRunId()`. Unlike `stateId` (a fresh `defaultRandom()` uuid that can
34
+ * differ if a replay-from-top mints a new `journey_states` row for a terminal
35
+ * prior enrollment), the run id is preserved across replays of the SAME logical
36
+ * durable run — so both dedup layers' keys collide as intended on a replay.
37
+ * Falls back to `stateId` when absent (a non-durable test/harness context).
38
+ */
39
+ runAnchor: string;
40
+ /**
41
+ * The nearest authored wait/checkpoint label, captured as side effects run so
42
+ * a same-template send inherits the label of the `ctx.sleep`/`waitForEvent`/
43
+ * `checkpoint` that immediately preceded it. This is the "site" discriminant
44
+ * that lets two sends of the SAME template on different branches derive
45
+ * distinct keys for free, without an explicit authoring label.
46
+ */
47
+ currentLabel: string | undefined;
48
+ /**
49
+ * Every derived key seen so far in THIS run. A second side effect that derives
50
+ * an already-seen key would silently over-dedup (the second email/trigger
51
+ * would be suppressed), so {@link registerKey} throws loudly instead — the
52
+ * footgun is a fail-fast dev error, never a silently-dropped message.
53
+ */
54
+ seenKeys: Set<string>;
55
+ /**
56
+ * Layer-1 fast path: run `fn` through Hatchet's durable `memo` keyed by
57
+ * `deps` when (and only when) the engine supports eviction; otherwise fall
58
+ * through to a bare `fn()`. Belt-and-suspenders over the Layer-2 DB key — when
59
+ * eviction is live the provider/ingest call is skipped before the DB is even
60
+ * touched; when it isn't, Layer 2 still guarantees exactly-once.
61
+ */
62
+ memoize<T>(deps: unknown[], fn: () => Promise<T> | T): Promise<T>;
63
+ }
64
+
65
+ const journeyBoundaryAls = new AsyncLocalStorage<JourneyBoundary>();
66
+
67
+ /**
68
+ * The active journey boundary, or `undefined` when not inside a journey `run()`
69
+ * (e.g. an admin bulk send or a `POST /v1/emails` call). Standalone helpers read
70
+ * this to decide whether to auto-key + memoize their side effect.
71
+ */
72
+ export function getJourneyBoundary(): JourneyBoundary | undefined {
73
+ return journeyBoundaryAls.getStore();
74
+ }
75
+
76
+ /** Run `fn` with `boundary` installed as the active journey boundary. */
77
+ export function runWithJourneyBoundary<T>(
78
+ boundary: JourneyBoundary,
79
+ fn: () => Promise<T>,
80
+ ): Promise<T> {
81
+ return journeyBoundaryAls.run(boundary, fn);
82
+ }
83
+
84
+ /**
85
+ * Build the {@link JourneyBoundary.memoize} closure for a given Hatchet context.
86
+ * Probes `memo`/`supportsEviction` at call time (not construction time) so a
87
+ * degraded engine cleanly falls through to `fn()`. Accepts `unknown` and casts
88
+ * to {@link HatchetMemoCtx} internally: the SDK's `memo` is TS-`private`, so the
89
+ * real `DurableContext` is not assignable to a public-`memo` type — keeping the
90
+ * cast HERE contains the one brittle SDK-private access to this single helper.
91
+ */
92
+ export function createMemoize(hatchetCtx: unknown): JourneyBoundary["memoize"] {
93
+ const ctx = hatchetCtx as HatchetMemoCtx;
94
+ return async <T>(deps: unknown[], fn: () => Promise<T> | T): Promise<T> => {
95
+ const memo = ctx.memo;
96
+ if (typeof memo === "function" && ctx.supportsEviction === true) {
97
+ return memo(fn, deps);
98
+ }
99
+ return fn();
100
+ };
101
+ }
102
+
103
+ /**
104
+ * Read `supportsEviction` off a Hatchet context for boot-time logging, tolerating
105
+ * the same private-member assignability gap as {@link createMemoize}.
106
+ */
107
+ export function supportsEviction(hatchetCtx: unknown): boolean {
108
+ return (hatchetCtx as HatchetMemoCtx).supportsEviction === true;
109
+ }
110
+
111
+ /** The kind of side effect a key is derived for — keeps the namespaces apart. */
112
+ export type JourneyKeyKind = "send" | "trigger" | "connector";
113
+
114
+ const KEY_PREFIX: Record<JourneyKeyKind, string> = {
115
+ send: "journeySend",
116
+ trigger: "journeyTrigger",
117
+ connector: "journeyConnector",
118
+ };
119
+
120
+ /**
121
+ * Derive the deterministic, branch-stable idempotency key shared by BOTH defense
122
+ * layers. Content/site-derived (NEVER a positional counter), so a replay that
123
+ * reaches a given logical side effect re-derives the SAME key and a genuinely
124
+ * different side effect derives a DIFFERENT key — in both branch directions.
125
+ *
126
+ * send → `journeySend:<anchor>:<site>:<discriminant>`
127
+ * trigger → `journeyTrigger:<anchor>:<site>:<discriminant>`
128
+ * connector → `journeyConnector:<anchor>:<site>:<discriminant>`
129
+ *
130
+ * `anchor` is the REPLAY-STABLE per-enrollment id (`boundary.runAnchor`, the
131
+ * Hatchet `workflowRunId()`), NOT the freshly-minted `journeyStates.id`: a
132
+ * replay-from-top that mints a new state row for a terminal prior enrollment
133
+ * would otherwise derive a NON-colliding key and re-deliver. `site` is the
134
+ * explicit `idempotencyLabel` ?? the nearest authored wait label ?? the
135
+ * discriminant itself (templateKey for sends, event name for triggers). The key
136
+ * contains NO wall-clock and NO live-DB-read result, so it is stable across the
137
+ * clock/branch divergence a replay can introduce.
138
+ */
139
+ export function deriveJourneyKey(opts: {
140
+ kind: JourneyKeyKind;
141
+ /** The replay-stable per-enrollment anchor (`boundary.runAnchor`). */
142
+ anchor: string;
143
+ site: string;
144
+ discriminant: string;
145
+ }): string {
146
+ return `${KEY_PREFIX[opts.kind]}:${opts.anchor}:${opts.site}:${opts.discriminant}`;
147
+ }
148
+
149
+ /**
150
+ * Record `key` as used in this run, throwing loudly if it collides with a key an
151
+ * earlier side effect in the same run already derived. A collision means two
152
+ * genuinely-different sends/triggers resolved to the same key (e.g. the SAME
153
+ * template sent twice under the SAME nearest label) — under the unique index the
154
+ * second would be silently suppressed, so we fail fast at dev time instead. The
155
+ * fix is to pass a distinct `idempotencyLabel` to one of them.
156
+ */
157
+ export function registerKey(boundary: JourneyBoundary, key: string): void {
158
+ if (boundary.seenKeys.has(key)) {
159
+ throw new Error(
160
+ `journey replay-safety: duplicate idempotency key "${key}" derived ` +
161
+ "twice in one journey run. Two distinct sends/triggers collided on " +
162
+ "the same key — pass a distinct `idempotencyLabel` to one of them.",
163
+ );
164
+ }
165
+ boundary.seenKeys.add(key);
166
+ }