@hogsend/engine 0.40.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -8
- package/src/buckets/check-membership.ts +6 -1
- package/src/campaigns/cohort-sql.ts +198 -0
- package/src/campaigns/define-campaign.ts +207 -28
- package/src/campaigns/reconcile.ts +105 -11
- package/src/campaigns/steps.ts +48 -0
- package/src/connectors/define-action.ts +70 -0
- package/src/container.ts +124 -42
- package/src/destinations/presets/posthog.ts +7 -1
- package/src/index.ts +96 -2
- package/src/journeys/constants.ts +10 -0
- package/src/journeys/define-journey.ts +14 -405
- package/src/journeys/execute-journey-run.ts +641 -0
- package/src/journeys/graph/build-graph.ts +212 -56
- package/src/journeys/journey-boundary.ts +54 -0
- package/src/journeys/journey-context.ts +406 -59
- package/src/journeys/record-once.ts +110 -0
- package/src/lib/blueprint-lock.ts +21 -0
- package/src/lib/blueprints.ts +803 -0
- package/src/lib/campaign-send-key.ts +58 -0
- package/src/lib/connector-actions.ts +203 -19
- package/src/lib/contacts.ts +54 -0
- package/src/lib/email-service-types.ts +4 -1
- package/src/lib/email.ts +13 -2
- package/src/lib/event-names.ts +143 -0
- package/src/lib/feed.ts +77 -57
- package/src/lib/frequency-cap.ts +45 -15
- package/src/lib/hatchet-duration.ts +22 -0
- package/src/lib/html.ts +1 -0
- package/src/lib/ingestion.ts +145 -4
- package/src/lib/journey-suppress.ts +87 -0
- package/src/lib/links.ts +204 -18
- package/src/lib/outbound.ts +20 -0
- package/src/lib/preferences.ts +80 -1
- package/src/lib/tracked.ts +44 -29
- package/src/lib/tracking-event-names.ts +8 -0
- package/src/lib/tracking.ts +1 -5
- package/src/lib/webhook-signing.ts +6 -1
- package/src/lists/channels.ts +75 -0
- package/src/lists/define-list.ts +34 -0
- package/src/lists/registry.ts +29 -2
- package/src/mcp/authoring-guide.ts +160 -0
- package/src/mcp/blueprint-tools.ts +390 -0
- package/src/middleware/require-admin.ts +13 -0
- package/src/routes/admin/api-keys.ts +72 -0
- package/src/routes/admin/blueprints.ts +795 -0
- package/src/routes/admin/campaigns.ts +179 -3
- package/src/routes/admin/emails.ts +13 -0
- package/src/routes/admin/events.ts +55 -0
- package/src/routes/admin/index.ts +4 -0
- package/src/routes/admin/journeys.ts +4 -2
- package/src/routes/admin/links.ts +303 -36
- package/src/routes/admin/lists.ts +51 -0
- package/src/routes/campaigns/index.ts +44 -6
- package/src/routes/email/preferences.ts +50 -13
- package/src/routes/feed/recipient.ts +2 -43
- package/src/routes/index.ts +6 -0
- package/src/routes/lists/index.ts +138 -18
- package/src/routes/tracking/arrive.ts +260 -0
- package/src/routes/tracking/click-pipeline.ts +329 -0
- package/src/routes/tracking/click.ts +5 -270
- package/src/routes/tracking/index.ts +2 -0
- package/src/routes/tracking/vanity.ts +64 -0
- package/src/worker.ts +5 -0
- package/src/workflows/bucket-backfill.ts +4 -2
- package/src/workflows/bucket-reconcile.ts +11 -4
- package/src/workflows/check-alerts.ts +130 -1
- package/src/workflows/journey-blueprint-interpreter.ts +656 -0
- package/src/workflows/send-campaign.ts +574 -180
|
@@ -24,10 +24,15 @@ import {
|
|
|
24
24
|
type SendWindow,
|
|
25
25
|
} from "@hogsend/core/schedule";
|
|
26
26
|
import type {
|
|
27
|
+
DigestEvent,
|
|
28
|
+
DigestOptions,
|
|
29
|
+
DigestResult,
|
|
27
30
|
IfPast,
|
|
28
31
|
JourneyContext,
|
|
29
32
|
PropertyCondition,
|
|
30
33
|
RecentEvent,
|
|
34
|
+
ThrottleOptions,
|
|
35
|
+
ThrottleResult,
|
|
31
36
|
TimeOfDayBuilder,
|
|
32
37
|
WaitForEventResult,
|
|
33
38
|
Weekday,
|
|
@@ -39,8 +44,21 @@ import {
|
|
|
39
44
|
journeyStates,
|
|
40
45
|
userEvents,
|
|
41
46
|
} from "@hogsend/db";
|
|
42
|
-
import {
|
|
47
|
+
import {
|
|
48
|
+
and,
|
|
49
|
+
asc,
|
|
50
|
+
count,
|
|
51
|
+
desc,
|
|
52
|
+
eq,
|
|
53
|
+
gte,
|
|
54
|
+
lte,
|
|
55
|
+
max,
|
|
56
|
+
notInArray,
|
|
57
|
+
sql,
|
|
58
|
+
} from "drizzle-orm";
|
|
43
59
|
import { checkEmailPreferences } from "../lib/enrollment-guards.js";
|
|
60
|
+
import { countRecentSends } from "../lib/frequency-cap.js";
|
|
61
|
+
import { toSleepDuration } from "../lib/hatchet-duration.js";
|
|
44
62
|
import { ingestEvent } from "../lib/ingestion.js";
|
|
45
63
|
import type { Logger } from "../lib/logger.js";
|
|
46
64
|
import {
|
|
@@ -52,8 +70,10 @@ import {
|
|
|
52
70
|
deriveJourneyKey,
|
|
53
71
|
getJourneyBoundary,
|
|
54
72
|
registerKey,
|
|
73
|
+
registerRecordLabel,
|
|
55
74
|
} from "./journey-boundary.js";
|
|
56
75
|
import { logTransition } from "./journey-log.js";
|
|
76
|
+
import { recordOnce } from "./record-once.js";
|
|
57
77
|
|
|
58
78
|
/** Journey statuses that are terminal — a journey in any of these must never be
|
|
59
79
|
* resurrected back to "active" by a wait resuming. Exported so the durable task
|
|
@@ -63,6 +83,20 @@ export const TERMINAL_STATUSES = ["completed", "failed", "exited"] as const;
|
|
|
63
83
|
/** Upper bound for a `waitForEvent` timeout — the journey task's executionTimeout. */
|
|
64
84
|
const MAX_WAIT_MS = durationToMs({ hours: JOURNEY_EXECUTION_TIMEOUT_HOURS });
|
|
65
85
|
|
|
86
|
+
/** Default cap on events a single digest returns AND records. */
|
|
87
|
+
const DIGEST_DEFAULT_MAX_EVENTS = 100;
|
|
88
|
+
/** Hard ceiling on the digest event cap — bounds the recorded jsonb payload. */
|
|
89
|
+
const DIGEST_MAX_EVENTS_CEILING = 500;
|
|
90
|
+
/** Default backward widening so the enrolling event is caught by the scan. */
|
|
91
|
+
const DIGEST_DEFAULT_LOOKBACK: DurationObject = { minutes: 15 };
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* `journeyId:nodeId:reason` keys already warned about a digest/definition
|
|
95
|
+
* interplay this process. Warn-once so a replay-from-top (which re-runs the
|
|
96
|
+
* whole `run()`) doesn't spam the log with the same authoring advisory.
|
|
97
|
+
*/
|
|
98
|
+
const digestWarned = new Set<string>();
|
|
99
|
+
|
|
66
100
|
/**
|
|
67
101
|
* Quote a string as a CEL single-quoted string literal, escaping backslashes
|
|
68
102
|
* then single quotes. Used to embed an externally-supplied userId into a CEL
|
|
@@ -72,13 +106,35 @@ function celStringLiteral(value: string): string {
|
|
|
72
106
|
return `'${value.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
|
|
73
107
|
}
|
|
74
108
|
|
|
109
|
+
/** SQL eq-pushdown predicates for a `where` — a COARSE narrowing pre-filter,
|
|
110
|
+
* never the verdict: jsonb `->>` extracts TEXT (a stored string "50"
|
|
111
|
+
* text-matches `.eq(50)`), so every consumer MUST re-verify fetched rows with
|
|
112
|
+
* the strict condition engine in JS. A null-eq is NOT pushdownable — `->> p =
|
|
113
|
+
* 'null'` never matches a jsonb null (SQL NULL) — so it is excluded here and
|
|
114
|
+
* resolved entirely by that JS re-verify. */
|
|
115
|
+
const eqPushdownPreds = (where: PropertyCondition[]) =>
|
|
116
|
+
where
|
|
117
|
+
.filter(
|
|
118
|
+
(c) => c.operator === "eq" && c.value !== undefined && c.value !== null,
|
|
119
|
+
)
|
|
120
|
+
.map(
|
|
121
|
+
(c) =>
|
|
122
|
+
sql`${userEvents.properties} ->> ${c.property} = ${String(c.value)}`,
|
|
123
|
+
);
|
|
124
|
+
|
|
75
125
|
interface JourneyContextConfig {
|
|
76
126
|
db: Database;
|
|
77
127
|
hatchet: HatchetClient;
|
|
78
128
|
hatchetCtx: {
|
|
79
|
-
// Hatchet's real `sleepFor` accepts a
|
|
80
|
-
//
|
|
81
|
-
|
|
129
|
+
// Hatchet's real `sleepFor` accepts a Go duration string, a DurationObject,
|
|
130
|
+
// or a number (ms). We always pass a normalized whole-seconds string (see
|
|
131
|
+
// `toSleepDuration`) — the multi-unit strings the SDK derives from a raw ms
|
|
132
|
+
// number are silently no-op'd by some hatchet-lite versions. The param is the
|
|
133
|
+
// exact `${number}s` literal (a subtype of the SDK's `Duration`) so the real
|
|
134
|
+
// `DurableContext` stays assignable to this structural stub.
|
|
135
|
+
sleepFor: (
|
|
136
|
+
duration: DurationObject | number | `${number}s`,
|
|
137
|
+
) => Promise<unknown>;
|
|
82
138
|
// The forwarded object is the real Hatchet `DurableContext`, which also has
|
|
83
139
|
// `waitFor` (used by `waitForEvent`). Param mirrors the SDK signature so the
|
|
84
140
|
// real context is assignable; we read back the envelope as a plain record.
|
|
@@ -102,6 +158,18 @@ interface JourneyContextConfig {
|
|
|
102
158
|
resolvedTimezone: string;
|
|
103
159
|
/** The client default send window, auto-applied by `ctx.when`. */
|
|
104
160
|
defaultSendWindow?: SendWindow;
|
|
161
|
+
/** The journey's trigger event — `ctx.digest`'s default `event`. */
|
|
162
|
+
triggerEvent?: string;
|
|
163
|
+
/** The journey's already-normalized `trigger.where` — `ctx.digest` applies it
|
|
164
|
+
* when digesting the trigger event with no explicit `where` (honors the
|
|
165
|
+
* trigger contract). */
|
|
166
|
+
triggerWhere?: PropertyCondition[];
|
|
167
|
+
/** The journey id — keys the digest definition-interplay warn-once. */
|
|
168
|
+
journeyId?: string;
|
|
169
|
+
/** The journey's entry limit — drives `ctx.digest` interplay warnings. */
|
|
170
|
+
entryLimit?: "once" | "once_per_period" | "unlimited";
|
|
171
|
+
/** The journey's entry period (for `once_per_period`). */
|
|
172
|
+
entryPeriod?: DurationObject;
|
|
105
173
|
}
|
|
106
174
|
|
|
107
175
|
/**
|
|
@@ -275,15 +343,17 @@ export function createJourneyContext(
|
|
|
275
343
|
};
|
|
276
344
|
|
|
277
345
|
// Durable sleep with the guarded waiting → active lifecycle. `sleep` passes a
|
|
278
|
-
// DurationObject; `sleepUntil`
|
|
279
|
-
// `
|
|
346
|
+
// DurationObject; `sleepUntil`/`digest` pass a precomputed ms delay. Both are
|
|
347
|
+
// normalized to a whole-seconds Go string (see `toSleepDuration`) — a raw ms
|
|
348
|
+
// number would be rendered by the SDK as a multi-unit string some hatchet-lite
|
|
349
|
+
// versions silently no-op, resolving the wait instantly.
|
|
280
350
|
const performSleep = async (
|
|
281
351
|
durationOrMs: DurationObject | number,
|
|
282
352
|
nodeId: string,
|
|
283
353
|
): Promise<{ sleptAt: string; resumedAt: string }> => {
|
|
284
354
|
const sleptAt = new Date().toISOString();
|
|
285
355
|
await enterWait(nodeId, "sleep");
|
|
286
|
-
await hatchetCtx.sleepFor(durationOrMs);
|
|
356
|
+
await hatchetCtx.sleepFor(toSleepDuration(durationOrMs));
|
|
287
357
|
const resumedAt = new Date().toISOString();
|
|
288
358
|
await resumeFromWait(nodeId);
|
|
289
359
|
// Refresh the memoized clock snapshot so a `ctx.when` chain used right after
|
|
@@ -362,14 +432,8 @@ export function createJourneyContext(
|
|
|
362
432
|
// into SQL so the LIMIT below bounds MATCHING rows, not all same-name rows —
|
|
363
433
|
// otherwise a chatty user emitting >LIMIT other same-name events could bury
|
|
364
434
|
// the matching row past the cutoff and the wait would falsely time out.
|
|
365
|
-
//
|
|
366
|
-
|
|
367
|
-
const eqPreds = where
|
|
368
|
-
.filter((c) => c.operator === "eq" && c.value !== undefined)
|
|
369
|
-
.map(
|
|
370
|
-
(c) =>
|
|
371
|
-
sql`${userEvents.properties} ->> ${c.property} = ${String(c.value)}`,
|
|
372
|
-
);
|
|
435
|
+
// Narrowing only — `scanForMatch` re-verifies every row in JS.
|
|
436
|
+
const eqPreds = eqPushdownPreds(where);
|
|
373
437
|
|
|
374
438
|
const scanForMatch = async (): Promise<NonNullable<
|
|
375
439
|
WaitForEventResult["properties"]
|
|
@@ -443,7 +507,7 @@ export function createJourneyContext(
|
|
|
443
507
|
`input.userId == ${celStringLiteral(userId)}`,
|
|
444
508
|
"event",
|
|
445
509
|
),
|
|
446
|
-
new SleepCondition(remainingMs, "timeout"),
|
|
510
|
+
new SleepCondition(toSleepDuration(remainingMs), "timeout"),
|
|
447
511
|
),
|
|
448
512
|
);
|
|
449
513
|
const fired = (("CREATE" in armed ? armed.CREATE : armed) ??
|
|
@@ -564,7 +628,7 @@ export function createJourneyContext(
|
|
|
564
628
|
`input.userId == ${celStringLiteral(userId)}`,
|
|
565
629
|
"event",
|
|
566
630
|
),
|
|
567
|
-
new SleepCondition(
|
|
631
|
+
new SleepCondition(toSleepDuration(timeout), "timeout"),
|
|
568
632
|
),
|
|
569
633
|
);
|
|
570
634
|
|
|
@@ -609,6 +673,287 @@ export function createJourneyContext(
|
|
|
609
673
|
return { timedOut, ...(properties ? { properties } : {}) };
|
|
610
674
|
};
|
|
611
675
|
|
|
676
|
+
// Scan the digest window ONCE at flush: all `event` rows for THIS user in
|
|
677
|
+
// [scanSince, flushInstant], Studio debug events excluded, capped at `cap`.
|
|
678
|
+
// Pure read — no status flips, no writes — returning a self-consistent,
|
|
679
|
+
// JSON-round-trip-safe DigestResult ready to be recorded set-once.
|
|
680
|
+
const scanDigestWindow = async (opts: {
|
|
681
|
+
event: string;
|
|
682
|
+
scanSince: Date;
|
|
683
|
+
flushInstant: Date;
|
|
684
|
+
where: PropertyCondition[];
|
|
685
|
+
cap: number;
|
|
686
|
+
}): Promise<DigestResult> => {
|
|
687
|
+
const { event, scanSince, flushInstant, where, cap } = opts;
|
|
688
|
+
|
|
689
|
+
// Narrow the fetch so a chatty user emitting >LIMIT other same-name events
|
|
690
|
+
// can't bury a match past the cutoff; the JS re-verify below is the verdict.
|
|
691
|
+
const eqPreds = eqPushdownPreds(where);
|
|
692
|
+
|
|
693
|
+
// Only the where-less path can trust the fetch as final, so `cap + 1` detects
|
|
694
|
+
// truncation exactly there. ANY `where` re-verifies in JS below (the SQL fetch
|
|
695
|
+
// is then a superset), so pull cap-relative headroom — sized to the cap, not a
|
|
696
|
+
// flat 100, since the chatty-user trap scales with the digest cap.
|
|
697
|
+
const fetchLimit = where.length === 0 ? cap + 1 : Math.min(cap * 10, 2000);
|
|
698
|
+
|
|
699
|
+
const rows = await db
|
|
700
|
+
.select({
|
|
701
|
+
id: userEvents.id,
|
|
702
|
+
properties: userEvents.properties,
|
|
703
|
+
occurredAt: userEvents.occurredAt,
|
|
704
|
+
})
|
|
705
|
+
.from(userEvents)
|
|
706
|
+
.where(
|
|
707
|
+
and(
|
|
708
|
+
eq(userEvents.userId, userId),
|
|
709
|
+
eq(userEvents.event, event),
|
|
710
|
+
gte(userEvents.occurredAt, scanSince),
|
|
711
|
+
lte(userEvents.occurredAt, flushInstant),
|
|
712
|
+
// Studio debug events must never pollute a customer digest; EVERY other
|
|
713
|
+
// source (api, webhook source ids, connector ids, journey) is
|
|
714
|
+
// deliberately included so the digest sees the full real event spine.
|
|
715
|
+
sql`${userEvents.source} IS DISTINCT FROM 'studio'`,
|
|
716
|
+
...eqPreds,
|
|
717
|
+
),
|
|
718
|
+
)
|
|
719
|
+
// Deterministic tie-break for bulk backfills sharing a timestamp, so a
|
|
720
|
+
// pre-record rescan on replay orders identically.
|
|
721
|
+
.orderBy(asc(userEvents.occurredAt), asc(userEvents.id))
|
|
722
|
+
.limit(fetchLimit);
|
|
723
|
+
|
|
724
|
+
// Re-verify EVERY fetched row against the FULL predicate with the canonical
|
|
725
|
+
// strict-equality engine whenever a `where` is present — the SQL eq pushdown
|
|
726
|
+
// is a TEXT narrowing filter only, so a stored string "50" that SQL-matched
|
|
727
|
+
// `.eq(50)` is rejected here, and an unpushdownable null-eq is matched here.
|
|
728
|
+
const matched =
|
|
729
|
+
where.length > 0
|
|
730
|
+
? rows.filter((row) =>
|
|
731
|
+
evaluatePropertyConditions({
|
|
732
|
+
conditions: where,
|
|
733
|
+
properties: (row.properties ?? {}) as Record<string, unknown>,
|
|
734
|
+
}),
|
|
735
|
+
)
|
|
736
|
+
: rows;
|
|
737
|
+
|
|
738
|
+
const events: DigestEvent[] = matched.slice(0, cap).map((row) => ({
|
|
739
|
+
properties: row.properties ? narrowScalars(row.properties) : null,
|
|
740
|
+
occurredAt:
|
|
741
|
+
row.occurredAt instanceof Date
|
|
742
|
+
? row.occurredAt.toISOString()
|
|
743
|
+
: String(row.occurredAt),
|
|
744
|
+
}));
|
|
745
|
+
|
|
746
|
+
// Truncated when more matched than the cap OR the raw fetch itself hit its
|
|
747
|
+
// ceiling (further matches may exist past the fetched slice).
|
|
748
|
+
const truncated = matched.length > cap || rows.length >= fetchLimit;
|
|
749
|
+
|
|
750
|
+
return {
|
|
751
|
+
events,
|
|
752
|
+
count: events.length,
|
|
753
|
+
truncated,
|
|
754
|
+
flushedAt: flushInstant.toISOString(),
|
|
755
|
+
};
|
|
756
|
+
};
|
|
757
|
+
|
|
758
|
+
// Aggregate multiple trigger events over a fixed window into ONE execution.
|
|
759
|
+
// Each numbered step is a replay-safety requirement — do NOT reorder them.
|
|
760
|
+
const performDigest = async (opts: DigestOptions): Promise<DigestResult> => {
|
|
761
|
+
// 1. VALIDATE — throw BEFORE any durable work (no db/hatchet touched yet).
|
|
762
|
+
const eventName = opts.event ?? config.triggerEvent;
|
|
763
|
+
if (!eventName) {
|
|
764
|
+
throw new TypeError(
|
|
765
|
+
"ctx.digest: no `event` given and the journey has no trigger event",
|
|
766
|
+
);
|
|
767
|
+
}
|
|
768
|
+
const windowMs = durationToMs(opts.window);
|
|
769
|
+
if (windowMs <= 0) {
|
|
770
|
+
throw new RangeError("ctx.digest window must be a positive duration");
|
|
771
|
+
}
|
|
772
|
+
if (windowMs > MAX_WAIT_MS) {
|
|
773
|
+
throw new RangeError(
|
|
774
|
+
`ctx.digest window exceeds the journey execution limit (${JOURNEY_EXECUTION_TIMEOUT})`,
|
|
775
|
+
);
|
|
776
|
+
}
|
|
777
|
+
const cap = Math.min(
|
|
778
|
+
Math.max(1, opts.maxEvents ?? DIGEST_DEFAULT_MAX_EVENTS),
|
|
779
|
+
DIGEST_MAX_EVENTS_CEILING,
|
|
780
|
+
);
|
|
781
|
+
const lookbackMs = durationToMs(opts.lookback ?? DIGEST_DEFAULT_LOOKBACK);
|
|
782
|
+
const nodeId = opts.label ?? `digest:${eventName}`;
|
|
783
|
+
// `where`: an explicit predicate wins; otherwise the trigger contract applies
|
|
784
|
+
// ONLY when digesting the trigger event itself (FM-16), so a digest of the
|
|
785
|
+
// journey's own trigger honors `trigger.where` without restating it.
|
|
786
|
+
const where =
|
|
787
|
+
opts.where !== undefined
|
|
788
|
+
? (normalizeWhere(opts.where) ?? [])
|
|
789
|
+
: eventName === config.triggerEvent
|
|
790
|
+
? (config.triggerWhere ?? [])
|
|
791
|
+
: [];
|
|
792
|
+
|
|
793
|
+
// 2. REGISTER the site label — reusing a label in one run throws loudly (two
|
|
794
|
+
// digests sharing a durable record would silently over-collapse). No-op
|
|
795
|
+
// outside a durable run (a unit-test context has no boundary).
|
|
796
|
+
registerRecordLabel(getJourneyBoundary(), nodeId);
|
|
797
|
+
|
|
798
|
+
// 3. DEFINITION-INTERPLAY WARNINGS — warn-once per process per journeyId:nodeId.
|
|
799
|
+
if (eventName === config.triggerEvent && config.entryLimit === "once") {
|
|
800
|
+
const key = `${config.journeyId ?? ""}:${nodeId}:once`;
|
|
801
|
+
if (!digestWarned.has(key)) {
|
|
802
|
+
digestWarned.add(key);
|
|
803
|
+
logger.warn(
|
|
804
|
+
`ctx.digest: journey "${config.journeyId ?? nodeId}" has entryLimit ` +
|
|
805
|
+
'"once" and digests its own trigger event — it digests exactly ONE ' +
|
|
806
|
+
"window ever; trigger events after the first enrollment are dropped, " +
|
|
807
|
+
'not digested. Use entryLimit "unlimited" for a rolling digest.',
|
|
808
|
+
);
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
if (
|
|
812
|
+
config.entryLimit === "once_per_period" &&
|
|
813
|
+
config.entryPeriod &&
|
|
814
|
+
durationToMs(config.entryPeriod) > windowMs
|
|
815
|
+
) {
|
|
816
|
+
const key = `${config.journeyId ?? ""}:${nodeId}:period`;
|
|
817
|
+
if (!digestWarned.has(key)) {
|
|
818
|
+
digestWarned.add(key);
|
|
819
|
+
logger.warn(
|
|
820
|
+
`ctx.digest: journey "${config.journeyId ?? nodeId}" entryPeriod is ` +
|
|
821
|
+
"longer than the digest window — events arriving in the gap between " +
|
|
822
|
+
"the flush and the next enrollment are absorbed but never digested " +
|
|
823
|
+
"(an enrollment-gap loss band).",
|
|
824
|
+
);
|
|
825
|
+
}
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
// 4. DURABLE DEADLINE — DB-only read-first / set-once (issues NO durable
|
|
829
|
+
// journal node: `compute` reads the seeded `latestNow` synchronously). The
|
|
830
|
+
// deadline drives `scanSince`, the stranded-waiting alert, and observability
|
|
831
|
+
// — it is NOT the sleep duration (see step 5). Never cleared; the result
|
|
832
|
+
// record is the terminal mark. Flat `<label>:deadline` / `<label>:result`
|
|
833
|
+
// subkeys keep set-once applying per subkey.
|
|
834
|
+
const deadlineIso = await recordOnce({
|
|
835
|
+
db,
|
|
836
|
+
stateId,
|
|
837
|
+
namespace: "__digest__",
|
|
838
|
+
// Read the seeded memoized snapshot SYNCHRONOUSLY rather than awaiting
|
|
839
|
+
// hatchetCtx.now(): recordOnce freezes this value (a replay reads the
|
|
840
|
+
// recorded deadline back), so the live-clock read never diverges — AND it
|
|
841
|
+
// adds no journal node inside a conditionally-executed compute (which a
|
|
842
|
+
// replay would skip, misaligning the positional journal).
|
|
843
|
+
key: `${nodeId}:deadline`,
|
|
844
|
+
compute: () => new Date(latestNow.getTime() + windowMs).toISOString(),
|
|
845
|
+
});
|
|
846
|
+
const deadline = new Date(deadlineIso);
|
|
847
|
+
|
|
848
|
+
// 5. SLEEP THE CONSTANT WINDOW — Hatchet's durable journal is POSITIONAL: on
|
|
849
|
+
// a replay every durable call must be re-issued in the identical order with
|
|
850
|
+
// identical args, and an already-fired node instant-resolves. So we ALWAYS
|
|
851
|
+
// issue `performSleep(windowMs)` — never a remainder (`deadline − now`), whose
|
|
852
|
+
// arg would drift ("87s" vs the journaled "120s") and trip the determinism
|
|
853
|
+
// checker (the exact live-smoke kill). Since `deadline := latestNow + windowMs`
|
|
854
|
+
// and the memoized seed returns the same `latestNow`, the duration is byte-
|
|
855
|
+
// identical every replay. A mid-window re-dispatch re-issues the same sleep and
|
|
856
|
+
// BLOCKS on the original server-side deadline; a post-fire replay instant-
|
|
857
|
+
// resolves. performSleep's enterWait/resumeFromWait lifecycle IS the terminal
|
|
858
|
+
// backstop (exitOn flipped the row terminal → 0 rows → JourneyExitedError),
|
|
859
|
+
// and it advances the boundary label + refreshes the memoized clock.
|
|
860
|
+
//
|
|
861
|
+
// ACCEPTED SEMANTIC: if a crash lands between recording the deadline and this
|
|
862
|
+
// sleep being journaled, the re-dispatch arms a FRESH full window from then —
|
|
863
|
+
// the collection period extends by the crash gap. Rare and harmless (the scan
|
|
864
|
+
// window simply collects more), and the price of journal determinism.
|
|
865
|
+
await performSleep(windowMs, nodeId);
|
|
866
|
+
|
|
867
|
+
// 6. FLUSH ONCE, RECORD ONCE — `flushInstant` is issued here on EVERY replay
|
|
868
|
+
// (positionally stable, right after the sleep), scanning up to it (NOT the
|
|
869
|
+
// deadline: a straggler landing during wake latency is still in-window;
|
|
870
|
+
// anything after belongs to the documented straggler band). On a replay-after-
|
|
871
|
+
// flush recordOnce's READ-FIRST returns the recorded result verbatim WITHOUT
|
|
872
|
+
// re-running the scan — the verbatim-replay guarantee lives entirely in
|
|
873
|
+
// recordOnce now (peek fast path removed). The read-back also means a zombie
|
|
874
|
+
// double-writer returns the FIRST-committed snapshot.
|
|
875
|
+
const flushInstant = await refreshNow();
|
|
876
|
+
const scanSince = new Date(deadline.getTime() - windowMs - lookbackMs);
|
|
877
|
+
return recordOnce({
|
|
878
|
+
db,
|
|
879
|
+
stateId,
|
|
880
|
+
namespace: "__digest__",
|
|
881
|
+
key: `${nodeId}:result`,
|
|
882
|
+
compute: () =>
|
|
883
|
+
scanDigestWindow({
|
|
884
|
+
event: eventName,
|
|
885
|
+
scanSince,
|
|
886
|
+
flushInstant,
|
|
887
|
+
where,
|
|
888
|
+
cap,
|
|
889
|
+
}),
|
|
890
|
+
});
|
|
891
|
+
};
|
|
892
|
+
|
|
893
|
+
// Advisory, RECORDED-once frequency check the author branches on. The verdict
|
|
894
|
+
// is frozen into the state row the FIRST time this site runs and replayed
|
|
895
|
+
// verbatim thereafter: a live re-count on replay is guaranteed to diverge
|
|
896
|
+
// because the run's OWN send lands in the counting window (check allowed →
|
|
897
|
+
// send → crash → replay re-counts → now blocked → different branch → different
|
|
898
|
+
// template → different idempotency key — the exact divergence class this
|
|
899
|
+
// engine's replay-safety design forbids). recordOnce under `__throttle__`
|
|
900
|
+
// gives that durability on ANY engine; countRecentSends is the SAME COUNT the
|
|
901
|
+
// mailer-level cap enforces on, keyed by recipient EMAIL (userId is NOT the
|
|
902
|
+
// cap key), so advisory and enforcement agree on what they count.
|
|
903
|
+
const performThrottle = async (
|
|
904
|
+
opts: ThrottleOptions,
|
|
905
|
+
): Promise<ThrottleResult> => {
|
|
906
|
+
// 1. VALIDATE — throw BEFORE any durable work (no db/hatchet touched yet).
|
|
907
|
+
if (!Number.isInteger(opts.limit) || opts.limit < 1) {
|
|
908
|
+
throw new RangeError("ctx.throttle limit must be an integer >= 1");
|
|
909
|
+
}
|
|
910
|
+
const windowMs = durationToMs(opts.window);
|
|
911
|
+
if (windowMs <= 0) {
|
|
912
|
+
throw new RangeError("ctx.throttle window must be a positive duration");
|
|
913
|
+
}
|
|
914
|
+
|
|
915
|
+
// 2. SITE — the same "site" rule the send auto-keys use: explicit label ??
|
|
916
|
+
// nearest authored wait label ?? "start" (outside any wait).
|
|
917
|
+
const site = opts.label ?? getJourneyBoundary()?.currentLabel ?? "start";
|
|
918
|
+
|
|
919
|
+
// 3. REGISTER + DERIVE KEY. The `throttle:` prefix keeps throttle sites from
|
|
920
|
+
// colliding with digest labels in the shared boundary label set; the key
|
|
921
|
+
// folds in category + limit/window so two DISTINCT throttle configs at one
|
|
922
|
+
// site don't share a record. Reusing a label throws the loud collision error
|
|
923
|
+
// (pass a distinct `label` to re-check).
|
|
924
|
+
const key = `${site}:${opts.category ?? "*"}:${opts.limit}/${windowMs}`;
|
|
925
|
+
registerRecordLabel(getJourneyBoundary(), `throttle:${key}`);
|
|
926
|
+
|
|
927
|
+
// 4. RECORD ONCE — the verdict is computed on the first winning writer and
|
|
928
|
+
// replayed verbatim after (recordOnce's read-back means a zombie
|
|
929
|
+
// double-writer returns the first-committed verdict too).
|
|
930
|
+
return recordOnce({
|
|
931
|
+
db,
|
|
932
|
+
stateId,
|
|
933
|
+
namespace: "__throttle__",
|
|
934
|
+
key,
|
|
935
|
+
compute: async () => {
|
|
936
|
+
// Read `latestNow` SYNCHRONOUSLY — never `await refreshNow()` (a memo).
|
|
937
|
+
// This compute is conditionally executed (a replay with a recorded
|
|
938
|
+
// verdict skips it), so a memo inside it would misalign the POSITIONAL
|
|
939
|
+
// journal for every durable call after. With this, ctx.throttle issues
|
|
940
|
+
// ZERO durable calls — it is positionally invisible.
|
|
941
|
+
const since = new Date(latestNow.getTime() - windowMs);
|
|
942
|
+
const count = await countRecentSends({
|
|
943
|
+
db,
|
|
944
|
+
to: userEmail,
|
|
945
|
+
since,
|
|
946
|
+
...(opts.category ? { category: opts.category } : {}),
|
|
947
|
+
});
|
|
948
|
+
return {
|
|
949
|
+
allowed: count < opts.limit,
|
|
950
|
+
count,
|
|
951
|
+
remaining: Math.max(0, opts.limit - count),
|
|
952
|
+
};
|
|
953
|
+
},
|
|
954
|
+
});
|
|
955
|
+
};
|
|
956
|
+
|
|
612
957
|
return {
|
|
613
958
|
when: createWhenBuilder({
|
|
614
959
|
timezone: resolvedTimezone,
|
|
@@ -744,55 +1089,57 @@ export function createJourneyContext(
|
|
|
744
1089
|
});
|
|
745
1090
|
},
|
|
746
1091
|
|
|
1092
|
+
async exit(reason?: string): Promise<never> {
|
|
1093
|
+
// Terminal "exited": a guarded flip (never clobber an already-terminal
|
|
1094
|
+
// row) plus the SAME JourneyExitedError control-flow signal a mid-wait
|
|
1095
|
+
// exitOn raises — the run lifecycle maps it to { status: "exited" } with
|
|
1096
|
+
// no "failed"/"completed" write and no journey:* event. This is the
|
|
1097
|
+
// single mechanism the blueprint interpreter's end-exited node drives too
|
|
1098
|
+
// (and the form promote-to-code emits), so there is one path, not two.
|
|
1099
|
+
const now = new Date();
|
|
1100
|
+
const [exited] = await db
|
|
1101
|
+
.update(journeyStates)
|
|
1102
|
+
.set({ status: "exited", exitedAt: now, updatedAt: now })
|
|
1103
|
+
.where(
|
|
1104
|
+
and(
|
|
1105
|
+
eq(journeyStates.id, stateId),
|
|
1106
|
+
notInArray(journeyStates.status, [...TERMINAL_STATUSES]),
|
|
1107
|
+
),
|
|
1108
|
+
)
|
|
1109
|
+
.returning({ id: journeyStates.id });
|
|
1110
|
+
if (exited) {
|
|
1111
|
+
// Fire-and-forget exit transition (best-effort; never affects the run).
|
|
1112
|
+
logTransition({
|
|
1113
|
+
db,
|
|
1114
|
+
journeyStateId: stateId,
|
|
1115
|
+
from: getJourneyBoundary()?.currentLabel ?? null,
|
|
1116
|
+
to: "end-exited",
|
|
1117
|
+
action: "exited",
|
|
1118
|
+
...(reason ? { detail: { reason } } : {}),
|
|
1119
|
+
});
|
|
1120
|
+
}
|
|
1121
|
+
throw new JourneyExitedError(stateId);
|
|
1122
|
+
},
|
|
1123
|
+
|
|
747
1124
|
async now() {
|
|
748
1125
|
return refreshNow();
|
|
749
1126
|
},
|
|
750
1127
|
|
|
751
1128
|
async once<T>(key: string, compute: () => Promise<T> | T): Promise<T> {
|
|
752
|
-
//
|
|
753
|
-
//
|
|
754
|
-
//
|
|
755
|
-
//
|
|
756
|
-
//
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
// clobbered.
|
|
760
|
-
const read = async (): Promise<Record<string, unknown>> => {
|
|
761
|
-
const row = await db.query.journeyStates.findFirst({
|
|
762
|
-
where: eq(journeyStates.id, stateId),
|
|
763
|
-
columns: { context: true },
|
|
764
|
-
});
|
|
765
|
-
const ctxBag = (row?.context ?? {}) as Record<string, unknown>;
|
|
766
|
-
return (ctxBag.__once__ ?? {}) as Record<string, unknown>;
|
|
767
|
-
};
|
|
1129
|
+
// Durable, engine-agnostic record-once. Delegated to the standalone
|
|
1130
|
+
// `recordOnce` (shared with the digest/throttle primitives + the mailer):
|
|
1131
|
+
// read-first fast path, FIRST-writer-wins jsonb merge under the reserved
|
|
1132
|
+
// `__once__` namespace, read-back return so a zombie double-writer cannot
|
|
1133
|
+
// clobber the value the winner already handed to author code.
|
|
1134
|
+
return recordOnce({ db, stateId, namespace: "__once__", key, compute });
|
|
1135
|
+
},
|
|
768
1136
|
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
}
|
|
1137
|
+
digest(opts) {
|
|
1138
|
+
return performDigest(opts);
|
|
1139
|
+
},
|
|
773
1140
|
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
// create_missing=true cannot create a NESTED key whose parent object is
|
|
777
|
-
// absent (a fresh '{}' has no '__once__'), so we set the TOP-LEVEL
|
|
778
|
-
// '__once__' to (its existing bag) MERGED with the single new key via `||`.
|
|
779
|
-
// This creates '__once__' when missing AND preserves sibling once-keys.
|
|
780
|
-
// `key` and the value are bound parameters (jsonb_build_object), so the
|
|
781
|
-
// write is injection-safe.
|
|
782
|
-
await db
|
|
783
|
-
.update(journeyStates)
|
|
784
|
-
.set({
|
|
785
|
-
context: sql`jsonb_set(
|
|
786
|
-
coalesce(${journeyStates.context}, '{}'::jsonb),
|
|
787
|
-
'{__once__}',
|
|
788
|
-
coalesce(${journeyStates.context} -> '__once__', '{}'::jsonb)
|
|
789
|
-
|| jsonb_build_object(${key}::text, ${JSON.stringify(value ?? null)}::jsonb),
|
|
790
|
-
true
|
|
791
|
-
)`,
|
|
792
|
-
updatedAt: new Date(),
|
|
793
|
-
})
|
|
794
|
-
.where(eq(journeyStates.id, stateId));
|
|
795
|
-
return value;
|
|
1141
|
+
throttle(opts) {
|
|
1142
|
+
return performThrottle(opts);
|
|
796
1143
|
},
|
|
797
1144
|
|
|
798
1145
|
guard: {
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { type Database, journeyStates } from "@hogsend/db";
|
|
2
|
+
import { eq, sql } from "drizzle-orm";
|
|
3
|
+
|
|
4
|
+
/** The reserved `journey_states.context` sub-bags a record-once value can live
|
|
5
|
+
* under. A closed union — the ONLY values reachable at the SQL path, so the
|
|
6
|
+
* token can be interpolated into the jsonb path (see {@link recordOnce}) while
|
|
7
|
+
* `key`/value stay bound parameters. */
|
|
8
|
+
export type RecordNamespace = "__once__" | "__digest__" | "__throttle__";
|
|
9
|
+
|
|
10
|
+
// Closed-union → validated path token. NOT user input: only these three literals
|
|
11
|
+
// reach the interpolated jsonb path below, so it is injection-safe. `key` and the
|
|
12
|
+
// JSON-stringified value are ALWAYS bound parameters regardless.
|
|
13
|
+
const NAMESPACE_TOKEN: Record<RecordNamespace, string> = {
|
|
14
|
+
__once__: "__once__",
|
|
15
|
+
__digest__: "__digest__",
|
|
16
|
+
__throttle__: "__throttle__",
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
/** Extract the `(namespace)` sub-bag from an already-fetched context, `{}` when
|
|
20
|
+
* absent — {@link recordOnce}'s read-first fast path. */
|
|
21
|
+
function namespaceBag(
|
|
22
|
+
context: unknown,
|
|
23
|
+
namespace: RecordNamespace,
|
|
24
|
+
): Record<string, unknown> {
|
|
25
|
+
const ctxBag = (context ?? {}) as Record<string, unknown>;
|
|
26
|
+
return (ctxBag[namespace] ?? {}) as Record<string, unknown>;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Durable set-once for a single `(stateId, namespace, key)` — the replay-safety
|
|
31
|
+
* primitive shared by `ctx.once`, `ctx.digest`, `ctx.throttle`, and the
|
|
32
|
+
* journey-suppress guard. Guarantees:
|
|
33
|
+
*
|
|
34
|
+
* - **Set-once**: the first committed write for a `(stateId, namespace, key)`
|
|
35
|
+
* wins; `compute` runs at most once per winning writer and never again once a
|
|
36
|
+
* value is recorded (a replay-from-top re-reads the stored value).
|
|
37
|
+
* - **First-writer-wins under a concurrent race**: a zombie double-writer (a
|
|
38
|
+
* partitioned worker's original execution racing its replay) CANNOT clobber a
|
|
39
|
+
* value the winner already returned to author code. The persist uses a
|
|
40
|
+
* FIRST-writer-wins jsonb merge and returns the value from the UPDATE's
|
|
41
|
+
* RETURNING (the post-merge committed row), so both racers observe the SAME
|
|
42
|
+
* (first-committed) value even if the loser computed something different.
|
|
43
|
+
*
|
|
44
|
+
* Values MUST be JSON-serializable (stored in a jsonb column, round-tripped
|
|
45
|
+
* through `JSON.stringify`/Postgres jsonb).
|
|
46
|
+
*/
|
|
47
|
+
export async function recordOnce<T>(opts: {
|
|
48
|
+
db: Database;
|
|
49
|
+
stateId: string;
|
|
50
|
+
namespace: RecordNamespace;
|
|
51
|
+
key: string;
|
|
52
|
+
compute: () => Promise<T> | T;
|
|
53
|
+
}): Promise<T> {
|
|
54
|
+
const { db, stateId, namespace, key, compute } = opts;
|
|
55
|
+
if (!Object.hasOwn(NAMESPACE_TOKEN, namespace)) {
|
|
56
|
+
throw new Error(`recordOnce: unknown namespace "${namespace}"`);
|
|
57
|
+
}
|
|
58
|
+
const token = NAMESPACE_TOKEN[namespace];
|
|
59
|
+
|
|
60
|
+
// Read the current state row's context and return `context[namespace][key]`
|
|
61
|
+
// when already recorded (an earlier writer, or a replay-from-top) WITHOUT
|
|
62
|
+
// re-running `compute`.
|
|
63
|
+
const row = await db.query.journeyStates.findFirst({
|
|
64
|
+
where: eq(journeyStates.id, stateId),
|
|
65
|
+
columns: { context: true },
|
|
66
|
+
});
|
|
67
|
+
const bag = namespaceBag(row?.context, namespace);
|
|
68
|
+
if (Object.hasOwn(bag, key)) {
|
|
69
|
+
return bag[key] as T;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const value = await compute();
|
|
73
|
+
// Persist under context.<namespace>.<key>. jsonb_set with create_missing=true
|
|
74
|
+
// cannot create a NESTED key whose parent object is absent (a fresh '{}' has no
|
|
75
|
+
// '<namespace>'), so we set the TOP-LEVEL '<namespace>' to the single new key
|
|
76
|
+
// MERGED with its existing bag — creating '<namespace>' when missing AND
|
|
77
|
+
// preserving sibling keys. The merge is FIRST-writer-wins: `a || b` lets b's
|
|
78
|
+
// keys win on conflict, so the EXISTING bag sits on the RIGHT — a concurrent
|
|
79
|
+
// writer that committed `key` first is NOT clobbered by this write. `key` and
|
|
80
|
+
// the value are bound parameters (jsonb_build_object), so the write is
|
|
81
|
+
// injection-safe; the namespace token is a validated closed-union literal.
|
|
82
|
+
//
|
|
83
|
+
// RETURNING hands back the POST-merge committed row in the same round-trip: a
|
|
84
|
+
// blocked concurrent loser re-evaluates the SET against the winner's committed
|
|
85
|
+
// tuple (existing bag wins the merge) and RETURNING yields the winner's value —
|
|
86
|
+
// byte-identical to a separate read-back, one query cheaper.
|
|
87
|
+
const [updated] = await db
|
|
88
|
+
.update(journeyStates)
|
|
89
|
+
.set({
|
|
90
|
+
context: sql`jsonb_set(
|
|
91
|
+
coalesce(${journeyStates.context}, '{}'::jsonb),
|
|
92
|
+
${sql.raw(`'{${token}}'`)},
|
|
93
|
+
jsonb_build_object(${key}::text, ${JSON.stringify(value ?? null)}::jsonb)
|
|
94
|
+
|| coalesce(${journeyStates.context} -> ${sql.raw(`'${token}'`)}, '{}'::jsonb),
|
|
95
|
+
true
|
|
96
|
+
)`,
|
|
97
|
+
updatedAt: new Date(),
|
|
98
|
+
})
|
|
99
|
+
.where(eq(journeyStates.id, stateId))
|
|
100
|
+
.returning({ context: journeyStates.context });
|
|
101
|
+
|
|
102
|
+
// Return the committed value — never the locally computed one. Fall back to
|
|
103
|
+
// the computed value only if the row vanished mid-flight (deleted
|
|
104
|
+
// concurrently: 0 rows updated).
|
|
105
|
+
const after = namespaceBag(updated?.context, namespace);
|
|
106
|
+
if (Object.hasOwn(after, key)) {
|
|
107
|
+
return after[key] as T;
|
|
108
|
+
}
|
|
109
|
+
return value;
|
|
110
|
+
}
|