@hogsend/engine 0.1.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +6 -6
- package/src/buckets/check-membership.ts +490 -0
- package/src/buckets/define-bucket.ts +52 -0
- package/src/buckets/membership-epoch.ts +186 -0
- package/src/buckets/registry-singleton.ts +21 -0
- package/src/buckets/registry.ts +62 -0
- package/src/container.ts +27 -1
- package/src/env.ts +6 -0
- package/src/index.ts +39 -1
- package/src/lib/bucket-emit.ts +107 -0
- package/src/lib/bucket-posthog-sync.ts +63 -0
- package/src/lib/ingestion.ts +25 -0
- package/src/routes/admin/buckets.ts +462 -0
- package/src/routes/admin/index.ts +2 -0
- package/src/routes/admin/metrics.ts +255 -0
- package/src/worker.ts +37 -0
- package/src/workflows/bucket-backfill.ts +593 -0
- package/src/workflows/bucket-reconcile.ts +1010 -0
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type BucketMeta,
|
|
3
|
+
type ConditionEval,
|
|
4
|
+
type DurationObject,
|
|
5
|
+
durationToMs,
|
|
6
|
+
type EventCondition,
|
|
7
|
+
} from "@hogsend/core";
|
|
8
|
+
import { bucketMemberships, type Database } from "@hogsend/db";
|
|
9
|
+
import { and, eq, sql } from "drizzle-orm";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* The reserved prefix every bucket transition event carries. Single source of
|
|
13
|
+
* truth shared by the ingest recursion guard (`checkBucketMembership`) and the
|
|
14
|
+
* fast-expiry arming event (`bucket:arm-expiry`).
|
|
15
|
+
*/
|
|
16
|
+
export const BUCKET_EVENT_PREFIX = "bucket:";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* The count of ALL prior memberships (active + left, NO status filter) for a
|
|
20
|
+
* (userId, bucketId) pair. This is the single source for the entryCount-ordinal
|
|
21
|
+
* rule (Section 6.3 / 8.2): the next epoch is `1 + countPriorMemberships(...)`,
|
|
22
|
+
* and the same count drives the entryLimit gate. Both the real-time join path
|
|
23
|
+
* (`handleJoin`) and the reconcile-discovered join path (`reconcileJoinOne`)
|
|
24
|
+
* call this so the ordinal can never drift between the two writers.
|
|
25
|
+
*
|
|
26
|
+
* The predicate MUST stay status-agnostic — narrowing it (e.g. to active-only)
|
|
27
|
+
* would corrupt entryCount and the entryLimit cooldown.
|
|
28
|
+
*/
|
|
29
|
+
export async function countPriorMemberships(
|
|
30
|
+
db: Database,
|
|
31
|
+
bucketId: string,
|
|
32
|
+
userId: string,
|
|
33
|
+
): Promise<number> {
|
|
34
|
+
const [counted] = await db
|
|
35
|
+
.select({ priorCount: sql<number>`count(*)::int` })
|
|
36
|
+
.from(bucketMemberships)
|
|
37
|
+
.where(
|
|
38
|
+
and(
|
|
39
|
+
eq(bucketMemberships.userId, userId),
|
|
40
|
+
eq(bucketMemberships.bucketId, bucketId),
|
|
41
|
+
),
|
|
42
|
+
);
|
|
43
|
+
return Number(counted?.priorCount ?? 0);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Find the first `EventCondition.within` rolling window in a criteria tree
|
|
48
|
+
* (depth-first). Returns null when no event leg carries a window. The single
|
|
49
|
+
* source for the membership-expiry / fastExpiry deadline math — shared so the
|
|
50
|
+
* three membership writers (real-time join, reconcile join, backfill join) can
|
|
51
|
+
* never disagree on which window drives `expiresAt`.
|
|
52
|
+
*/
|
|
53
|
+
export function firstWithin(criteria: ConditionEval): DurationObject | null {
|
|
54
|
+
if (criteria.type === "event" && criteria.within) {
|
|
55
|
+
return criteria.within;
|
|
56
|
+
}
|
|
57
|
+
if (criteria.type === "composite") {
|
|
58
|
+
for (const child of criteria.conditions) {
|
|
59
|
+
const found = firstWithin(child);
|
|
60
|
+
if (found) return found;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* The persisted membership-expiry / fastExpiry arming deadline. Non-time-based,
|
|
68
|
+
* non-fastExpiry buckets have no deadline (returns null). Time-based / fastExpiry
|
|
69
|
+
* buckets that carry a single `within` window get `now + within`; the reconcile
|
|
70
|
+
* cron and fastExpiry timer own the actual leave.
|
|
71
|
+
*
|
|
72
|
+
* Centralized so the real-time join (check-membership.ts), the reconcile-discovered
|
|
73
|
+
* join (bucket-reconcile.ts), and the backfill join (bucket-backfill.ts) compute
|
|
74
|
+
* the SAME deadline — a divergence here would let a membership-writer arm a window
|
|
75
|
+
* the cron/timer disagrees with.
|
|
76
|
+
*/
|
|
77
|
+
export function computeExpiresAt(bucket: BucketMeta): Date | null {
|
|
78
|
+
if (!bucket.criteria) return null;
|
|
79
|
+
if (!bucket.timeBased && !bucket.fastExpiry) return null;
|
|
80
|
+
const within = firstWithin(bucket.criteria);
|
|
81
|
+
if (!within) return null;
|
|
82
|
+
return new Date(Date.now() + durationToMs(within));
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* The unconditional max-dwell TTL deadline, stamped once on JOIN. null when the
|
|
87
|
+
* bucket has no `maxDwell`; the TTL sweep filters `isNotNull(maxDwellAt)`, so an
|
|
88
|
+
* unset value is never force-left. Shared by all three join writers so the TTL
|
|
89
|
+
* stamp is computed identically.
|
|
90
|
+
*/
|
|
91
|
+
export function computeMaxDwellAt(bucket: BucketMeta): Date | null {
|
|
92
|
+
return bucket.maxDwell
|
|
93
|
+
? new Date(Date.now() + durationToMs(bucket.maxDwell))
|
|
94
|
+
: null;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* The shared windowed event-count operator comparison kernel (`gt/gte/lt/lte/eq`).
|
|
99
|
+
* Returns the boolean of `count <operator> value`, or null for an unrecognized
|
|
100
|
+
* operator so each caller keeps its own default (the leave/match wrappers below).
|
|
101
|
+
* This is the single source for the count-operator math the reconcile SHOULD-LEAVE
|
|
102
|
+
* and the backfill MATCH paths both depend on.
|
|
103
|
+
*/
|
|
104
|
+
function compareCount(
|
|
105
|
+
operator: NonNullable<EventCondition["operator"]>,
|
|
106
|
+
count: number,
|
|
107
|
+
value: number,
|
|
108
|
+
): boolean | null {
|
|
109
|
+
switch (operator) {
|
|
110
|
+
case "gt":
|
|
111
|
+
return count > value;
|
|
112
|
+
case "gte":
|
|
113
|
+
return count >= value;
|
|
114
|
+
case "lt":
|
|
115
|
+
return count < value;
|
|
116
|
+
case "lte":
|
|
117
|
+
return count <= value;
|
|
118
|
+
case "eq":
|
|
119
|
+
return count === value;
|
|
120
|
+
default:
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* True when a windowed `count` satisfies the (exists/not_exists/count) event
|
|
127
|
+
* criterion — the POSITIVE "is a member by this windowed count" decision. Shared
|
|
128
|
+
* by the backfill matcher path (selectEventMatchers' positive branch). Behavior is
|
|
129
|
+
* identical to the prior local `matchesCount`: a `count` check with no
|
|
130
|
+
* operator/value (or an unrecognized operator) falls back to `count > 0` / `false`
|
|
131
|
+
* respectively.
|
|
132
|
+
*/
|
|
133
|
+
export function matchesEventCount(
|
|
134
|
+
criteria: EventCondition,
|
|
135
|
+
count: number,
|
|
136
|
+
): boolean {
|
|
137
|
+
switch (criteria.check) {
|
|
138
|
+
case "exists":
|
|
139
|
+
return count > 0;
|
|
140
|
+
case "not_exists":
|
|
141
|
+
return count === 0;
|
|
142
|
+
case "count": {
|
|
143
|
+
if (!criteria.operator || criteria.value === undefined) return count > 0;
|
|
144
|
+
const result = compareCount(criteria.operator, count, criteria.value);
|
|
145
|
+
return result ?? false;
|
|
146
|
+
}
|
|
147
|
+
default:
|
|
148
|
+
return false;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* The SHOULD-LEAVE decision from a windowed count, per criterion shape — a member
|
|
154
|
+
* leaves when the criterion is NO LONGER satisfied. Shared by the reconcile
|
|
155
|
+
* cron's set-based leave path. Behavior is identical to the prior local
|
|
156
|
+
* `shouldLeaveByCount`: it is the per-shape NEGATION of {@link matchesEventCount}
|
|
157
|
+
* for `exists`/`count`, an event-reappeared check for `not_exists`, and preserves
|
|
158
|
+
* the `count` `default → false` for an unrecognized operator.
|
|
159
|
+
*/
|
|
160
|
+
export function shouldLeaveByCount(
|
|
161
|
+
criteria: EventCondition,
|
|
162
|
+
windowedCount: number,
|
|
163
|
+
): boolean {
|
|
164
|
+
switch (criteria.check) {
|
|
165
|
+
case "not_exists":
|
|
166
|
+
// Absence bucket: SHOULD LEAVE when an event REAPPEARS in the window.
|
|
167
|
+
return windowedCount > 0;
|
|
168
|
+
case "exists":
|
|
169
|
+
// Positive existence: SHOULD LEAVE when NOT EXISTS in the window.
|
|
170
|
+
return windowedCount === 0;
|
|
171
|
+
case "count": {
|
|
172
|
+
// SHOULD LEAVE when the windowed count NO LONGER satisfies the operator.
|
|
173
|
+
if (!criteria.operator || criteria.value === undefined) {
|
|
174
|
+
return windowedCount === 0;
|
|
175
|
+
}
|
|
176
|
+
const result = compareCount(
|
|
177
|
+
criteria.operator,
|
|
178
|
+
windowedCount,
|
|
179
|
+
criteria.value,
|
|
180
|
+
);
|
|
181
|
+
return result == null ? false : !result;
|
|
182
|
+
}
|
|
183
|
+
default:
|
|
184
|
+
return false;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { BucketRegistry } from "@hogsend/core/registry";
|
|
2
|
+
|
|
3
|
+
let _registry: BucketRegistry | undefined;
|
|
4
|
+
|
|
5
|
+
export function setBucketRegistry(registry: BucketRegistry): void {
|
|
6
|
+
_registry = registry;
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export function getBucketRegistrySingleton(): BucketRegistry {
|
|
10
|
+
if (!_registry) {
|
|
11
|
+
throw new Error(
|
|
12
|
+
"Bucket registry not initialized. Call setBucketRegistry() at startup.",
|
|
13
|
+
);
|
|
14
|
+
}
|
|
15
|
+
return _registry;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
/** Reset the singleton — only for test cleanup. */
|
|
19
|
+
export function resetBucketRegistry(): void {
|
|
20
|
+
_registry = undefined;
|
|
21
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { BucketRegistry } from "@hogsend/core/registry";
|
|
2
|
+
import { parseEnabledFilter } from "../journeys/registry.js";
|
|
3
|
+
import { bucketExpiryTask } from "../workflows/bucket-reconcile.js";
|
|
4
|
+
import type { DefinedBucket } from "./define-bucket.js";
|
|
5
|
+
import { setBucketRegistry } from "./registry-singleton.js";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Build a {@link BucketRegistry} from an injected array of buckets, applying the
|
|
9
|
+
* enabled filter, and install it as the process singleton (so the real-time
|
|
10
|
+
* ingest path and the reconcile cron can resolve it). Returns the registry.
|
|
11
|
+
*
|
|
12
|
+
* `parseEnabledFilter` (journeys/registry.ts) is reused as-is — `ENABLED_BUCKETS`
|
|
13
|
+
* honours the same `"*"`-or-csv contract as `ENABLED_JOURNEYS` (Section 9.3).
|
|
14
|
+
* `BucketRegistry.register()` runs `bucketMetaSchema.parse()` internally, so no
|
|
15
|
+
* separate validation step is needed here.
|
|
16
|
+
*/
|
|
17
|
+
export function buildBucketRegistry(
|
|
18
|
+
buckets: DefinedBucket[],
|
|
19
|
+
enabledFilter?: string,
|
|
20
|
+
): BucketRegistry {
|
|
21
|
+
const registry = new BucketRegistry();
|
|
22
|
+
const enabled = parseEnabledFilter(enabledFilter);
|
|
23
|
+
|
|
24
|
+
for (const bucket of buckets) {
|
|
25
|
+
if (enabled === "*" || enabled.has(bucket.meta.id)) {
|
|
26
|
+
registry.register(bucket.meta);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
setBucketRegistry(registry);
|
|
31
|
+
return registry;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Select the Hatchet durable tasks for the enabled buckets. This is the SINGLE
|
|
36
|
+
* place a bucket's per-user fast-expiry timer task is constructed (Section
|
|
37
|
+
* 4.3/9.4) — task construction happens at worker build, AFTER the registry has
|
|
38
|
+
* validated every meta, never at module-load.
|
|
39
|
+
*
|
|
40
|
+
* Only `meta.fastExpiry` buckets contribute a task; the engine-wide
|
|
41
|
+
* `bucketReconcileTask` (registered separately in `baseWorkflows`) owns
|
|
42
|
+
* time-based leaves regardless. The fast-expiry timer is a single shared
|
|
43
|
+
* `durableTask` keyed on `bucket:arm-expiry` (per-bucket arming is by event
|
|
44
|
+
* payload, not per-bucket task instances), so it is registered once if ANY
|
|
45
|
+
* enabled bucket opts in.
|
|
46
|
+
*/
|
|
47
|
+
export function selectBucketTasks(
|
|
48
|
+
buckets: DefinedBucket[],
|
|
49
|
+
enabledFilter?: string,
|
|
50
|
+
): NonNullable<DefinedBucket["task"]>[] {
|
|
51
|
+
const enabled = parseEnabledFilter(enabledFilter);
|
|
52
|
+
const hasFastExpiry = buckets.some(
|
|
53
|
+
(b) =>
|
|
54
|
+
(enabled === "*" || enabled.has(b.meta.id)) && b.meta.fastExpiry === true,
|
|
55
|
+
);
|
|
56
|
+
if (!hasFastExpiry) return [];
|
|
57
|
+
|
|
58
|
+
// The single shared `bucket:arm-expiry` durableTask, registered ONCE because
|
|
59
|
+
// any enabled bucket opts in (per-bucket arming is by event payload). Cast to
|
|
60
|
+
// the DefinedBucket task shape — both are `hatchet.durableTask` returns.
|
|
61
|
+
return [bucketExpiryTask as NonNullable<DefinedBucket["task"]>];
|
|
62
|
+
}
|
package/src/container.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
|
|
2
2
|
import type { TimeZone } from "@hogsend/core";
|
|
3
|
-
import type { JourneyRegistry } from "@hogsend/core/registry";
|
|
3
|
+
import type { BucketRegistry, JourneyRegistry } from "@hogsend/core/registry";
|
|
4
4
|
import type { SendWindow } from "@hogsend/core/schedule";
|
|
5
5
|
import {
|
|
6
6
|
createDatabase,
|
|
@@ -16,6 +16,8 @@ import {
|
|
|
16
16
|
type EmailProvider,
|
|
17
17
|
} from "@hogsend/plugin-resend";
|
|
18
18
|
import type { Resend } from "resend";
|
|
19
|
+
import type { DefinedBucket } from "./buckets/define-bucket.js";
|
|
20
|
+
import { buildBucketRegistry } from "./buckets/registry.js";
|
|
19
21
|
import { env } from "./env.js";
|
|
20
22
|
import { setClientScheduleDefaults } from "./journeys/client-defaults-singleton.js";
|
|
21
23
|
import type { DefinedJourney } from "./journeys/define-journey.js";
|
|
@@ -58,6 +60,13 @@ export interface HogsendClient {
|
|
|
58
60
|
templates: TemplateRegistry;
|
|
59
61
|
analytics?: PostHogService;
|
|
60
62
|
registry: JourneyRegistry;
|
|
63
|
+
/**
|
|
64
|
+
* The bucket registry (id map + event/property inverted indexes for candidate
|
|
65
|
+
* narrowing). Built and installed as the process singleton at client build;
|
|
66
|
+
* the real-time ingest path reads it via `getBucketRegistrySingleton()`.
|
|
67
|
+
* Empty when no buckets are wired.
|
|
68
|
+
*/
|
|
69
|
+
bucketRegistry: BucketRegistry;
|
|
61
70
|
hatchet: HatchetClient;
|
|
62
71
|
/**
|
|
63
72
|
* The client repo's migration journal (`migrations/meta/_journal.json`),
|
|
@@ -77,6 +86,8 @@ export interface HogsendClient {
|
|
|
77
86
|
export interface HogsendClientOptions {
|
|
78
87
|
/** Journeys to register in the {@link JourneyRegistry}. Defaults to none. */
|
|
79
88
|
journeys?: DefinedJourney[];
|
|
89
|
+
/** Buckets to register in the {@link BucketRegistry}. Defaults to none. */
|
|
90
|
+
buckets?: DefinedBucket[];
|
|
80
91
|
/**
|
|
81
92
|
* Email is a first-class channel. Its config is grouped here rather than
|
|
82
93
|
* spread across top-level args — the engine owns the cohesive email pipeline
|
|
@@ -112,6 +123,11 @@ export interface HogsendClientOptions {
|
|
|
112
123
|
* `env.ENABLED_JOURNEYS`.
|
|
113
124
|
*/
|
|
114
125
|
enabledJourneys?: string;
|
|
126
|
+
/**
|
|
127
|
+
* Comma-separated ids (or `*`) controlling which buckets load. Defaults to
|
|
128
|
+
* `env.ENABLED_BUCKETS`.
|
|
129
|
+
*/
|
|
130
|
+
enabledBuckets?: string;
|
|
115
131
|
/**
|
|
116
132
|
* The client repo's migration journal for the `schema.client` health block.
|
|
117
133
|
* Defaults to `{ entries: [] }` (empty client track ⇒ trivially in sync).
|
|
@@ -182,6 +198,14 @@ export function createHogsendClient(
|
|
|
182
198
|
opts.enabledJourneys ?? env.ENABLED_JOURNEYS,
|
|
183
199
|
);
|
|
184
200
|
|
|
201
|
+
// Installs the bucket registry singleton in BOTH the API and worker processes
|
|
202
|
+
// (both call createHogsendClient); the real-time ingest path reads it via
|
|
203
|
+
// getBucketRegistrySingleton().
|
|
204
|
+
const bucketRegistry = buildBucketRegistry(
|
|
205
|
+
opts.buckets ?? [],
|
|
206
|
+
opts.enabledBuckets ?? env.ENABLED_BUCKETS,
|
|
207
|
+
);
|
|
208
|
+
|
|
185
209
|
const provider =
|
|
186
210
|
opts.email?.provider ??
|
|
187
211
|
createResendProvider({
|
|
@@ -228,6 +252,7 @@ export function createHogsendClient(
|
|
|
228
252
|
const analytics = opts.analytics ?? getPostHog();
|
|
229
253
|
|
|
230
254
|
logger.info(`Journey registry loaded: ${registry.count()} journeys`);
|
|
255
|
+
logger.info(`Bucket registry loaded: ${bucketRegistry.count()} buckets`);
|
|
231
256
|
|
|
232
257
|
return {
|
|
233
258
|
env,
|
|
@@ -240,6 +265,7 @@ export function createHogsendClient(
|
|
|
240
265
|
templates,
|
|
241
266
|
analytics,
|
|
242
267
|
registry,
|
|
268
|
+
bucketRegistry,
|
|
243
269
|
hatchet: opts.overrides?.hatchet ?? hatchet,
|
|
244
270
|
clientJournal: opts.clientJournal ?? { entries: [] },
|
|
245
271
|
defaults,
|
package/src/env.ts
CHANGED
|
@@ -54,6 +54,12 @@ export const env = createEnv({
|
|
|
54
54
|
ADMIN_API_KEY: z.string().min(1).optional(),
|
|
55
55
|
API_PUBLIC_URL: z.string().url().default("http://localhost:3002"),
|
|
56
56
|
ENABLED_JOURNEYS: z.string().default("*"),
|
|
57
|
+
// Buckets: same `"*"`-or-csv contract as ENABLED_JOURNEYS (Section 9.3).
|
|
58
|
+
// Evaluated at worker boot — a toggle requires a worker restart; only the
|
|
59
|
+
// bucket_configs DB override is hot.
|
|
60
|
+
ENABLED_BUCKETS: z.string().default("*"),
|
|
61
|
+
// Cadence for the engine-owned bucket reconcile cron (time-based leaves).
|
|
62
|
+
BUCKET_RECONCILE_CRON: z.string().default("*/5 * * * *"),
|
|
57
63
|
},
|
|
58
64
|
runtimeEnv: process.env,
|
|
59
65
|
emptyStringAsUndefined: true,
|
package/src/index.ts
CHANGED
|
@@ -6,7 +6,10 @@
|
|
|
6
6
|
// Core helpers used by content journeys (days/hours/minutes, condition + journey
|
|
7
7
|
// types) so content can import everything from `@hogsend/engine`.
|
|
8
8
|
export * from "@hogsend/core";
|
|
9
|
-
export {
|
|
9
|
+
export {
|
|
10
|
+
BucketRegistry,
|
|
11
|
+
JourneyRegistry,
|
|
12
|
+
} from "@hogsend/core/registry";
|
|
10
13
|
// --- Re-exports for content ---
|
|
11
14
|
// Schema/version helpers used by the boot guard and the /v1/health route.
|
|
12
15
|
export {
|
|
@@ -19,6 +22,25 @@ export {
|
|
|
19
22
|
} from "@hogsend/db";
|
|
20
23
|
// --- App / container / worker factories ---
|
|
21
24
|
export { type AppEnv, type CreateAppOptions, createApp } from "./app.js";
|
|
25
|
+
// --- Buckets ---
|
|
26
|
+
export {
|
|
27
|
+
type BucketTransition,
|
|
28
|
+
type BucketTransitionKind,
|
|
29
|
+
checkBucketMembership,
|
|
30
|
+
} from "./buckets/check-membership.js";
|
|
31
|
+
export {
|
|
32
|
+
type DefinedBucket,
|
|
33
|
+
defineBucket,
|
|
34
|
+
} from "./buckets/define-bucket.js";
|
|
35
|
+
export {
|
|
36
|
+
buildBucketRegistry,
|
|
37
|
+
selectBucketTasks,
|
|
38
|
+
} from "./buckets/registry.js";
|
|
39
|
+
export {
|
|
40
|
+
getBucketRegistrySingleton,
|
|
41
|
+
resetBucketRegistry,
|
|
42
|
+
setBucketRegistry,
|
|
43
|
+
} from "./buckets/registry-singleton.js";
|
|
22
44
|
export {
|
|
23
45
|
createHogsendClient,
|
|
24
46
|
type HogsendClient,
|
|
@@ -50,6 +72,11 @@ export {
|
|
|
50
72
|
type BatchedBackfillResult,
|
|
51
73
|
runBatchedBackfill,
|
|
52
74
|
} from "./lib/backfill.js";
|
|
75
|
+
// --- Bucket transition emission (shared by real-time / cron / fast-expiry) ---
|
|
76
|
+
export {
|
|
77
|
+
type BucketTransitionSource,
|
|
78
|
+
emitBucketTransition,
|
|
79
|
+
} from "./lib/bucket-emit.js";
|
|
53
80
|
// --- Infrastructure singletons ---
|
|
54
81
|
export { getDb } from "./lib/db.js";
|
|
55
82
|
// --- Email ---
|
|
@@ -121,6 +148,17 @@ export {
|
|
|
121
148
|
createWorker,
|
|
122
149
|
type Worker,
|
|
123
150
|
} from "./worker.js";
|
|
151
|
+
export {
|
|
152
|
+
type BucketBackfillInput,
|
|
153
|
+
bucketBackfillTask,
|
|
154
|
+
computeCriteriaHash,
|
|
155
|
+
enqueueBucketBackfills,
|
|
156
|
+
} from "./workflows/bucket-backfill.js";
|
|
157
|
+
export {
|
|
158
|
+
type BucketArmExpiryInput,
|
|
159
|
+
bucketExpiryTask,
|
|
160
|
+
bucketReconcileTask,
|
|
161
|
+
} from "./workflows/bucket-reconcile.js";
|
|
124
162
|
export { checkAlertsTask } from "./workflows/check-alerts.js";
|
|
125
163
|
export { importContactsTask } from "./workflows/import-contacts.js";
|
|
126
164
|
// --- Built-in Hatchet workflow tasks ---
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
|
|
2
|
+
import type { BucketMeta } from "@hogsend/core";
|
|
3
|
+
import type { JourneyRegistry } from "@hogsend/core/registry";
|
|
4
|
+
import type { Database } from "@hogsend/db";
|
|
5
|
+
import { syncBucketToPostHog } from "./bucket-posthog-sync.js";
|
|
6
|
+
import { ingestEvent } from "./ingestion.js";
|
|
7
|
+
import type { Logger } from "./logger.js";
|
|
8
|
+
|
|
9
|
+
export type BucketTransitionKind = "entered" | "left";
|
|
10
|
+
|
|
11
|
+
/** Where a transition originated — carried on the emitted event properties. */
|
|
12
|
+
export type BucketTransitionSource =
|
|
13
|
+
| "event"
|
|
14
|
+
| "reconcile"
|
|
15
|
+
| "backfill"
|
|
16
|
+
| "manual";
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Emit a bucket transition back through `ingestEvent` (the `ctx.trigger`
|
|
20
|
+
* precedent) — shared by ALL three producers (real-time `checkBucketMembership`,
|
|
21
|
+
* the reconcile cron, and the fast-expiry timer) so they compute byte-identical
|
|
22
|
+
* `idempotencyKey`s for the same transition and converge to ONE emission
|
|
23
|
+
* (Section 6.3 worked example).
|
|
24
|
+
*
|
|
25
|
+
* Persists to `userEvents`, pushes to Hatchet (routing to journeys), and runs
|
|
26
|
+
* `checkExits`. Emits the per-bucket ALIAS (`bucket:<kind>:<id>`) by default; the
|
|
27
|
+
* generic `bucket:<kind>` is emitted ONLY when a generic-bound journey actually
|
|
28
|
+
* exists (aliased-only default — Section 8.5). `epoch` is the winning membership
|
|
29
|
+
* row's `entryCount`, read off the single winning mutation by the caller.
|
|
30
|
+
*/
|
|
31
|
+
export async function emitBucketTransition(opts: {
|
|
32
|
+
db: Database;
|
|
33
|
+
/** The JOURNEY registry — forwarded into the recursive emit ingestEvent. */
|
|
34
|
+
registry: JourneyRegistry;
|
|
35
|
+
hatchet: HatchetClient;
|
|
36
|
+
logger: Logger;
|
|
37
|
+
kind: BucketTransitionKind;
|
|
38
|
+
bucket: BucketMeta;
|
|
39
|
+
userId: string;
|
|
40
|
+
userEmail: string | null;
|
|
41
|
+
epoch: number;
|
|
42
|
+
source?: BucketTransitionSource;
|
|
43
|
+
}): Promise<void> {
|
|
44
|
+
const {
|
|
45
|
+
db,
|
|
46
|
+
registry,
|
|
47
|
+
hatchet,
|
|
48
|
+
logger,
|
|
49
|
+
kind,
|
|
50
|
+
bucket,
|
|
51
|
+
userId,
|
|
52
|
+
userEmail,
|
|
53
|
+
epoch,
|
|
54
|
+
source = "event",
|
|
55
|
+
} = opts;
|
|
56
|
+
|
|
57
|
+
const properties: Record<string, unknown> = {
|
|
58
|
+
bucketId: bucket.id,
|
|
59
|
+
bucketName: bucket.name,
|
|
60
|
+
userId,
|
|
61
|
+
transition: kind,
|
|
62
|
+
source,
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
// Optional PostHog person-property mirror (Section 12). Off by default; a
|
|
66
|
+
// no-op without POSTHOG_API_KEY. Wired here, the single transition path shared
|
|
67
|
+
// by all three producers (real-time / reconcile / fast-expiry), so the sync
|
|
68
|
+
// fires exactly once per emitted transition. Best-effort — it never blocks the
|
|
69
|
+
// event emit below.
|
|
70
|
+
syncBucketToPostHog({ logger, kind, bucket, userId });
|
|
71
|
+
|
|
72
|
+
// Per-bucket alias — the recommended, narrowly-routed binding. The
|
|
73
|
+
// deterministic idempotencyKey rides the userEvents dedup short-circuit as
|
|
74
|
+
// defense-in-depth (Section 6.3).
|
|
75
|
+
await ingestEvent({
|
|
76
|
+
db,
|
|
77
|
+
registry,
|
|
78
|
+
hatchet,
|
|
79
|
+
logger,
|
|
80
|
+
event: {
|
|
81
|
+
event: `bucket:${kind}:${bucket.id}`,
|
|
82
|
+
userId,
|
|
83
|
+
userEmail: userEmail ?? "",
|
|
84
|
+
properties,
|
|
85
|
+
idempotencyKey: `bucket:${bucket.id}:${userId}:${kind}:${epoch}`,
|
|
86
|
+
},
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
// Generic form — emitted ONLY if a journey actually binds to it, so the
|
|
90
|
+
// recursion-guarded generic event is not written for nothing (Section 8.5).
|
|
91
|
+
const genericEvent = `bucket:${kind}`;
|
|
92
|
+
if (registry.getByTriggerEvent(genericEvent).length > 0) {
|
|
93
|
+
await ingestEvent({
|
|
94
|
+
db,
|
|
95
|
+
registry,
|
|
96
|
+
hatchet,
|
|
97
|
+
logger,
|
|
98
|
+
event: {
|
|
99
|
+
event: genericEvent,
|
|
100
|
+
userId,
|
|
101
|
+
userEmail: userEmail ?? "",
|
|
102
|
+
properties,
|
|
103
|
+
idempotencyKey: `bucket:${bucket.id}:${userId}:${kind}:${epoch}:generic`,
|
|
104
|
+
},
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import type { BucketMeta } from "@hogsend/core";
|
|
2
|
+
import type { Logger } from "./logger.js";
|
|
3
|
+
import { getPostHog } from "./posthog.js";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Optional PostHog person-property mirror for a bucket transition (Section 12).
|
|
7
|
+
*
|
|
8
|
+
* OFF BY DEFAULT — a no-op unless `meta.syncToPostHog === true`. Also a no-op
|
|
9
|
+
* without `POSTHOG_API_KEY` (`getPostHog()` returns undefined), so self-host
|
|
10
|
+
* setups that omit PostHog silently do nothing — documented, not broken.
|
|
11
|
+
*
|
|
12
|
+
* On JOIN it `$set`s a boolean person property `true`; on LEAVE it `$unset`s the
|
|
13
|
+
* same key. `$unset` (NOT `$set false`) is the recommended default: a cohort
|
|
14
|
+
* `key = true` excludes a false value, but a cohort `key is set` STILL matches a
|
|
15
|
+
* false value, so `$unset` is the only form where both cohort idioms behave
|
|
16
|
+
* correctly. The property key defaults to `hogsend_bucket_<id>`, overridable via
|
|
17
|
+
* `meta.postHogPropertyKey`.
|
|
18
|
+
*
|
|
19
|
+
* This reuses the existing `plugin-posthog` capture path (the same one
|
|
20
|
+
* `identify()` uses at journey-context.ts) — it adds no new integration surface
|
|
21
|
+
* and never pushes to any non-PostHog destination (the Section 2.4 anti-CDP
|
|
22
|
+
* invariant). Best-effort: a capture failure is logged and swallowed so a sync
|
|
23
|
+
* error never blocks a transition emit.
|
|
24
|
+
*/
|
|
25
|
+
export function syncBucketToPostHog(opts: {
|
|
26
|
+
logger: Logger;
|
|
27
|
+
kind: "entered" | "left";
|
|
28
|
+
bucket: BucketMeta;
|
|
29
|
+
userId: string;
|
|
30
|
+
}): void {
|
|
31
|
+
const { logger, kind, bucket, userId } = opts;
|
|
32
|
+
|
|
33
|
+
if (!bucket.syncToPostHog) return;
|
|
34
|
+
|
|
35
|
+
const posthog = getPostHog();
|
|
36
|
+
if (!posthog) return; // no POSTHOG_API_KEY → silent no-op
|
|
37
|
+
|
|
38
|
+
const propertyKey =
|
|
39
|
+
bucket.postHogPropertyKey ?? `hogsend_bucket_${bucket.id}`;
|
|
40
|
+
|
|
41
|
+
try {
|
|
42
|
+
if (kind === "entered") {
|
|
43
|
+
// $set { key: true } — mirrors plugin-posthog identify() ($set path).
|
|
44
|
+
posthog.identify(userId, { [propertyKey]: true });
|
|
45
|
+
} else {
|
|
46
|
+
// $unset [key] — RECOMMENDED on leave (Section 12). The property is absent
|
|
47
|
+
// unless the user is currently a member, so both `key = true` and
|
|
48
|
+
// `key is set` cohorts behave correctly.
|
|
49
|
+
posthog.captureEvent({
|
|
50
|
+
distinctId: userId,
|
|
51
|
+
event: "$set",
|
|
52
|
+
properties: { $unset: [propertyKey] },
|
|
53
|
+
});
|
|
54
|
+
}
|
|
55
|
+
} catch (err) {
|
|
56
|
+
logger.warn("Bucket PostHog sync failed (best-effort)", {
|
|
57
|
+
bucketId: bucket.id,
|
|
58
|
+
userId,
|
|
59
|
+
kind,
|
|
60
|
+
error: err instanceof Error ? err.message : String(err),
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
}
|
package/src/lib/ingestion.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { evaluatePropertyConditions } from "@hogsend/core";
|
|
|
3
3
|
import type { JourneyRegistry } from "@hogsend/core/registry";
|
|
4
4
|
import { type Database, journeyStates, userEvents } from "@hogsend/db";
|
|
5
5
|
import { and, eq, inArray, isNull } from "drizzle-orm";
|
|
6
|
+
import { checkBucketMembership } from "../buckets/check-membership.js";
|
|
6
7
|
import { upsertContact } from "./contacts.js";
|
|
7
8
|
import type { Logger } from "./logger.js";
|
|
8
9
|
|
|
@@ -93,6 +94,30 @@ export async function ingestEvent(opts: {
|
|
|
93
94
|
}),
|
|
94
95
|
]);
|
|
95
96
|
|
|
97
|
+
// Real-time bucket membership re-evaluation (Section 6.1). NOT part of the
|
|
98
|
+
// Promise.all above: its property eval reads MERGED contact state, and its
|
|
99
|
+
// bucket:entered/left emissions recurse back into ingestEvent (the recursion
|
|
100
|
+
// guard in checkBucketMembership bounds them). Best-effort: a bucket failure
|
|
101
|
+
// must not fail the ingest of the originating event.
|
|
102
|
+
try {
|
|
103
|
+
await checkBucketMembership({
|
|
104
|
+
db,
|
|
105
|
+
registry,
|
|
106
|
+
hatchet,
|
|
107
|
+
logger,
|
|
108
|
+
userId: event.userId,
|
|
109
|
+
userEmail: event.userEmail || null,
|
|
110
|
+
event: event.event,
|
|
111
|
+
properties: event.properties,
|
|
112
|
+
});
|
|
113
|
+
} catch (err) {
|
|
114
|
+
logger.warn("Bucket membership check failed", {
|
|
115
|
+
event: event.event,
|
|
116
|
+
userId: event.userId,
|
|
117
|
+
error: err instanceof Error ? err.message : String(err),
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
|
|
96
121
|
logger.info("Event ingested", {
|
|
97
122
|
event: event.event,
|
|
98
123
|
userId: event.userId,
|