@hogsend/engine 0.46.0 → 0.50.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/README.md +6 -0
  2. package/package.json +13 -10
  3. package/src/connectors/presets/index.ts +8 -2
  4. package/src/container.ts +149 -7
  5. package/src/destinations/presets/posthog.ts +7 -0
  6. package/src/destinations/presets/segment.ts +5 -0
  7. package/src/env.ts +38 -0
  8. package/src/flags/reconcile.ts +195 -0
  9. package/src/index.ts +79 -0
  10. package/src/journeys/authoring.ts +32 -0
  11. package/src/journeys/define-journey.ts +45 -40
  12. package/src/journeys/execute-journey-run.ts +73 -32
  13. package/src/journeys/journey-boundary.ts +76 -0
  14. package/src/journeys/journey-context.ts +128 -70
  15. package/src/journeys/journey-task-factory.ts +31 -0
  16. package/src/journeys/journey-task-runtime.ts +31 -0
  17. package/src/journeys/journey-version.test.ts +236 -0
  18. package/src/journeys/journey-version.ts +158 -0
  19. package/src/journeys/record-once.ts +28 -2
  20. package/src/lib/connector-actions.test.ts +36 -0
  21. package/src/lib/connector-actions.ts +42 -4
  22. package/src/lib/contacts.ts +49 -0
  23. package/src/lib/conversions.test.ts +33 -0
  24. package/src/lib/conversions.ts +8 -0
  25. package/src/lib/email.ts +23 -2
  26. package/src/lib/enrollment-policy.ts +51 -0
  27. package/src/lib/feed.ts +55 -3
  28. package/src/lib/flags-registry.ts +48 -0
  29. package/src/lib/flags.ts +498 -0
  30. package/src/lib/fx.ts +374 -0
  31. package/src/lib/global-control-readout.ts +166 -0
  32. package/src/lib/groups.ts +362 -0
  33. package/src/lib/ingestion.ts +38 -0
  34. package/src/lib/journey-lift.ts +142 -0
  35. package/src/lib/operator-settings.ts +71 -0
  36. package/src/lib/outbound.ts +147 -0
  37. package/src/lib/preferences.ts +6 -80
  38. package/src/lib/recipient-preferences.ts +53 -0
  39. package/src/lib/sms.ts +21 -2
  40. package/src/lib/stable-stringify.test.ts +42 -0
  41. package/src/lib/stable-stringify.ts +23 -0
  42. package/src/lib/variant.ts +99 -0
  43. package/src/lib/webhook-signing.ts +16 -1
  44. package/src/lists/registry.ts +6 -2
  45. package/src/lists/subscription.ts +13 -0
  46. package/src/routes/admin/contacts.ts +53 -4
  47. package/src/routes/admin/conversions.ts +137 -0
  48. package/src/routes/admin/flags.ts +369 -0
  49. package/src/routes/admin/groups.ts +866 -0
  50. package/src/routes/admin/impact-schemas.ts +44 -0
  51. package/src/routes/admin/impact-wire.test.ts +181 -0
  52. package/src/routes/admin/impact.ts +486 -0
  53. package/src/routes/admin/index.ts +12 -0
  54. package/src/routes/admin/journey-impact.ts +446 -0
  55. package/src/routes/admin/journeys.ts +70 -100
  56. package/src/routes/admin/readiness.ts +8 -8
  57. package/src/routes/admin/settings.ts +145 -0
  58. package/src/routes/admin/targeting.ts +256 -0
  59. package/src/routes/events/index.ts +11 -0
  60. package/src/routes/flags/index.ts +120 -0
  61. package/src/routes/groups/index.ts +417 -0
  62. package/src/routes/index.ts +23 -1
  63. package/src/testing.ts +25 -0
  64. package/src/webhook-sources/presets/index.ts +5 -1
  65. package/src/webhook-sources/presets/intercom.ts +230 -0
  66. package/src/webhook-sources/presets/segment.ts +56 -4
  67. package/src/worker.ts +3 -0
  68. package/src/workflows/bucket-backfill.ts +1 -14
  69. package/src/workflows/impact-digest.ts +761 -0
  70. package/src/workflows/journey-blueprint-interpreter.ts +18 -1
@@ -0,0 +1,256 @@
1
+ import { CANONICAL_STAGES, flagTargetingSchema } from "@hogsend/core";
2
+ import { campaigns, contacts } from "@hogsend/db";
3
+ import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
4
+ import { desc, isNull, sql } from "drizzle-orm";
5
+ import type { AppEnv } from "../../app.js";
6
+ import {
7
+ eventNameEntrySchema,
8
+ listEventNameVocabulary,
9
+ } from "../../lib/event-names.js";
10
+ import { evaluateTargeting, loadTargetingSnapshot } from "../../lib/flags.js";
11
+
12
+ /**
13
+ * Reusable targeting-builder catalog + a live match-count estimator. The
14
+ * catalog is the raw material a Studio condition builder needs to compose EVERY
15
+ * targeting leaf — the distinct contact-property keys and operator vocabulary
16
+ * (property leaves) plus the id/name lists a builder picks from for the richer
17
+ * snapshot leaves (buckets, journeys, deal stages) and the scan leaves (event
18
+ * names) and campaigns. Named generically (`/targeting/catalog`, not
19
+ * `/flags/catalog`) because the same vocabulary feeds any targeting UI.
20
+ * `POST /targeting/count` estimates how many live contacts a targeting tree
21
+ * matches, over a bounded sample. Both inherit the admin router's
22
+ * `requireAdmin` guard.
23
+ */
24
+
25
+ /** Cap on distinct property keys returned — a builder combobox, not an export. */
26
+ const MAX_PROPERTY_KEYS = 200;
27
+
28
+ /** Cap on event names returned in the catalog (observed + declared vocabulary). */
29
+ const MAX_EVENT_NAMES = 200;
30
+
31
+ /**
32
+ * Cap on contacts SAMPLED for key discovery. The catalog only needs the shape
33
+ * of the property vocabulary, not every key that ever existed, so we bound the
34
+ * SCAN (not just the output): without this, `distinct` + `order by` must expand
35
+ * every live contact's property object before `limit` can trim, i.e. a full
36
+ * sequential scan on every fetch. Sampling the most-recently-updated slice
37
+ * keeps the work bounded and the vocabulary current.
38
+ */
39
+ const MAX_SAMPLED_CONTACTS = 2000;
40
+
41
+ /**
42
+ * Cap on contacts SAMPLED for a match-count estimate. Mirrors the catalog's
43
+ * ~2000 most-recently-updated sampling so the estimate is bounded regardless of
44
+ * table size — each sampled contact costs one targeting snapshot (a fixed set
45
+ * of indexed queries), so this endpoint is O(sample), never O(all contacts).
46
+ */
47
+ const MAX_COUNT_SAMPLE = 2000;
48
+
49
+ /**
50
+ * How many sampled contacts to evaluate concurrently. Each evaluation loads a
51
+ * snapshot (a few indexed queries) and, in server mode, may run scan leaves; a
52
+ * small fixed fan-out bounds in-flight DB work instead of firing all
53
+ * `MAX_COUNT_SAMPLE` at once.
54
+ */
55
+ const COUNT_CONCURRENCY = 25;
56
+
57
+ /**
58
+ * The 9 property operators (see `PropertyCondition`) with human labels. `unary`
59
+ * operators (`exists`/`not_exists`) take no comparison value, so the builder
60
+ * hides the value input for them.
61
+ */
62
+ const OPERATORS: Array<{ value: string; label: string; unary: boolean }> = [
63
+ { value: "eq", label: "equals", unary: false },
64
+ { value: "neq", label: "does not equal", unary: false },
65
+ { value: "gt", label: "greater than", unary: false },
66
+ { value: "gte", label: "greater than or equal to", unary: false },
67
+ { value: "lt", label: "less than", unary: false },
68
+ { value: "lte", label: "less than or equal to", unary: false },
69
+ { value: "contains", label: "contains", unary: false },
70
+ { value: "exists", label: "is set", unary: true },
71
+ { value: "not_exists", label: "is not set", unary: true },
72
+ ];
73
+
74
+ /**
75
+ * The deal-stage vocabulary a `deal` leaf's `stage` predicate picks from: the
76
+ * deployment's canonical ladder plus the reserved terminal `"lost"`.
77
+ */
78
+ const DEAL_STAGES: string[] = [...CANONICAL_STAGES, "lost"];
79
+
80
+ const idNameSchema = z.object({ id: z.string(), name: z.string() });
81
+
82
+ const catalogSchema = z.object({
83
+ properties: z.array(z.string()),
84
+ operators: z.array(
85
+ z.object({
86
+ value: z.string(),
87
+ label: z.string(),
88
+ unary: z.boolean(),
89
+ }),
90
+ ),
91
+ buckets: z.array(idNameSchema),
92
+ journeys: z.array(idNameSchema),
93
+ dealStages: z.array(z.string()),
94
+ events: z.array(eventNameEntrySchema),
95
+ campaigns: z.array(idNameSchema),
96
+ });
97
+
98
+ const catalogRoute = createRoute({
99
+ method: "get",
100
+ path: "/catalog",
101
+ tags: ["Admin — Targeting"],
102
+ summary: "Targeting builder catalog (every leaf's vocabulary)",
103
+ responses: {
104
+ 200: {
105
+ content: { "application/json": { schema: catalogSchema } },
106
+ description:
107
+ "Property keys + operators, plus buckets, journeys, deal stages, event names, and campaigns",
108
+ },
109
+ },
110
+ });
111
+
112
+ const countBodySchema = z.object({
113
+ /** One condition set's targeting tree (or the legacy bare property array). */
114
+ targeting: flagTargetingSchema,
115
+ });
116
+
117
+ const countResponseSchema = z.object({
118
+ /** Sampled contacts the tree matched. */
119
+ matched: z.number(),
120
+ /** Sampled contacts evaluated (≤ MAX_COUNT_SAMPLE, ≤ live contacts). */
121
+ sampled: z.number(),
122
+ /**
123
+ * Honest estimate of matching LIVE contacts: `matched/sampled` scaled by the
124
+ * live-contact count. Exact when the sample covers every live contact.
125
+ */
126
+ estimatedTotal: z.number(),
127
+ });
128
+
129
+ const countRoute = createRoute({
130
+ method: "post",
131
+ path: "/count",
132
+ tags: ["Admin — Targeting"],
133
+ summary: "Estimate how many live contacts a targeting tree matches",
134
+ request: {
135
+ body: {
136
+ content: { "application/json": { schema: countBodySchema } },
137
+ },
138
+ },
139
+ responses: {
140
+ 200: {
141
+ content: { "application/json": { schema: countResponseSchema } },
142
+ description: "Sampled match count + scaled estimate",
143
+ },
144
+ },
145
+ });
146
+
147
+ export const targetingRouter = new OpenAPIHono<AppEnv>()
148
+ .openapi(catalogRoute, async (c) => {
149
+ const { db, registry, bucketRegistry } = c.get("container");
150
+
151
+ // Distinct top-level property keys across a bounded SAMPLE of the most
152
+ // recently-updated live contacts, sorted + capped. Sampling first (inner
153
+ // limit) keeps `jsonb_object_keys` — which fans each contact's properties
154
+ // object out to one row per key — from expanding the whole table on every
155
+ // fetch. `coalesce` guards NULL/absent property maps. The property scan and
156
+ // the event/campaign lookups are independent, so they go out together.
157
+ const [propertyRows, events, campaignRows] = await Promise.all([
158
+ db.execute<{ key: string }>(sql`
159
+ select distinct key
160
+ from (
161
+ select properties
162
+ from contacts
163
+ where deleted_at is null
164
+ order by updated_at desc
165
+ limit ${MAX_SAMPLED_CONTACTS}
166
+ ) as sampled,
167
+ jsonb_object_keys(coalesce(properties, '{}'::jsonb)) as key
168
+ order by key
169
+ limit ${MAX_PROPERTY_KEYS}
170
+ `),
171
+ listEventNameVocabulary({ db, registry, limit: MAX_EVENT_NAMES }),
172
+ db
173
+ .select({ id: campaigns.id, name: campaigns.name })
174
+ .from(campaigns)
175
+ .orderBy(desc(campaigns.createdAt))
176
+ .limit(MAX_PROPERTY_KEYS),
177
+ ]);
178
+
179
+ const properties = Array.from(propertyRows, (r) => r.key);
180
+ const buckets = bucketRegistry
181
+ .getAll()
182
+ .map((b) => ({ id: b.id, name: b.name }));
183
+ const journeys = registry.getAll().map((j) => ({ id: j.id, name: j.name }));
184
+
185
+ return c.json(
186
+ {
187
+ properties,
188
+ operators: OPERATORS,
189
+ buckets,
190
+ journeys,
191
+ dealStages: DEAL_STAGES,
192
+ events: events.events,
193
+ campaigns: campaignRows,
194
+ },
195
+ 200,
196
+ );
197
+ })
198
+ .openapi(countRoute, async (c) => {
199
+ const { db } = c.get("container");
200
+ const { targeting } = c.req.valid("json");
201
+
202
+ // Bounded, most-recently-updated sample + the live-contact total for
203
+ // scaling. `id` is the trusted uuid for property/deal reads; the logical
204
+ // `contactKey` (external_id ?? anonymous_id ?? id) keys the
205
+ // bucket/journey/event reads — the same identity split the flag readers use.
206
+ const [sample, totalRows] = await Promise.all([
207
+ db
208
+ .select({
209
+ id: contacts.id,
210
+ externalId: contacts.externalId,
211
+ anonymousId: contacts.anonymousId,
212
+ })
213
+ .from(contacts)
214
+ .where(isNull(contacts.deletedAt))
215
+ .orderBy(desc(contacts.updatedAt))
216
+ .limit(MAX_COUNT_SAMPLE),
217
+ db
218
+ .select({ total: sql<number>`count(*)`.mapWith(Number) })
219
+ .from(contacts)
220
+ .where(isNull(contacts.deletedAt)),
221
+ ]);
222
+
223
+ const liveTotal = totalRows[0]?.total ?? 0;
224
+
225
+ const evaluateOne = async (row: (typeof sample)[number]) => {
226
+ const contactKey = row.externalId ?? row.anonymousId ?? row.id;
227
+ const snapshot = await loadTargetingSnapshot({
228
+ db,
229
+ contactKey,
230
+ contactId: row.id,
231
+ });
232
+ // Server mode so bucket/journey/deal AND the scan leaves
233
+ // (event/email_engagement) all resolve for the estimate.
234
+ return evaluateTargeting(targeting, {
235
+ snapshot,
236
+ mode: "server",
237
+ db,
238
+ userId: contactKey,
239
+ });
240
+ };
241
+
242
+ // Bounded fan-out: evaluate in fixed-size chunks so at most
243
+ // COUNT_CONCURRENCY snapshots are loading at once.
244
+ let matched = 0;
245
+ for (let i = 0; i < sample.length; i += COUNT_CONCURRENCY) {
246
+ const chunk = sample.slice(i, i + COUNT_CONCURRENCY);
247
+ const results = await Promise.all(chunk.map(evaluateOne));
248
+ for (const hit of results) if (hit) matched += 1;
249
+ }
250
+
251
+ const sampled = sample.length;
252
+ const estimatedTotal =
253
+ sampled === 0 ? 0 : Math.round((matched / sampled) * liveTotal);
254
+
255
+ return c.json({ matched, sampled, estimatedTotal }, 200);
256
+ });
@@ -31,6 +31,14 @@ const eventRequestSchema = z.object({
31
31
  .regex(/^[A-Za-z]{3}$/)
32
32
  .optional(),
33
33
  lists: z.record(z.string(), z.boolean()).optional(),
34
+ // groupType → groupKey association map (e.g. `{ company: "acme.com" }`). The
35
+ // event is recorded against each group (`user_events.groups`), its membership
36
+ // is ensured, and the map is forwarded to analytics as `$groups`. This is
37
+ // ASSOCIATION-ONLY: the ingest path writes NO group properties, so a
38
+ // publishable (pk_) key can reference a group key here but can never write
39
+ // group properties or read groups (those live on the secret-only
40
+ // `/v1/groups` router).
41
+ groups: z.record(z.string().min(1), z.string().min(1)).optional(),
34
42
  idempotencyKey: z.string().optional(),
35
43
  timestamp: z.string().datetime().optional(),
36
44
  // Publishable-key identity assertion (§Phase 1). A pk_ key is anon-only
@@ -133,6 +141,9 @@ export const eventsRouter = new OpenAPIHono<AppEnv>().openapi(
133
141
  contactProperties: body.contactProperties,
134
142
  value: body.value,
135
143
  currency: body.currency,
144
+ // groupType→groupKey association (association-only; no property write).
145
+ // Forwarded as `$groups` on the mirrored analytics capture.
146
+ groups: body.groups,
136
147
  idempotencyKey,
137
148
  // §2.5: caller-supplied event time (backfill/replay). The validated
138
149
  // ISO string is coerced to a Date inside ingestEvent.
@@ -0,0 +1,120 @@
1
+ import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
2
+ import type { AppEnv } from "../../app.js";
3
+ import { evaluateFlagsForContact } from "../../lib/flags.js";
4
+ import { errorSchema } from "../../lib/schemas.js";
5
+ import { resolveFeedRecipient } from "../feed/recipient.js";
6
+
7
+ // The evaluated flag map — values are arbitrary JSON (boolean flags → true, the
8
+ // default; multivariate → the arm's `value`).
9
+ const flagsMapSchema = z.object({
10
+ flags: z.record(z.string(), z.unknown()),
11
+ });
12
+
13
+ // ---------------------------------------------------------------------------
14
+ // GET /v1/flags — the BROWSER read (publishable OR secret-ingest tier).
15
+ // Identity is recipient-scoped SERVER-SIDE via `resolveFeedRecipient` (the same
16
+ // leak boundary as the in-app feed): a userToken-verified userId, a secret
17
+ // key's trusted userId/email, or a publishable caller's OWN anon id — a pk_
18
+ // `anonymousId` that collides with an IDENTIFIED contact's canonical key is
19
+ // rejected (403), so a browser key can never read another user's flags.
20
+ // ---------------------------------------------------------------------------
21
+ const listQuerySchema = z.object({
22
+ userToken: z.string().optional(),
23
+ anonymousId: z.string().optional(),
24
+ userId: z.string().optional(),
25
+ email: z.string().optional(),
26
+ });
27
+
28
+ const listRoute = createRoute({
29
+ method: "get",
30
+ path: "/",
31
+ tags: ["Flags"],
32
+ summary: "Evaluate all flags for the resolved recipient",
33
+ description:
34
+ "Recipient-scoped server-side. A publishable key reads its own anon flags (anonymousId) or a token-verified userId; a secret key may pass userId/email directly. NEVER reads the contact key from the request.",
35
+ request: { query: listQuerySchema },
36
+ responses: {
37
+ 200: {
38
+ content: { "application/json": { schema: flagsMapSchema } },
39
+ description: "The recipient's evaluated flag map",
40
+ },
41
+ 400: {
42
+ content: { "application/json": { schema: errorSchema } },
43
+ description: "Missing identity",
44
+ },
45
+ 403: {
46
+ content: { "application/json": { schema: errorSchema } },
47
+ description: "Invalid userToken or non-addressable anonymousId",
48
+ },
49
+ },
50
+ });
51
+
52
+ // ---------------------------------------------------------------------------
53
+ // POST /v1/flags/evaluate — the SERVER SDK read (secret-key only; guarded by
54
+ // `requireApiKey` + `requireScope("ingest")` in routes/index.ts). The canonical
55
+ // contact key is resolved SERVER-TRUSTED from userId/email via the same
56
+ // `resolveFeedRecipient` (a secret caller is never `publishable`, so its
57
+ // userId/email is trusted directly).
58
+ // ---------------------------------------------------------------------------
59
+ const evaluateBodySchema = z.object({
60
+ userId: z.string().optional(),
61
+ email: z.string().optional(),
62
+ });
63
+
64
+ const evaluateRoute = createRoute({
65
+ method: "post",
66
+ path: "/evaluate",
67
+ tags: ["Flags"],
68
+ summary: "Evaluate all flags for a contact (server SDK)",
69
+ description:
70
+ "Secret-key only. Resolves the canonical contact key server-trusted from userId/email and returns the evaluated flag map.",
71
+ request: {
72
+ body: {
73
+ content: { "application/json": { schema: evaluateBodySchema } },
74
+ },
75
+ },
76
+ responses: {
77
+ 200: {
78
+ content: { "application/json": { schema: flagsMapSchema } },
79
+ description: "The contact's evaluated flag map",
80
+ },
81
+ 400: {
82
+ content: { "application/json": { schema: errorSchema } },
83
+ description: "Missing identity",
84
+ },
85
+ 403: {
86
+ content: { "application/json": { schema: errorSchema } },
87
+ description: "Invalid userToken",
88
+ },
89
+ },
90
+ });
91
+
92
+ export const flagsRouter = new OpenAPIHono<AppEnv>()
93
+ .openapi(listRoute, async (c) => {
94
+ const { db } = c.get("container");
95
+ const rec = await resolveFeedRecipient(c, c.req.valid("query"));
96
+ if (!rec.ok) return c.json({ error: rec.error }, rec.status);
97
+ const flags = await evaluateFlagsForContact({
98
+ db,
99
+ contactKey: rec.recipientKey,
100
+ contactId: rec.contactId,
101
+ // BROWSER path: server-only scan leaves (event/email_engagement)
102
+ // short-circuit to false — NO per-flag DB query (the O(1) invariant).
103
+ mode: "browser",
104
+ });
105
+ return c.json({ flags }, 200);
106
+ })
107
+ .openapi(evaluateRoute, async (c) => {
108
+ const { db } = c.get("container");
109
+ const rec = await resolveFeedRecipient(c, c.req.valid("json"));
110
+ if (!rec.ok) return c.json({ error: rec.error }, rec.status);
111
+ const flags = await evaluateFlagsForContact({
112
+ db,
113
+ contactKey: rec.recipientKey,
114
+ contactId: rec.contactId,
115
+ // SERVER path (secret-key only): event/email_engagement leaves resolve
116
+ // server-side via evaluateCondition.
117
+ mode: "server",
118
+ });
119
+ return c.json({ flags }, 200);
120
+ });