@hogsend/engine 0.46.1 → 0.50.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/README.md +6 -0
  2. package/package.json +13 -10
  3. package/src/connectors/presets/index.ts +8 -2
  4. package/src/container.ts +149 -7
  5. package/src/destinations/presets/posthog.ts +7 -0
  6. package/src/destinations/presets/segment.ts +5 -0
  7. package/src/env.ts +38 -0
  8. package/src/flags/reconcile.ts +195 -0
  9. package/src/index.ts +79 -0
  10. package/src/journeys/authoring.ts +32 -0
  11. package/src/journeys/define-journey.ts +45 -40
  12. package/src/journeys/execute-journey-run.ts +73 -32
  13. package/src/journeys/journey-boundary.ts +76 -0
  14. package/src/journeys/journey-context.ts +128 -70
  15. package/src/journeys/journey-task-factory.ts +31 -0
  16. package/src/journeys/journey-task-runtime.ts +31 -0
  17. package/src/journeys/journey-version.test.ts +236 -0
  18. package/src/journeys/journey-version.ts +158 -0
  19. package/src/journeys/record-once.ts +28 -2
  20. package/src/lib/connector-actions.test.ts +36 -0
  21. package/src/lib/connector-actions.ts +42 -4
  22. package/src/lib/contacts.ts +49 -0
  23. package/src/lib/conversions.test.ts +33 -0
  24. package/src/lib/conversions.ts +8 -0
  25. package/src/lib/email.ts +23 -2
  26. package/src/lib/enrollment-policy.ts +51 -0
  27. package/src/lib/feed.ts +55 -3
  28. package/src/lib/flags-registry.ts +48 -0
  29. package/src/lib/flags.ts +498 -0
  30. package/src/lib/fx.ts +374 -0
  31. package/src/lib/global-control-readout.ts +166 -0
  32. package/src/lib/groups.ts +362 -0
  33. package/src/lib/ingestion.ts +38 -0
  34. package/src/lib/journey-lift.ts +142 -0
  35. package/src/lib/operator-settings.ts +71 -0
  36. package/src/lib/outbound.ts +147 -0
  37. package/src/lib/preferences.ts +6 -80
  38. package/src/lib/recipient-preferences.ts +53 -0
  39. package/src/lib/sms.ts +21 -2
  40. package/src/lib/stable-stringify.test.ts +42 -0
  41. package/src/lib/stable-stringify.ts +23 -0
  42. package/src/lib/variant.ts +99 -0
  43. package/src/lib/webhook-signing.ts +16 -1
  44. package/src/lists/registry.ts +6 -2
  45. package/src/lists/subscription.ts +13 -0
  46. package/src/routes/admin/contacts.ts +53 -4
  47. package/src/routes/admin/conversions.ts +137 -0
  48. package/src/routes/admin/flags.ts +369 -0
  49. package/src/routes/admin/groups.ts +866 -0
  50. package/src/routes/admin/impact-schemas.ts +44 -0
  51. package/src/routes/admin/impact-wire.test.ts +181 -0
  52. package/src/routes/admin/impact.ts +486 -0
  53. package/src/routes/admin/index.ts +12 -0
  54. package/src/routes/admin/journey-impact.ts +446 -0
  55. package/src/routes/admin/journeys.ts +70 -100
  56. package/src/routes/admin/settings.ts +145 -0
  57. package/src/routes/admin/targeting.ts +256 -0
  58. package/src/routes/events/index.ts +11 -0
  59. package/src/routes/flags/index.ts +120 -0
  60. package/src/routes/groups/index.ts +417 -0
  61. package/src/routes/index.ts +23 -1
  62. package/src/testing.ts +25 -0
  63. package/src/webhook-sources/presets/index.ts +5 -1
  64. package/src/webhook-sources/presets/intercom.ts +230 -0
  65. package/src/webhook-sources/presets/segment.ts +56 -4
  66. package/src/worker.ts +3 -0
  67. package/src/workflows/bucket-backfill.ts +1 -14
  68. package/src/workflows/impact-digest.ts +761 -0
  69. package/src/workflows/journey-blueprint-interpreter.ts +18 -1
@@ -0,0 +1,761 @@
1
+ import { ConcurrencyLimitStrategy } from "@hatchet-dev/typescript-sdk/v1/index.js";
2
+ import {
3
+ createDatabase,
4
+ type Database,
5
+ webhookDeliveries,
6
+ webhookEndpoints,
7
+ } from "@hogsend/db";
8
+ import { and, desc, eq, isNull, sql } from "drizzle-orm";
9
+ import { getJourneyRegistrySingleton } from "../journeys/registry-singleton.js";
10
+ import { hatchet } from "../lib/hatchet.js";
11
+ import { computeJourneyLift } from "../lib/journey-lift.js";
12
+ import { createLogger, type Logger } from "../lib/logger.js";
13
+ import {
14
+ emitOutbound,
15
+ type ImpactDigestEntry,
16
+ type ImpactDigestLiftEntry,
17
+ type ImpactDigestShippedEntry,
18
+ type ImpactVersionCohort,
19
+ } from "../lib/outbound.js";
20
+
21
+ const DEFAULT_WIN_PROB_THRESHOLD = 0.95;
22
+ const DEFAULT_LOOKBACK_DAYS = 7; // first-ever run window
23
+ const MAX_LOOKBACK_DAYS = 30; // clamp on the watermark
24
+ const LIFT_WINDOW_DAYS = 90; // mirrors the lift route default
25
+ const ENTRY_CAP = 50;
26
+ const CANDIDATE_CAP = 200;
27
+ const LIFT_CONCURRENCY = 5; // pool for computeJourneyLift pairs
28
+ const DAY_MS = 86_400_000;
29
+
30
+ /**
31
+ * Structural slice of the journey registry the digest reads (goal + name).
32
+ * `JourneyRegistry.get` (@hogsend/core registry/index.ts:32-34) satisfies
33
+ * it; tests hand in a plain object. A registry MISS (blueprint journey,
34
+ * removed journey) degrades to nulls — never a crash.
35
+ */
36
+ export interface DigestRegistryLike {
37
+ get(id: string): { goal?: string; name?: string } | undefined;
38
+ }
39
+
40
+ /**
41
+ * Cron input. `now` is a TEST SEAM: the Hatchet cron always pushes `{}`
42
+ * and the task falls back to `Date.now()` — legal here (this is a cron
43
+ * task, not a journey; determinism is delivered by the daily dedupeKey).
44
+ */
45
+ export interface ImpactDigestInput {
46
+ now?: string;
47
+ }
48
+
49
+ /** postgres-js returns Date for timestamptz, but raw execute rows are
50
+ * typed defensively — normalize either representation. */
51
+ function toDate(value: Date | string): Date {
52
+ return value instanceof Date ? value : new Date(value);
53
+ }
54
+
55
+ export function subDays(date: Date, days: number): Date {
56
+ return new Date(date.getTime() - days * DAY_MS);
57
+ }
58
+
59
+ function message(err: unknown): string {
60
+ return err instanceof Error ? err.message : String(err);
61
+ }
62
+
63
+ /**
64
+ * Watermark → window. `since` = the last impact.digest delivery's
65
+ * created_at (max), defaulting to a 7-day window on the first-ever run,
66
+ * clamped to at most 30 days back. Self-healing: pruned delivery rows
67
+ * widen the window at most to the clamp.
68
+ */
69
+ export function deriveDigestWindow(opts: {
70
+ lastDeliveryAt: Date | null;
71
+ now: Date;
72
+ }): { since: Date; until: Date } {
73
+ const until = opts.now;
74
+ const floor = new Date(until.getTime() - MAX_LOOKBACK_DAYS * DAY_MS);
75
+ const fallback = new Date(until.getTime() - DEFAULT_LOOKBACK_DAYS * DAY_MS);
76
+ const raw = opts.lastDeliveryAt ?? fallback;
77
+ const since = raw.getTime() < floor.getTime() ? floor : raw;
78
+ return { since, until };
79
+ }
80
+
81
+ /**
82
+ * Payload ordering + cap: lift entries first (desc |winProbability − 0.5|
83
+ * — strongest evidence up top), then shipped (desc firstSeenAt — newest
84
+ * change first); deterministic journeyId tiebreak; capped with a
85
+ * `truncated` flag.
86
+ */
87
+ export function assembleDigestEntries(opts: {
88
+ lift: ImpactDigestLiftEntry[];
89
+ shipped: ImpactDigestShippedEntry[];
90
+ cap: number;
91
+ }): { entries: ImpactDigestEntry[]; truncated: boolean } {
92
+ const lift = [...opts.lift].sort((a, b) => {
93
+ const delta =
94
+ Math.abs(b.winProbability - 0.5) - Math.abs(a.winProbability - 0.5);
95
+ if (delta !== 0) return delta;
96
+ return a.journeyId.localeCompare(b.journeyId);
97
+ });
98
+ const shipped = [...opts.shipped].sort((a, b) => {
99
+ // ISO-8601 strings sort chronologically.
100
+ const delta = b.firstSeenAt.localeCompare(a.firstSeenAt);
101
+ if (delta !== 0) return delta;
102
+ return a.journeyId.localeCompare(b.journeyId);
103
+ });
104
+ const all: ImpactDigestEntry[] = [...lift, ...shipped];
105
+ return { entries: all.slice(0, opts.cap), truncated: all.length > opts.cap };
106
+ }
107
+
108
+ /** Tiny dependency-free promise pool (LIFT_CONCURRENCY budget). */
109
+ async function mapWithConcurrency<T>(
110
+ items: readonly T[],
111
+ limit: number,
112
+ fn: (item: T) => Promise<void>,
113
+ ): Promise<void> {
114
+ let next = 0;
115
+ const workers = Array.from(
116
+ { length: Math.min(limit, items.length) },
117
+ async () => {
118
+ while (next < items.length) {
119
+ const index = next++;
120
+ await fn(items[index] as T);
121
+ }
122
+ },
123
+ );
124
+ await Promise.all(workers);
125
+ }
126
+
127
+ type HashFirstSeenRow = {
128
+ journey_id: string;
129
+ journey_version_hash: string;
130
+ version_label: string | null;
131
+ first_seen_at: Date | string;
132
+ };
133
+
134
+ /**
135
+ * All-time cohort of one (journey, hash) version. Reuses the lift route's
136
+ * outcome semantics exactly: treatment-only (`status != 'held_out'`),
137
+ * EXISTS with the per-row ITT clock (`occurred_at >= created_at`),
138
+ * goal-conditional `definition_id` append, snapshot-bounded at `until`.
139
+ * Label pick is latest-by-created_at (the array_agg form — max() is
140
+ * lexicographic and shows stale labels after a label-only rename).
141
+ * Soft-deleted journey_states rows are INCLUDED throughout this module —
142
+ * the /lift and /impact cohort SQL (journey-lift.ts, journey-impact.ts)
143
+ * never filters deleted_at, so the digest counting them too keeps every
144
+ * readout surface reporting the same enrollments for one cohort.
145
+ */
146
+ async function versionCohort(opts: {
147
+ db: Database;
148
+ journeyId: string;
149
+ hash: string;
150
+ goal: string | null;
151
+ until: Date;
152
+ }): Promise<ImpactVersionCohort> {
153
+ const { db, journeyId, hash, goal, until } = opts;
154
+ const untilTs = sql`${until.toISOString()}::timestamptz`;
155
+ const goalCond = goal === null ? sql`` : sql` and c.definition_id = ${goal}`;
156
+ const rows = [
157
+ ...(await db.execute<{
158
+ enrollments: number;
159
+ converters: number;
160
+ first_seen_at: Date | string | null;
161
+ version_label: string | null;
162
+ }>(sql`
163
+ select
164
+ count(distinct js.user_id)
165
+ filter (where js.status != 'held_out')::int as enrollments,
166
+ (count(distinct js.user_id)
167
+ filter (where js.status != 'held_out' and exists (
168
+ select 1 from conversions c
169
+ where c.user_key = js.user_id
170
+ and c.occurred_at >= js.created_at
171
+ and c.occurred_at <= ${untilTs}${goalCond}
172
+ )))::int as converters,
173
+ min(js.created_at) as first_seen_at,
174
+ (array_agg(js.journey_version_label order by js.created_at desc)
175
+ filter (where js.journey_version_label is not null))[1]
176
+ as version_label
177
+ from journey_states js
178
+ where js.journey_id = ${journeyId}
179
+ and js.journey_version_hash = ${hash}
180
+ `)),
181
+ ];
182
+ const row = rows[0];
183
+ const enrollments = Number(row?.enrollments ?? 0);
184
+ const converters = Number(row?.converters ?? 0);
185
+ const firstSeen = row?.first_seen_at ? toDate(row.first_seen_at) : until;
186
+ return {
187
+ versionHash: hash,
188
+ versionLabel: row?.version_label ?? null,
189
+ enrollmentsAllTime: enrollments,
190
+ converters,
191
+ conversionRate: enrollments > 0 ? converters / enrollments : 0,
192
+ firstSeenAt: firstSeen.toISOString(),
193
+ exposureDays: Math.max(
194
+ 0,
195
+ Math.floor((until.getTime() - firstSeen.getTime()) / DAY_MS),
196
+ ),
197
+ };
198
+ }
199
+
200
+ /**
201
+ * Detection A — "you shipped a change" (causal: false). Two queries, not
202
+ * one (a window-filtered GROUP BY cannot supply earlier hashes' first-seen),
203
+ * plus the label pass: a (journey, label) pair first observed in-window
204
+ * whose hash first-seen PREDATES the window is a label-only change — the
205
+ * first-class "shipped" signal for template reworks the hash cannot see.
206
+ * Inert until Decision A's columns fill (the IS NOT NULL filter); blueprint
207
+ * enrollments are covered automatically (this reads journey_states, not the
208
+ * code registry). Consumers MUST treat a new hash as "possible new
209
+ * version" — toolchain bumps can fork it with zero content change.
210
+ */
211
+ export async function detectShippedVersions(opts: {
212
+ db: Database;
213
+ since: Date;
214
+ until: Date;
215
+ registry?: DigestRegistryLike;
216
+ logger?: Logger;
217
+ }): Promise<{ entries: ImpactDigestShippedEntry[] }> {
218
+ const { db, since, until, registry } = opts;
219
+ const sinceTs = sql`${since.toISOString()}::timestamptz`;
220
+ const untilTs = sql`${until.toISOString()}::timestamptz`;
221
+
222
+ // 1. (journey, hash) pairs first observed inside the window.
223
+ const inWindow = [
224
+ ...(await db.execute<HashFirstSeenRow>(sql`
225
+ select journey_id, journey_version_hash,
226
+ (array_agg(journey_version_label order by created_at desc)
227
+ filter (where journey_version_label is not null))[1]
228
+ as version_label,
229
+ min(created_at) as first_seen_at
230
+ from journey_states
231
+ where journey_version_hash is not null
232
+ group by journey_id, journey_version_hash
233
+ having min(created_at) >= ${sinceTs} and min(created_at) < ${untilTs}
234
+ `)),
235
+ ];
236
+
237
+ // 1b. (journey, label) pairs first observed inside the window, with the
238
+ // hash carried by the EARLIEST row of the pair (label-only-change probe).
239
+ const labelRows = [
240
+ ...(await db.execute<{
241
+ journey_id: string;
242
+ version_label: string;
243
+ first_seen_at: Date | string;
244
+ hash: string | null;
245
+ }>(sql`
246
+ select journey_id, journey_version_label as version_label,
247
+ min(created_at) as first_seen_at,
248
+ (array_agg(journey_version_hash order by created_at asc))[1]
249
+ as hash
250
+ from journey_states
251
+ where journey_version_label is not null
252
+ and journey_version_hash is not null
253
+
254
+ group by journey_id, journey_version_label
255
+ having min(created_at) >= ${sinceTs} and min(created_at) < ${untilTs}
256
+ `)),
257
+ ];
258
+
259
+ const affectedIds = [
260
+ ...new Set([
261
+ ...inWindow.map((r) => r.journey_id),
262
+ ...labelRows.map((r) => r.journey_id),
263
+ ]),
264
+ ];
265
+ if (affectedIds.length === 0) return { entries: [] };
266
+
267
+ // 2. All-time first-seen per (journey, hash) for the affected journeys —
268
+ // classifies new_journey vs new_version and picks `previous`.
269
+ const idList = sql.join(
270
+ affectedIds.map((id) => sql`${id}`),
271
+ sql`, `,
272
+ );
273
+ const allTime = [
274
+ ...(await db.execute<HashFirstSeenRow>(sql`
275
+ select journey_id, journey_version_hash,
276
+ (array_agg(journey_version_label order by created_at desc)
277
+ filter (where journey_version_label is not null))[1]
278
+ as version_label,
279
+ min(created_at) as first_seen_at
280
+ from journey_states
281
+ where journey_id in (${idList})
282
+ and journey_version_hash is not null
283
+
284
+ group by journey_id, journey_version_hash
285
+ `)),
286
+ ];
287
+
288
+ const entries: ImpactDigestShippedEntry[] = [];
289
+
290
+ // Hash pass: new_journey / new_version.
291
+ for (const row of inWindow) {
292
+ const meta = registry?.get(row.journey_id);
293
+ const goal = meta?.goal ?? null;
294
+ const firstSeen = toDate(row.first_seen_at);
295
+ const earlier = allTime
296
+ .filter(
297
+ (a) =>
298
+ a.journey_id === row.journey_id &&
299
+ a.journey_version_hash !== row.journey_version_hash &&
300
+ toDate(a.first_seen_at).getTime() < firstSeen.getTime(),
301
+ )
302
+ .sort(
303
+ (a, b) =>
304
+ toDate(b.first_seen_at).getTime() - toDate(a.first_seen_at).getTime(),
305
+ );
306
+ const previousHash = earlier[0]?.journey_version_hash ?? null;
307
+ const current = await versionCohort({
308
+ db,
309
+ journeyId: row.journey_id,
310
+ hash: row.journey_version_hash,
311
+ goal,
312
+ until,
313
+ });
314
+ const previous =
315
+ previousHash === null
316
+ ? null
317
+ : await versionCohort({
318
+ db,
319
+ journeyId: row.journey_id,
320
+ hash: previousHash,
321
+ goal,
322
+ until,
323
+ });
324
+ entries.push({
325
+ kind: "shipped",
326
+ causal: false,
327
+ journeyId: row.journey_id,
328
+ journeyName: meta?.name ?? null,
329
+ versionHash: row.journey_version_hash,
330
+ versionLabel: row.version_label ?? null,
331
+ change: previousHash === null ? "new_journey" : "new_version",
332
+ previousVersionLabel: null,
333
+ firstSeenAt: firstSeen.toISOString(),
334
+ goalDefinitionId: goal,
335
+ current,
336
+ previous,
337
+ });
338
+ }
339
+
340
+ // Label pass: new_label (same content hash, fresh label).
341
+ for (const row of labelRows) {
342
+ if (row.hash === null) continue;
343
+ const hashFirst = allTime.find(
344
+ (a) =>
345
+ a.journey_id === row.journey_id && a.journey_version_hash === row.hash,
346
+ );
347
+ // Hash unseen, or itself new in-window → already reported above.
348
+ if (
349
+ !hashFirst ||
350
+ toDate(hashFirst.first_seen_at).getTime() >= since.getTime()
351
+ ) {
352
+ continue;
353
+ }
354
+ const meta = registry?.get(row.journey_id);
355
+ const goal = meta?.goal ?? null;
356
+ const prevLabelRows = [
357
+ ...(await db.execute<{ prev_label: string | null }>(sql`
358
+ select (array_agg(journey_version_label order by created_at desc)
359
+ filter (where journey_version_label is not null))[1]
360
+ as prev_label
361
+ from journey_states
362
+ where journey_id = ${row.journey_id}
363
+ and created_at < ${sinceTs}
364
+
365
+ `)),
366
+ ];
367
+ entries.push({
368
+ kind: "shipped",
369
+ causal: false,
370
+ journeyId: row.journey_id,
371
+ journeyName: meta?.name ?? null,
372
+ versionHash: row.hash,
373
+ versionLabel: row.version_label,
374
+ change: "new_label",
375
+ previousVersionLabel: prevLabelRows[0]?.prev_label ?? null,
376
+ firstSeenAt: toDate(row.first_seen_at).toISOString(),
377
+ goalDefinitionId: goal,
378
+ current: await versionCohort({
379
+ db,
380
+ journeyId: row.journey_id,
381
+ hash: row.hash,
382
+ goal,
383
+ until,
384
+ }),
385
+ previous: null,
386
+ });
387
+ }
388
+
389
+ entries.sort(
390
+ (a, b) =>
391
+ a.journeyId.localeCompare(b.journeyId) ||
392
+ a.versionHash.localeCompare(b.versionHash),
393
+ );
394
+ return { entries };
395
+ }
396
+
397
+ /**
398
+ * Detection B — "working / hurting" (causal: true). Candidates = journeys
399
+ * with held-out rows inside the 90-day lift window, capped at
400
+ * CANDIDATE_CAP (warned when hit). Per candidate the unified helper runs
401
+ * two snapshots through a LIFT_CONCURRENCY pool: `now` (asOf = until) and
402
+ * — only when the frozen payload has no answer — `prev` (asOf = since).
403
+ *
404
+ * FROZEN-PAYLOAD OVERRIDE (the drift hole): `prev` is the LAST digest's
405
+ * as-REPORTED winProbability when available. Late-arriving/backfilled
406
+ * conversions can otherwise retroactively flip last week's probability
407
+ * across T, silently swallowing a real crossing or re-reporting one
408
+ * already sent. The recompute remains the fallback for journeys absent
409
+ * from the last payload.
410
+ *
411
+ * Crossing (not level) semantics — no weekly re-nag; the down side is
412
+ * included deliberately. Suppression is absolute; smallSample rides.
413
+ */
414
+ export async function detectLiftCrossings(opts: {
415
+ db: Database;
416
+ since: Date;
417
+ until: Date;
418
+ threshold: number;
419
+ registry?: DigestRegistryLike;
420
+ previousWinProbabilities?: Map<string, number>;
421
+ logger?: Logger;
422
+ }): Promise<{ entries: ImpactDigestLiftEntry[] }> {
423
+ const {
424
+ db,
425
+ since,
426
+ until,
427
+ threshold,
428
+ registry,
429
+ previousWinProbabilities,
430
+ logger,
431
+ } = opts;
432
+
433
+ const candidateRows = [
434
+ ...(await db.execute<{ journey_id: string }>(sql`
435
+ select distinct journey_id
436
+ from journey_states
437
+ where status = 'held_out'
438
+ and created_at >= ${subDays(until, LIFT_WINDOW_DAYS).toISOString()}::timestamptz
439
+ order by journey_id asc
440
+ limit ${CANDIDATE_CAP + 1}
441
+ `)),
442
+ ];
443
+ let candidates = candidateRows.map((r) => r.journey_id);
444
+ if (candidates.length > CANDIDATE_CAP) {
445
+ logger?.warn(
446
+ "impact-digest: lift candidate cap hit — ids beyond the cap are skipped this run",
447
+ { cap: CANDIDATE_CAP },
448
+ );
449
+ candidates = candidates.slice(0, CANDIDATE_CAP);
450
+ }
451
+
452
+ const entries: ImpactDigestLiftEntry[] = [];
453
+ await mapWithConcurrency(candidates, LIFT_CONCURRENCY, async (journeyId) => {
454
+ const meta = registry?.get(journeyId);
455
+ const goal = meta?.goal;
456
+ const now = await computeJourneyLift({
457
+ db,
458
+ journeyId,
459
+ definitionId: goal,
460
+ since: subDays(until, LIFT_WINDOW_DAYS),
461
+ asOf: until,
462
+ });
463
+ const winProbability = now.verdict.winProbability;
464
+ // Suppression is absolute; and a probability strictly inside
465
+ // (1−T, T) cannot be a crossing — skip before paying for `prev`.
466
+ if (now.verdict.suppressed || winProbability === null) return;
467
+ if (winProbability < threshold && winProbability > 1 - threshold) {
468
+ return;
469
+ }
470
+
471
+ let prev: number | null;
472
+ if (previousWinProbabilities?.has(journeyId)) {
473
+ prev = previousWinProbabilities.get(journeyId) ?? null;
474
+ } else {
475
+ const prevResult = await computeJourneyLift({
476
+ db,
477
+ journeyId,
478
+ definitionId: goal,
479
+ since: subDays(since, LIFT_WINDOW_DAYS),
480
+ asOf: since,
481
+ });
482
+ prev = prevResult.verdict.winProbability;
483
+ }
484
+
485
+ let direction: "up" | "down" | null = null;
486
+ if (winProbability >= threshold && (prev === null || prev < threshold)) {
487
+ direction = "up";
488
+ } else if (
489
+ winProbability <= 1 - threshold &&
490
+ (prev === null || prev > 1 - threshold)
491
+ ) {
492
+ direction = "down";
493
+ }
494
+ if (direction === null) return;
495
+
496
+ entries.push({
497
+ kind: "lift",
498
+ causal: true,
499
+ journeyId,
500
+ journeyName: meta?.name ?? null,
501
+ goalDefinitionId: goal ?? null,
502
+ windowDays: LIFT_WINDOW_DAYS,
503
+ direction,
504
+ treatment: now.treatment,
505
+ control: now.control,
506
+ liftPercent: now.verdict.liftPercent,
507
+ winProbability,
508
+ previousWinProbability: prev,
509
+ smallSample: now.verdict.smallSample,
510
+ });
511
+ });
512
+
513
+ // Pool completion order is nondeterministic — stabilize.
514
+ entries.sort((a, b) => a.journeyId.localeCompare(b.journeyId));
515
+ return { entries };
516
+ }
517
+
518
+ /**
519
+ * Compose Detection A (+ label pass) and Detection B, then order + cap.
520
+ * Each detection degrades independently (per-section try/catch — the
521
+ * check-alerts posture): a failed section logs and yields zero entries of
522
+ * its kind rather than killing the whole digest.
523
+ */
524
+ export async function buildImpactDigest(opts: {
525
+ db: Database;
526
+ since: Date;
527
+ until: Date;
528
+ threshold: number;
529
+ registry?: DigestRegistryLike;
530
+ previousWinProbabilities?: Map<string, number>;
531
+ logger?: Logger;
532
+ }): Promise<{ entries: ImpactDigestEntry[]; truncated: boolean }> {
533
+ let shipped: ImpactDigestShippedEntry[] = [];
534
+ try {
535
+ shipped = (await detectShippedVersions(opts)).entries;
536
+ } catch (err) {
537
+ opts.logger?.warn(
538
+ "impact-digest: shipped-version detection failed — degrading to lift entries only",
539
+ { error: message(err) },
540
+ );
541
+ }
542
+ let lift: ImpactDigestLiftEntry[] = [];
543
+ try {
544
+ lift = (await detectLiftCrossings(opts)).entries;
545
+ } catch (err) {
546
+ opts.logger?.warn(
547
+ "impact-digest: lift-crossing detection failed — degrading to shipped entries only",
548
+ { error: message(err) },
549
+ );
550
+ }
551
+ return assembleDigestEntries({ lift, shipped, cap: ENTRY_CAP });
552
+ }
553
+
554
+ /**
555
+ * The as-REPORTED winProbability set from the latest impact.digest
556
+ * delivery — webhook_deliveries.payload holds the frozen envelope
557
+ * (lib/outbound.ts:390-408), so this reads what subscribers were actually
558
+ * told last time. detectLiftCrossings falls back to a live recompute only
559
+ * for journeys absent from it.
560
+ */
561
+ async function readPreviousWinProbabilities(
562
+ db: Database,
563
+ ): Promise<Map<string, number>> {
564
+ const map = new Map<string, number>();
565
+ const rows = await db
566
+ .select({ payload: webhookDeliveries.payload })
567
+ .from(webhookDeliveries)
568
+ .where(eq(webhookDeliveries.eventType, "impact.digest"))
569
+ .orderBy(desc(webhookDeliveries.createdAt))
570
+ .limit(1);
571
+ const envelope = rows[0]?.payload as
572
+ | { data?: { entries?: unknown[] } }
573
+ | undefined;
574
+ for (const entry of envelope?.data?.entries ?? []) {
575
+ if (!entry || typeof entry !== "object") continue;
576
+ const candidate = entry as {
577
+ kind?: unknown;
578
+ journeyId?: unknown;
579
+ winProbability?: unknown;
580
+ };
581
+ if (candidate.kind !== "lift") continue;
582
+ if (
583
+ typeof candidate.journeyId === "string" &&
584
+ typeof candidate.winProbability === "number"
585
+ ) {
586
+ map.set(candidate.journeyId, candidate.winProbability);
587
+ }
588
+ }
589
+ return map;
590
+ }
591
+
592
+ /**
593
+ * The weekly impact digest cron (impact experiments D5). Read-only except
594
+ * the emit; no email/SMS (rendering the digest is a subscriber choice);
595
+ * inert without subscribers — the opt-in is an endpoint subscribing to
596
+ * impact.digest, NOT an env var. Registered in worker.ts baseWorkflows.
597
+ *
598
+ * REPLAY-LAW NOTE: this is a CRON task, not a journey — Date.now() is
599
+ * legal; determinism is delivered by the UTC-day dedupeKey plus
600
+ * emitOutbound's onConflictDoNothing fan-out on (endpointId, dedupeKey).
601
+ * That index is a plain uniqueIndex relying on NULL-distinctness, NOT a
602
+ * partial index — do not add a WHERE predicate to it.
603
+ *
604
+ * Documented consequence of the day-keyed dedupe: at most one digest per
605
+ * endpoint per UTC day, even on a sub-daily cron. The cross-midnight
606
+ * retry edge is self-healing (a retry re-reads the watermark; the window
607
+ * collapses to minutes and is almost surely empty → no emit).
608
+ */
609
+ export const impactDigestTask = hatchet.task({
610
+ name: "impact-digest",
611
+ // Mondays 09:00 UTC by default. Read raw off process.env at module load
612
+ // (the bucket-reconcile pattern); declared in env.ts for the
613
+ // validated-env contract.
614
+ onCrons: [process.env.IMPACT_DIGEST_CRON ?? "0 9 * * 1"],
615
+ retries: 1,
616
+ // Budget: 200 candidates × 2 lift snapshots × 2 count queries ≈ 800
617
+ // queries at pool 5 — fits with room.
618
+ executionTimeout: "300s",
619
+ concurrency: {
620
+ expression: "'impact-digest'",
621
+ maxRuns: 1,
622
+ limitStrategy: ConcurrencyLimitStrategy.GROUP_ROUND_ROBIN,
623
+ },
624
+ fn: async (
625
+ input: ImpactDigestInput = {},
626
+ ): Promise<{
627
+ emitted: boolean;
628
+ reason?: string;
629
+ entries?: number;
630
+ since?: string;
631
+ until?: string;
632
+ }> => {
633
+ // Cron runs have no request container — self-bootstrap (check-alerts).
634
+ const { db } = createDatabase({ url: process.env.DATABASE_URL ?? "" });
635
+ const logger = createLogger(process.env.LOG_LEVEL ?? "info");
636
+
637
+ // (1) Subscriber pre-check — THE opt-in gate, one indexed query with
638
+ // the EXACT predicate emitOutbound uses (lib/outbound.ts:376-385).
639
+ let subscribers: Array<{ id: string }>;
640
+ try {
641
+ subscribers = await db
642
+ .select({ id: webhookEndpoints.id })
643
+ .from(webhookEndpoints)
644
+ .where(
645
+ and(
646
+ eq(webhookEndpoints.disabled, false),
647
+ isNull(webhookEndpoints.organizationId),
648
+ sql`${webhookEndpoints.eventTypes} @> ${JSON.stringify([
649
+ "impact.digest",
650
+ ])}::jsonb`,
651
+ ),
652
+ )
653
+ .limit(1);
654
+ } catch (err) {
655
+ logger.warn("impact-digest: subscriber pre-check failed", {
656
+ error: message(err),
657
+ });
658
+ return { emitted: false, reason: "subscriber_check_failed" };
659
+ }
660
+ if (subscribers.length === 0) {
661
+ return { emitted: false, reason: "no_subscribers" };
662
+ }
663
+
664
+ // (2) Watermark off our own delivery rows (no new storage).
665
+ // `input.now` is a test seam; the cron always pushes {}.
666
+ const parsedNow = input.now ? new Date(input.now) : new Date();
667
+ const now = Number.isNaN(parsedNow.getTime()) ? new Date() : parsedNow;
668
+ let lastDeliveryAt: Date | null = null;
669
+ try {
670
+ const rows = await db
671
+ .select({
672
+ last: sql<Date | string | null>`max(${webhookDeliveries.createdAt})`,
673
+ })
674
+ .from(webhookDeliveries)
675
+ .where(eq(webhookDeliveries.eventType, "impact.digest"));
676
+ const raw = rows[0]?.last ?? null;
677
+ lastDeliveryAt = raw ? toDate(raw) : null;
678
+ } catch (err) {
679
+ logger.warn("impact-digest: watermark read failed", {
680
+ error: message(err),
681
+ });
682
+ return { emitted: false, reason: "watermark_failed" };
683
+ }
684
+ const { since, until } = deriveDigestWindow({ lastDeliveryAt, now });
685
+
686
+ // (3) Registry singleton (set by createHogsendClient in both API and
687
+ // worker); degrade to unregistered lookups, never crash.
688
+ let registry: DigestRegistryLike | undefined;
689
+ try {
690
+ registry = getJourneyRegistrySingleton();
691
+ } catch {
692
+ registry = undefined;
693
+ }
694
+
695
+ // (4) Frozen prev-winProbability set from the LAST digest delivery.
696
+ let previousWinProbabilities: Map<string, number> | undefined;
697
+ try {
698
+ previousWinProbabilities = await readPreviousWinProbabilities(db);
699
+ } catch (err) {
700
+ logger.warn("impact-digest: previous payload read failed", {
701
+ error: message(err),
702
+ });
703
+ previousWinProbabilities = undefined;
704
+ }
705
+
706
+ const threshold = Math.min(
707
+ 0.999,
708
+ Math.max(
709
+ 0.5,
710
+ Number(process.env.IMPACT_DIGEST_WIN_PROB) ||
711
+ DEFAULT_WIN_PROB_THRESHOLD,
712
+ ),
713
+ );
714
+
715
+ // (5) Detections (each degrades internally).
716
+ const { entries, truncated } = await buildImpactDigest({
717
+ db,
718
+ since,
719
+ until,
720
+ threshold,
721
+ registry,
722
+ previousWinProbabilities,
723
+ logger,
724
+ });
725
+
726
+ // (6) An empty digest is never emitted; the watermark intentionally
727
+ // does not advance (no delivery row is written).
728
+ if (entries.length === 0) {
729
+ return {
730
+ emitted: false,
731
+ reason: "no_entries",
732
+ since: since.toISOString(),
733
+ until: until.toISOString(),
734
+ };
735
+ }
736
+
737
+ // (7) Emit through the spine. emitOutbound never throws; per-endpoint
738
+ // fan-out dedupe is onConflictDoNothing on (endpointId, dedupeKey).
739
+ const periodKey = until.toISOString().slice(0, 10);
740
+ await emitOutbound({
741
+ db,
742
+ hatchet,
743
+ logger,
744
+ event: "impact.digest",
745
+ payload: {
746
+ periodKey,
747
+ since: since.toISOString(),
748
+ until: until.toISOString(),
749
+ entries,
750
+ truncated,
751
+ },
752
+ dedupeKey: `impact.digest:${periodKey}`,
753
+ });
754
+ return {
755
+ emitted: true,
756
+ entries: entries.length,
757
+ since: since.toISOString(),
758
+ until: until.toISOString(),
759
+ };
760
+ },
761
+ });