@hogsend/engine 0.43.0 → 0.45.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/package.json +12 -9
  2. package/src/container.ts +289 -13
  3. package/src/destinations/define-destination.ts +7 -3
  4. package/src/destinations/presets/posthog.ts +31 -0
  5. package/src/index.ts +99 -0
  6. package/src/journeys/execute-journey-run.ts +93 -0
  7. package/src/journeys/journey-context.ts +4 -0
  8. package/src/lib/analytics-providers-from-env.ts +106 -43
  9. package/src/lib/attribution-backfill.ts +285 -0
  10. package/src/lib/attribution.ts +239 -0
  11. package/src/lib/contacts.ts +71 -0
  12. package/src/lib/conversion-dispatch.ts +326 -0
  13. package/src/lib/conversions.ts +197 -0
  14. package/src/lib/crm-deals.ts +320 -0
  15. package/src/lib/crm-ingest.ts +341 -0
  16. package/src/lib/crm-provider-registry.ts +41 -0
  17. package/src/lib/crm-registry-singleton.ts +22 -0
  18. package/src/lib/deal-money-events.ts +85 -0
  19. package/src/lib/email-service-types.ts +16 -1
  20. package/src/lib/event-names.ts +1 -1
  21. package/src/lib/funnel-progress.ts +76 -0
  22. package/src/lib/funnel-registry.ts +108 -0
  23. package/src/lib/funnel-transitions.ts +280 -0
  24. package/src/lib/holdout.ts +62 -0
  25. package/src/lib/ingestion.ts +218 -9
  26. package/src/lib/lift-stats.ts +125 -0
  27. package/src/lib/mailer.ts +1 -0
  28. package/src/lib/operator.ts +31 -0
  29. package/src/lib/outbound.ts +51 -0
  30. package/src/lib/posthog-scopes.ts +30 -6
  31. package/src/lib/revenue.ts +116 -0
  32. package/src/lib/sms-service-types.ts +7 -1
  33. package/src/lib/sms-tracked.ts +35 -0
  34. package/src/lib/tracked.ts +40 -0
  35. package/src/lib/tracking-events.ts +26 -0
  36. package/src/lib/unconfigured-email-provider.ts +45 -0
  37. package/src/lib/webhook-signing.ts +7 -0
  38. package/src/routes/admin/analytics.ts +14 -5
  39. package/src/routes/admin/attribution.ts +414 -0
  40. package/src/routes/admin/contacts.ts +71 -11
  41. package/src/routes/admin/conversions.ts +247 -0
  42. package/src/routes/admin/deals.ts +472 -0
  43. package/src/routes/admin/funnels.ts +228 -0
  44. package/src/routes/admin/index.ts +8 -0
  45. package/src/routes/admin/journeys.ts +158 -1
  46. package/src/routes/admin/readiness.ts +97 -2
  47. package/src/routes/admin/templates.ts +28 -1
  48. package/src/routes/events/index.ts +12 -0
  49. package/src/routes/tracking/click-pipeline.ts +26 -18
  50. package/src/routes/webhooks/crm-provider.ts +117 -0
  51. package/src/routes/webhooks/index.ts +5 -1
  52. package/src/sources/define-contact-source.ts +159 -0
  53. package/src/sources/registry.ts +71 -0
  54. package/src/sources/webhook-source.ts +109 -0
  55. package/src/worker.ts +4 -0
  56. package/src/workflows/crm-reconcile.ts +129 -0
  57. package/src/workflows/dispatch-conversion.ts +31 -0
  58. package/src/workflows/send-campaign.ts +5 -0
package/src/index.ts CHANGED
@@ -367,11 +367,51 @@ export {
367
367
  } from "./lib/connector-state.js";
368
368
  // --- Contacts identity (resolve/create — used by connector member-link) ---
369
369
  export { resolveOrCreateContact } from "./lib/contacts.js";
370
+ // --- Conversion dispatch (plan §5.2): destinations registry + delivery ---
371
+ export {
372
+ ConversionDestinationRegistry,
373
+ conversionEventId,
374
+ createConversionDispatches,
375
+ deliverConversionDispatch,
376
+ enqueueConversionDispatches,
377
+ getConversionDestinations,
378
+ recoverClickContext,
379
+ resetConversionDestinations,
380
+ setConversionDestinations,
381
+ } from "./lib/conversion-dispatch.js";
382
+ // --- Conversion points (plan §5.1): registry + ingest-time evaluation ---
383
+ export {
384
+ ConversionRegistry,
385
+ evaluateConversionsAtIngest,
386
+ type FiredConversion,
387
+ getConversionRegistry,
388
+ resetConversionRegistry,
389
+ setConversionRegistry,
390
+ } from "./lib/conversions.js";
370
391
  export {
371
392
  AdminAlreadyExistsError,
372
393
  type CreatedAdmin,
373
394
  createAdminUser,
374
395
  } from "./lib/create-admin.js";
396
+ // --- CRM sync (provider registry + the stage-event spine sink) ---
397
+ export {
398
+ type AppliedStageChange,
399
+ applyCrmStageEvent,
400
+ ensureCrmLinks,
401
+ resolveCrmLinkedContact,
402
+ } from "./lib/crm-deals.js";
403
+ export {
404
+ DEAL_QUOTED,
405
+ DEAL_SOLD,
406
+ FUNNEL_STAGE_CHANGED,
407
+ ingestCrmStageEvents,
408
+ } from "./lib/crm-ingest.js";
409
+ export { CrmProviderRegistry } from "./lib/crm-provider-registry.js";
410
+ export {
411
+ getCrmSyncConfig,
412
+ resetCrmSyncConfig,
413
+ setCrmSyncConfig,
414
+ } from "./lib/crm-registry-singleton.js";
375
415
  // --- Infrastructure singletons ---
376
416
  export { getDb } from "./lib/db.js";
377
417
  // --- Discord gateway-worker liveness heartbeat (Studio status) ---
@@ -420,7 +460,14 @@ export {
420
460
  } from "./lib/feed.js";
421
461
  export { countRecentSends, isFrequencyCapped } from "./lib/frequency-cap.js";
422
462
  export { addrSpecOf, hostOfFromAddress } from "./lib/from-address.js";
463
+ export { FunnelRegistry } from "./lib/funnel-registry.js";
423
464
  export { hatchet } from "./lib/hatchet.js";
465
+ export {
466
+ globalControlPercent,
467
+ holdoutBucket,
468
+ isGlobalControl,
469
+ isHeldOut,
470
+ } from "./lib/holdout.js";
424
471
  // --- Identity service (resolve/merge + analytics merge propagation, §7) ---
425
472
  export {
426
473
  createIdentityService,
@@ -448,6 +495,14 @@ export {
448
495
  releaseLeaderLease,
449
496
  renewLeaderLease,
450
497
  } from "./lib/leader-lease.js";
498
+ // --- Holdout lift statistics (impact plan §4.2) ---
499
+ export {
500
+ betaWinProbability,
501
+ computeLift,
502
+ type LiftVerdict,
503
+ MIN_COMBINED_CONVERSIONS,
504
+ SMALL_SAMPLE_FLOOR,
505
+ } from "./lib/lift-stats.js";
451
506
  // --- Managed tracked links (channel-agnostic mint — Studio/Discord/share) ---
452
507
  // NOTE: the QR scan-row plumbing (ensureQrTrackedLink, the canonical-row
453
508
  // filter, the 'qr' source marker) is deliberately NOT exported — it is
@@ -516,6 +571,12 @@ export {
516
571
  } from "./lib/redis.js";
517
572
  // --- Self-service password reset (engine-owned, self-contained email) ---
518
573
  export { sendResetPasswordEmail } from "./lib/reset-email.js";
574
+ // --- Revenue rollups (the event spine's value/currency columns) ---
575
+ export {
576
+ type ContactRevenue,
577
+ type ContactRevenueTotal,
578
+ getContactRevenue,
579
+ } from "./lib/revenue.js";
519
580
  // --- PostHog destination seed (idempotent; ENABLE_POSTHOG_DESTINATION) ---
520
581
  export { seedPostHogDestination } from "./lib/seed-posthog-destination.js";
521
582
  export {
@@ -583,6 +644,14 @@ export {
583
644
  prepareTrackedHtml,
584
645
  rewriteLinks,
585
646
  } from "./lib/tracking.js";
647
+ // --- First-party tracking event names (touchpoint-classed in @hogsend/core) ---
648
+ export {
649
+ EMAIL_LINK_CLICKED,
650
+ EMAIL_OPENED,
651
+ LINK_ARRIVED,
652
+ LINK_CLICKED,
653
+ SMS_LINK_CLICKED,
654
+ } from "./lib/tracking-event-names.js";
586
655
  export {
587
656
  pushSmsTrackingEvent,
588
657
  pushTrackingEvent,
@@ -677,6 +746,32 @@ export {
677
746
  } from "./middleware/rate-limit.js";
678
747
  // --- Middleware (consumer-mounted routes, e.g. the @hogsend/mcp hosted route) ---
679
748
  export { requireAdmin } from "./middleware/require-admin.js";
749
+ // --- Contact sources (Clay/Attio/generic-webhook → cold prospects) ---
750
+ export {
751
+ type ColdChannelPosture,
752
+ type ColdPosture,
753
+ type ContactSourceMeta,
754
+ type ContactWriteBack,
755
+ contactSourceToWebhookSource,
756
+ type DefinedContactSource,
757
+ defaultColdPosture,
758
+ defineContactSource,
759
+ isColdChannelAllowed,
760
+ resolveColdPosture,
761
+ } from "./sources/define-contact-source.js";
762
+ export {
763
+ buildContactSourceRegistry,
764
+ ContactSourceRegistry,
765
+ getContactSourceRegistry,
766
+ setContactSourceRegistry,
767
+ } from "./sources/registry.js";
768
+ export {
769
+ normalizeWebhookContactEvent,
770
+ type WebhookContactPayload,
771
+ type WebhookContactSourceOptions,
772
+ webhookContactPayloadSchema,
773
+ webhookContactSource,
774
+ } from "./sources/webhook-source.js";
680
775
  // --- Webhook sources ---
681
776
  export {
682
777
  type DefinedWebhookSource,
@@ -717,6 +812,10 @@ export {
717
812
  checkAlertsTask,
718
813
  surfaceStrandedWaiting,
719
814
  } from "./workflows/check-alerts.js";
815
+ export {
816
+ crmReconcileTask,
817
+ runCrmReconcile,
818
+ } from "./workflows/crm-reconcile.js";
720
819
  // --- Outbound webhooks: durable delivery task + reaper (Section 1.5) ---
721
820
  export {
722
821
  deliverWebhookTask,
@@ -25,6 +25,8 @@ import {
25
25
  checkEntryLimit,
26
26
  } from "../lib/enrollment-guards.js";
27
27
  import { hatchet } from "../lib/hatchet.js";
28
+ import { isHeldOut } from "../lib/holdout.js";
29
+ import { ingestEvent } from "../lib/ingestion.js";
28
30
  import { createLogger } from "../lib/logger.js";
29
31
  import { emitOutbound } from "../lib/outbound.js";
30
32
  import { resolveTimezoneWithSource } from "../lib/timezone.js";
@@ -303,6 +305,97 @@ export async function executeJourneyRun(
303
305
  return { status: "skipped", reason: "user_unsubscribed" };
304
306
  }
305
307
 
308
+ // HOLDOUT DIVERSION (impact plan §4.1) — deliberately LAST in the guard
309
+ // chain: a contact who'd have been blocked by any gate above is never
310
+ // counted as held out (intent-to-treat — the control must mirror the
311
+ // would-have-entered population, not the triggered one). Assignment is a
312
+ // deterministic hash (replay law: no RNG in durable paths), so the same
313
+ // contact diverts identically on every trigger AND every replay. One
314
+ // held_out row per (user, journey), ever; the spine event's idempotency
315
+ // key carries the same once-ever semantics for fan-out/analytics.
316
+ if (meta.holdout && meta.holdout.percent > 0) {
317
+ const diverted = isHeldOut({
318
+ userId,
319
+ journeyId: meta.id,
320
+ percent: meta.holdout.percent,
321
+ salt: meta.holdout.salt,
322
+ });
323
+ if (diverted) {
324
+ const priorHoldout = await db.query.journeyStates.findFirst({
325
+ where: and(
326
+ eq(journeyStates.userId, userId),
327
+ eq(journeyStates.journeyId, meta.id),
328
+ eq(journeyStates.status, "held_out"),
329
+ ),
330
+ });
331
+ if (!priorHoldout) {
332
+ const heldOutAt = new Date();
333
+ const inserted = await db
334
+ .insert(journeyStates)
335
+ .values({
336
+ userId,
337
+ userEmail,
338
+ journeyId: meta.id,
339
+ currentNodeId: "held-out",
340
+ status: "held_out",
341
+ context: properties,
342
+ exitedAt: heldOutAt,
343
+ })
344
+ .returning({ id: journeyStates.id });
345
+ const holdoutStateId = inserted[0]?.id;
346
+ if (holdoutStateId) {
347
+ // The counterfactual as data (the Iterable Send Skip pattern):
348
+ // queryable on the spine, fan-out-able to journeys/destinations.
349
+ // Once-ever per (user, journey) via the idempotency key — a
350
+ // concurrent double-diversion dedups here even if it raced the
351
+ // row existence check above.
352
+ try {
353
+ await ingestEvent({
354
+ db,
355
+ registry: getJourneyRegistrySingleton(),
356
+ hatchet,
357
+ logger,
358
+ event: {
359
+ event: "journey.heldout",
360
+ userId,
361
+ userEmail,
362
+ eventProperties: {
363
+ journeyId: meta.id,
364
+ journeyName: meta.name,
365
+ holdoutPercent: meta.holdout.percent,
366
+ },
367
+ source: "journey",
368
+ idempotencyKey: `journey:heldout:${meta.id}:${userId}`,
369
+ },
370
+ });
371
+ } catch (err) {
372
+ logger.warn("journey.heldout ingest failed", {
373
+ journeyId: meta.id,
374
+ userId,
375
+ error: err instanceof Error ? err.message : String(err),
376
+ });
377
+ }
378
+ void emitOutbound({
379
+ db,
380
+ hatchet,
381
+ logger,
382
+ event: "journey.heldout",
383
+ dedupeKey: `journey.heldout:${holdoutStateId}`,
384
+ payload: {
385
+ journeyId: meta.id,
386
+ journeyName: meta.name,
387
+ stateId: holdoutStateId,
388
+ userId,
389
+ userEmail,
390
+ heldOutAt: heldOutAt.toISOString(),
391
+ },
392
+ }).catch(logger.warn);
393
+ }
394
+ }
395
+ return { status: "skipped", reason: "held_out" };
396
+ }
397
+ }
398
+
306
399
  const activeState = await db.query.journeyStates.findFirst({
307
400
  where: and(
308
401
  eq(journeyStates.userId, userId),
@@ -1026,6 +1026,8 @@ export function createJourneyContext(
1026
1026
  userId: targetUserId,
1027
1027
  userEmail: targetEmail,
1028
1028
  properties,
1029
+ value,
1030
+ currency,
1029
1031
  idempotencyLabel,
1030
1032
  }) {
1031
1033
  // Keep the PUBLIC `TriggerOptions.properties` field name (decision #13 —
@@ -1065,6 +1067,8 @@ export function createJourneyContext(
1065
1067
  userId: targetUserId,
1066
1068
  userEmail: targetEmail ?? userEmail,
1067
1069
  eventProperties: properties ?? {},
1070
+ value,
1071
+ currency,
1068
1072
  // Cross-journey trigger (ctx.trigger).
1069
1073
  source: "journey",
1070
1074
  ...(idempotencyKey ? { idempotencyKey } : {}),
@@ -2,11 +2,13 @@ import type { AnalyticsProvider } from "@hogsend/core";
2
2
  import type { Database } from "@hogsend/db";
3
3
  import {
4
4
  createPostHogProvider,
5
+ deriveIngestHost,
5
6
  type PostHogAuthTokenAccessor,
6
7
  } from "@hogsend/plugin-posthog";
7
8
  import type { env as envSchema } from "../env.js";
8
9
  import type { Logger } from "./logger.js";
9
10
  import { createTokenManager } from "./oauth-token-manager.js";
11
+ import { getDerivedCredential } from "./provider-credentials.js";
10
12
  import { getRedis } from "./redis.js";
11
13
 
12
14
  /**
@@ -28,52 +30,113 @@ export function analyticsProvidersFromEnv(
28
30
  const providers: AnalyticsProvider[] = [];
29
31
 
30
32
  if (env.POSTHOG_API_KEY) {
31
- // Token-manager-backed accessor: the manager re-checks the DB (30s
32
- // negative cache), so a credential stored at RUNTIME via
33
- // `hogsend connect posthog` comes alive without a restart.
34
- let authToken: PostHogAuthTokenAccessor | undefined;
35
- if (deps?.db) {
36
- const tokenManager = createTokenManager({
37
- db: deps.db,
38
- providerId: "posthog",
39
- logger: deps.logger,
40
- });
41
- // Load-only warm-up (no refresh, never blocks construction). The
42
- // person-reads nudge logs HERE, after the load settles — the container
43
- // can't log it truthfully at boot because capabilities resolve async
44
- // for OAuth-capable providers (a connected instance would otherwise
45
- // log "DISABLED" once on every boot).
46
- const personalKeySet = Boolean(env.POSTHOG_PERSONAL_API_KEY);
47
- void tokenManager
48
- .prime()
49
- .then(() => {
50
- if (!personalKeySet && tokenManager.credentialState() !== "present") {
51
- deps.logger?.info(
52
- 'analytics provider "posthog" has person reads DISABLED — ' +
53
- "timezone resolution falls back to contact properties. Set " +
54
- "POSTHOG_PERSONAL_API_KEY or run `hogsend connect posthog`. " +
55
- "Docs: https://hogsend.com/docs/guides/analytics-access",
56
- );
57
- }
58
- })
59
- .catch(() => {});
60
- authToken = {
61
- getToken: () => tokenManager.getAccessToken(),
62
- isAvailable: () => tokenManager.credentialState() === "present",
63
- };
64
- }
65
33
  providers.push(
66
- createPostHogProvider({
67
- apiKey: env.POSTHOG_API_KEY,
68
- host: env.POSTHOG_HOST,
69
- personalApiKey: env.POSTHOG_PERSONAL_API_KEY,
70
- projectId: env.POSTHOG_PROJECT_ID,
71
- privateHost: env.POSTHOG_PRIVATE_HOST,
72
- redis: getRedis(),
73
- authToken,
74
- }),
34
+ buildPosthogProvider(env, deps, { apiKey: env.POSTHOG_API_KEY }),
75
35
  );
76
36
  }
77
37
 
78
38
  return providers;
79
39
  }
40
+
41
+ /**
42
+ * Construct the PostHog provider with the token-manager-backed OAuth accessor.
43
+ * Shared by the env preset above (`POSTHOG_API_KEY`) and the stored-credential
44
+ * activation below (`hogsend connect posthog`'s persisted phc_). Env values
45
+ * win over `source` overrides so a hand-set var still takes precedence.
46
+ */
47
+ function buildPosthogProvider(
48
+ env: typeof envSchema,
49
+ deps: { db?: Database; logger?: Logger } | undefined,
50
+ source: {
51
+ apiKey: string;
52
+ host?: string;
53
+ projectId?: string;
54
+ privateHost?: string;
55
+ },
56
+ ): AnalyticsProvider {
57
+ // Token-manager-backed accessor: the manager re-checks the DB (30s
58
+ // negative cache), so a credential stored at RUNTIME via
59
+ // `hogsend connect posthog` comes alive without a restart.
60
+ let authToken: PostHogAuthTokenAccessor | undefined;
61
+ if (deps?.db) {
62
+ const tokenManager = createTokenManager({
63
+ db: deps.db,
64
+ providerId: "posthog",
65
+ logger: deps.logger,
66
+ });
67
+ // Load-only warm-up (no refresh, never blocks construction). The
68
+ // person-reads nudge logs HERE, after the load settles — the container
69
+ // can't log it truthfully at boot because capabilities resolve async
70
+ // for OAuth-capable providers (a connected instance would otherwise
71
+ // log "DISABLED" once on every boot).
72
+ const personalKeySet = Boolean(env.POSTHOG_PERSONAL_API_KEY);
73
+ void tokenManager
74
+ .prime()
75
+ .then(() => {
76
+ if (!personalKeySet && tokenManager.credentialState() !== "present") {
77
+ deps.logger?.info(
78
+ 'analytics provider "posthog" has person reads DISABLED — ' +
79
+ "timezone resolution falls back to contact properties. Set " +
80
+ "POSTHOG_PERSONAL_API_KEY or run `hogsend connect posthog`. " +
81
+ "Docs: https://hogsend.com/docs/guides/analytics-access",
82
+ );
83
+ }
84
+ })
85
+ .catch(() => {});
86
+ authToken = {
87
+ getToken: () => tokenManager.getAccessToken(),
88
+ isAvailable: () => tokenManager.credentialState() === "present",
89
+ };
90
+ }
91
+ return createPostHogProvider({
92
+ apiKey: source.apiKey,
93
+ host: env.POSTHOG_HOST ?? source.host,
94
+ personalApiKey: env.POSTHOG_PERSONAL_API_KEY,
95
+ projectId: env.POSTHOG_PROJECT_ID ?? source.projectId,
96
+ privateHost: env.POSTHOG_PRIVATE_HOST ?? source.privateHost,
97
+ redis: getRedis(),
98
+ authToken,
99
+ });
100
+ }
101
+
102
+ /**
103
+ * Boot-time reader for the phc_ persisted by `hogsend connect posthog`
104
+ * (provider_credentials kind="derived", `projectApiKey`): a PURE builder —
105
+ * reads the stored credential and constructs the PostHog provider from it, or
106
+ * resolves null when there is nothing stored (or it can't be decrypted, which
107
+ * must behave exactly like "unconfigured"). ALL activation side effects
108
+ * (registry, `client.analytics`, the module singleton, identity rebind) live
109
+ * at the single call site in `container.ts`. Without this reader the
110
+ * persisted key is dead weight and outbound capture silently keeps requiring
111
+ * a hand-pasted env var.
112
+ *
113
+ * The capture host resolves env-first, then the ingestion host derived from
114
+ * the stored PRIVATE host (`eu.posthog.com` → `eu.i.posthog.com`) so an
115
+ * EU-connected instance never captures to the US default.
116
+ */
117
+ export async function buildStoredPosthogProvider(opts: {
118
+ env: typeof envSchema;
119
+ db: Database;
120
+ logger: Logger;
121
+ }): Promise<AnalyticsProvider | null> {
122
+ let derived: Awaited<ReturnType<typeof getDerivedCredential>>;
123
+ try {
124
+ derived = await getDerivedCredential(opts.db, "posthog");
125
+ } catch {
126
+ return null;
127
+ }
128
+ if (!derived?.projectApiKey) return null;
129
+
130
+ return buildPosthogProvider(
131
+ opts.env,
132
+ { db: opts.db, logger: opts.logger },
133
+ {
134
+ apiKey: derived.projectApiKey,
135
+ projectId: derived.projectId,
136
+ privateHost: derived.privateHost,
137
+ host: derived.privateHost
138
+ ? deriveIngestHost(derived.privateHost)
139
+ : undefined,
140
+ },
141
+ );
142
+ }