@hogsend/engine 0.42.0 → 0.44.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/package.json +10 -7
  2. package/src/container.ts +419 -4
  3. package/src/destinations/define-destination.ts +7 -3
  4. package/src/destinations/presets/posthog.ts +31 -0
  5. package/src/env.ts +28 -0
  6. package/src/index.ts +138 -0
  7. package/src/journeys/execute-journey-run.ts +93 -0
  8. package/src/journeys/journey-boundary.ts +5 -1
  9. package/src/journeys/journey-context.ts +30 -0
  10. package/src/lib/attribution-backfill.ts +285 -0
  11. package/src/lib/attribution.ts +239 -0
  12. package/src/lib/contacts.ts +71 -0
  13. package/src/lib/conversion-dispatch.ts +326 -0
  14. package/src/lib/conversions.ts +197 -0
  15. package/src/lib/crm-deals.ts +320 -0
  16. package/src/lib/crm-ingest.ts +341 -0
  17. package/src/lib/crm-provider-registry.ts +41 -0
  18. package/src/lib/crm-registry-singleton.ts +22 -0
  19. package/src/lib/deal-money-events.ts +85 -0
  20. package/src/lib/email-service-types.ts +16 -1
  21. package/src/lib/event-names.ts +1 -1
  22. package/src/lib/funnel-progress.ts +76 -0
  23. package/src/lib/funnel-registry.ts +108 -0
  24. package/src/lib/funnel-transitions.ts +280 -0
  25. package/src/lib/holdout.ts +62 -0
  26. package/src/lib/ingestion.ts +218 -9
  27. package/src/lib/journey-suppress.ts +33 -2
  28. package/src/lib/lift-stats.ts +125 -0
  29. package/src/lib/mailer.ts +1 -0
  30. package/src/lib/operator.ts +31 -0
  31. package/src/lib/outbound.ts +109 -0
  32. package/src/lib/phone.ts +19 -0
  33. package/src/lib/preferences.ts +32 -0
  34. package/src/lib/revenue.ts +116 -0
  35. package/src/lib/sms-frequency-cap.ts +47 -0
  36. package/src/lib/sms-inbound.ts +200 -0
  37. package/src/lib/sms-link-tracking.ts +160 -0
  38. package/src/lib/sms-mailer.ts +338 -0
  39. package/src/lib/sms-provider-registry.ts +35 -0
  40. package/src/lib/sms-providers-from-env.ts +100 -0
  41. package/src/lib/sms-service-types.ts +175 -0
  42. package/src/lib/sms-tracked.ts +625 -0
  43. package/src/lib/sms.ts +117 -0
  44. package/src/lib/tracked.ts +40 -0
  45. package/src/lib/tracking-event-names.ts +6 -0
  46. package/src/lib/tracking-events.ts +133 -1
  47. package/src/lib/tracking.ts +2 -1
  48. package/src/lib/webhook-signing.ts +24 -1
  49. package/src/lists/channels.ts +21 -0
  50. package/src/routes/admin/attribution.ts +414 -0
  51. package/src/routes/admin/contacts.ts +71 -11
  52. package/src/routes/admin/conversions.ts +247 -0
  53. package/src/routes/admin/deals.ts +472 -0
  54. package/src/routes/admin/funnels.ts +228 -0
  55. package/src/routes/admin/index.ts +8 -0
  56. package/src/routes/admin/journeys.ts +158 -1
  57. package/src/routes/admin/readiness.ts +97 -2
  58. package/src/routes/admin/templates.ts +28 -1
  59. package/src/routes/email/unsubscribe.ts +1 -0
  60. package/src/routes/events/index.ts +12 -0
  61. package/src/routes/index.ts +6 -0
  62. package/src/routes/lists/index.ts +54 -7
  63. package/src/routes/tracking/click-pipeline.ts +105 -16
  64. package/src/routes/tracking/short.ts +61 -0
  65. package/src/routes/webhooks/crm-provider.ts +117 -0
  66. package/src/routes/webhooks/index.ts +9 -1
  67. package/src/routes/webhooks/sms-provider.ts +110 -0
  68. package/src/sources/define-contact-source.ts +159 -0
  69. package/src/sources/registry.ts +71 -0
  70. package/src/sources/webhook-source.ts +109 -0
  71. package/src/worker.ts +4 -0
  72. package/src/workflows/crm-reconcile.ts +129 -0
  73. package/src/workflows/dispatch-conversion.ts +31 -0
  74. package/src/workflows/import-contacts.ts +61 -4
  75. package/src/workflows/send-campaign.ts +5 -0
package/src/index.ts CHANGED
@@ -367,11 +367,51 @@ export {
367
367
  } from "./lib/connector-state.js";
368
368
  // --- Contacts identity (resolve/create — used by connector member-link) ---
369
369
  export { resolveOrCreateContact } from "./lib/contacts.js";
370
+ // --- Conversion dispatch (plan §5.2): destinations registry + delivery ---
371
+ export {
372
+ ConversionDestinationRegistry,
373
+ conversionEventId,
374
+ createConversionDispatches,
375
+ deliverConversionDispatch,
376
+ enqueueConversionDispatches,
377
+ getConversionDestinations,
378
+ recoverClickContext,
379
+ resetConversionDestinations,
380
+ setConversionDestinations,
381
+ } from "./lib/conversion-dispatch.js";
382
+ // --- Conversion points (plan §5.1): registry + ingest-time evaluation ---
383
+ export {
384
+ ConversionRegistry,
385
+ evaluateConversionsAtIngest,
386
+ type FiredConversion,
387
+ getConversionRegistry,
388
+ resetConversionRegistry,
389
+ setConversionRegistry,
390
+ } from "./lib/conversions.js";
370
391
  export {
371
392
  AdminAlreadyExistsError,
372
393
  type CreatedAdmin,
373
394
  createAdminUser,
374
395
  } from "./lib/create-admin.js";
396
+ // --- CRM sync (provider registry + the stage-event spine sink) ---
397
+ export {
398
+ type AppliedStageChange,
399
+ applyCrmStageEvent,
400
+ ensureCrmLinks,
401
+ resolveCrmLinkedContact,
402
+ } from "./lib/crm-deals.js";
403
+ export {
404
+ DEAL_QUOTED,
405
+ DEAL_SOLD,
406
+ FUNNEL_STAGE_CHANGED,
407
+ ingestCrmStageEvents,
408
+ } from "./lib/crm-ingest.js";
409
+ export { CrmProviderRegistry } from "./lib/crm-provider-registry.js";
410
+ export {
411
+ getCrmSyncConfig,
412
+ resetCrmSyncConfig,
413
+ setCrmSyncConfig,
414
+ } from "./lib/crm-registry-singleton.js";
375
415
  // --- Infrastructure singletons ---
376
416
  export { getDb } from "./lib/db.js";
377
417
  // --- Discord gateway-worker liveness heartbeat (Studio status) ---
@@ -420,7 +460,14 @@ export {
420
460
  } from "./lib/feed.js";
421
461
  export { countRecentSends, isFrequencyCapped } from "./lib/frequency-cap.js";
422
462
  export { addrSpecOf, hostOfFromAddress } from "./lib/from-address.js";
463
+ export { FunnelRegistry } from "./lib/funnel-registry.js";
423
464
  export { hatchet } from "./lib/hatchet.js";
465
+ export {
466
+ globalControlPercent,
467
+ holdoutBucket,
468
+ isGlobalControl,
469
+ isHeldOut,
470
+ } from "./lib/holdout.js";
424
471
  // --- Identity service (resolve/merge + analytics merge propagation, §7) ---
425
472
  export {
426
473
  createIdentityService,
@@ -448,6 +495,14 @@ export {
448
495
  releaseLeaderLease,
449
496
  renewLeaderLease,
450
497
  } from "./lib/leader-lease.js";
498
+ // --- Holdout lift statistics (impact plan §4.2) ---
499
+ export {
500
+ betaWinProbability,
501
+ computeLift,
502
+ type LiftVerdict,
503
+ MIN_COMBINED_CONVERSIONS,
504
+ SMALL_SAMPLE_FLOOR,
505
+ } from "./lib/lift-stats.js";
451
506
  // --- Managed tracked links (channel-agnostic mint — Studio/Discord/share) ---
452
507
  // NOTE: the QR scan-row plumbing (ensureQrTrackedLink, the canonical-row
453
508
  // filter, the 'qr' source marker) is deliberately NOT exported — it is
@@ -483,6 +538,7 @@ export {
483
538
  type OutboundEventName,
484
539
  type OutboundPayloads,
485
540
  } from "./lib/outbound.js";
541
+ export { isE164, normalizePhone } from "./lib/phone.js";
486
542
  export { getPostHog } from "./lib/posthog.js";
487
543
  // --- PostHog OAuth scopes (front-loaded set; gap-detector source of truth) ---
488
544
  export { EXPECTED_POSTHOG_SCOPES } from "./lib/posthog-scopes.js";
@@ -515,6 +571,12 @@ export {
515
571
  } from "./lib/redis.js";
516
572
  // --- Self-service password reset (engine-owned, self-contained email) ---
517
573
  export { sendResetPasswordEmail } from "./lib/reset-email.js";
574
+ // --- Revenue rollups (the event spine's value/currency columns) ---
575
+ export {
576
+ type ContactRevenue,
577
+ type ContactRevenueTotal,
578
+ getContactRevenue,
579
+ } from "./lib/revenue.js";
518
580
  // --- PostHog destination seed (idempotent; ENABLE_POSTHOG_DESTINATION) ---
519
581
  export { seedPostHogDestination } from "./lib/seed-posthog-destination.js";
520
582
  export {
@@ -524,6 +586,42 @@ export {
524
586
  SEMANTIC_BURST_DISTINCT_LINKS,
525
587
  SEMANTIC_BURST_WINDOW_MS,
526
588
  } from "./lib/semantic-click.js";
589
+ // --- SMS ---
590
+ export {
591
+ getSmsService,
592
+ type SendSmsOptions as SendSmsJourneyOptions,
593
+ type SendSmsResult,
594
+ sendSms,
595
+ setSmsService,
596
+ } from "./lib/sms.js";
597
+ // --- SMS inbound consent helpers (phone-track grant/opt-out) ---
598
+ export {
599
+ grantPhoneConsent,
600
+ recordPhoneOptOut,
601
+ } from "./lib/sms-inbound.js";
602
+ // --- SMS link shortening (pure rewriter; the insert rides the send txn) ---
603
+ export {
604
+ generateShortCode,
605
+ isShortCodeCollision,
606
+ type PendingSmsLink,
607
+ planSmsLinkRewrite,
608
+ SHORT_CODE_ALPHABET,
609
+ SHORT_CODE_LENGTH,
610
+ type SmsLinkRewritePlan,
611
+ } from "./lib/sms-link-tracking.js";
612
+ // --- SMS mailer factory (the engine-owned tracked sender; mirrors createTrackedMailer) ---
613
+ export { createTrackedSmsSender } from "./lib/sms-mailer.js";
614
+ // --- SMS provider registry (container-held, keyed by meta.id) ---
615
+ export { SmsProviderRegistry } from "./lib/sms-provider-registry.js";
616
+ // --- SMS service (engine-owned tracked SMS sender) ---
617
+ export type {
618
+ SmsService,
619
+ SmsServiceConfig,
620
+ SmsServiceSendOptions,
621
+ SmsServiceWebhookResult,
622
+ SmsTrackedSendResult,
623
+ } from "./lib/sms-service-types.js";
624
+ export { SMS_CHANNEL_ID } from "./lib/sms-tracked.js";
527
625
  export { type MountStudioResult, mountStudio } from "./lib/studio.js";
528
626
  // --- In-app survey/rating (sendSurvey — producer sugar over sendFeedItem) ---
529
627
  export { type SendSurveyOptions, sendSurvey } from "./lib/survey.js";
@@ -546,7 +644,16 @@ export {
546
644
  prepareTrackedHtml,
547
645
  rewriteLinks,
548
646
  } from "./lib/tracking.js";
647
+ // --- First-party tracking event names (touchpoint-classed in @hogsend/core) ---
549
648
  export {
649
+ EMAIL_LINK_CLICKED,
650
+ EMAIL_OPENED,
651
+ LINK_ARRIVED,
652
+ LINK_CLICKED,
653
+ SMS_LINK_CLICKED,
654
+ } from "./lib/tracking-event-names.js";
655
+ export {
656
+ pushSmsTrackingEvent,
550
657
  pushTrackingEvent,
551
658
  resolveEmailSendContext,
552
659
  resolveEmailSendContextByMessageId,
@@ -555,6 +662,7 @@ export {
555
662
  * {@link resolveEmailSendContextByMessageId}.
556
663
  */
557
664
  resolveEmailSendContextByResendId,
665
+ resolveSmsSendContext,
558
666
  } from "./lib/tracking-events.js";
559
667
  /**
560
668
  * Publishable-key `userToken` mint/verify helpers.
@@ -638,6 +746,32 @@ export {
638
746
  } from "./middleware/rate-limit.js";
639
747
  // --- Middleware (consumer-mounted routes, e.g. the @hogsend/mcp hosted route) ---
640
748
  export { requireAdmin } from "./middleware/require-admin.js";
749
+ // --- Contact sources (Clay/Attio/generic-webhook → cold prospects) ---
750
+ export {
751
+ type ColdChannelPosture,
752
+ type ColdPosture,
753
+ type ContactSourceMeta,
754
+ type ContactWriteBack,
755
+ contactSourceToWebhookSource,
756
+ type DefinedContactSource,
757
+ defaultColdPosture,
758
+ defineContactSource,
759
+ isColdChannelAllowed,
760
+ resolveColdPosture,
761
+ } from "./sources/define-contact-source.js";
762
+ export {
763
+ buildContactSourceRegistry,
764
+ ContactSourceRegistry,
765
+ getContactSourceRegistry,
766
+ setContactSourceRegistry,
767
+ } from "./sources/registry.js";
768
+ export {
769
+ normalizeWebhookContactEvent,
770
+ type WebhookContactPayload,
771
+ type WebhookContactSourceOptions,
772
+ webhookContactPayloadSchema,
773
+ webhookContactSource,
774
+ } from "./sources/webhook-source.js";
641
775
  // --- Webhook sources ---
642
776
  export {
643
777
  type DefinedWebhookSource,
@@ -678,6 +812,10 @@ export {
678
812
  checkAlertsTask,
679
813
  surfaceStrandedWaiting,
680
814
  } from "./workflows/check-alerts.js";
815
+ export {
816
+ crmReconcileTask,
817
+ runCrmReconcile,
818
+ } from "./workflows/crm-reconcile.js";
681
819
  // --- Outbound webhooks: durable delivery task + reaper (Section 1.5) ---
682
820
  export {
683
821
  deliverWebhookTask,
@@ -25,6 +25,8 @@ import {
25
25
  checkEntryLimit,
26
26
  } from "../lib/enrollment-guards.js";
27
27
  import { hatchet } from "../lib/hatchet.js";
28
+ import { isHeldOut } from "../lib/holdout.js";
29
+ import { ingestEvent } from "../lib/ingestion.js";
28
30
  import { createLogger } from "../lib/logger.js";
29
31
  import { emitOutbound } from "../lib/outbound.js";
30
32
  import { resolveTimezoneWithSource } from "../lib/timezone.js";
@@ -303,6 +305,97 @@ export async function executeJourneyRun(
303
305
  return { status: "skipped", reason: "user_unsubscribed" };
304
306
  }
305
307
 
308
+ // HOLDOUT DIVERSION (impact plan §4.1) — deliberately LAST in the guard
309
+ // chain: a contact who'd have been blocked by any gate above is never
310
+ // counted as held out (intent-to-treat — the control must mirror the
311
+ // would-have-entered population, not the triggered one). Assignment is a
312
+ // deterministic hash (replay law: no RNG in durable paths), so the same
313
+ // contact diverts identically on every trigger AND every replay. One
314
+ // held_out row per (user, journey), ever; the spine event's idempotency
315
+ // key carries the same once-ever semantics for fan-out/analytics.
316
+ if (meta.holdout && meta.holdout.percent > 0) {
317
+ const diverted = isHeldOut({
318
+ userId,
319
+ journeyId: meta.id,
320
+ percent: meta.holdout.percent,
321
+ salt: meta.holdout.salt,
322
+ });
323
+ if (diverted) {
324
+ const priorHoldout = await db.query.journeyStates.findFirst({
325
+ where: and(
326
+ eq(journeyStates.userId, userId),
327
+ eq(journeyStates.journeyId, meta.id),
328
+ eq(journeyStates.status, "held_out"),
329
+ ),
330
+ });
331
+ if (!priorHoldout) {
332
+ const heldOutAt = new Date();
333
+ const inserted = await db
334
+ .insert(journeyStates)
335
+ .values({
336
+ userId,
337
+ userEmail,
338
+ journeyId: meta.id,
339
+ currentNodeId: "held-out",
340
+ status: "held_out",
341
+ context: properties,
342
+ exitedAt: heldOutAt,
343
+ })
344
+ .returning({ id: journeyStates.id });
345
+ const holdoutStateId = inserted[0]?.id;
346
+ if (holdoutStateId) {
347
+ // The counterfactual as data (the Iterable Send Skip pattern):
348
+ // queryable on the spine, fan-out-able to journeys/destinations.
349
+ // Once-ever per (user, journey) via the idempotency key — a
350
+ // concurrent double-diversion dedups here even if it raced the
351
+ // row existence check above.
352
+ try {
353
+ await ingestEvent({
354
+ db,
355
+ registry: getJourneyRegistrySingleton(),
356
+ hatchet,
357
+ logger,
358
+ event: {
359
+ event: "journey.heldout",
360
+ userId,
361
+ userEmail,
362
+ eventProperties: {
363
+ journeyId: meta.id,
364
+ journeyName: meta.name,
365
+ holdoutPercent: meta.holdout.percent,
366
+ },
367
+ source: "journey",
368
+ idempotencyKey: `journey:heldout:${meta.id}:${userId}`,
369
+ },
370
+ });
371
+ } catch (err) {
372
+ logger.warn("journey.heldout ingest failed", {
373
+ journeyId: meta.id,
374
+ userId,
375
+ error: err instanceof Error ? err.message : String(err),
376
+ });
377
+ }
378
+ void emitOutbound({
379
+ db,
380
+ hatchet,
381
+ logger,
382
+ event: "journey.heldout",
383
+ dedupeKey: `journey.heldout:${holdoutStateId}`,
384
+ payload: {
385
+ journeyId: meta.id,
386
+ journeyName: meta.name,
387
+ stateId: holdoutStateId,
388
+ userId,
389
+ userEmail,
390
+ heldOutAt: heldOutAt.toISOString(),
391
+ },
392
+ }).catch(logger.warn);
393
+ }
394
+ }
395
+ return { status: "skipped", reason: "held_out" };
396
+ }
397
+ }
398
+
306
399
  const activeState = await db.query.journeyStates.findFirst({
307
400
  where: and(
308
401
  eq(journeyStates.userId, userId),
@@ -146,10 +146,14 @@ export function supportsEviction(hatchetCtx: unknown): boolean {
146
146
  }
147
147
 
148
148
  /** The kind of side effect a key is derived for — keeps the namespaces apart. */
149
- export type JourneyKeyKind = "send" | "trigger" | "connector";
149
+ export type JourneyKeyKind = "send" | "smsSend" | "trigger" | "connector";
150
150
 
151
151
  const KEY_PREFIX: Record<JourneyKeyKind, string> = {
152
152
  send: "journeySend",
153
+ // Disjoint from `send` so a `sendEmail("x")` and a `sendSms("x")` under the
154
+ // SAME nearest wait label derive DIFFERENT keys and never collide in
155
+ // registerKey — email + SMS of the same template on one branch is legitimate.
156
+ smsSend: "journeySmsSend",
153
157
  trigger: "journeyTrigger",
154
158
  connector: "journeyConnector",
155
159
  };
@@ -42,6 +42,7 @@ import {
42
42
  type Database,
43
43
  emailSends,
44
44
  journeyStates,
45
+ smsSends,
45
46
  userEvents,
46
47
  } from "@hogsend/db";
47
48
  import {
@@ -1025,6 +1026,8 @@ export function createJourneyContext(
1025
1026
  userId: targetUserId,
1026
1027
  userEmail: targetEmail,
1027
1028
  properties,
1029
+ value,
1030
+ currency,
1028
1031
  idempotencyLabel,
1029
1032
  }) {
1030
1033
  // Keep the PUBLIC `TriggerOptions.properties` field name (decision #13 —
@@ -1064,6 +1067,8 @@ export function createJourneyContext(
1064
1067
  userId: targetUserId,
1065
1068
  userEmail: targetEmail ?? userEmail,
1066
1069
  eventProperties: properties ?? {},
1070
+ value,
1071
+ currency,
1067
1072
  // Cross-journey trigger (ctx.trigger).
1068
1073
  source: "journey",
1069
1074
  ...(idempotencyKey ? { idempotencyKey } : {}),
@@ -1212,6 +1217,31 @@ export function createJourneyContext(
1212
1217
  };
1213
1218
  },
1214
1219
 
1220
+ async sms({ phone, template }) {
1221
+ const [result] = await db
1222
+ .select({
1223
+ count: count(),
1224
+ lastSentAt: max(smsSends.sentAt),
1225
+ })
1226
+ .from(smsSends)
1227
+ .where(
1228
+ and(
1229
+ eq(smsSends.toPhone, phone),
1230
+ eq(smsSends.templateKey, template),
1231
+ ),
1232
+ );
1233
+
1234
+ const total = result?.count ?? 0;
1235
+ return {
1236
+ sent: total > 0,
1237
+ lastSentAt:
1238
+ result?.lastSentAt instanceof Date
1239
+ ? result.lastSentAt.toISOString()
1240
+ : null,
1241
+ count: total,
1242
+ };
1243
+ },
1244
+
1215
1245
  async events({
1216
1246
  userId: targetUserId,
1217
1247
  event,
@@ -0,0 +1,285 @@
1
+ import {
2
+ attributionCredits,
3
+ contacts,
4
+ conversions,
5
+ type Database,
6
+ userEvents,
7
+ } from "@hogsend/db";
8
+ import {
9
+ and,
10
+ asc,
11
+ eq,
12
+ gt,
13
+ gte,
14
+ inArray,
15
+ isNotNull,
16
+ or,
17
+ sql,
18
+ } from "drizzle-orm";
19
+ import { recordAttributionCredits } from "./attribution.js";
20
+ import type { ConversionRegistry } from "./conversions.js";
21
+ import { evaluateConversionsAtIngest } from "./conversions.js";
22
+ import type { Logger } from "./logger.js";
23
+
24
+ /**
25
+ * Attribution backfill (impact plan §5.1) — replays HISTORY through the
26
+ * same idempotent machinery ingest uses, so an existing deploy that
27
+ * upgrades gets its whole event history credited, and a definition/window
28
+ * change gets a deliberate, logged recompute path.
29
+ *
30
+ * Two stages behind one cursor (`events:<id>` → `credits:<id>`):
31
+ *
32
+ * 1. `events` — scan historical `user_events` matching the registry's
33
+ * trigger set and run conversion evaluation. The unique
34
+ * (definition, event) index makes replays no-ops; only genuinely new
35
+ * conversions are minted. Ad-platform dispatches are deliberately NOT
36
+ * enqueued — historical conversions must never re-fire to destinations.
37
+ * 2. `credits` — scan conversions rows (of the scoped definitions) that
38
+ * have NO credit rows yet and write their ledger entries under the
39
+ * definition's CURRENT window config.
40
+ *
41
+ * `recompute: true` (requires `definitionId`) deletes the definition's
42
+ * existing credit rows up front, then stage 2 re-credits everything — the
43
+ * ONE sanctioned exception to the ledger's write-once philosophy, guarded
44
+ * and logged. Conversion rows themselves are never deleted.
45
+ *
46
+ * Batch-shaped by design: each call processes up to `limit` rows of the
47
+ * current stage and returns `nextCursor` (null = done). The CLI loops it;
48
+ * an admin can curl it. Contacts are matched only — a historical event
49
+ * whose user never became a contact is skipped, never minted.
50
+ */
51
+ export interface BackfillBatchResult {
52
+ stage: "events" | "credits";
53
+ processed: number;
54
+ conversionsFired: number;
55
+ creditsWritten: number;
56
+ nextCursor: string | null;
57
+ }
58
+
59
+ const UUID_RE =
60
+ /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
61
+
62
+ export async function backfillAttributionBatch(opts: {
63
+ db: Database;
64
+ logger: Logger;
65
+ registry: ConversionRegistry | undefined;
66
+ definitionId?: string;
67
+ since?: Date;
68
+ cursor?: string;
69
+ limit?: number;
70
+ recompute?: boolean;
71
+ }): Promise<BackfillBatchResult> {
72
+ const { db, logger, registry, definitionId, since } = opts;
73
+ const limit = Math.min(2000, Math.max(1, opts.limit ?? 500));
74
+
75
+ const defs = (registry?.getAll() ?? []).filter(
76
+ (def) => !definitionId || def.meta.id === definitionId,
77
+ );
78
+ if (defs.length === 0) {
79
+ return {
80
+ stage: "credits",
81
+ processed: 0,
82
+ conversionsFired: 0,
83
+ creditsWritten: 0,
84
+ nextCursor: null,
85
+ };
86
+ }
87
+ const defIds = defs.map((def) => def.meta.id);
88
+ const defById = new Map(defs.map((def) => [def.meta.id, def]));
89
+ const namedEvents = [
90
+ ...new Set(
91
+ defs
92
+ .map((def) => def.meta.trigger.event)
93
+ .filter((event) => event !== "*"),
94
+ ),
95
+ ];
96
+ const hasWildcard = defs.some((def) => def.meta.trigger.event === "*");
97
+
98
+ const [stage, lastId] = opts.cursor
99
+ ? (opts.cursor.split(":", 2) as ["events" | "credits", string])
100
+ : (["events", ""] as const);
101
+
102
+ // Recompute (guarded): first call only, one definition only.
103
+ if (opts.recompute && !opts.cursor) {
104
+ if (!definitionId) {
105
+ throw new Error("recompute requires a definitionId — never blanket");
106
+ }
107
+ const deleted = await db
108
+ .delete(attributionCredits)
109
+ .where(
110
+ inArray(
111
+ attributionCredits.conversionId,
112
+ db
113
+ .select({ id: conversions.id })
114
+ .from(conversions)
115
+ .where(eq(conversions.definitionId, definitionId)),
116
+ ),
117
+ )
118
+ .returning({ id: attributionCredits.id });
119
+ logger.warn("attribution recompute: credit rows deleted for refill", {
120
+ definitionId,
121
+ deleted: deleted.length,
122
+ });
123
+ }
124
+
125
+ if (stage === "events") {
126
+ const eventMatch = or(
127
+ namedEvents.length > 0
128
+ ? inArray(userEvents.event, namedEvents)
129
+ : sql`false`,
130
+ hasWildcard ? isNotNull(userEvents.value) : sql`false`,
131
+ );
132
+ const rows = await db
133
+ .select({
134
+ id: userEvents.id,
135
+ event: userEvents.event,
136
+ userId: userEvents.userId,
137
+ properties: userEvents.properties,
138
+ value: userEvents.value,
139
+ currency: userEvents.currency,
140
+ source: userEvents.source,
141
+ occurredAt: userEvents.occurredAt,
142
+ })
143
+ .from(userEvents)
144
+ .where(
145
+ and(
146
+ eventMatch,
147
+ ...(since ? [gte(userEvents.occurredAt, since)] : []),
148
+ ...(lastId ? [gt(userEvents.id, lastId)] : []),
149
+ ),
150
+ )
151
+ .orderBy(asc(userEvents.id))
152
+ .limit(limit);
153
+
154
+ // Batch contact resolution: canonical keys are externalId or the
155
+ // contact row uuid. MATCH ONLY — backfill never mints contacts.
156
+ const keys = [...new Set(rows.map((row) => row.userId))];
157
+ const uuidKeys = keys.filter((key) => UUID_RE.test(key));
158
+ const contactRows =
159
+ keys.length > 0
160
+ ? await db
161
+ .select({ id: contacts.id, externalId: contacts.externalId })
162
+ .from(contacts)
163
+ .where(
164
+ or(
165
+ inArray(contacts.externalId, keys),
166
+ uuidKeys.length > 0
167
+ ? inArray(contacts.id, uuidKeys)
168
+ : sql`false`,
169
+ ),
170
+ )
171
+ : [];
172
+ const contactByKey = new Map<string, string>();
173
+ for (const contact of contactRows) {
174
+ contactByKey.set(contact.id, contact.id);
175
+ if (contact.externalId) contactByKey.set(contact.externalId, contact.id);
176
+ }
177
+
178
+ let conversionsFired = 0;
179
+ let creditsWritten = 0;
180
+ for (const row of rows) {
181
+ const contactId = contactByKey.get(row.userId);
182
+ if (!contactId) continue;
183
+ const fired = await evaluateConversionsAtIngest({
184
+ db,
185
+ logger,
186
+ registry,
187
+ event: {
188
+ name: row.event,
189
+ source: row.source ?? null,
190
+ properties: (row.properties ?? {}) as Record<string, unknown>,
191
+ value: row.value,
192
+ currency: row.currency,
193
+ occurredAt: row.occurredAt,
194
+ },
195
+ eventRowId: row.id,
196
+ contactId,
197
+ userKey: row.userId,
198
+ });
199
+ for (const conversion of fired) {
200
+ if (definitionId && conversion.definition.meta.id !== definitionId) {
201
+ continue;
202
+ }
203
+ conversionsFired++;
204
+ const { touchpoints } = await recordAttributionCredits({
205
+ db,
206
+ logger,
207
+ conversionId: conversion.conversionId,
208
+ userKey: row.userId,
209
+ value: conversion.value,
210
+ currency: conversion.currency,
211
+ occurredAt: row.occurredAt,
212
+ windowDays: conversion.definition.meta.attributionWindowDays ?? 90,
213
+ windows: conversion.definition.meta.windows,
214
+ });
215
+ if (touchpoints > 0) creditsWritten++;
216
+ }
217
+ }
218
+
219
+ const nextCursor =
220
+ rows.length < limit
221
+ ? "credits:"
222
+ : `events:${rows[rows.length - 1]?.id ?? ""}`;
223
+ return {
224
+ stage: "events",
225
+ processed: rows.length,
226
+ conversionsFired,
227
+ creditsWritten,
228
+ nextCursor,
229
+ };
230
+ }
231
+
232
+ // Stage 2: credit conversions that have no ledger rows yet (after a
233
+ // recompute delete, that is all of the definition's conversions).
234
+ const uncredited = sql`not exists (
235
+ select 1 from ${attributionCredits}
236
+ where ${attributionCredits.conversionId} = ${conversions.id}
237
+ )`;
238
+ const rows = await db
239
+ .select({
240
+ id: conversions.id,
241
+ definitionId: conversions.definitionId,
242
+ userKey: conversions.userKey,
243
+ value: conversions.value,
244
+ currency: conversions.currency,
245
+ occurredAt: conversions.occurredAt,
246
+ })
247
+ .from(conversions)
248
+ .where(
249
+ and(
250
+ inArray(conversions.definitionId, defIds),
251
+ uncredited,
252
+ ...(since ? [gte(conversions.occurredAt, since)] : []),
253
+ ...(lastId ? [gt(conversions.id, lastId)] : []),
254
+ ),
255
+ )
256
+ .orderBy(asc(conversions.id))
257
+ .limit(limit);
258
+
259
+ let creditsWritten = 0;
260
+ for (const row of rows) {
261
+ const def = defById.get(row.definitionId);
262
+ if (!def) continue;
263
+ const { touchpoints } = await recordAttributionCredits({
264
+ db,
265
+ logger,
266
+ conversionId: row.id,
267
+ userKey: row.userKey,
268
+ value: row.value,
269
+ currency: row.currency,
270
+ occurredAt: row.occurredAt,
271
+ windowDays: def.meta.attributionWindowDays ?? 90,
272
+ windows: def.meta.windows,
273
+ });
274
+ if (touchpoints > 0) creditsWritten++;
275
+ }
276
+
277
+ return {
278
+ stage: "credits",
279
+ processed: rows.length,
280
+ conversionsFired: 0,
281
+ creditsWritten,
282
+ nextCursor:
283
+ rows.length < limit ? null : `credits:${rows[rows.length - 1]?.id ?? ""}`,
284
+ };
285
+ }