@hogsend/engine 0.42.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +10 -7
- package/src/container.ts +419 -4
- package/src/destinations/define-destination.ts +7 -3
- package/src/destinations/presets/posthog.ts +31 -0
- package/src/env.ts +28 -0
- package/src/index.ts +138 -0
- package/src/journeys/execute-journey-run.ts +93 -0
- package/src/journeys/journey-boundary.ts +5 -1
- package/src/journeys/journey-context.ts +30 -0
- package/src/lib/attribution-backfill.ts +285 -0
- package/src/lib/attribution.ts +239 -0
- package/src/lib/contacts.ts +71 -0
- package/src/lib/conversion-dispatch.ts +326 -0
- package/src/lib/conversions.ts +197 -0
- package/src/lib/crm-deals.ts +320 -0
- package/src/lib/crm-ingest.ts +341 -0
- package/src/lib/crm-provider-registry.ts +41 -0
- package/src/lib/crm-registry-singleton.ts +22 -0
- package/src/lib/deal-money-events.ts +85 -0
- package/src/lib/email-service-types.ts +16 -1
- package/src/lib/event-names.ts +1 -1
- package/src/lib/funnel-progress.ts +76 -0
- package/src/lib/funnel-registry.ts +108 -0
- package/src/lib/funnel-transitions.ts +280 -0
- package/src/lib/holdout.ts +62 -0
- package/src/lib/ingestion.ts +218 -9
- package/src/lib/journey-suppress.ts +33 -2
- package/src/lib/lift-stats.ts +125 -0
- package/src/lib/mailer.ts +1 -0
- package/src/lib/operator.ts +31 -0
- package/src/lib/outbound.ts +109 -0
- package/src/lib/phone.ts +19 -0
- package/src/lib/preferences.ts +32 -0
- package/src/lib/revenue.ts +116 -0
- package/src/lib/sms-frequency-cap.ts +47 -0
- package/src/lib/sms-inbound.ts +200 -0
- package/src/lib/sms-link-tracking.ts +160 -0
- package/src/lib/sms-mailer.ts +338 -0
- package/src/lib/sms-provider-registry.ts +35 -0
- package/src/lib/sms-providers-from-env.ts +100 -0
- package/src/lib/sms-service-types.ts +175 -0
- package/src/lib/sms-tracked.ts +625 -0
- package/src/lib/sms.ts +117 -0
- package/src/lib/tracked.ts +40 -0
- package/src/lib/tracking-event-names.ts +6 -0
- package/src/lib/tracking-events.ts +133 -1
- package/src/lib/tracking.ts +2 -1
- package/src/lib/webhook-signing.ts +24 -1
- package/src/lists/channels.ts +21 -0
- package/src/routes/admin/attribution.ts +414 -0
- package/src/routes/admin/contacts.ts +71 -11
- package/src/routes/admin/conversions.ts +247 -0
- package/src/routes/admin/deals.ts +472 -0
- package/src/routes/admin/funnels.ts +228 -0
- package/src/routes/admin/index.ts +8 -0
- package/src/routes/admin/journeys.ts +158 -1
- package/src/routes/admin/readiness.ts +97 -2
- package/src/routes/admin/templates.ts +28 -1
- package/src/routes/email/unsubscribe.ts +1 -0
- package/src/routes/events/index.ts +12 -0
- package/src/routes/index.ts +6 -0
- package/src/routes/lists/index.ts +54 -7
- package/src/routes/tracking/click-pipeline.ts +105 -16
- package/src/routes/tracking/short.ts +61 -0
- package/src/routes/webhooks/crm-provider.ts +117 -0
- package/src/routes/webhooks/index.ts +9 -1
- package/src/routes/webhooks/sms-provider.ts +110 -0
- package/src/sources/define-contact-source.ts +159 -0
- package/src/sources/registry.ts +71 -0
- package/src/sources/webhook-source.ts +109 -0
- package/src/worker.ts +4 -0
- package/src/workflows/crm-reconcile.ts +129 -0
- package/src/workflows/dispatch-conversion.ts +31 -0
- package/src/workflows/import-contacts.ts +61 -4
- package/src/workflows/send-campaign.ts +5 -0
package/src/index.ts
CHANGED
|
@@ -367,11 +367,51 @@ export {
|
|
|
367
367
|
} from "./lib/connector-state.js";
|
|
368
368
|
// --- Contacts identity (resolve/create — used by connector member-link) ---
|
|
369
369
|
export { resolveOrCreateContact } from "./lib/contacts.js";
|
|
370
|
+
// --- Conversion dispatch (plan §5.2): destinations registry + delivery ---
|
|
371
|
+
export {
|
|
372
|
+
ConversionDestinationRegistry,
|
|
373
|
+
conversionEventId,
|
|
374
|
+
createConversionDispatches,
|
|
375
|
+
deliverConversionDispatch,
|
|
376
|
+
enqueueConversionDispatches,
|
|
377
|
+
getConversionDestinations,
|
|
378
|
+
recoverClickContext,
|
|
379
|
+
resetConversionDestinations,
|
|
380
|
+
setConversionDestinations,
|
|
381
|
+
} from "./lib/conversion-dispatch.js";
|
|
382
|
+
// --- Conversion points (plan §5.1): registry + ingest-time evaluation ---
|
|
383
|
+
export {
|
|
384
|
+
ConversionRegistry,
|
|
385
|
+
evaluateConversionsAtIngest,
|
|
386
|
+
type FiredConversion,
|
|
387
|
+
getConversionRegistry,
|
|
388
|
+
resetConversionRegistry,
|
|
389
|
+
setConversionRegistry,
|
|
390
|
+
} from "./lib/conversions.js";
|
|
370
391
|
export {
|
|
371
392
|
AdminAlreadyExistsError,
|
|
372
393
|
type CreatedAdmin,
|
|
373
394
|
createAdminUser,
|
|
374
395
|
} from "./lib/create-admin.js";
|
|
396
|
+
// --- CRM sync (provider registry + the stage-event spine sink) ---
|
|
397
|
+
export {
|
|
398
|
+
type AppliedStageChange,
|
|
399
|
+
applyCrmStageEvent,
|
|
400
|
+
ensureCrmLinks,
|
|
401
|
+
resolveCrmLinkedContact,
|
|
402
|
+
} from "./lib/crm-deals.js";
|
|
403
|
+
export {
|
|
404
|
+
DEAL_QUOTED,
|
|
405
|
+
DEAL_SOLD,
|
|
406
|
+
FUNNEL_STAGE_CHANGED,
|
|
407
|
+
ingestCrmStageEvents,
|
|
408
|
+
} from "./lib/crm-ingest.js";
|
|
409
|
+
export { CrmProviderRegistry } from "./lib/crm-provider-registry.js";
|
|
410
|
+
export {
|
|
411
|
+
getCrmSyncConfig,
|
|
412
|
+
resetCrmSyncConfig,
|
|
413
|
+
setCrmSyncConfig,
|
|
414
|
+
} from "./lib/crm-registry-singleton.js";
|
|
375
415
|
// --- Infrastructure singletons ---
|
|
376
416
|
export { getDb } from "./lib/db.js";
|
|
377
417
|
// --- Discord gateway-worker liveness heartbeat (Studio status) ---
|
|
@@ -420,7 +460,14 @@ export {
|
|
|
420
460
|
} from "./lib/feed.js";
|
|
421
461
|
export { countRecentSends, isFrequencyCapped } from "./lib/frequency-cap.js";
|
|
422
462
|
export { addrSpecOf, hostOfFromAddress } from "./lib/from-address.js";
|
|
463
|
+
export { FunnelRegistry } from "./lib/funnel-registry.js";
|
|
423
464
|
export { hatchet } from "./lib/hatchet.js";
|
|
465
|
+
export {
|
|
466
|
+
globalControlPercent,
|
|
467
|
+
holdoutBucket,
|
|
468
|
+
isGlobalControl,
|
|
469
|
+
isHeldOut,
|
|
470
|
+
} from "./lib/holdout.js";
|
|
424
471
|
// --- Identity service (resolve/merge + analytics merge propagation, §7) ---
|
|
425
472
|
export {
|
|
426
473
|
createIdentityService,
|
|
@@ -448,6 +495,14 @@ export {
|
|
|
448
495
|
releaseLeaderLease,
|
|
449
496
|
renewLeaderLease,
|
|
450
497
|
} from "./lib/leader-lease.js";
|
|
498
|
+
// --- Holdout lift statistics (impact plan §4.2) ---
|
|
499
|
+
export {
|
|
500
|
+
betaWinProbability,
|
|
501
|
+
computeLift,
|
|
502
|
+
type LiftVerdict,
|
|
503
|
+
MIN_COMBINED_CONVERSIONS,
|
|
504
|
+
SMALL_SAMPLE_FLOOR,
|
|
505
|
+
} from "./lib/lift-stats.js";
|
|
451
506
|
// --- Managed tracked links (channel-agnostic mint — Studio/Discord/share) ---
|
|
452
507
|
// NOTE: the QR scan-row plumbing (ensureQrTrackedLink, the canonical-row
|
|
453
508
|
// filter, the 'qr' source marker) is deliberately NOT exported — it is
|
|
@@ -483,6 +538,7 @@ export {
|
|
|
483
538
|
type OutboundEventName,
|
|
484
539
|
type OutboundPayloads,
|
|
485
540
|
} from "./lib/outbound.js";
|
|
541
|
+
export { isE164, normalizePhone } from "./lib/phone.js";
|
|
486
542
|
export { getPostHog } from "./lib/posthog.js";
|
|
487
543
|
// --- PostHog OAuth scopes (front-loaded set; gap-detector source of truth) ---
|
|
488
544
|
export { EXPECTED_POSTHOG_SCOPES } from "./lib/posthog-scopes.js";
|
|
@@ -515,6 +571,12 @@ export {
|
|
|
515
571
|
} from "./lib/redis.js";
|
|
516
572
|
// --- Self-service password reset (engine-owned, self-contained email) ---
|
|
517
573
|
export { sendResetPasswordEmail } from "./lib/reset-email.js";
|
|
574
|
+
// --- Revenue rollups (the event spine's value/currency columns) ---
|
|
575
|
+
export {
|
|
576
|
+
type ContactRevenue,
|
|
577
|
+
type ContactRevenueTotal,
|
|
578
|
+
getContactRevenue,
|
|
579
|
+
} from "./lib/revenue.js";
|
|
518
580
|
// --- PostHog destination seed (idempotent; ENABLE_POSTHOG_DESTINATION) ---
|
|
519
581
|
export { seedPostHogDestination } from "./lib/seed-posthog-destination.js";
|
|
520
582
|
export {
|
|
@@ -524,6 +586,42 @@ export {
|
|
|
524
586
|
SEMANTIC_BURST_DISTINCT_LINKS,
|
|
525
587
|
SEMANTIC_BURST_WINDOW_MS,
|
|
526
588
|
} from "./lib/semantic-click.js";
|
|
589
|
+
// --- SMS ---
|
|
590
|
+
export {
|
|
591
|
+
getSmsService,
|
|
592
|
+
type SendSmsOptions as SendSmsJourneyOptions,
|
|
593
|
+
type SendSmsResult,
|
|
594
|
+
sendSms,
|
|
595
|
+
setSmsService,
|
|
596
|
+
} from "./lib/sms.js";
|
|
597
|
+
// --- SMS inbound consent helpers (phone-track grant/opt-out) ---
|
|
598
|
+
export {
|
|
599
|
+
grantPhoneConsent,
|
|
600
|
+
recordPhoneOptOut,
|
|
601
|
+
} from "./lib/sms-inbound.js";
|
|
602
|
+
// --- SMS link shortening (pure rewriter; the insert rides the send txn) ---
|
|
603
|
+
export {
|
|
604
|
+
generateShortCode,
|
|
605
|
+
isShortCodeCollision,
|
|
606
|
+
type PendingSmsLink,
|
|
607
|
+
planSmsLinkRewrite,
|
|
608
|
+
SHORT_CODE_ALPHABET,
|
|
609
|
+
SHORT_CODE_LENGTH,
|
|
610
|
+
type SmsLinkRewritePlan,
|
|
611
|
+
} from "./lib/sms-link-tracking.js";
|
|
612
|
+
// --- SMS mailer factory (the engine-owned tracked sender; mirrors createTrackedMailer) ---
|
|
613
|
+
export { createTrackedSmsSender } from "./lib/sms-mailer.js";
|
|
614
|
+
// --- SMS provider registry (container-held, keyed by meta.id) ---
|
|
615
|
+
export { SmsProviderRegistry } from "./lib/sms-provider-registry.js";
|
|
616
|
+
// --- SMS service (engine-owned tracked SMS sender) ---
|
|
617
|
+
export type {
|
|
618
|
+
SmsService,
|
|
619
|
+
SmsServiceConfig,
|
|
620
|
+
SmsServiceSendOptions,
|
|
621
|
+
SmsServiceWebhookResult,
|
|
622
|
+
SmsTrackedSendResult,
|
|
623
|
+
} from "./lib/sms-service-types.js";
|
|
624
|
+
export { SMS_CHANNEL_ID } from "./lib/sms-tracked.js";
|
|
527
625
|
export { type MountStudioResult, mountStudio } from "./lib/studio.js";
|
|
528
626
|
// --- In-app survey/rating (sendSurvey — producer sugar over sendFeedItem) ---
|
|
529
627
|
export { type SendSurveyOptions, sendSurvey } from "./lib/survey.js";
|
|
@@ -546,7 +644,16 @@ export {
|
|
|
546
644
|
prepareTrackedHtml,
|
|
547
645
|
rewriteLinks,
|
|
548
646
|
} from "./lib/tracking.js";
|
|
647
|
+
// --- First-party tracking event names (touchpoint-classed in @hogsend/core) ---
|
|
549
648
|
export {
|
|
649
|
+
EMAIL_LINK_CLICKED,
|
|
650
|
+
EMAIL_OPENED,
|
|
651
|
+
LINK_ARRIVED,
|
|
652
|
+
LINK_CLICKED,
|
|
653
|
+
SMS_LINK_CLICKED,
|
|
654
|
+
} from "./lib/tracking-event-names.js";
|
|
655
|
+
export {
|
|
656
|
+
pushSmsTrackingEvent,
|
|
550
657
|
pushTrackingEvent,
|
|
551
658
|
resolveEmailSendContext,
|
|
552
659
|
resolveEmailSendContextByMessageId,
|
|
@@ -555,6 +662,7 @@ export {
|
|
|
555
662
|
* {@link resolveEmailSendContextByMessageId}.
|
|
556
663
|
*/
|
|
557
664
|
resolveEmailSendContextByResendId,
|
|
665
|
+
resolveSmsSendContext,
|
|
558
666
|
} from "./lib/tracking-events.js";
|
|
559
667
|
/**
|
|
560
668
|
* Publishable-key `userToken` mint/verify helpers.
|
|
@@ -638,6 +746,32 @@ export {
|
|
|
638
746
|
} from "./middleware/rate-limit.js";
|
|
639
747
|
// --- Middleware (consumer-mounted routes, e.g. the @hogsend/mcp hosted route) ---
|
|
640
748
|
export { requireAdmin } from "./middleware/require-admin.js";
|
|
749
|
+
// --- Contact sources (Clay/Attio/generic-webhook → cold prospects) ---
|
|
750
|
+
export {
|
|
751
|
+
type ColdChannelPosture,
|
|
752
|
+
type ColdPosture,
|
|
753
|
+
type ContactSourceMeta,
|
|
754
|
+
type ContactWriteBack,
|
|
755
|
+
contactSourceToWebhookSource,
|
|
756
|
+
type DefinedContactSource,
|
|
757
|
+
defaultColdPosture,
|
|
758
|
+
defineContactSource,
|
|
759
|
+
isColdChannelAllowed,
|
|
760
|
+
resolveColdPosture,
|
|
761
|
+
} from "./sources/define-contact-source.js";
|
|
762
|
+
export {
|
|
763
|
+
buildContactSourceRegistry,
|
|
764
|
+
ContactSourceRegistry,
|
|
765
|
+
getContactSourceRegistry,
|
|
766
|
+
setContactSourceRegistry,
|
|
767
|
+
} from "./sources/registry.js";
|
|
768
|
+
export {
|
|
769
|
+
normalizeWebhookContactEvent,
|
|
770
|
+
type WebhookContactPayload,
|
|
771
|
+
type WebhookContactSourceOptions,
|
|
772
|
+
webhookContactPayloadSchema,
|
|
773
|
+
webhookContactSource,
|
|
774
|
+
} from "./sources/webhook-source.js";
|
|
641
775
|
// --- Webhook sources ---
|
|
642
776
|
export {
|
|
643
777
|
type DefinedWebhookSource,
|
|
@@ -678,6 +812,10 @@ export {
|
|
|
678
812
|
checkAlertsTask,
|
|
679
813
|
surfaceStrandedWaiting,
|
|
680
814
|
} from "./workflows/check-alerts.js";
|
|
815
|
+
export {
|
|
816
|
+
crmReconcileTask,
|
|
817
|
+
runCrmReconcile,
|
|
818
|
+
} from "./workflows/crm-reconcile.js";
|
|
681
819
|
// --- Outbound webhooks: durable delivery task + reaper (Section 1.5) ---
|
|
682
820
|
export {
|
|
683
821
|
deliverWebhookTask,
|
|
@@ -25,6 +25,8 @@ import {
|
|
|
25
25
|
checkEntryLimit,
|
|
26
26
|
} from "../lib/enrollment-guards.js";
|
|
27
27
|
import { hatchet } from "../lib/hatchet.js";
|
|
28
|
+
import { isHeldOut } from "../lib/holdout.js";
|
|
29
|
+
import { ingestEvent } from "../lib/ingestion.js";
|
|
28
30
|
import { createLogger } from "../lib/logger.js";
|
|
29
31
|
import { emitOutbound } from "../lib/outbound.js";
|
|
30
32
|
import { resolveTimezoneWithSource } from "../lib/timezone.js";
|
|
@@ -303,6 +305,97 @@ export async function executeJourneyRun(
|
|
|
303
305
|
return { status: "skipped", reason: "user_unsubscribed" };
|
|
304
306
|
}
|
|
305
307
|
|
|
308
|
+
// HOLDOUT DIVERSION (impact plan §4.1) — deliberately LAST in the guard
|
|
309
|
+
// chain: a contact who'd have been blocked by any gate above is never
|
|
310
|
+
// counted as held out (intent-to-treat — the control must mirror the
|
|
311
|
+
// would-have-entered population, not the triggered one). Assignment is a
|
|
312
|
+
// deterministic hash (replay law: no RNG in durable paths), so the same
|
|
313
|
+
// contact diverts identically on every trigger AND every replay. One
|
|
314
|
+
// held_out row per (user, journey), ever; the spine event's idempotency
|
|
315
|
+
// key carries the same once-ever semantics for fan-out/analytics.
|
|
316
|
+
if (meta.holdout && meta.holdout.percent > 0) {
|
|
317
|
+
const diverted = isHeldOut({
|
|
318
|
+
userId,
|
|
319
|
+
journeyId: meta.id,
|
|
320
|
+
percent: meta.holdout.percent,
|
|
321
|
+
salt: meta.holdout.salt,
|
|
322
|
+
});
|
|
323
|
+
if (diverted) {
|
|
324
|
+
const priorHoldout = await db.query.journeyStates.findFirst({
|
|
325
|
+
where: and(
|
|
326
|
+
eq(journeyStates.userId, userId),
|
|
327
|
+
eq(journeyStates.journeyId, meta.id),
|
|
328
|
+
eq(journeyStates.status, "held_out"),
|
|
329
|
+
),
|
|
330
|
+
});
|
|
331
|
+
if (!priorHoldout) {
|
|
332
|
+
const heldOutAt = new Date();
|
|
333
|
+
const inserted = await db
|
|
334
|
+
.insert(journeyStates)
|
|
335
|
+
.values({
|
|
336
|
+
userId,
|
|
337
|
+
userEmail,
|
|
338
|
+
journeyId: meta.id,
|
|
339
|
+
currentNodeId: "held-out",
|
|
340
|
+
status: "held_out",
|
|
341
|
+
context: properties,
|
|
342
|
+
exitedAt: heldOutAt,
|
|
343
|
+
})
|
|
344
|
+
.returning({ id: journeyStates.id });
|
|
345
|
+
const holdoutStateId = inserted[0]?.id;
|
|
346
|
+
if (holdoutStateId) {
|
|
347
|
+
// The counterfactual as data (the Iterable Send Skip pattern):
|
|
348
|
+
// queryable on the spine, fan-out-able to journeys/destinations.
|
|
349
|
+
// Once-ever per (user, journey) via the idempotency key — a
|
|
350
|
+
// concurrent double-diversion dedups here even if it raced the
|
|
351
|
+
// row existence check above.
|
|
352
|
+
try {
|
|
353
|
+
await ingestEvent({
|
|
354
|
+
db,
|
|
355
|
+
registry: getJourneyRegistrySingleton(),
|
|
356
|
+
hatchet,
|
|
357
|
+
logger,
|
|
358
|
+
event: {
|
|
359
|
+
event: "journey.heldout",
|
|
360
|
+
userId,
|
|
361
|
+
userEmail,
|
|
362
|
+
eventProperties: {
|
|
363
|
+
journeyId: meta.id,
|
|
364
|
+
journeyName: meta.name,
|
|
365
|
+
holdoutPercent: meta.holdout.percent,
|
|
366
|
+
},
|
|
367
|
+
source: "journey",
|
|
368
|
+
idempotencyKey: `journey:heldout:${meta.id}:${userId}`,
|
|
369
|
+
},
|
|
370
|
+
});
|
|
371
|
+
} catch (err) {
|
|
372
|
+
logger.warn("journey.heldout ingest failed", {
|
|
373
|
+
journeyId: meta.id,
|
|
374
|
+
userId,
|
|
375
|
+
error: err instanceof Error ? err.message : String(err),
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
void emitOutbound({
|
|
379
|
+
db,
|
|
380
|
+
hatchet,
|
|
381
|
+
logger,
|
|
382
|
+
event: "journey.heldout",
|
|
383
|
+
dedupeKey: `journey.heldout:${holdoutStateId}`,
|
|
384
|
+
payload: {
|
|
385
|
+
journeyId: meta.id,
|
|
386
|
+
journeyName: meta.name,
|
|
387
|
+
stateId: holdoutStateId,
|
|
388
|
+
userId,
|
|
389
|
+
userEmail,
|
|
390
|
+
heldOutAt: heldOutAt.toISOString(),
|
|
391
|
+
},
|
|
392
|
+
}).catch(logger.warn);
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
return { status: "skipped", reason: "held_out" };
|
|
396
|
+
}
|
|
397
|
+
}
|
|
398
|
+
|
|
306
399
|
const activeState = await db.query.journeyStates.findFirst({
|
|
307
400
|
where: and(
|
|
308
401
|
eq(journeyStates.userId, userId),
|
|
@@ -146,10 +146,14 @@ export function supportsEviction(hatchetCtx: unknown): boolean {
|
|
|
146
146
|
}
|
|
147
147
|
|
|
148
148
|
/** The kind of side effect a key is derived for — keeps the namespaces apart. */
|
|
149
|
-
export type JourneyKeyKind = "send" | "trigger" | "connector";
|
|
149
|
+
export type JourneyKeyKind = "send" | "smsSend" | "trigger" | "connector";
|
|
150
150
|
|
|
151
151
|
const KEY_PREFIX: Record<JourneyKeyKind, string> = {
|
|
152
152
|
send: "journeySend",
|
|
153
|
+
// Disjoint from `send` so a `sendEmail("x")` and a `sendSms("x")` under the
|
|
154
|
+
// SAME nearest wait label derive DIFFERENT keys and never collide in
|
|
155
|
+
// registerKey — email + SMS of the same template on one branch is legitimate.
|
|
156
|
+
smsSend: "journeySmsSend",
|
|
153
157
|
trigger: "journeyTrigger",
|
|
154
158
|
connector: "journeyConnector",
|
|
155
159
|
};
|
|
@@ -42,6 +42,7 @@ import {
|
|
|
42
42
|
type Database,
|
|
43
43
|
emailSends,
|
|
44
44
|
journeyStates,
|
|
45
|
+
smsSends,
|
|
45
46
|
userEvents,
|
|
46
47
|
} from "@hogsend/db";
|
|
47
48
|
import {
|
|
@@ -1025,6 +1026,8 @@ export function createJourneyContext(
|
|
|
1025
1026
|
userId: targetUserId,
|
|
1026
1027
|
userEmail: targetEmail,
|
|
1027
1028
|
properties,
|
|
1029
|
+
value,
|
|
1030
|
+
currency,
|
|
1028
1031
|
idempotencyLabel,
|
|
1029
1032
|
}) {
|
|
1030
1033
|
// Keep the PUBLIC `TriggerOptions.properties` field name (decision #13 —
|
|
@@ -1064,6 +1067,8 @@ export function createJourneyContext(
|
|
|
1064
1067
|
userId: targetUserId,
|
|
1065
1068
|
userEmail: targetEmail ?? userEmail,
|
|
1066
1069
|
eventProperties: properties ?? {},
|
|
1070
|
+
value,
|
|
1071
|
+
currency,
|
|
1067
1072
|
// Cross-journey trigger (ctx.trigger).
|
|
1068
1073
|
source: "journey",
|
|
1069
1074
|
...(idempotencyKey ? { idempotencyKey } : {}),
|
|
@@ -1212,6 +1217,31 @@ export function createJourneyContext(
|
|
|
1212
1217
|
};
|
|
1213
1218
|
},
|
|
1214
1219
|
|
|
1220
|
+
async sms({ phone, template }) {
|
|
1221
|
+
const [result] = await db
|
|
1222
|
+
.select({
|
|
1223
|
+
count: count(),
|
|
1224
|
+
lastSentAt: max(smsSends.sentAt),
|
|
1225
|
+
})
|
|
1226
|
+
.from(smsSends)
|
|
1227
|
+
.where(
|
|
1228
|
+
and(
|
|
1229
|
+
eq(smsSends.toPhone, phone),
|
|
1230
|
+
eq(smsSends.templateKey, template),
|
|
1231
|
+
),
|
|
1232
|
+
);
|
|
1233
|
+
|
|
1234
|
+
const total = result?.count ?? 0;
|
|
1235
|
+
return {
|
|
1236
|
+
sent: total > 0,
|
|
1237
|
+
lastSentAt:
|
|
1238
|
+
result?.lastSentAt instanceof Date
|
|
1239
|
+
? result.lastSentAt.toISOString()
|
|
1240
|
+
: null,
|
|
1241
|
+
count: total,
|
|
1242
|
+
};
|
|
1243
|
+
},
|
|
1244
|
+
|
|
1215
1245
|
async events({
|
|
1216
1246
|
userId: targetUserId,
|
|
1217
1247
|
event,
|
|
@@ -0,0 +1,285 @@
|
|
|
1
|
+
import {
|
|
2
|
+
attributionCredits,
|
|
3
|
+
contacts,
|
|
4
|
+
conversions,
|
|
5
|
+
type Database,
|
|
6
|
+
userEvents,
|
|
7
|
+
} from "@hogsend/db";
|
|
8
|
+
import {
|
|
9
|
+
and,
|
|
10
|
+
asc,
|
|
11
|
+
eq,
|
|
12
|
+
gt,
|
|
13
|
+
gte,
|
|
14
|
+
inArray,
|
|
15
|
+
isNotNull,
|
|
16
|
+
or,
|
|
17
|
+
sql,
|
|
18
|
+
} from "drizzle-orm";
|
|
19
|
+
import { recordAttributionCredits } from "./attribution.js";
|
|
20
|
+
import type { ConversionRegistry } from "./conversions.js";
|
|
21
|
+
import { evaluateConversionsAtIngest } from "./conversions.js";
|
|
22
|
+
import type { Logger } from "./logger.js";
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Attribution backfill (impact plan §5.1) — replays HISTORY through the
|
|
26
|
+
* same idempotent machinery ingest uses, so an existing deploy that
|
|
27
|
+
* upgrades gets its whole event history credited, and a definition/window
|
|
28
|
+
* change gets a deliberate, logged recompute path.
|
|
29
|
+
*
|
|
30
|
+
* Two stages behind one cursor (`events:<id>` → `credits:<id>`):
|
|
31
|
+
*
|
|
32
|
+
* 1. `events` — scan historical `user_events` matching the registry's
|
|
33
|
+
* trigger set and run conversion evaluation. The unique
|
|
34
|
+
* (definition, event) index makes replays no-ops; only genuinely new
|
|
35
|
+
* conversions are minted. Ad-platform dispatches are deliberately NOT
|
|
36
|
+
* enqueued — historical conversions must never re-fire to destinations.
|
|
37
|
+
* 2. `credits` — scan conversions rows (of the scoped definitions) that
|
|
38
|
+
* have NO credit rows yet and write their ledger entries under the
|
|
39
|
+
* definition's CURRENT window config.
|
|
40
|
+
*
|
|
41
|
+
* `recompute: true` (requires `definitionId`) deletes the definition's
|
|
42
|
+
* existing credit rows up front, then stage 2 re-credits everything — the
|
|
43
|
+
* ONE sanctioned exception to the ledger's write-once philosophy, guarded
|
|
44
|
+
* and logged. Conversion rows themselves are never deleted.
|
|
45
|
+
*
|
|
46
|
+
* Batch-shaped by design: each call processes up to `limit` rows of the
|
|
47
|
+
* current stage and returns `nextCursor` (null = done). The CLI loops it;
|
|
48
|
+
* an admin can curl it. Contacts are matched only — a historical event
|
|
49
|
+
* whose user never became a contact is skipped, never minted.
|
|
50
|
+
*/
|
|
51
|
+
export interface BackfillBatchResult {
|
|
52
|
+
stage: "events" | "credits";
|
|
53
|
+
processed: number;
|
|
54
|
+
conversionsFired: number;
|
|
55
|
+
creditsWritten: number;
|
|
56
|
+
nextCursor: string | null;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const UUID_RE =
|
|
60
|
+
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
|
|
61
|
+
|
|
62
|
+
export async function backfillAttributionBatch(opts: {
|
|
63
|
+
db: Database;
|
|
64
|
+
logger: Logger;
|
|
65
|
+
registry: ConversionRegistry | undefined;
|
|
66
|
+
definitionId?: string;
|
|
67
|
+
since?: Date;
|
|
68
|
+
cursor?: string;
|
|
69
|
+
limit?: number;
|
|
70
|
+
recompute?: boolean;
|
|
71
|
+
}): Promise<BackfillBatchResult> {
|
|
72
|
+
const { db, logger, registry, definitionId, since } = opts;
|
|
73
|
+
const limit = Math.min(2000, Math.max(1, opts.limit ?? 500));
|
|
74
|
+
|
|
75
|
+
const defs = (registry?.getAll() ?? []).filter(
|
|
76
|
+
(def) => !definitionId || def.meta.id === definitionId,
|
|
77
|
+
);
|
|
78
|
+
if (defs.length === 0) {
|
|
79
|
+
return {
|
|
80
|
+
stage: "credits",
|
|
81
|
+
processed: 0,
|
|
82
|
+
conversionsFired: 0,
|
|
83
|
+
creditsWritten: 0,
|
|
84
|
+
nextCursor: null,
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
const defIds = defs.map((def) => def.meta.id);
|
|
88
|
+
const defById = new Map(defs.map((def) => [def.meta.id, def]));
|
|
89
|
+
const namedEvents = [
|
|
90
|
+
...new Set(
|
|
91
|
+
defs
|
|
92
|
+
.map((def) => def.meta.trigger.event)
|
|
93
|
+
.filter((event) => event !== "*"),
|
|
94
|
+
),
|
|
95
|
+
];
|
|
96
|
+
const hasWildcard = defs.some((def) => def.meta.trigger.event === "*");
|
|
97
|
+
|
|
98
|
+
const [stage, lastId] = opts.cursor
|
|
99
|
+
? (opts.cursor.split(":", 2) as ["events" | "credits", string])
|
|
100
|
+
: (["events", ""] as const);
|
|
101
|
+
|
|
102
|
+
// Recompute (guarded): first call only, one definition only.
|
|
103
|
+
if (opts.recompute && !opts.cursor) {
|
|
104
|
+
if (!definitionId) {
|
|
105
|
+
throw new Error("recompute requires a definitionId — never blanket");
|
|
106
|
+
}
|
|
107
|
+
const deleted = await db
|
|
108
|
+
.delete(attributionCredits)
|
|
109
|
+
.where(
|
|
110
|
+
inArray(
|
|
111
|
+
attributionCredits.conversionId,
|
|
112
|
+
db
|
|
113
|
+
.select({ id: conversions.id })
|
|
114
|
+
.from(conversions)
|
|
115
|
+
.where(eq(conversions.definitionId, definitionId)),
|
|
116
|
+
),
|
|
117
|
+
)
|
|
118
|
+
.returning({ id: attributionCredits.id });
|
|
119
|
+
logger.warn("attribution recompute: credit rows deleted for refill", {
|
|
120
|
+
definitionId,
|
|
121
|
+
deleted: deleted.length,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
if (stage === "events") {
|
|
126
|
+
const eventMatch = or(
|
|
127
|
+
namedEvents.length > 0
|
|
128
|
+
? inArray(userEvents.event, namedEvents)
|
|
129
|
+
: sql`false`,
|
|
130
|
+
hasWildcard ? isNotNull(userEvents.value) : sql`false`,
|
|
131
|
+
);
|
|
132
|
+
const rows = await db
|
|
133
|
+
.select({
|
|
134
|
+
id: userEvents.id,
|
|
135
|
+
event: userEvents.event,
|
|
136
|
+
userId: userEvents.userId,
|
|
137
|
+
properties: userEvents.properties,
|
|
138
|
+
value: userEvents.value,
|
|
139
|
+
currency: userEvents.currency,
|
|
140
|
+
source: userEvents.source,
|
|
141
|
+
occurredAt: userEvents.occurredAt,
|
|
142
|
+
})
|
|
143
|
+
.from(userEvents)
|
|
144
|
+
.where(
|
|
145
|
+
and(
|
|
146
|
+
eventMatch,
|
|
147
|
+
...(since ? [gte(userEvents.occurredAt, since)] : []),
|
|
148
|
+
...(lastId ? [gt(userEvents.id, lastId)] : []),
|
|
149
|
+
),
|
|
150
|
+
)
|
|
151
|
+
.orderBy(asc(userEvents.id))
|
|
152
|
+
.limit(limit);
|
|
153
|
+
|
|
154
|
+
// Batch contact resolution: canonical keys are externalId or the
|
|
155
|
+
// contact row uuid. MATCH ONLY — backfill never mints contacts.
|
|
156
|
+
const keys = [...new Set(rows.map((row) => row.userId))];
|
|
157
|
+
const uuidKeys = keys.filter((key) => UUID_RE.test(key));
|
|
158
|
+
const contactRows =
|
|
159
|
+
keys.length > 0
|
|
160
|
+
? await db
|
|
161
|
+
.select({ id: contacts.id, externalId: contacts.externalId })
|
|
162
|
+
.from(contacts)
|
|
163
|
+
.where(
|
|
164
|
+
or(
|
|
165
|
+
inArray(contacts.externalId, keys),
|
|
166
|
+
uuidKeys.length > 0
|
|
167
|
+
? inArray(contacts.id, uuidKeys)
|
|
168
|
+
: sql`false`,
|
|
169
|
+
),
|
|
170
|
+
)
|
|
171
|
+
: [];
|
|
172
|
+
const contactByKey = new Map<string, string>();
|
|
173
|
+
for (const contact of contactRows) {
|
|
174
|
+
contactByKey.set(contact.id, contact.id);
|
|
175
|
+
if (contact.externalId) contactByKey.set(contact.externalId, contact.id);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
let conversionsFired = 0;
|
|
179
|
+
let creditsWritten = 0;
|
|
180
|
+
for (const row of rows) {
|
|
181
|
+
const contactId = contactByKey.get(row.userId);
|
|
182
|
+
if (!contactId) continue;
|
|
183
|
+
const fired = await evaluateConversionsAtIngest({
|
|
184
|
+
db,
|
|
185
|
+
logger,
|
|
186
|
+
registry,
|
|
187
|
+
event: {
|
|
188
|
+
name: row.event,
|
|
189
|
+
source: row.source ?? null,
|
|
190
|
+
properties: (row.properties ?? {}) as Record<string, unknown>,
|
|
191
|
+
value: row.value,
|
|
192
|
+
currency: row.currency,
|
|
193
|
+
occurredAt: row.occurredAt,
|
|
194
|
+
},
|
|
195
|
+
eventRowId: row.id,
|
|
196
|
+
contactId,
|
|
197
|
+
userKey: row.userId,
|
|
198
|
+
});
|
|
199
|
+
for (const conversion of fired) {
|
|
200
|
+
if (definitionId && conversion.definition.meta.id !== definitionId) {
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
conversionsFired++;
|
|
204
|
+
const { touchpoints } = await recordAttributionCredits({
|
|
205
|
+
db,
|
|
206
|
+
logger,
|
|
207
|
+
conversionId: conversion.conversionId,
|
|
208
|
+
userKey: row.userId,
|
|
209
|
+
value: conversion.value,
|
|
210
|
+
currency: conversion.currency,
|
|
211
|
+
occurredAt: row.occurredAt,
|
|
212
|
+
windowDays: conversion.definition.meta.attributionWindowDays ?? 90,
|
|
213
|
+
windows: conversion.definition.meta.windows,
|
|
214
|
+
});
|
|
215
|
+
if (touchpoints > 0) creditsWritten++;
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
const nextCursor =
|
|
220
|
+
rows.length < limit
|
|
221
|
+
? "credits:"
|
|
222
|
+
: `events:${rows[rows.length - 1]?.id ?? ""}`;
|
|
223
|
+
return {
|
|
224
|
+
stage: "events",
|
|
225
|
+
processed: rows.length,
|
|
226
|
+
conversionsFired,
|
|
227
|
+
creditsWritten,
|
|
228
|
+
nextCursor,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// Stage 2: credit conversions that have no ledger rows yet (after a
|
|
233
|
+
// recompute delete, that is all of the definition's conversions).
|
|
234
|
+
const uncredited = sql`not exists (
|
|
235
|
+
select 1 from ${attributionCredits}
|
|
236
|
+
where ${attributionCredits.conversionId} = ${conversions.id}
|
|
237
|
+
)`;
|
|
238
|
+
const rows = await db
|
|
239
|
+
.select({
|
|
240
|
+
id: conversions.id,
|
|
241
|
+
definitionId: conversions.definitionId,
|
|
242
|
+
userKey: conversions.userKey,
|
|
243
|
+
value: conversions.value,
|
|
244
|
+
currency: conversions.currency,
|
|
245
|
+
occurredAt: conversions.occurredAt,
|
|
246
|
+
})
|
|
247
|
+
.from(conversions)
|
|
248
|
+
.where(
|
|
249
|
+
and(
|
|
250
|
+
inArray(conversions.definitionId, defIds),
|
|
251
|
+
uncredited,
|
|
252
|
+
...(since ? [gte(conversions.occurredAt, since)] : []),
|
|
253
|
+
...(lastId ? [gt(conversions.id, lastId)] : []),
|
|
254
|
+
),
|
|
255
|
+
)
|
|
256
|
+
.orderBy(asc(conversions.id))
|
|
257
|
+
.limit(limit);
|
|
258
|
+
|
|
259
|
+
let creditsWritten = 0;
|
|
260
|
+
for (const row of rows) {
|
|
261
|
+
const def = defById.get(row.definitionId);
|
|
262
|
+
if (!def) continue;
|
|
263
|
+
const { touchpoints } = await recordAttributionCredits({
|
|
264
|
+
db,
|
|
265
|
+
logger,
|
|
266
|
+
conversionId: row.id,
|
|
267
|
+
userKey: row.userKey,
|
|
268
|
+
value: row.value,
|
|
269
|
+
currency: row.currency,
|
|
270
|
+
occurredAt: row.occurredAt,
|
|
271
|
+
windowDays: def.meta.attributionWindowDays ?? 90,
|
|
272
|
+
windows: def.meta.windows,
|
|
273
|
+
});
|
|
274
|
+
if (touchpoints > 0) creditsWritten++;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
return {
|
|
278
|
+
stage: "credits",
|
|
279
|
+
processed: rows.length,
|
|
280
|
+
conversionsFired: 0,
|
|
281
|
+
creditsWritten,
|
|
282
|
+
nextCursor:
|
|
283
|
+
rows.length < limit ? null : `credits:${rows[rows.length - 1]?.id ?? ""}`,
|
|
284
|
+
};
|
|
285
|
+
}
|