@hogsend/engine 0.43.0 → 0.45.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +12 -9
- package/src/container.ts +289 -13
- package/src/destinations/define-destination.ts +7 -3
- package/src/destinations/presets/posthog.ts +31 -0
- package/src/index.ts +99 -0
- package/src/journeys/execute-journey-run.ts +93 -0
- package/src/journeys/journey-context.ts +4 -0
- package/src/lib/analytics-providers-from-env.ts +106 -43
- package/src/lib/attribution-backfill.ts +285 -0
- package/src/lib/attribution.ts +239 -0
- package/src/lib/contacts.ts +71 -0
- package/src/lib/conversion-dispatch.ts +326 -0
- package/src/lib/conversions.ts +197 -0
- package/src/lib/crm-deals.ts +320 -0
- package/src/lib/crm-ingest.ts +341 -0
- package/src/lib/crm-provider-registry.ts +41 -0
- package/src/lib/crm-registry-singleton.ts +22 -0
- package/src/lib/deal-money-events.ts +85 -0
- package/src/lib/email-service-types.ts +16 -1
- package/src/lib/event-names.ts +1 -1
- package/src/lib/funnel-progress.ts +76 -0
- package/src/lib/funnel-registry.ts +108 -0
- package/src/lib/funnel-transitions.ts +280 -0
- package/src/lib/holdout.ts +62 -0
- package/src/lib/ingestion.ts +218 -9
- package/src/lib/lift-stats.ts +125 -0
- package/src/lib/mailer.ts +1 -0
- package/src/lib/operator.ts +31 -0
- package/src/lib/outbound.ts +51 -0
- package/src/lib/posthog-scopes.ts +30 -6
- package/src/lib/revenue.ts +116 -0
- package/src/lib/sms-service-types.ts +7 -1
- package/src/lib/sms-tracked.ts +35 -0
- package/src/lib/tracked.ts +40 -0
- package/src/lib/tracking-events.ts +26 -0
- package/src/lib/unconfigured-email-provider.ts +45 -0
- package/src/lib/webhook-signing.ts +7 -0
- package/src/routes/admin/analytics.ts +14 -5
- package/src/routes/admin/attribution.ts +414 -0
- package/src/routes/admin/contacts.ts +71 -11
- package/src/routes/admin/conversions.ts +247 -0
- package/src/routes/admin/deals.ts +472 -0
- package/src/routes/admin/funnels.ts +228 -0
- package/src/routes/admin/index.ts +8 -0
- package/src/routes/admin/journeys.ts +158 -1
- package/src/routes/admin/readiness.ts +97 -2
- package/src/routes/admin/templates.ts +28 -1
- package/src/routes/events/index.ts +12 -0
- package/src/routes/tracking/click-pipeline.ts +26 -18
- package/src/routes/webhooks/crm-provider.ts +117 -0
- package/src/routes/webhooks/index.ts +5 -1
- package/src/sources/define-contact-source.ts +159 -0
- package/src/sources/registry.ts +71 -0
- package/src/sources/webhook-source.ts +109 -0
- package/src/worker.ts +4 -0
- package/src/workflows/crm-reconcile.ts +129 -0
- package/src/workflows/dispatch-conversion.ts +31 -0
- package/src/workflows/send-campaign.ts +5 -0
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import type { z } from "zod";
|
|
2
|
+
import type { IngestEvent } from "../lib/ingestion.js";
|
|
3
|
+
import {
|
|
4
|
+
type DefinedWebhookSource,
|
|
5
|
+
defineWebhookSource,
|
|
6
|
+
type WebhookSourceAuth,
|
|
7
|
+
type WebhookSourceCtx,
|
|
8
|
+
} from "../webhook-sources/define-webhook-source.js";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* A CONTACT SOURCE is an origin of *sourced* contacts — Clay, Attio, a generic
|
|
12
|
+
* webhook, any CRM/enrichment tool. It is thin sugar over
|
|
13
|
+
* {@link defineWebhookSource}: the same inbound `transform` (raw payload →
|
|
14
|
+
* {@link IngestEvent}) served on the existing `POST /v1/webhooks/:sourceId`
|
|
15
|
+
* route, PLUS three things a plain webhook source has no opinion on:
|
|
16
|
+
*
|
|
17
|
+
* 1. **Provenance is automatic.** The webhook route ingests with
|
|
18
|
+
* `source = meta.id`, and the engine stamps that onto `contacts.source` on
|
|
19
|
+
* create (first-touch). So a source `id: "clay"` mints contacts with
|
|
20
|
+
* `source: "clay"` — a cold **prospect** — with NO extra wiring. Registering
|
|
21
|
+
* the id in the {@link ContactSourceRegistry} is what lets the rest of the
|
|
22
|
+
* engine *classify* such a contact as a prospect.
|
|
23
|
+
* 2. **A cold-consent posture** ({@link ColdPosture}) — which channels a cold,
|
|
24
|
+
* no-consent contact may receive before they opt in. Legal-safe default:
|
|
25
|
+
* email `allow`, every other channel `block`. This is the deliberate,
|
|
26
|
+
* per-source "unflick" knob (see `docs/.../consent-and-legal.md`).
|
|
27
|
+
* 3. **An optional {@link ContactWriteBack}** — pushing journey/engagement
|
|
28
|
+
* status back to the source CRM (implemented per-source; e.g. Attio).
|
|
29
|
+
*
|
|
30
|
+
* A contact source is NOT a new ingestion surface — {@link contactSourceToWebhookSource}
|
|
31
|
+
* lifts it back onto the webhook-source umbrella for the container/route to
|
|
32
|
+
* register, so it rides the exact battle-tested path every webhook source uses.
|
|
33
|
+
*/
|
|
34
|
+
export type ColdChannelPosture = "allow" | "block";
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Per-channel cold posture. Keys are channel ids (`"email"`, `"sms"`, a
|
|
38
|
+
* connector id like `"discord"`). Any channel not named defaults to `block`,
|
|
39
|
+
* EXCEPT `email`, which defaults to `allow` (cold email is lawful with
|
|
40
|
+
* identification + unsubscribe, which the mailer already provides). Opening a
|
|
41
|
+
* non-email channel to `"allow"` for cold contacts is a deliberate, auditable
|
|
42
|
+
* act — see {@link resolveColdPosture} / {@link isColdChannelAllowed}.
|
|
43
|
+
*/
|
|
44
|
+
export type ColdPosture = Record<string, ColdChannelPosture>;
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The legal-safe default posture applied when a source declares none: cold
|
|
48
|
+
* email allowed, everything else blocked. Kept as a function (not a shared
|
|
49
|
+
* object) so callers can never mutate a shared default.
|
|
50
|
+
*/
|
|
51
|
+
export function defaultColdPosture(): ColdPosture {
|
|
52
|
+
return { email: "allow" };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Merge a source's declared posture over the safe default. `email` stays
|
|
57
|
+
* `allow` unless the source explicitly blocks it; every other channel stays
|
|
58
|
+
* `block` unless the source explicitly allows it.
|
|
59
|
+
*/
|
|
60
|
+
export function resolveColdPosture(declared?: ColdPosture): ColdPosture {
|
|
61
|
+
return { ...defaultColdPosture(), ...(declared ?? {}) };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Is `channelId` deliverable to a COLD (sourced, no-consent) contact under this
|
|
66
|
+
* posture? `email` → allowed unless explicitly blocked; any other channel →
|
|
67
|
+
* blocked unless explicitly allowed. This is the single source of truth the
|
|
68
|
+
* cold gate consults, so the default is fail-closed for everything but email.
|
|
69
|
+
*/
|
|
70
|
+
export function isColdChannelAllowed(
|
|
71
|
+
posture: ColdPosture,
|
|
72
|
+
channelId: string,
|
|
73
|
+
): boolean {
|
|
74
|
+
if (channelId === "email") return (posture.email ?? "allow") === "allow";
|
|
75
|
+
return posture[channelId] === "allow";
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export interface ContactSourceMeta {
|
|
79
|
+
id: string;
|
|
80
|
+
name: string;
|
|
81
|
+
description?: string;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Optional per-source write-back — push engagement/lifecycle status from a
|
|
86
|
+
* journey milestone back to the origin CRM. Best-effort and provider-specific
|
|
87
|
+
* (e.g. Attio's `PUT /v2/objects/people/records` + a note). Implemented in the
|
|
88
|
+
* source-adapter phases; declared here so the primitive owns the contract.
|
|
89
|
+
*/
|
|
90
|
+
export interface ContactWriteBack {
|
|
91
|
+
syncStatus(args: {
|
|
92
|
+
/** The contact's email (the cross-system business key), if known. */
|
|
93
|
+
email: string | null;
|
|
94
|
+
/** The contact's external id, if known. */
|
|
95
|
+
externalId: string | null;
|
|
96
|
+
/** A short lifecycle status ("emailed", "replied", "opted_in", …). */
|
|
97
|
+
status: string;
|
|
98
|
+
/** Optional structured attributes to upsert on the CRM record. */
|
|
99
|
+
properties?: Record<string, unknown>;
|
|
100
|
+
}): Promise<void>;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export interface DefinedContactSource<T = unknown> {
|
|
104
|
+
meta: ContactSourceMeta;
|
|
105
|
+
/** Inbound-request verification — identical shape to a webhook source. */
|
|
106
|
+
auth: WebhookSourceAuth;
|
|
107
|
+
/** Optional Zod schema validating the payload BEFORE transform. */
|
|
108
|
+
schema?: z.ZodSchema<T>;
|
|
109
|
+
/** Raw CRM/enrichment payload → a sourcing {@link IngestEvent} (or null). */
|
|
110
|
+
transform(payload: T, ctx: WebhookSourceCtx): Promise<IngestEvent | null>;
|
|
111
|
+
/** RESOLVED cold posture (safe defaults already merged in). */
|
|
112
|
+
coldPosture: ColdPosture;
|
|
113
|
+
/** Optional write-back adapter. */
|
|
114
|
+
writeBack?: ContactWriteBack;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Author a contact source. Identity/validating function (mirrors
|
|
119
|
+
* {@link defineWebhookSource}) that additionally resolves the cold posture to
|
|
120
|
+
* its safe-default-merged form so downstream reads never re-derive it.
|
|
121
|
+
*/
|
|
122
|
+
export function defineContactSource<T>(def: {
|
|
123
|
+
meta: ContactSourceMeta;
|
|
124
|
+
auth: WebhookSourceAuth;
|
|
125
|
+
schema?: z.ZodSchema<T>;
|
|
126
|
+
transform(payload: T, ctx: WebhookSourceCtx): Promise<IngestEvent | null>;
|
|
127
|
+
coldPosture?: ColdPosture;
|
|
128
|
+
writeBack?: ContactWriteBack;
|
|
129
|
+
}): DefinedContactSource<T> {
|
|
130
|
+
if (!def.meta.id) {
|
|
131
|
+
throw new Error("defineContactSource: meta.id is required");
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
meta: def.meta,
|
|
135
|
+
auth: def.auth,
|
|
136
|
+
schema: def.schema,
|
|
137
|
+
transform: def.transform,
|
|
138
|
+
coldPosture: resolveColdPosture(def.coldPosture),
|
|
139
|
+
writeBack: def.writeBack,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Lift a contact source back onto the {@link DefinedWebhookSource} umbrella so
|
|
145
|
+
* the container/route can register it exactly like any webhook source (it rides
|
|
146
|
+
* the `POST /v1/webhooks/:sourceId` path, and the route stamps `source = id`,
|
|
147
|
+
* giving provenance for free). The cold posture / write-back live only on the
|
|
148
|
+
* {@link ContactSourceRegistry}; the webhook path needs neither.
|
|
149
|
+
*/
|
|
150
|
+
export function contactSourceToWebhookSource<T>(
|
|
151
|
+
source: DefinedContactSource<T>,
|
|
152
|
+
): DefinedWebhookSource<T> {
|
|
153
|
+
return defineWebhookSource<T>({
|
|
154
|
+
meta: source.meta,
|
|
155
|
+
auth: source.auth,
|
|
156
|
+
schema: source.schema,
|
|
157
|
+
transform: source.transform,
|
|
158
|
+
});
|
|
159
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { DefinedContactSource } from "./define-contact-source.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* In-process index of the registered contact sources (Clay, Attio, generic
|
|
5
|
+
* webhook, …), keyed by `meta.id`. Mirrors {@link ListRegistry} /
|
|
6
|
+
* `ConnectorRegistry`: a plain id-keyed map plus the one predicate the rest of
|
|
7
|
+
* the engine needs — {@link ContactSourceRegistry.isProspectSource}, which
|
|
8
|
+
* decides whether a contact's stamped `contacts.source` names a Contact Source
|
|
9
|
+
* (⇒ a cold **prospect**) versus an ordinary inbound origin ("api"/"posthog").
|
|
10
|
+
* The cold posture + write-back adapter travel with each entry so the cold gate
|
|
11
|
+
* and the write-back step can resolve them by source id.
|
|
12
|
+
*/
|
|
13
|
+
export class ContactSourceRegistry {
|
|
14
|
+
private sources: Map<string, DefinedContactSource> = new Map();
|
|
15
|
+
|
|
16
|
+
register(source: DefinedContactSource): void {
|
|
17
|
+
this.sources.set(source.meta.id, source);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
get(id: string): DefinedContactSource | undefined {
|
|
21
|
+
return this.sources.get(id);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
getAll(): DefinedContactSource[] {
|
|
25
|
+
return Array.from(this.sources.values());
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
has(id: string): boolean {
|
|
29
|
+
return this.sources.has(id);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
count(): number {
|
|
33
|
+
return this.sources.size;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Does `source` (a `contacts.source` value) name a registered Contact Source?
|
|
38
|
+
* A contact whose provenance is a Contact Source id is a cold **prospect**;
|
|
39
|
+
* one stamped with a plain pipeline origin ("api"/"posthog"/…) or null is not.
|
|
40
|
+
* Null/undefined ⇒ false (an un-sourced contact is never a prospect).
|
|
41
|
+
*/
|
|
42
|
+
isProspectSource(source: string | null | undefined): boolean {
|
|
43
|
+
return source != null && this.sources.has(source);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// The process singleton — installed by `createHogsendClient` at container build
|
|
48
|
+
// so the cold gate + write-back can resolve a source without threading the
|
|
49
|
+
// container. Defaults to an empty registry (no sources ⇒ nothing is a prospect
|
|
50
|
+
// source), mirroring the lists registry-singleton.
|
|
51
|
+
let currentRegistry: ContactSourceRegistry | undefined;
|
|
52
|
+
|
|
53
|
+
export function setContactSourceRegistry(
|
|
54
|
+
registry: ContactSourceRegistry,
|
|
55
|
+
): void {
|
|
56
|
+
currentRegistry = registry;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function getContactSourceRegistry(): ContactSourceRegistry {
|
|
60
|
+
return currentRegistry ?? new ContactSourceRegistry();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Build a {@link ContactSourceRegistry} from sources and install the singleton. */
|
|
64
|
+
export function buildContactSourceRegistry(
|
|
65
|
+
sources: DefinedContactSource[] = [],
|
|
66
|
+
): ContactSourceRegistry {
|
|
67
|
+
const registry = new ContactSourceRegistry();
|
|
68
|
+
for (const source of sources) registry.register(source);
|
|
69
|
+
setContactSourceRegistry(registry);
|
|
70
|
+
return registry;
|
|
71
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { IngestEvent } from "../lib/ingestion.js";
|
|
3
|
+
import type { WebhookSourceAuth } from "../webhook-sources/define-webhook-source.js";
|
|
4
|
+
import {
|
|
5
|
+
type ColdPosture,
|
|
6
|
+
type ContactWriteBack,
|
|
7
|
+
type DefinedContactSource,
|
|
8
|
+
defineContactSource,
|
|
9
|
+
} from "./define-contact-source.js";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Normalized inbound payload for the built-in GENERIC webhook contact source.
|
|
13
|
+
* Anything that can POST JSON — a HubSpot/Salesforce workflow, Zapier/Make/n8n,
|
|
14
|
+
* a bespoke CRM — maps its record to this shape and gets a sourced prospect with
|
|
15
|
+
* no bespoke adapter. At least one of `email` / `external_id` is required (an
|
|
16
|
+
* event with no resolvable key can't create a contact).
|
|
17
|
+
*/
|
|
18
|
+
export const webhookContactPayloadSchema = z
|
|
19
|
+
.object({
|
|
20
|
+
/** The sourcing event name (e.g. "prospect.sourced"). */
|
|
21
|
+
event: z.string().min(1),
|
|
22
|
+
/** The anchor identity key for a sourced lead. */
|
|
23
|
+
email: z.string().email().optional(),
|
|
24
|
+
/** Alternative/again identity key (a CRM record id, etc.). */
|
|
25
|
+
external_id: z.string().min(1).optional(),
|
|
26
|
+
anonymous_id: z.string().min(1).optional(),
|
|
27
|
+
/** Enrichment → `contacts.properties` (merged additively). */
|
|
28
|
+
properties: z.record(z.string(), z.unknown()).optional(),
|
|
29
|
+
/** Event payload → `user_events` only. */
|
|
30
|
+
event_properties: z.record(z.string(), z.unknown()).optional(),
|
|
31
|
+
/** Dedup key so source retries / re-sends don't re-ingest (Layer 2). */
|
|
32
|
+
idempotency_key: z.string().min(1).optional(),
|
|
33
|
+
/** Caller-supplied event time (ISO-8601) for backfill/replay. */
|
|
34
|
+
occurred_at: z.string().datetime().optional(),
|
|
35
|
+
})
|
|
36
|
+
.refine((p) => Boolean(p.email || p.external_id), {
|
|
37
|
+
message: "at least one of email / external_id is required",
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
export type WebhookContactPayload = z.infer<typeof webhookContactPayloadSchema>;
|
|
41
|
+
|
|
42
|
+
export interface WebhookContactSourceOptions {
|
|
43
|
+
/** The source id — keys the registry AND the route `/v1/webhooks/:id`. */
|
|
44
|
+
id: string;
|
|
45
|
+
name?: string;
|
|
46
|
+
description?: string;
|
|
47
|
+
/** Header carrying the shared secret. Default `x-hogsend-secret`. */
|
|
48
|
+
header?: string;
|
|
49
|
+
/**
|
|
50
|
+
* Env var holding the shared secret the header is matched against. NOTE: a
|
|
51
|
+
* `match` auth is OPEN when the env value is unset — set the secret in
|
|
52
|
+
* production (this source mints identified contacts).
|
|
53
|
+
*/
|
|
54
|
+
envKey: string;
|
|
55
|
+
coldPosture?: ColdPosture;
|
|
56
|
+
writeBack?: ContactWriteBack;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Map the normalized payload to an {@link IngestEvent}. Pure + unit-tested; the
|
|
61
|
+
* `source` (provenance) is stamped by the webhook route from the source id, so
|
|
62
|
+
* it is intentionally NOT set here.
|
|
63
|
+
*/
|
|
64
|
+
export function normalizeWebhookContactEvent(
|
|
65
|
+
payload: WebhookContactPayload,
|
|
66
|
+
): IngestEvent {
|
|
67
|
+
return {
|
|
68
|
+
event: payload.event,
|
|
69
|
+
userEmail: payload.email,
|
|
70
|
+
userId: payload.external_id,
|
|
71
|
+
anonymousId: payload.anonymous_id,
|
|
72
|
+
eventProperties: payload.event_properties ?? {},
|
|
73
|
+
contactProperties: payload.properties,
|
|
74
|
+
idempotencyKey: payload.idempotency_key,
|
|
75
|
+
occurredAt: payload.occurred_at,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Build the built-in GENERIC webhook contact source — the day-one path for any
|
|
81
|
+
* CRM/tool that isn't Clay/Attio. Point its outbound webhook at
|
|
82
|
+
* `POST /v1/webhooks/:id` with the shared-secret header and the normalized
|
|
83
|
+
* {@link webhookContactPayloadSchema}. Auth is a shared-secret header MATCH
|
|
84
|
+
* ({@link WebhookSourceAuth} `type: "match"`); the cold posture defaults to
|
|
85
|
+
* email-only (see {@link defineContactSource}).
|
|
86
|
+
*/
|
|
87
|
+
export function webhookContactSource(
|
|
88
|
+
opts: WebhookContactSourceOptions,
|
|
89
|
+
): DefinedContactSource<WebhookContactPayload> {
|
|
90
|
+
const auth: WebhookSourceAuth = {
|
|
91
|
+
type: "match",
|
|
92
|
+
header: opts.header ?? "x-hogsend-secret",
|
|
93
|
+
envKey: opts.envKey,
|
|
94
|
+
};
|
|
95
|
+
return defineContactSource<WebhookContactPayload>({
|
|
96
|
+
meta: {
|
|
97
|
+
id: opts.id,
|
|
98
|
+
name: opts.name ?? "Webhook",
|
|
99
|
+
description:
|
|
100
|
+
opts.description ??
|
|
101
|
+
"Generic webhook contact source (normalized CRM/tool payload).",
|
|
102
|
+
},
|
|
103
|
+
auth,
|
|
104
|
+
schema: webhookContactPayloadSchema,
|
|
105
|
+
coldPosture: opts.coldPosture,
|
|
106
|
+
writeBack: opts.writeBack,
|
|
107
|
+
transform: async (payload) => normalizeWebhookContactEvent(payload),
|
|
108
|
+
});
|
|
109
|
+
}
|
package/src/worker.ts
CHANGED
|
@@ -24,10 +24,12 @@ import {
|
|
|
24
24
|
import { bucketReconcileTask } from "./workflows/bucket-reconcile.js";
|
|
25
25
|
import { checkAlertsTask } from "./workflows/check-alerts.js";
|
|
26
26
|
import { confirmSemanticClickTask } from "./workflows/confirm-semantic-click.js";
|
|
27
|
+
import { crmReconcileTask } from "./workflows/crm-reconcile.js";
|
|
27
28
|
import {
|
|
28
29
|
deliverWebhookTask,
|
|
29
30
|
reapDueWebhookDeliveriesTask,
|
|
30
31
|
} from "./workflows/deliver-webhook.js";
|
|
32
|
+
import { dispatchConversionTask } from "./workflows/dispatch-conversion.js";
|
|
31
33
|
import { importContactsTask } from "./workflows/import-contacts.js";
|
|
32
34
|
import { importSuppressionsTask } from "./workflows/import-suppressions.js";
|
|
33
35
|
import { journeyBlueprintInterpreter } from "./workflows/journey-blueprint-interpreter.js";
|
|
@@ -127,6 +129,8 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
|
|
|
127
129
|
checkAlertsTask,
|
|
128
130
|
bucketReconcileTask,
|
|
129
131
|
bucketBackfillTask,
|
|
132
|
+
crmReconcileTask,
|
|
133
|
+
dispatchConversionTask,
|
|
130
134
|
...journeyTasks,
|
|
131
135
|
...bucketTasks,
|
|
132
136
|
...bucketReactionTasks,
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { ConcurrencyLimitStrategy } from "@hatchet-dev/typescript-sdk/v1/index.js";
|
|
2
|
+
import { createDatabase, crmSyncCursors, type Database } from "@hogsend/db";
|
|
3
|
+
import { eq } from "drizzle-orm";
|
|
4
|
+
import { getJourneyRegistrySingleton } from "../journeys/registry-singleton.js";
|
|
5
|
+
import { getAnalytics } from "../lib/analytics-singleton.js";
|
|
6
|
+
import { ingestCrmStageEvents } from "../lib/crm-ingest.js";
|
|
7
|
+
import { getCrmSyncConfig } from "../lib/crm-registry-singleton.js";
|
|
8
|
+
import { hatchet } from "../lib/hatchet.js";
|
|
9
|
+
import { createLogger, type Logger } from "../lib/logger.js";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* CRM reconciliation poll (docs/revenue-attribution-plan.md §4.3) — the pull
|
|
13
|
+
* half of the webhook+poll hybrid. Every provider implementing `poll` is
|
|
14
|
+
* walked on a cron: read the persisted cursor, pull changed deals, feed them
|
|
15
|
+
* through the SAME `ingestCrmStageEvents` sink the webhook route uses (the
|
|
16
|
+
* spine's idempotency dedups the overlap), persist the next cursor. A failing
|
|
17
|
+
* provider records `lastError` on its cursor row and never blocks the others
|
|
18
|
+
* — webhooks give latency, this poll guarantees eventual consistency.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/** The sweep body — exported for direct testing; the cron task wraps it. */
|
|
22
|
+
export async function runCrmReconcile(deps: {
|
|
23
|
+
db: Database;
|
|
24
|
+
logger: Logger;
|
|
25
|
+
/** Test seam — defaults to the process Hatchet client. */
|
|
26
|
+
hatchet?: typeof hatchet;
|
|
27
|
+
}): Promise<{ polled: number; ingested: number }> {
|
|
28
|
+
const { db, logger } = deps;
|
|
29
|
+
const hatchetClient = deps.hatchet ?? hatchet;
|
|
30
|
+
const config = getCrmSyncConfig();
|
|
31
|
+
if (!config || config.registry.count() === 0) {
|
|
32
|
+
return { polled: 0, ingested: 0 };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const registry = getJourneyRegistrySingleton();
|
|
36
|
+
const analytics = getAnalytics();
|
|
37
|
+
|
|
38
|
+
let polled = 0;
|
|
39
|
+
let ingested = 0;
|
|
40
|
+
|
|
41
|
+
for (const provider of config.registry.getAll()) {
|
|
42
|
+
if (!provider.poll) continue;
|
|
43
|
+
const providerId = provider.meta.id;
|
|
44
|
+
|
|
45
|
+
const cursorRows = await db
|
|
46
|
+
.select()
|
|
47
|
+
.from(crmSyncCursors)
|
|
48
|
+
.where(eq(crmSyncCursors.provider, providerId))
|
|
49
|
+
.limit(1);
|
|
50
|
+
const cursor = cursorRows[0]?.cursor ?? null;
|
|
51
|
+
|
|
52
|
+
try {
|
|
53
|
+
const result = await provider.poll(cursor);
|
|
54
|
+
const sink = await ingestCrmStageEvents({
|
|
55
|
+
db,
|
|
56
|
+
registry,
|
|
57
|
+
hatchet: hatchetClient,
|
|
58
|
+
logger,
|
|
59
|
+
analytics,
|
|
60
|
+
providerId,
|
|
61
|
+
events: result.events,
|
|
62
|
+
funnels: config.funnels,
|
|
63
|
+
});
|
|
64
|
+
ingested += sink.ingested;
|
|
65
|
+
polled++;
|
|
66
|
+
|
|
67
|
+
await db
|
|
68
|
+
.insert(crmSyncCursors)
|
|
69
|
+
.values({
|
|
70
|
+
provider: providerId,
|
|
71
|
+
cursor: result.nextCursor,
|
|
72
|
+
lastPolledAt: new Date(),
|
|
73
|
+
lastError: null,
|
|
74
|
+
updatedAt: new Date(),
|
|
75
|
+
})
|
|
76
|
+
.onConflictDoUpdate({
|
|
77
|
+
target: crmSyncCursors.provider,
|
|
78
|
+
set: {
|
|
79
|
+
cursor: result.nextCursor,
|
|
80
|
+
lastPolledAt: new Date(),
|
|
81
|
+
lastError: null,
|
|
82
|
+
updatedAt: new Date(),
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
} catch (err) {
|
|
86
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
87
|
+
logger.warn("crm-reconcile provider poll failed", {
|
|
88
|
+
provider: providerId,
|
|
89
|
+
error: message,
|
|
90
|
+
});
|
|
91
|
+
await db
|
|
92
|
+
.insert(crmSyncCursors)
|
|
93
|
+
.values({
|
|
94
|
+
provider: providerId,
|
|
95
|
+
cursor,
|
|
96
|
+
lastError: { message, at: new Date().toISOString() },
|
|
97
|
+
updatedAt: new Date(),
|
|
98
|
+
})
|
|
99
|
+
.onConflictDoUpdate({
|
|
100
|
+
target: crmSyncCursors.provider,
|
|
101
|
+
set: {
|
|
102
|
+
lastError: { message, at: new Date().toISOString() },
|
|
103
|
+
updatedAt: new Date(),
|
|
104
|
+
},
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return { polled, ingested };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export const crmReconcileTask = hatchet.task({
|
|
113
|
+
name: "crm-reconcile",
|
|
114
|
+
onCrons: [process.env.CRM_RECONCILE_CRON ?? "*/10 * * * *"],
|
|
115
|
+
retries: 1,
|
|
116
|
+
executionTimeout: "300s",
|
|
117
|
+
concurrency: {
|
|
118
|
+
// Single global key → at most one sweep runs; the next QUEUES rather than
|
|
119
|
+
// cancelling the in-flight run.
|
|
120
|
+
expression: "'crm-reconcile'",
|
|
121
|
+
maxRuns: 1,
|
|
122
|
+
limitStrategy: ConcurrencyLimitStrategy.GROUP_ROUND_ROBIN,
|
|
123
|
+
},
|
|
124
|
+
fn: async () => {
|
|
125
|
+
const { db } = createDatabase({ url: process.env.DATABASE_URL ?? "" });
|
|
126
|
+
const logger = createLogger(process.env.LOG_LEVEL ?? "info");
|
|
127
|
+
return runCrmReconcile({ db, logger });
|
|
128
|
+
},
|
|
129
|
+
});
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { createDatabase } from "@hogsend/db";
|
|
2
|
+
import { deliverConversionDispatch } from "../lib/conversion-dispatch.js";
|
|
3
|
+
import { hatchet } from "../lib/hatchet.js";
|
|
4
|
+
import { createLogger } from "../lib/logger.js";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Durable delivery of ONE conversion dispatch row (plan §5.2). Retries ride
|
|
8
|
+
* Hatchet (the deterministic event_id makes platform-side dedup safe);
|
|
9
|
+
* `deliverConversionDispatch` marks the row failed once attempts exhaust and
|
|
10
|
+
* stops throwing, so the task run ends cleanly.
|
|
11
|
+
*/
|
|
12
|
+
export interface DispatchConversionInput {
|
|
13
|
+
dispatchId: string;
|
|
14
|
+
[key: string]: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export const dispatchConversionTask = hatchet.task({
|
|
18
|
+
name: "dispatch-conversion",
|
|
19
|
+
retries: 4,
|
|
20
|
+
executionTimeout: "60s",
|
|
21
|
+
fn: async (input: DispatchConversionInput) => {
|
|
22
|
+
const { db } = createDatabase({ url: process.env.DATABASE_URL ?? "" });
|
|
23
|
+
const logger = createLogger(process.env.LOG_LEVEL ?? "info");
|
|
24
|
+
return deliverConversionDispatch({
|
|
25
|
+
db,
|
|
26
|
+
logger,
|
|
27
|
+
dispatchId: input.dispatchId,
|
|
28
|
+
maxAttempts: 5,
|
|
29
|
+
});
|
|
30
|
+
},
|
|
31
|
+
});
|
|
@@ -598,6 +598,11 @@ export const sendCampaignTask = hatchet.task({
|
|
|
598
598
|
// real, honored List-Unsubscribe target. The rule applies to
|
|
599
599
|
// EVERY wave, not just the first.
|
|
600
600
|
category: audienceKind === "bucket" ? undefined : audienceId,
|
|
601
|
+
// Queryable campaign identity on the email_sends row — the
|
|
602
|
+
// idempotency key below still encodes it for dedup, but
|
|
603
|
+
// suppressed sends write no key, so the column is the carrier
|
|
604
|
+
// stats + attribution can trust.
|
|
605
|
+
campaignId: input.campaignId,
|
|
601
606
|
// The idempotency key dedupes a retried send to its prior row.
|
|
602
607
|
// Legacy format for single-step campaigns; step-scoped for every
|
|
603
608
|
// step of a multi-step campaign (campaignSendKey contract).
|