@hogsend/engine 0.42.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +10 -7
- package/src/container.ts +419 -4
- package/src/destinations/define-destination.ts +7 -3
- package/src/destinations/presets/posthog.ts +31 -0
- package/src/env.ts +28 -0
- package/src/index.ts +138 -0
- package/src/journeys/execute-journey-run.ts +93 -0
- package/src/journeys/journey-boundary.ts +5 -1
- package/src/journeys/journey-context.ts +30 -0
- package/src/lib/attribution-backfill.ts +285 -0
- package/src/lib/attribution.ts +239 -0
- package/src/lib/contacts.ts +71 -0
- package/src/lib/conversion-dispatch.ts +326 -0
- package/src/lib/conversions.ts +197 -0
- package/src/lib/crm-deals.ts +320 -0
- package/src/lib/crm-ingest.ts +341 -0
- package/src/lib/crm-provider-registry.ts +41 -0
- package/src/lib/crm-registry-singleton.ts +22 -0
- package/src/lib/deal-money-events.ts +85 -0
- package/src/lib/email-service-types.ts +16 -1
- package/src/lib/event-names.ts +1 -1
- package/src/lib/funnel-progress.ts +76 -0
- package/src/lib/funnel-registry.ts +108 -0
- package/src/lib/funnel-transitions.ts +280 -0
- package/src/lib/holdout.ts +62 -0
- package/src/lib/ingestion.ts +218 -9
- package/src/lib/journey-suppress.ts +33 -2
- package/src/lib/lift-stats.ts +125 -0
- package/src/lib/mailer.ts +1 -0
- package/src/lib/operator.ts +31 -0
- package/src/lib/outbound.ts +109 -0
- package/src/lib/phone.ts +19 -0
- package/src/lib/preferences.ts +32 -0
- package/src/lib/revenue.ts +116 -0
- package/src/lib/sms-frequency-cap.ts +47 -0
- package/src/lib/sms-inbound.ts +200 -0
- package/src/lib/sms-link-tracking.ts +160 -0
- package/src/lib/sms-mailer.ts +338 -0
- package/src/lib/sms-provider-registry.ts +35 -0
- package/src/lib/sms-providers-from-env.ts +100 -0
- package/src/lib/sms-service-types.ts +175 -0
- package/src/lib/sms-tracked.ts +625 -0
- package/src/lib/sms.ts +117 -0
- package/src/lib/tracked.ts +40 -0
- package/src/lib/tracking-event-names.ts +6 -0
- package/src/lib/tracking-events.ts +133 -1
- package/src/lib/tracking.ts +2 -1
- package/src/lib/webhook-signing.ts +24 -1
- package/src/lists/channels.ts +21 -0
- package/src/routes/admin/attribution.ts +414 -0
- package/src/routes/admin/contacts.ts +71 -11
- package/src/routes/admin/conversions.ts +247 -0
- package/src/routes/admin/deals.ts +472 -0
- package/src/routes/admin/funnels.ts +228 -0
- package/src/routes/admin/index.ts +8 -0
- package/src/routes/admin/journeys.ts +158 -1
- package/src/routes/admin/readiness.ts +97 -2
- package/src/routes/admin/templates.ts +28 -1
- package/src/routes/email/unsubscribe.ts +1 -0
- package/src/routes/events/index.ts +12 -0
- package/src/routes/index.ts +6 -0
- package/src/routes/lists/index.ts +54 -7
- package/src/routes/tracking/click-pipeline.ts +105 -16
- package/src/routes/tracking/short.ts +61 -0
- package/src/routes/webhooks/crm-provider.ts +117 -0
- package/src/routes/webhooks/index.ts +9 -1
- package/src/routes/webhooks/sms-provider.ts +110 -0
- package/src/sources/define-contact-source.ts +159 -0
- package/src/sources/registry.ts +71 -0
- package/src/sources/webhook-source.ts +109 -0
- package/src/worker.ts +4 -0
- package/src/workflows/crm-reconcile.ts +129 -0
- package/src/workflows/dispatch-conversion.ts +31 -0
- package/src/workflows/import-contacts.ts +61 -4
- package/src/workflows/send-campaign.ts +5 -0
|
@@ -1,16 +1,27 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
|
-
import {
|
|
2
|
+
import {
|
|
3
|
+
emailSends,
|
|
4
|
+
linkClicks,
|
|
5
|
+
links,
|
|
6
|
+
smsSends,
|
|
7
|
+
trackedLinks,
|
|
8
|
+
} from "@hogsend/db";
|
|
3
9
|
import { and, eq, isNull, sql } from "drizzle-orm";
|
|
4
10
|
import type { Context } from "hono";
|
|
5
11
|
import type { AppEnv } from "../../app.js";
|
|
6
12
|
import { isBotOrPrefetch } from "../../lib/bot-prefetch.js";
|
|
7
13
|
import { generateIdentityToken } from "../../lib/identity-token.js";
|
|
8
14
|
import { emitOutbound } from "../../lib/outbound.js";
|
|
9
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
EMAIL_LINK_CLICKED,
|
|
17
|
+
SMS_LINK_CLICKED,
|
|
18
|
+
} from "../../lib/tracking-event-names.js";
|
|
10
19
|
import {
|
|
11
20
|
pushLinkClickEvent,
|
|
21
|
+
pushSmsTrackingEvent,
|
|
12
22
|
pushTrackingEvent,
|
|
13
23
|
resolveEmailSendContext,
|
|
24
|
+
resolveSmsSendContext,
|
|
14
25
|
} from "../../lib/tracking-events.js";
|
|
15
26
|
import { confirmSemanticClickTask } from "../../workflows/confirm-semantic-click.js";
|
|
16
27
|
|
|
@@ -21,6 +32,10 @@ export const clickSelection = {
|
|
|
21
32
|
id: trackedLinks.id,
|
|
22
33
|
originalUrl: trackedLinks.originalUrl,
|
|
23
34
|
emailSendId: trackedLinks.emailSendId,
|
|
35
|
+
// The SMS send this per-send short link belongs to (`/s/:code` rewrites).
|
|
36
|
+
// Selected on EVERY resolver, so an SMS row clicked via its raw
|
|
37
|
+
// `/v1/t/c/:id` URL still gets full SMS semantics.
|
|
38
|
+
smsSendId: trackedLinks.smsSendId,
|
|
24
39
|
distinctId: trackedLinks.distinctId,
|
|
25
40
|
source: trackedLinks.source,
|
|
26
41
|
// `event` gates the semantic-confirm dispatch; the confirm task re-reads the
|
|
@@ -41,6 +56,7 @@ export type ResolvedClickLink = {
|
|
|
41
56
|
id: string;
|
|
42
57
|
originalUrl: string;
|
|
43
58
|
emailSendId: string | null;
|
|
59
|
+
smsSendId: string | null;
|
|
44
60
|
distinctId: string | null;
|
|
45
61
|
source: string | null;
|
|
46
62
|
event: string | null;
|
|
@@ -127,6 +143,20 @@ export async function handleTrackedClick(
|
|
|
127
143
|
),
|
|
128
144
|
]
|
|
129
145
|
: []),
|
|
146
|
+
// SMS first-touch mirror: set sms_sends.clickedAt exactly once.
|
|
147
|
+
...(link.smsSendId
|
|
148
|
+
? [
|
|
149
|
+
db
|
|
150
|
+
.update(smsSends)
|
|
151
|
+
.set({
|
|
152
|
+
clickedAt: new Date(),
|
|
153
|
+
updatedAt: new Date(),
|
|
154
|
+
})
|
|
155
|
+
.where(
|
|
156
|
+
and(eq(smsSends.id, link.smsSendId), isNull(smsSends.clickedAt)),
|
|
157
|
+
),
|
|
158
|
+
]
|
|
159
|
+
: []),
|
|
130
160
|
]);
|
|
131
161
|
|
|
132
162
|
const { hatchet, registry, logger } = c.get("container");
|
|
@@ -235,21 +265,31 @@ export async function handleTrackedClick(
|
|
|
235
265
|
: resolveEmailSendContext(db, emailSendId)
|
|
236
266
|
)
|
|
237
267
|
.then(async (ctx) => {
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
268
|
+
// Bus re-ingest GATED on `!isBot` (impact plan §2.4 touch hygiene):
|
|
269
|
+
// `email.link_clicked` is a credit-bearing attribution touch AND a
|
|
270
|
+
// journey trigger, and inbox security scanners (Outlook SafeLinks,
|
|
271
|
+
// Barracuda, …) fetch email links with bot UAs — without the gate a
|
|
272
|
+
// scanner sweep mints touches and phantom journey triggers. The
|
|
273
|
+
// linkClicks row, clickCount, clickedAt, and the per-hit outbound
|
|
274
|
+
// emit below stay UNCONDITIONAL — stats record every hit; only the
|
|
275
|
+
// credit/journey spine is protected.
|
|
276
|
+
if (!isBot) {
|
|
277
|
+
await pushTrackingEvent({
|
|
278
|
+
db,
|
|
279
|
+
hatchet,
|
|
280
|
+
registry,
|
|
281
|
+
logger,
|
|
282
|
+
event: EMAIL_LINK_CLICKED,
|
|
283
|
+
emailSendId,
|
|
284
|
+
properties: { linkUrl: link.originalUrl, linkId: link.id },
|
|
285
|
+
resolvedContext: ctx,
|
|
286
|
+
}).catch((err) => {
|
|
287
|
+
logger.warn("Failed to push click tracking event", {
|
|
288
|
+
linkId: link.id,
|
|
289
|
+
error: err instanceof Error ? err.message : String(err),
|
|
290
|
+
});
|
|
251
291
|
});
|
|
252
|
-
}
|
|
292
|
+
}
|
|
253
293
|
|
|
254
294
|
// Only emit when the send-context resolved. A missing emailSends row
|
|
255
295
|
// (orphaned tracked link / deleted send) has no userId or recipient to
|
|
@@ -275,6 +315,55 @@ export async function handleTrackedClick(
|
|
|
275
315
|
}
|
|
276
316
|
})
|
|
277
317
|
.catch(logger.warn);
|
|
318
|
+
} else if (link.smsSendId) {
|
|
319
|
+
// SMS short link (`/s/:code`): re-ingest the first-party
|
|
320
|
+
// `sms.link_clicked` bus event and emit the per-hit `sms.clicked`
|
|
321
|
+
// outbound — the SMS sibling of the email branch above. The bus
|
|
322
|
+
// re-ingest is GATED on `!isBot` (iMessage/WhatsApp/RCS link-preview
|
|
323
|
+
// bots prefetch texted URLs) AND on a resolved contact key (phone is
|
|
324
|
+
// NOT a contact `Kind`; a userless raw send has no subject to ingest).
|
|
325
|
+
const smsSendId = link.smsSendId;
|
|
326
|
+
void resolveSmsSendContext(db, smsSendId)
|
|
327
|
+
.then(async (ctx) => {
|
|
328
|
+
if (!isBot && ctx?.userId) {
|
|
329
|
+
await pushSmsTrackingEvent({
|
|
330
|
+
db,
|
|
331
|
+
hatchet,
|
|
332
|
+
registry,
|
|
333
|
+
logger,
|
|
334
|
+
event: SMS_LINK_CLICKED,
|
|
335
|
+
smsSendId,
|
|
336
|
+
properties: { linkUrl: link.originalUrl, linkId: link.id },
|
|
337
|
+
resolvedContext: ctx,
|
|
338
|
+
}).catch((err) => {
|
|
339
|
+
logger.warn("Failed to push sms click tracking event", {
|
|
340
|
+
linkId: link.id,
|
|
341
|
+
error: err instanceof Error ? err.message : String(err),
|
|
342
|
+
});
|
|
343
|
+
});
|
|
344
|
+
}
|
|
345
|
+
// Per-hit outbound — mirrors the email click emit (every destination
|
|
346
|
+
// gets every click; NO dedupeKey). Gated on a resolved send row only.
|
|
347
|
+
if (ctx) {
|
|
348
|
+
await emitOutbound({
|
|
349
|
+
db,
|
|
350
|
+
hatchet,
|
|
351
|
+
logger,
|
|
352
|
+
event: "sms.clicked",
|
|
353
|
+
payload: {
|
|
354
|
+
smsSendId,
|
|
355
|
+
messageId: ctx.messageId ?? null,
|
|
356
|
+
templateKey: ctx.templateKey ?? null,
|
|
357
|
+
userId: ctx.userId ?? null,
|
|
358
|
+
to: ctx.to,
|
|
359
|
+
at: new Date().toISOString(),
|
|
360
|
+
linkUrl: link.originalUrl,
|
|
361
|
+
linkId: link.id,
|
|
362
|
+
},
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
})
|
|
366
|
+
.catch(logger.warn);
|
|
278
367
|
} else {
|
|
279
368
|
// Non-email tracked link: emit the catalogued `link.clicked` (NOT
|
|
280
369
|
// `email.clicked`). `userId` is the link's stitch subject when
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { links, trackedLinks } from "@hogsend/db";
|
|
2
|
+
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
+
import { eq } from "drizzle-orm";
|
|
4
|
+
import type { AppEnv } from "../../app.js";
|
|
5
|
+
import { clickSelection, handleTrackedClick } from "./click-pipeline.js";
|
|
6
|
+
|
|
7
|
+
// The SMS short path — `/s/:code` layered over `/v1/t/c/:id`, mirroring the
|
|
8
|
+
// vanity `/l/:slug` route. Mounted at the APP ROOT so the texted URL stays
|
|
9
|
+
// short (every character counts against the GSM-7 segment budget); `/s`
|
|
10
|
+
// collides with nothing (`/v1`, `/l`, `/studio`, `/docs`, `/api/auth`,
|
|
11
|
+
// `/connectors`, webhooks).
|
|
12
|
+
const shortRoute = createRoute({
|
|
13
|
+
method: "get",
|
|
14
|
+
path: "/s/:code",
|
|
15
|
+
tags: ["Tracking"],
|
|
16
|
+
summary: "SMS short link redirect",
|
|
17
|
+
request: {
|
|
18
|
+
params: z.object({
|
|
19
|
+
code: z.string(),
|
|
20
|
+
}),
|
|
21
|
+
},
|
|
22
|
+
responses: {
|
|
23
|
+
302: { description: "Redirect to original URL" },
|
|
24
|
+
},
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
// Lenient superset of minted codes (8-char lowercase base32): tolerate a
|
|
28
|
+
// hand-retyped code without rejecting future length changes.
|
|
29
|
+
const SHORT_CODE_RE = /^[a-z0-9]{4,32}$/;
|
|
30
|
+
|
|
31
|
+
export const shortLinkRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
32
|
+
shortRoute,
|
|
33
|
+
async (c) => {
|
|
34
|
+
const { code } = c.req.valid("param");
|
|
35
|
+
const { db, env } = c.get("container");
|
|
36
|
+
|
|
37
|
+
// Codes are minted lowercase; normalize a cased retype. A malformed code
|
|
38
|
+
// can't match anything — treat it like an unknown link (redirect home),
|
|
39
|
+
// mirroring the vanity route's malformed-slug behavior.
|
|
40
|
+
const normalized = code.trim().toLowerCase();
|
|
41
|
+
if (!SHORT_CODE_RE.test(normalized)) {
|
|
42
|
+
return c.redirect(env.API_PUBLIC_URL, 302);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// LEFT JOIN links purely to satisfy the shared clickSelection shape —
|
|
46
|
+
// SMS-minted rows have `link_id` NULL (mirrors the UUID route).
|
|
47
|
+
const rows = await db
|
|
48
|
+
.select(clickSelection)
|
|
49
|
+
.from(trackedLinks)
|
|
50
|
+
.leftJoin(links, eq(trackedLinks.linkId, links.id))
|
|
51
|
+
.where(eq(trackedLinks.shortCode, normalized))
|
|
52
|
+
.limit(1);
|
|
53
|
+
|
|
54
|
+
const link = rows[0];
|
|
55
|
+
if (!link) {
|
|
56
|
+
return c.redirect(env.API_PUBLIC_URL, 302);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
return handleTrackedClick(c, link);
|
|
60
|
+
},
|
|
61
|
+
);
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
import { WebhookHandshakeSignal } from "@hogsend/core";
|
|
2
|
+
import { createRoute, type OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
+
import type { Context } from "hono";
|
|
4
|
+
import type { AppEnv } from "../../app.js";
|
|
5
|
+
import { env } from "../../env.js";
|
|
6
|
+
import { ingestCrmStageEvents } from "../../lib/crm-ingest.js";
|
|
7
|
+
import { headersToRecord } from "../../lib/headers.js";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* CRM-provider webhook dispatch for `POST /v1/webhooks/crm/:providerId` — the
|
|
11
|
+
* CRM sibling of the email/SMS provider routes. Resolves the provider from
|
|
12
|
+
* the container's {@link CrmProviderRegistry} (404 on unknown id), verifies
|
|
13
|
+
* with the EXACT received bytes + canonical public URL, and lands the
|
|
14
|
+
* normalized {@link CrmStageEvent}s on the spine via
|
|
15
|
+
* {@link ingestCrmStageEvents} (idempotent against the reconciliation poll).
|
|
16
|
+
*/
|
|
17
|
+
export async function dispatchCrmProviderWebhook(
|
|
18
|
+
c: Context<AppEnv>,
|
|
19
|
+
providerId: string,
|
|
20
|
+
) {
|
|
21
|
+
const { crmProviders, funnels, db, registry, hatchet, logger, analytics } =
|
|
22
|
+
c.get("container");
|
|
23
|
+
|
|
24
|
+
const provider = crmProviders.get(providerId);
|
|
25
|
+
if (!provider) {
|
|
26
|
+
return c.json({ error: "Unknown CRM provider" }, 404);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
const payload = await c.req.text();
|
|
30
|
+
const headers = headersToRecord(c.req.raw.headers);
|
|
31
|
+
|
|
32
|
+
const requestUrl = new URL(c.req.url);
|
|
33
|
+
const base = env.API_PUBLIC_URL.replace(/\/+$/, "");
|
|
34
|
+
const url = `${base}${requestUrl.pathname}${requestUrl.search}`;
|
|
35
|
+
|
|
36
|
+
try {
|
|
37
|
+
const events = await provider.verifyWebhook({ payload, headers, url });
|
|
38
|
+
const result = await ingestCrmStageEvents({
|
|
39
|
+
db,
|
|
40
|
+
registry,
|
|
41
|
+
hatchet,
|
|
42
|
+
logger,
|
|
43
|
+
analytics,
|
|
44
|
+
providerId,
|
|
45
|
+
events,
|
|
46
|
+
funnels,
|
|
47
|
+
});
|
|
48
|
+
logger.info("CRM provider webhook processed", {
|
|
49
|
+
providerId,
|
|
50
|
+
events: events.length,
|
|
51
|
+
...result,
|
|
52
|
+
});
|
|
53
|
+
return c.json({ received: events.length, ...result }, 200);
|
|
54
|
+
} catch (err) {
|
|
55
|
+
if (err instanceof WebhookHandshakeSignal) {
|
|
56
|
+
logger.info("CRM webhook handshake", { providerId, action: err.action });
|
|
57
|
+
return c.json({ ok: true }, 200);
|
|
58
|
+
}
|
|
59
|
+
logger.warn("CRM provider webhook failed", {
|
|
60
|
+
providerId,
|
|
61
|
+
error: err instanceof Error ? err.message : String(err),
|
|
62
|
+
});
|
|
63
|
+
return c.json({ error: "Webhook verification failed" }, 401);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const crmProviderWebhookRoute = createRoute({
|
|
68
|
+
method: "post",
|
|
69
|
+
path: "/v1/webhooks/crm/{providerId}",
|
|
70
|
+
tags: ["Webhooks"],
|
|
71
|
+
summary:
|
|
72
|
+
"CRM provider webhook receiver (pipeline stage changes + deal values)",
|
|
73
|
+
request: {
|
|
74
|
+
params: z.object({ providerId: z.string() }),
|
|
75
|
+
body: {
|
|
76
|
+
content: {
|
|
77
|
+
// Providers post arbitrary JSON (object OR array); verification reads
|
|
78
|
+
// the raw bytes, so the schema is deliberately permissive.
|
|
79
|
+
"application/json": { schema: z.unknown() },
|
|
80
|
+
},
|
|
81
|
+
},
|
|
82
|
+
},
|
|
83
|
+
responses: {
|
|
84
|
+
200: {
|
|
85
|
+
content: {
|
|
86
|
+
"application/json": {
|
|
87
|
+
schema: z.object({
|
|
88
|
+
received: z.number().optional(),
|
|
89
|
+
ingested: z.number().optional(),
|
|
90
|
+
skipped: z.number().optional(),
|
|
91
|
+
ok: z.boolean().optional(),
|
|
92
|
+
}),
|
|
93
|
+
},
|
|
94
|
+
},
|
|
95
|
+
description: "Webhook processed",
|
|
96
|
+
},
|
|
97
|
+
401: {
|
|
98
|
+
content: {
|
|
99
|
+
"application/json": { schema: z.object({ error: z.string() }) },
|
|
100
|
+
},
|
|
101
|
+
description: "Missing or invalid webhook signature",
|
|
102
|
+
},
|
|
103
|
+
404: {
|
|
104
|
+
content: {
|
|
105
|
+
"application/json": { schema: z.object({ error: z.string() }) },
|
|
106
|
+
},
|
|
107
|
+
description: "Unknown CRM provider",
|
|
108
|
+
},
|
|
109
|
+
},
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
export function registerCrmProviderRoutes(app: OpenAPIHono<AppEnv>) {
|
|
113
|
+
app.openapi(crmProviderWebhookRoute, (c) => {
|
|
114
|
+
const { providerId } = c.req.valid("param");
|
|
115
|
+
return dispatchCrmProviderWebhook(c, providerId);
|
|
116
|
+
});
|
|
117
|
+
}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import type { OpenAPIHono } from "@hono/zod-openapi";
|
|
2
2
|
import type { AppEnv } from "../../app.js";
|
|
3
3
|
import type { DefinedConnector } from "../../connectors/define-connector.js";
|
|
4
|
+
import { registerCrmProviderRoutes } from "./crm-provider.js";
|
|
4
5
|
import { registerEmailProviderRoutes } from "./email-provider.js";
|
|
5
6
|
import { resendWebhookRouter } from "./resend.js";
|
|
7
|
+
import { registerSmsProviderRoutes } from "./sms-provider.js";
|
|
6
8
|
import { registerWebhookSourceRoutes } from "./sources.js";
|
|
7
9
|
|
|
8
10
|
export interface RegisterWebhookRoutesOptions {
|
|
@@ -17,8 +19,14 @@ export function registerWebhookRoutes(
|
|
|
17
19
|
// 1. the thin `/v1/webhooks/resend` alias (static),
|
|
18
20
|
// 2. the `/v1/webhooks/email/:providerId` id-dispatched route (static
|
|
19
21
|
// `email/` prefix — MUST come before the catch-all),
|
|
20
|
-
// 3. the `/v1/webhooks/:
|
|
22
|
+
// 3. the `/v1/webhooks/sms/:providerId` id-dispatched route (static `sms/`
|
|
23
|
+
// prefix — MUST come before the catch-all),
|
|
24
|
+
// 4. the `/v1/webhooks/crm/:providerId` id-dispatched route (static `crm/`
|
|
25
|
+
// prefix — MUST come before the catch-all),
|
|
26
|
+
// 5. the `/v1/webhooks/:sourceId` consumer-source catch-all (LAST).
|
|
21
27
|
app.route("/v1/webhooks", resendWebhookRouter);
|
|
22
28
|
registerEmailProviderRoutes(app);
|
|
29
|
+
registerSmsProviderRoutes(app);
|
|
30
|
+
registerCrmProviderRoutes(app);
|
|
23
31
|
registerWebhookSourceRoutes(app, opts.webhookConnectors);
|
|
24
32
|
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { WebhookHandshakeSignal } from "@hogsend/core";
|
|
2
|
+
import { createRoute, type OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
+
import type { Context } from "hono";
|
|
4
|
+
import type { AppEnv } from "../../app.js";
|
|
5
|
+
import { env } from "../../env.js";
|
|
6
|
+
import { headersToRecord } from "../../lib/headers.js";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Shared SMS-provider webhook dispatch for
|
|
10
|
+
* `POST /v1/webhooks/sms/:providerId`. Resolves the provider from the
|
|
11
|
+
* container's {@link SmsProviderRegistry} (404 on unknown id), reads the raw
|
|
12
|
+
* body as the EXACT received bytes, builds the canonical PUBLIC url (Twilio
|
|
13
|
+
* signs `API_PUBLIC_URL + path + query`, NOT `c.req.url` which has the wrong
|
|
14
|
+
* host behind a proxy), verifies + dispatches the normalized {@link SmsEvent},
|
|
15
|
+
* 200s a {@link WebhookHandshakeSignal} (intermediate status / unrecognized),
|
|
16
|
+
* and 401s a verification error.
|
|
17
|
+
*
|
|
18
|
+
* Responds with an empty TwiML document — Twilio's inbound MO webhook expects
|
|
19
|
+
* TwiML; status callbacks accept any 2xx.
|
|
20
|
+
*/
|
|
21
|
+
export async function dispatchSmsProviderWebhook(
|
|
22
|
+
c: Context<AppEnv>,
|
|
23
|
+
providerId: string,
|
|
24
|
+
) {
|
|
25
|
+
const { smsProviders, smsService, logger } = c.get("container");
|
|
26
|
+
|
|
27
|
+
const provider = smsProviders.get(providerId);
|
|
28
|
+
if (!provider) {
|
|
29
|
+
return c.json({ error: "Unknown SMS provider" }, 404);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const payload = await c.req.text();
|
|
33
|
+
const headers = headersToRecord(c.req.raw.headers);
|
|
34
|
+
|
|
35
|
+
// Canonical public URL the provider signed: API_PUBLIC_URL + path (+ query).
|
|
36
|
+
// Twilio's HMAC covers the URL string byte-for-byte, so normalize the one
|
|
37
|
+
// operator-controlled degree of freedom — a trailing slash on
|
|
38
|
+
// API_PUBLIC_URL would otherwise 401 EVERY callback and inbound STOP.
|
|
39
|
+
const requestUrl = new URL(c.req.url);
|
|
40
|
+
const base = env.API_PUBLIC_URL.replace(/\/+$/, "");
|
|
41
|
+
const url = `${base}${requestUrl.pathname}${requestUrl.search}`;
|
|
42
|
+
|
|
43
|
+
try {
|
|
44
|
+
const event = await provider.verifyWebhook({ payload, headers, url });
|
|
45
|
+
const result = await smsService.handleWebhook(event, providerId);
|
|
46
|
+
logger.info("SMS provider webhook processed", {
|
|
47
|
+
providerId,
|
|
48
|
+
type: event.type,
|
|
49
|
+
handled: result.handled,
|
|
50
|
+
});
|
|
51
|
+
return c.body(
|
|
52
|
+
'<?xml version="1.0" encoding="UTF-8"?><Response></Response>',
|
|
53
|
+
200,
|
|
54
|
+
{ "Content-Type": "text/xml" },
|
|
55
|
+
);
|
|
56
|
+
} catch (err) {
|
|
57
|
+
if (err instanceof WebhookHandshakeSignal) {
|
|
58
|
+
logger.info("SMS webhook handshake", { providerId, action: err.action });
|
|
59
|
+
return c.body(
|
|
60
|
+
'<?xml version="1.0" encoding="UTF-8"?><Response></Response>',
|
|
61
|
+
200,
|
|
62
|
+
{ "Content-Type": "text/xml" },
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
logger.warn("SMS provider webhook failed", {
|
|
66
|
+
providerId,
|
|
67
|
+
error: err instanceof Error ? err.message : String(err),
|
|
68
|
+
});
|
|
69
|
+
return c.json({ error: "Webhook verification failed" }, 401);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const smsProviderWebhookRoute = createRoute({
|
|
74
|
+
method: "post",
|
|
75
|
+
path: "/v1/webhooks/sms/{providerId}",
|
|
76
|
+
tags: ["Webhooks"],
|
|
77
|
+
summary: "SMS provider webhook receiver (status callbacks + inbound STOP)",
|
|
78
|
+
request: {
|
|
79
|
+
params: z.object({ providerId: z.string() }),
|
|
80
|
+
body: {
|
|
81
|
+
content: {
|
|
82
|
+
"application/x-www-form-urlencoded": {
|
|
83
|
+
schema: z.record(z.string(), z.unknown()),
|
|
84
|
+
},
|
|
85
|
+
},
|
|
86
|
+
},
|
|
87
|
+
},
|
|
88
|
+
responses: {
|
|
89
|
+
200: { description: "Webhook processed (empty TwiML)" },
|
|
90
|
+
401: {
|
|
91
|
+
content: {
|
|
92
|
+
"application/json": { schema: z.object({ error: z.string() }) },
|
|
93
|
+
},
|
|
94
|
+
description: "Missing or invalid webhook signature",
|
|
95
|
+
},
|
|
96
|
+
404: {
|
|
97
|
+
content: {
|
|
98
|
+
"application/json": { schema: z.object({ error: z.string() }) },
|
|
99
|
+
},
|
|
100
|
+
description: "Unknown SMS provider",
|
|
101
|
+
},
|
|
102
|
+
},
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
export function registerSmsProviderRoutes(app: OpenAPIHono<AppEnv>) {
|
|
106
|
+
app.openapi(smsProviderWebhookRoute, (c) => {
|
|
107
|
+
const { providerId } = c.req.valid("param");
|
|
108
|
+
return dispatchSmsProviderWebhook(c, providerId);
|
|
109
|
+
});
|
|
110
|
+
}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
import type { z } from "zod";
|
|
2
|
+
import type { IngestEvent } from "../lib/ingestion.js";
|
|
3
|
+
import {
|
|
4
|
+
type DefinedWebhookSource,
|
|
5
|
+
defineWebhookSource,
|
|
6
|
+
type WebhookSourceAuth,
|
|
7
|
+
type WebhookSourceCtx,
|
|
8
|
+
} from "../webhook-sources/define-webhook-source.js";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* A CONTACT SOURCE is an origin of *sourced* contacts — Clay, Attio, a generic
|
|
12
|
+
* webhook, any CRM/enrichment tool. It is thin sugar over
|
|
13
|
+
* {@link defineWebhookSource}: the same inbound `transform` (raw payload →
|
|
14
|
+
* {@link IngestEvent}) served on the existing `POST /v1/webhooks/:sourceId`
|
|
15
|
+
* route, PLUS three things a plain webhook source has no opinion on:
|
|
16
|
+
*
|
|
17
|
+
* 1. **Provenance is automatic.** The webhook route ingests with
|
|
18
|
+
* `source = meta.id`, and the engine stamps that onto `contacts.source` on
|
|
19
|
+
* create (first-touch). So a source `id: "clay"` mints contacts with
|
|
20
|
+
* `source: "clay"` — a cold **prospect** — with NO extra wiring. Registering
|
|
21
|
+
* the id in the {@link ContactSourceRegistry} is what lets the rest of the
|
|
22
|
+
* engine *classify* such a contact as a prospect.
|
|
23
|
+
* 2. **A cold-consent posture** ({@link ColdPosture}) — which channels a cold,
|
|
24
|
+
* no-consent contact may receive before they opt in. Legal-safe default:
|
|
25
|
+
* email `allow`, every other channel `block`. This is the deliberate,
|
|
26
|
+
* per-source "unflick" knob (see `docs/.../consent-and-legal.md`).
|
|
27
|
+
* 3. **An optional {@link ContactWriteBack}** — pushing journey/engagement
|
|
28
|
+
* status back to the source CRM (implemented per-source; e.g. Attio).
|
|
29
|
+
*
|
|
30
|
+
* A contact source is NOT a new ingestion surface — {@link contactSourceToWebhookSource}
|
|
31
|
+
* lifts it back onto the webhook-source umbrella for the container/route to
|
|
32
|
+
* register, so it rides the exact battle-tested path every webhook source uses.
|
|
33
|
+
*/
|
|
34
|
+
export type ColdChannelPosture = "allow" | "block";
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Per-channel cold posture. Keys are channel ids (`"email"`, `"sms"`, a
|
|
38
|
+
* connector id like `"discord"`). Any channel not named defaults to `block`,
|
|
39
|
+
* EXCEPT `email`, which defaults to `allow` (cold email is lawful with
|
|
40
|
+
* identification + unsubscribe, which the mailer already provides). Opening a
|
|
41
|
+
* non-email channel to `"allow"` for cold contacts is a deliberate, auditable
|
|
42
|
+
* act — see {@link resolveColdPosture} / {@link isColdChannelAllowed}.
|
|
43
|
+
*/
|
|
44
|
+
export type ColdPosture = Record<string, ColdChannelPosture>;
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The legal-safe default posture applied when a source declares none: cold
|
|
48
|
+
* email allowed, everything else blocked. Kept as a function (not a shared
|
|
49
|
+
* object) so callers can never mutate a shared default.
|
|
50
|
+
*/
|
|
51
|
+
export function defaultColdPosture(): ColdPosture {
|
|
52
|
+
return { email: "allow" };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Merge a source's declared posture over the safe default. `email` stays
|
|
57
|
+
* `allow` unless the source explicitly blocks it; every other channel stays
|
|
58
|
+
* `block` unless the source explicitly allows it.
|
|
59
|
+
*/
|
|
60
|
+
export function resolveColdPosture(declared?: ColdPosture): ColdPosture {
|
|
61
|
+
return { ...defaultColdPosture(), ...(declared ?? {}) };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Is `channelId` deliverable to a COLD (sourced, no-consent) contact under this
|
|
66
|
+
* posture? `email` → allowed unless explicitly blocked; any other channel →
|
|
67
|
+
* blocked unless explicitly allowed. This is the single source of truth the
|
|
68
|
+
* cold gate consults, so the default is fail-closed for everything but email.
|
|
69
|
+
*/
|
|
70
|
+
export function isColdChannelAllowed(
|
|
71
|
+
posture: ColdPosture,
|
|
72
|
+
channelId: string,
|
|
73
|
+
): boolean {
|
|
74
|
+
if (channelId === "email") return (posture.email ?? "allow") === "allow";
|
|
75
|
+
return posture[channelId] === "allow";
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export interface ContactSourceMeta {
|
|
79
|
+
id: string;
|
|
80
|
+
name: string;
|
|
81
|
+
description?: string;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Optional per-source write-back — push engagement/lifecycle status from a
|
|
86
|
+
* journey milestone back to the origin CRM. Best-effort and provider-specific
|
|
87
|
+
* (e.g. Attio's `PUT /v2/objects/people/records` + a note). Implemented in the
|
|
88
|
+
* source-adapter phases; declared here so the primitive owns the contract.
|
|
89
|
+
*/
|
|
90
|
+
export interface ContactWriteBack {
|
|
91
|
+
syncStatus(args: {
|
|
92
|
+
/** The contact's email (the cross-system business key), if known. */
|
|
93
|
+
email: string | null;
|
|
94
|
+
/** The contact's external id, if known. */
|
|
95
|
+
externalId: string | null;
|
|
96
|
+
/** A short lifecycle status ("emailed", "replied", "opted_in", …). */
|
|
97
|
+
status: string;
|
|
98
|
+
/** Optional structured attributes to upsert on the CRM record. */
|
|
99
|
+
properties?: Record<string, unknown>;
|
|
100
|
+
}): Promise<void>;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export interface DefinedContactSource<T = unknown> {
|
|
104
|
+
meta: ContactSourceMeta;
|
|
105
|
+
/** Inbound-request verification — identical shape to a webhook source. */
|
|
106
|
+
auth: WebhookSourceAuth;
|
|
107
|
+
/** Optional Zod schema validating the payload BEFORE transform. */
|
|
108
|
+
schema?: z.ZodSchema<T>;
|
|
109
|
+
/** Raw CRM/enrichment payload → a sourcing {@link IngestEvent} (or null). */
|
|
110
|
+
transform(payload: T, ctx: WebhookSourceCtx): Promise<IngestEvent | null>;
|
|
111
|
+
/** RESOLVED cold posture (safe defaults already merged in). */
|
|
112
|
+
coldPosture: ColdPosture;
|
|
113
|
+
/** Optional write-back adapter. */
|
|
114
|
+
writeBack?: ContactWriteBack;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Author a contact source. Identity/validating function (mirrors
|
|
119
|
+
* {@link defineWebhookSource}) that additionally resolves the cold posture to
|
|
120
|
+
* its safe-default-merged form so downstream reads never re-derive it.
|
|
121
|
+
*/
|
|
122
|
+
export function defineContactSource<T>(def: {
|
|
123
|
+
meta: ContactSourceMeta;
|
|
124
|
+
auth: WebhookSourceAuth;
|
|
125
|
+
schema?: z.ZodSchema<T>;
|
|
126
|
+
transform(payload: T, ctx: WebhookSourceCtx): Promise<IngestEvent | null>;
|
|
127
|
+
coldPosture?: ColdPosture;
|
|
128
|
+
writeBack?: ContactWriteBack;
|
|
129
|
+
}): DefinedContactSource<T> {
|
|
130
|
+
if (!def.meta.id) {
|
|
131
|
+
throw new Error("defineContactSource: meta.id is required");
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
meta: def.meta,
|
|
135
|
+
auth: def.auth,
|
|
136
|
+
schema: def.schema,
|
|
137
|
+
transform: def.transform,
|
|
138
|
+
coldPosture: resolveColdPosture(def.coldPosture),
|
|
139
|
+
writeBack: def.writeBack,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Lift a contact source back onto the {@link DefinedWebhookSource} umbrella so
|
|
145
|
+
* the container/route can register it exactly like any webhook source (it rides
|
|
146
|
+
* the `POST /v1/webhooks/:sourceId` path, and the route stamps `source = id`,
|
|
147
|
+
* giving provenance for free). The cold posture / write-back live only on the
|
|
148
|
+
* {@link ContactSourceRegistry}; the webhook path needs neither.
|
|
149
|
+
*/
|
|
150
|
+
export function contactSourceToWebhookSource<T>(
|
|
151
|
+
source: DefinedContactSource<T>,
|
|
152
|
+
): DefinedWebhookSource<T> {
|
|
153
|
+
return defineWebhookSource<T>({
|
|
154
|
+
meta: source.meta,
|
|
155
|
+
auth: source.auth,
|
|
156
|
+
schema: source.schema,
|
|
157
|
+
transform: source.transform,
|
|
158
|
+
});
|
|
159
|
+
}
|