@hogsend/engine 0.31.1 → 0.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -7
- package/src/cold-connect/index.ts +1 -0
- package/src/connectors/define-connector.ts +12 -2
- package/src/connectors/runtime.ts +10 -8
- package/src/index.ts +8 -0
- package/src/journeys/define-journey.ts +111 -41
- package/src/journeys/journey-boundary.ts +166 -0
- package/src/journeys/journey-context.ts +391 -23
- package/src/lib/boot.ts +8 -4
- package/src/lib/bot-prefetch.ts +48 -0
- package/src/lib/bucket-emit.ts +2 -0
- package/src/lib/connector-actions.ts +197 -5
- package/src/lib/domain-status.ts +27 -12
- package/src/lib/email.ts +63 -5
- package/src/lib/ingestion.ts +71 -1
- package/src/lib/links.ts +1 -1
- package/src/lib/tracked.ts +111 -37
- package/src/lib/tracking-event-names.ts +9 -0
- package/src/lib/tracking-events.ts +73 -0
- package/src/routes/admin/bulk.ts +2 -0
- package/src/routes/admin/events.ts +51 -11
- package/src/routes/admin/index.ts +2 -0
- package/src/routes/admin/journeys.ts +95 -1
- package/src/routes/admin/links.ts +51 -8
- package/src/routes/admin/readiness.ts +181 -0
- package/src/routes/connectors/index.ts +22 -5
- package/src/routes/events/index.ts +2 -0
- package/src/routes/tracking/click.ts +51 -1
- package/src/routes/webhooks/sources.ts +31 -4
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { apiKeys } from "@hogsend/db";
|
|
2
|
+
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
+
import { count, isNull } from "drizzle-orm";
|
|
4
|
+
import type { AppEnv } from "../../app.js";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* First-time setup readiness — a NON-BLOCKING checklist for Studio's Setup view.
|
|
8
|
+
* It reports per-area configuration state ("done", "needs action", "optional")
|
|
9
|
+
* so a fresh operator sees what's left to wire (email key, sending domain,
|
|
10
|
+
* data-plane key, analytics) without anything erroring or gating the UI. It is
|
|
11
|
+
* deliberately read-only and best-effort: a single failing probe degrades that
|
|
12
|
+
* one row, never the whole response.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
const DOCS = "https://docs.hogsend.com";
|
|
16
|
+
/** The env.example placeholder — a present-but-unset Resend key. */
|
|
17
|
+
const RESEND_PLACEHOLDER = "re_your_api_key_here";
|
|
18
|
+
|
|
19
|
+
const checkSchema = z.object({
|
|
20
|
+
id: z.string(),
|
|
21
|
+
label: z.string(),
|
|
22
|
+
// "ok" satisfied · "action" recommended + not done · "optional" nice-to-have
|
|
23
|
+
status: z.enum(["ok", "action", "optional"]),
|
|
24
|
+
detail: z.string(),
|
|
25
|
+
docsUrl: z.string().optional(),
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
type Check = z.infer<typeof checkSchema>;
|
|
29
|
+
|
|
30
|
+
const readinessSchema = z.object({
|
|
31
|
+
/** true when nothing is left in the "action" state (optional rows may remain). */
|
|
32
|
+
ready: z.boolean(),
|
|
33
|
+
doneCount: z.number(),
|
|
34
|
+
totalCount: z.number(),
|
|
35
|
+
checks: z.array(checkSchema),
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const getReadinessRoute = createRoute({
|
|
39
|
+
method: "get",
|
|
40
|
+
path: "/",
|
|
41
|
+
tags: ["Admin — Readiness"],
|
|
42
|
+
summary: "First-time setup readiness checklist (non-blocking)",
|
|
43
|
+
responses: {
|
|
44
|
+
200: {
|
|
45
|
+
content: { "application/json": { schema: readinessSchema } },
|
|
46
|
+
description: "Per-area setup readiness for the Studio Setup checklist",
|
|
47
|
+
},
|
|
48
|
+
},
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
export const readinessRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
52
|
+
getReadinessRoute,
|
|
53
|
+
async (c) => {
|
|
54
|
+
const { env, db, emailProvider, analytics, domainStatus } =
|
|
55
|
+
c.get("container");
|
|
56
|
+
const checks: Check[] = [];
|
|
57
|
+
|
|
58
|
+
// 1. Studio admin — you must be one to reach this guarded route, so it's
|
|
59
|
+
// trivially satisfied; surfaced anyway as reassurance in the checklist.
|
|
60
|
+
checks.push({
|
|
61
|
+
id: "studio_admin",
|
|
62
|
+
label: "Studio admin",
|
|
63
|
+
status: "ok",
|
|
64
|
+
detail: "Signed in.",
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
// 2. Hatchet — required at boot, so reaching here means it's connected.
|
|
68
|
+
checks.push({
|
|
69
|
+
id: "hatchet",
|
|
70
|
+
label: "Hatchet worker engine",
|
|
71
|
+
status: env.HATCHET_CLIENT_TOKEN ? "ok" : "action",
|
|
72
|
+
detail: env.HATCHET_CLIENT_TOKEN
|
|
73
|
+
? `Connected (${env.HATCHET_CLIENT_HOST_PORT}).`
|
|
74
|
+
: "Set HATCHET_CLIENT_TOKEN so the worker can run your journeys.",
|
|
75
|
+
docsUrl: DOCS,
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
// 3. Email provider key — sends fail without it.
|
|
79
|
+
const providerId = emailProvider.meta?.id ?? "email";
|
|
80
|
+
const providerKeyPresent =
|
|
81
|
+
providerId === "resend"
|
|
82
|
+
? Boolean(env.RESEND_API_KEY) &&
|
|
83
|
+
env.RESEND_API_KEY !== RESEND_PLACEHOLDER
|
|
84
|
+
: providerId === "postmark"
|
|
85
|
+
? Boolean(env.POSTMARK_SERVER_TOKEN)
|
|
86
|
+
: // A custom provider was constructed with its own credentials.
|
|
87
|
+
true;
|
|
88
|
+
checks.push({
|
|
89
|
+
id: "email_provider",
|
|
90
|
+
label: "Email provider",
|
|
91
|
+
status: providerKeyPresent ? "ok" : "action",
|
|
92
|
+
detail: providerKeyPresent
|
|
93
|
+
? `${providerId} key configured.`
|
|
94
|
+
: `No ${providerId} API key yet — sends will fail until one is set.`,
|
|
95
|
+
docsUrl: DOCS,
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
// 4. Data-plane API key — any non-revoked key lets your app code ingest.
|
|
99
|
+
let dataPlaneKeys = 0;
|
|
100
|
+
try {
|
|
101
|
+
const [row] = await db
|
|
102
|
+
.select({ n: count() })
|
|
103
|
+
.from(apiKeys)
|
|
104
|
+
.where(isNull(apiKeys.revokedAt));
|
|
105
|
+
dataPlaneKeys = Number(row?.n ?? 0);
|
|
106
|
+
} catch {
|
|
107
|
+
// DB hiccup — degrade this one row, don't fail the whole checklist.
|
|
108
|
+
}
|
|
109
|
+
checks.push({
|
|
110
|
+
id: "data_plane_key",
|
|
111
|
+
label: "Data-plane API key",
|
|
112
|
+
status: dataPlaneKeys > 0 ? "ok" : "action",
|
|
113
|
+
detail:
|
|
114
|
+
dataPlaneKeys > 0
|
|
115
|
+
? `${dataPlaneKeys} active key${dataPlaneKeys === 1 ? "" : "s"}.`
|
|
116
|
+
: "Mint one (run bootstrap, or POST /v1/admin/api-keys) so your app can send events.",
|
|
117
|
+
docsUrl: DOCS,
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
// 5. Sending domain — derive from the (now graceful) domain status. A bad
|
|
121
|
+
// provider key no longer throws here; it surfaces as status === null.
|
|
122
|
+
try {
|
|
123
|
+
const dom = await domainStatus.getStatus();
|
|
124
|
+
if (!dom.domain) {
|
|
125
|
+
checks.push({
|
|
126
|
+
id: "sending_domain",
|
|
127
|
+
label: "Sending domain",
|
|
128
|
+
status: "optional",
|
|
129
|
+
detail:
|
|
130
|
+
"No EMAIL_DOMAIN set — sends use the provider default. Configure one to send from your own domain.",
|
|
131
|
+
docsUrl: DOCS,
|
|
132
|
+
});
|
|
133
|
+
} else if (dom.status?.state === "verified") {
|
|
134
|
+
checks.push({
|
|
135
|
+
id: "sending_domain",
|
|
136
|
+
label: "Sending domain",
|
|
137
|
+
status: "ok",
|
|
138
|
+
detail: `${dom.domain} verified.`,
|
|
139
|
+
});
|
|
140
|
+
} else {
|
|
141
|
+
checks.push({
|
|
142
|
+
id: "sending_domain",
|
|
143
|
+
label: "Sending domain",
|
|
144
|
+
status: "action",
|
|
145
|
+
detail:
|
|
146
|
+
dom.status === null
|
|
147
|
+
? `${dom.domain}: can't read status — confirm your ${providerId} key can read domains.`
|
|
148
|
+
: `${dom.domain} is ${dom.status.state.replace("_", " ")} — verify it below.`,
|
|
149
|
+
docsUrl: DOCS,
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
} catch {
|
|
153
|
+
checks.push({
|
|
154
|
+
id: "sending_domain",
|
|
155
|
+
label: "Sending domain",
|
|
156
|
+
status: "action",
|
|
157
|
+
detail: "Couldn't read domain status — check your email provider key.",
|
|
158
|
+
docsUrl: DOCS,
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// 6. PostHog analytics — optional.
|
|
163
|
+
const analyticsConfigured =
|
|
164
|
+
Boolean(env.POSTHOG_API_KEY) || Boolean(analytics);
|
|
165
|
+
checks.push({
|
|
166
|
+
id: "analytics",
|
|
167
|
+
label: "PostHog analytics",
|
|
168
|
+
status: analyticsConfigured ? "ok" : "optional",
|
|
169
|
+
detail: analyticsConfigured
|
|
170
|
+
? "Connected."
|
|
171
|
+
: "Optional — connect PostHog to capture events and person properties.",
|
|
172
|
+
docsUrl: DOCS,
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
const doneCount = checks.filter((ch) => ch.status === "ok").length;
|
|
176
|
+
// "ready" ignores optional rows — only outstanding "action" items block it.
|
|
177
|
+
const ready = checks.every((ch) => ch.status !== "action");
|
|
178
|
+
|
|
179
|
+
return c.json({ ready, doneCount, totalCount: checks.length, checks }, 200);
|
|
180
|
+
},
|
|
181
|
+
);
|
|
@@ -4,7 +4,7 @@ import type { DefinedConnector } from "../../connectors/define-connector.js";
|
|
|
4
4
|
import { getConnectorRegistry } from "../../connectors/registry-singleton.js";
|
|
5
5
|
import { verifyConnectorState } from "../../lib/connector-state.js";
|
|
6
6
|
import { headersToRecord } from "../../lib/headers.js";
|
|
7
|
-
import { ingestEvent } from "../../lib/ingestion.js";
|
|
7
|
+
import { ingestEvent, ingestTransformResult } from "../../lib/ingestion.js";
|
|
8
8
|
import type { Logger } from "../../lib/logger.js";
|
|
9
9
|
import { getRedisIfConnected } from "../../lib/redis.js";
|
|
10
10
|
import { clientIpKey, createRateLimit } from "../../middleware/rate-limit.js";
|
|
@@ -213,7 +213,7 @@ export function registerConnectorRoutes(app: OpenAPIHono<AppEnv>) {
|
|
|
213
213
|
registry,
|
|
214
214
|
hatchet,
|
|
215
215
|
logger,
|
|
216
|
-
event: result.event,
|
|
216
|
+
event: { ...result.event, source: "connector" },
|
|
217
217
|
});
|
|
218
218
|
return c.json({ ok: true }, 200);
|
|
219
219
|
}
|
|
@@ -254,18 +254,35 @@ export function registerConnectorRoutes(app: OpenAPIHono<AppEnv>) {
|
|
|
254
254
|
return c.json({ error: "Unauthorized" }, 401);
|
|
255
255
|
}
|
|
256
256
|
const payload = await c.req.json();
|
|
257
|
-
const
|
|
257
|
+
const transformed = await connector.transform(payload, {
|
|
258
258
|
db,
|
|
259
259
|
logger,
|
|
260
260
|
transport: "gateway",
|
|
261
261
|
});
|
|
262
|
-
if (!
|
|
262
|
+
if (!transformed) return c.json({ ok: true, skipped: true }, 200);
|
|
263
|
+
// A transform may fan one dispatch into an ARRAY (dual-side reactions);
|
|
264
|
+
// each element ingests independently.
|
|
265
|
+
if (Array.isArray(transformed)) {
|
|
266
|
+
const { exits } = await ingestTransformResult({
|
|
267
|
+
result: transformed,
|
|
268
|
+
db,
|
|
269
|
+
registry,
|
|
270
|
+
hatchet,
|
|
271
|
+
logger,
|
|
272
|
+
source: "connector",
|
|
273
|
+
});
|
|
274
|
+
return c.json(
|
|
275
|
+
{ ok: true, event: transformed[0]?.event ?? "batch", exits },
|
|
276
|
+
200,
|
|
277
|
+
);
|
|
278
|
+
}
|
|
279
|
+
const event = transformed;
|
|
263
280
|
const result = await ingestEvent({
|
|
264
281
|
db,
|
|
265
282
|
registry,
|
|
266
283
|
hatchet,
|
|
267
284
|
logger,
|
|
268
|
-
event,
|
|
285
|
+
event: { ...event, source: "connector" },
|
|
269
286
|
});
|
|
270
287
|
// INTENTIONALLY `result.exits.length` (a number) — a deliberate
|
|
271
288
|
// divergence from the `/v1/webhooks/:sourceId` route, which returns the
|
|
@@ -108,6 +108,8 @@ export const eventsRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
108
108
|
// §2.5: caller-supplied event time (backfill/replay). The validated ISO
|
|
109
109
|
// string is coerced to a Date inside ingestEvent.
|
|
110
110
|
occurredAt: body.timestamp,
|
|
111
|
+
// Public data-plane ingest (the `@hogsend/client` SDK / your app code).
|
|
112
|
+
source: "api",
|
|
111
113
|
},
|
|
112
114
|
});
|
|
113
115
|
|
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import { emailSends, linkClicks, trackedLinks } from "@hogsend/db";
|
|
1
|
+
import { emailSends, linkClicks, links, trackedLinks } from "@hogsend/db";
|
|
2
2
|
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
3
|
import { and, eq, isNull, sql } from "drizzle-orm";
|
|
4
4
|
import type { AppEnv } from "../../app.js";
|
|
5
|
+
import { isBotOrPrefetch } from "../../lib/bot-prefetch.js";
|
|
5
6
|
import { generateIdentityToken } from "../../lib/identity-token.js";
|
|
6
7
|
import { emitOutbound } from "../../lib/outbound.js";
|
|
7
8
|
import { EMAIL_LINK_CLICKED } from "../../lib/tracking-event-names.js";
|
|
8
9
|
import {
|
|
10
|
+
pushLinkClickEvent,
|
|
9
11
|
pushTrackingEvent,
|
|
10
12
|
resolveEmailSendContext,
|
|
11
13
|
} from "../../lib/tracking-events.js";
|
|
@@ -42,8 +44,16 @@ export const clickRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
42
44
|
source: trackedLinks.source,
|
|
43
45
|
event: trackedLinks.event,
|
|
44
46
|
eventProperties: trackedLinks.eventProperties,
|
|
47
|
+
// Managed-link provenance (NULL for email-rewritten links, whose
|
|
48
|
+
// `tracked_links.link_id` is NULL). `linkId` is the durable `links.id`
|
|
49
|
+
// a journey filters on; it rides ONLY the bus re-ingest (A5), never the
|
|
50
|
+
// unchanged outbound payloads which keep `trackedLinks.id` as `linkId`.
|
|
51
|
+
linkId: links.id,
|
|
52
|
+
campaign: links.campaign,
|
|
53
|
+
linkType: links.type,
|
|
45
54
|
})
|
|
46
55
|
.from(trackedLinks)
|
|
56
|
+
.leftJoin(links, eq(trackedLinks.linkId, links.id))
|
|
47
57
|
.where(eq(trackedLinks.id, id))
|
|
48
58
|
.limit(1);
|
|
49
59
|
|
|
@@ -58,6 +68,18 @@ export const clickRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
58
68
|
null;
|
|
59
69
|
const userAgent = c.req.header("user-agent") ?? null;
|
|
60
70
|
|
|
71
|
+
// Unfurl-bot / prefetch detection. A link-preview bot (Discord/Slack/…)
|
|
72
|
+
// auto-fetches a DM'd link BEFORE a human clicks; we still record the click
|
|
73
|
+
// + 302, but must NOT re-ingest it onto the journey bus (it would phantom
|
|
74
|
+
// enroll the recipient). Affects ONLY the non-email bus re-ingest below.
|
|
75
|
+
const isBot = isBotOrPrefetch({
|
|
76
|
+
userAgent,
|
|
77
|
+
purpose: c.req.header("purpose"),
|
|
78
|
+
xPurpose: c.req.header("x-purpose"),
|
|
79
|
+
xMozPrefetch: c.req.header("x-moz"),
|
|
80
|
+
secPurpose: c.req.header("sec-purpose"),
|
|
81
|
+
});
|
|
82
|
+
|
|
61
83
|
// The linkClicks insert + clickCount increment stay UNCONDITIONAL — every
|
|
62
84
|
// tracked link counts clicks, email or not.
|
|
63
85
|
//
|
|
@@ -251,6 +273,34 @@ export const clickRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
251
273
|
at: new Date().toISOString(),
|
|
252
274
|
},
|
|
253
275
|
}).catch(logger.warn);
|
|
276
|
+
|
|
277
|
+
// BUS re-ingest: the first-party `link.clicked` event for a NON-email
|
|
278
|
+
// managed link, so journeys can trigger / `ctx.waitForEvent` on a click of
|
|
279
|
+
// a SPECIFIC managed link (filter by `linkId`/`campaign`). GATED on:
|
|
280
|
+
// • `!isBot` — an unfurl/prefetch bot fetched the link, not a human.
|
|
281
|
+
// • `link.distinctId` — a broadcast/public link carries no person, and
|
|
282
|
+
// `resolveOrCreateContact` THROWS on a zero-key event. Personal links
|
|
283
|
+
// only; the resolved subject is the survivor contact (not the raw
|
|
284
|
+
// mint key). Per-second idempotencyKey dedupes the userEvents INSERT on
|
|
285
|
+
// retry only — `entryLimit` is the journey-level throttle.
|
|
286
|
+
if (!isBot && link.distinctId) {
|
|
287
|
+
void pushLinkClickEvent({
|
|
288
|
+
db,
|
|
289
|
+
hatchet,
|
|
290
|
+
registry,
|
|
291
|
+
logger,
|
|
292
|
+
linkId: link.linkId ?? null,
|
|
293
|
+
trackedLinkId: link.id,
|
|
294
|
+
campaign: link.campaign ?? null,
|
|
295
|
+
source: link.source ?? null,
|
|
296
|
+
linkType: link.linkType ?? null,
|
|
297
|
+
linkUrl: link.originalUrl,
|
|
298
|
+
distinctId: link.distinctId,
|
|
299
|
+
idempotencyKey: `link:click:${link.id}:${Math.floor(
|
|
300
|
+
Date.now() / 1000,
|
|
301
|
+
)}`,
|
|
302
|
+
}).catch(logger.warn);
|
|
303
|
+
}
|
|
254
304
|
}
|
|
255
305
|
|
|
256
306
|
return c.redirect(redirectUrl, 302);
|
|
@@ -3,7 +3,7 @@ import { createRoute, type OpenAPIHono, z } from "@hono/zod-openapi";
|
|
|
3
3
|
import type { AppEnv } from "../../app.js";
|
|
4
4
|
import type { DefinedConnector } from "../../connectors/define-connector.js";
|
|
5
5
|
import { headersToRecord } from "../../lib/headers.js";
|
|
6
|
-
import { ingestEvent } from "../../lib/ingestion.js";
|
|
6
|
+
import { ingestEvent, ingestTransformResult } from "../../lib/ingestion.js";
|
|
7
7
|
import type { Logger } from "../../lib/logger.js";
|
|
8
8
|
import { getDerivedCredential } from "../../lib/provider-credentials.js";
|
|
9
9
|
import { verifySignature } from "../../webhook-sources/verify.js";
|
|
@@ -224,19 +224,46 @@ export function registerWebhookSourceRoutes(
|
|
|
224
224
|
payload = parsed.data;
|
|
225
225
|
}
|
|
226
226
|
|
|
227
|
-
const
|
|
227
|
+
const transformed = await source.transform(payload, {
|
|
228
228
|
db,
|
|
229
229
|
logger,
|
|
230
230
|
transport: "webhook",
|
|
231
231
|
rawBody,
|
|
232
232
|
headers,
|
|
233
233
|
});
|
|
234
|
-
if (!
|
|
234
|
+
if (!transformed) {
|
|
235
235
|
logger.info("Webhook event skipped", { source: sourceId });
|
|
236
236
|
return c.json({ ok: true, skipped: true });
|
|
237
237
|
}
|
|
238
|
+
// A transform MAY fan into an array (contract parity with connectors); no
|
|
239
|
+
// webhook source does today, but handle it so the union stays honest.
|
|
240
|
+
if (Array.isArray(transformed)) {
|
|
241
|
+
const { exits } = await ingestTransformResult({
|
|
242
|
+
result: transformed,
|
|
243
|
+
db,
|
|
244
|
+
registry,
|
|
245
|
+
hatchet,
|
|
246
|
+
logger,
|
|
247
|
+
source: sourceId,
|
|
248
|
+
});
|
|
249
|
+
return c.json({
|
|
250
|
+
ok: true,
|
|
251
|
+
event: transformed[0]?.event ?? "batch",
|
|
252
|
+
userId: transformed[0]?.userId,
|
|
253
|
+
exits,
|
|
254
|
+
});
|
|
255
|
+
}
|
|
256
|
+
const event = transformed;
|
|
238
257
|
|
|
239
|
-
|
|
258
|
+
// Stamp the origin with the webhook source id ("posthog", "stripe", …) so
|
|
259
|
+
// the Events feed can show + filter where it came from.
|
|
260
|
+
const result = await ingestEvent({
|
|
261
|
+
db,
|
|
262
|
+
registry,
|
|
263
|
+
hatchet,
|
|
264
|
+
logger,
|
|
265
|
+
event: { ...event, source: sourceId },
|
|
266
|
+
});
|
|
240
267
|
|
|
241
268
|
return c.json({
|
|
242
269
|
ok: true,
|