@hogsend/engine 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +93 -0
- package/README.md +18 -0
- package/package.json +58 -0
- package/src/app.ts +102 -0
- package/src/container.ts +172 -0
- package/src/env.ts +56 -0
- package/src/index.ts +114 -0
- package/src/journeys/define-journey.ts +188 -0
- package/src/journeys/journey-context.ts +179 -0
- package/src/journeys/registry-singleton.ts +21 -0
- package/src/journeys/registry.ts +53 -0
- package/src/lib/alerting.ts +205 -0
- package/src/lib/api-key-hash.ts +19 -0
- package/src/lib/auth.ts +39 -0
- package/src/lib/backfill.ts +84 -0
- package/src/lib/contacts.ts +68 -0
- package/src/lib/db.ts +13 -0
- package/src/lib/email-service-types.ts +115 -0
- package/src/lib/email-stats.ts +33 -0
- package/src/lib/email.ts +94 -0
- package/src/lib/enrollment-guards.ts +56 -0
- package/src/lib/hatchet.ts +20 -0
- package/src/lib/html.ts +25 -0
- package/src/lib/ingestion.ts +162 -0
- package/src/lib/logger.ts +32 -0
- package/src/lib/mailer.ts +266 -0
- package/src/lib/notifications.ts +61 -0
- package/src/lib/posthog.ts +19 -0
- package/src/lib/redis.ts +30 -0
- package/src/lib/schemas.ts +8 -0
- package/src/lib/tracked.ts +175 -0
- package/src/lib/tracking-event-names.ts +5 -0
- package/src/lib/tracking-events.ts +84 -0
- package/src/lib/tracking.ts +78 -0
- package/src/middleware/api-key.ts +129 -0
- package/src/middleware/audit.ts +47 -0
- package/src/middleware/auth.ts +24 -0
- package/src/middleware/error-handler.ts +22 -0
- package/src/middleware/rate-limit.ts +65 -0
- package/src/middleware/request-logger.ts +19 -0
- package/src/routes/admin/alerts.ts +347 -0
- package/src/routes/admin/api-keys.ts +211 -0
- package/src/routes/admin/audit-logs.ts +102 -0
- package/src/routes/admin/bulk.ts +503 -0
- package/src/routes/admin/contacts.ts +342 -0
- package/src/routes/admin/dlq.ts +202 -0
- package/src/routes/admin/emails.ts +269 -0
- package/src/routes/admin/events.ts +132 -0
- package/src/routes/admin/index.ts +36 -0
- package/src/routes/admin/journey-logs.ts +117 -0
- package/src/routes/admin/journeys.ts +677 -0
- package/src/routes/admin/metrics.ts +559 -0
- package/src/routes/admin/preferences.ts +165 -0
- package/src/routes/admin/timeline.ts +221 -0
- package/src/routes/email/index.ts +8 -0
- package/src/routes/email/preferences.ts +144 -0
- package/src/routes/email/unsubscribe.ts +161 -0
- package/src/routes/health.ts +131 -0
- package/src/routes/index.ts +32 -0
- package/src/routes/ingest.ts +71 -0
- package/src/routes/tracking/click.ts +103 -0
- package/src/routes/tracking/index.ts +9 -0
- package/src/routes/tracking/open.ts +71 -0
- package/src/routes/webhooks/index.ts +17 -0
- package/src/routes/webhooks/resend.ts +68 -0
- package/src/routes/webhooks/sources.ts +97 -0
- package/src/webhook-sources/define-webhook-source.ts +34 -0
- package/src/worker.ts +64 -0
- package/src/workflows/check-alerts.ts +24 -0
- package/src/workflows/import-contacts.ts +134 -0
- package/src/workflows/send-email.ts +54 -0
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
import type { Database } from "@hogsend/db";
|
|
2
|
+
import { trackedLinks } from "@hogsend/db";
|
|
3
|
+
|
|
4
|
+
const HREF_RE = /href="(https?:\/\/[^"]+)"/gi;
|
|
5
|
+
|
|
6
|
+
const SKIP_PATTERNS = ["/v1/email/unsubscribe", "/v1/email/preferences"];
|
|
7
|
+
|
|
8
|
+
function shouldSkipUrl(url: string): boolean {
|
|
9
|
+
return SKIP_PATTERNS.some((pattern) => url.includes(pattern));
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export async function rewriteLinks(opts: {
|
|
13
|
+
html: string;
|
|
14
|
+
emailSendId: string;
|
|
15
|
+
baseUrl: string;
|
|
16
|
+
db: Database;
|
|
17
|
+
}): Promise<string> {
|
|
18
|
+
const { html, emailSendId, baseUrl, db } = opts;
|
|
19
|
+
|
|
20
|
+
const uniqueUrls = new Set<string>();
|
|
21
|
+
|
|
22
|
+
for (const match of html.matchAll(HREF_RE)) {
|
|
23
|
+
const url = match[1];
|
|
24
|
+
if (url && !shouldSkipUrl(url)) {
|
|
25
|
+
uniqueUrls.add(url);
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
if (uniqueUrls.size === 0) return html;
|
|
30
|
+
|
|
31
|
+
const urlList = [...uniqueUrls];
|
|
32
|
+
const rows = await db
|
|
33
|
+
.insert(trackedLinks)
|
|
34
|
+
.values(urlList.map((url) => ({ emailSendId, originalUrl: url })))
|
|
35
|
+
.returning({ id: trackedLinks.id, originalUrl: trackedLinks.originalUrl });
|
|
36
|
+
|
|
37
|
+
const urlToId = new Map<string, string>();
|
|
38
|
+
for (const row of rows) {
|
|
39
|
+
urlToId.set(row.originalUrl, row.id);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
return html.replace(HREF_RE, (full, url: string) => {
|
|
43
|
+
if (shouldSkipUrl(url)) return full;
|
|
44
|
+
const linkId = urlToId.get(url);
|
|
45
|
+
return linkId ? `href="${baseUrl}/v1/t/c/${linkId}"` : full;
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function injectOpenPixel(opts: {
|
|
50
|
+
html: string;
|
|
51
|
+
emailSendId: string;
|
|
52
|
+
baseUrl: string;
|
|
53
|
+
}): string {
|
|
54
|
+
const { html, emailSendId, baseUrl } = opts;
|
|
55
|
+
const pixel = `<img src="${baseUrl}/v1/t/o/${emailSendId}" width="1" height="1" alt="" style="display:none" />`;
|
|
56
|
+
|
|
57
|
+
const bodyCloseIdx = html.lastIndexOf("</body>");
|
|
58
|
+
if (bodyCloseIdx !== -1) {
|
|
59
|
+
return html.slice(0, bodyCloseIdx) + pixel + html.slice(bodyCloseIdx);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return html + pixel;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export async function prepareTrackedHtml(opts: {
|
|
66
|
+
html: string;
|
|
67
|
+
emailSendId: string;
|
|
68
|
+
baseUrl: string;
|
|
69
|
+
db: Database;
|
|
70
|
+
}): Promise<string> {
|
|
71
|
+
let result = await rewriteLinks(opts);
|
|
72
|
+
result = injectOpenPixel({
|
|
73
|
+
html: result,
|
|
74
|
+
emailSendId: opts.emailSendId,
|
|
75
|
+
baseUrl: opts.baseUrl,
|
|
76
|
+
});
|
|
77
|
+
return result;
|
|
78
|
+
}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { apiKeys } from "@hogsend/db";
|
|
2
|
+
import { and, eq, isNull } from "drizzle-orm";
|
|
3
|
+
import { createMiddleware } from "hono/factory";
|
|
4
|
+
import type { AppEnv } from "../app.js";
|
|
5
|
+
import { hashApiKey } from "../lib/api-key-hash.js";
|
|
6
|
+
|
|
7
|
+
export interface ApiKeyContext {
|
|
8
|
+
id: string;
|
|
9
|
+
name: string;
|
|
10
|
+
scopes: string[];
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const SCOPE_HIERARCHY: Record<string, number> = {
|
|
14
|
+
read: 0,
|
|
15
|
+
"journey-admin": 1,
|
|
16
|
+
"full-admin": 2,
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
const KEY_CACHE = new Map<
|
|
20
|
+
string,
|
|
21
|
+
{
|
|
22
|
+
data: ApiKeyContext & { expiresAt: Date | null; lastUsedAt: Date | null };
|
|
23
|
+
cachedAt: number;
|
|
24
|
+
}
|
|
25
|
+
>();
|
|
26
|
+
const CACHE_TTL = 60_000;
|
|
27
|
+
const LAST_USED_DEBOUNCE = 5 * 60_000;
|
|
28
|
+
|
|
29
|
+
export const requireApiKey = createMiddleware<AppEnv>(async (c, next) => {
|
|
30
|
+
const { env, db } = c.get("container");
|
|
31
|
+
|
|
32
|
+
const header = c.req.header("authorization");
|
|
33
|
+
const provided = header?.startsWith("Bearer ") ? header.slice(7) : undefined;
|
|
34
|
+
|
|
35
|
+
if (!provided) {
|
|
36
|
+
return c.json({ error: "Unauthorized" }, 401);
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
if (env.ADMIN_API_KEY && provided === env.ADMIN_API_KEY) {
|
|
40
|
+
c.set("apiKey", {
|
|
41
|
+
id: "legacy",
|
|
42
|
+
name: "legacy",
|
|
43
|
+
scopes: ["full-admin"],
|
|
44
|
+
});
|
|
45
|
+
return next();
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const keyHash = hashApiKey(provided);
|
|
49
|
+
|
|
50
|
+
const cached = KEY_CACHE.get(keyHash);
|
|
51
|
+
if (cached && Date.now() - cached.cachedAt < CACHE_TTL) {
|
|
52
|
+
if (cached.data.expiresAt && cached.data.expiresAt < new Date()) {
|
|
53
|
+
return c.json({ error: "API key expired" }, 401);
|
|
54
|
+
}
|
|
55
|
+
c.set("apiKey", {
|
|
56
|
+
id: cached.data.id,
|
|
57
|
+
name: cached.data.name,
|
|
58
|
+
scopes: cached.data.scopes,
|
|
59
|
+
});
|
|
60
|
+
return next();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const rows = await db
|
|
64
|
+
.select()
|
|
65
|
+
.from(apiKeys)
|
|
66
|
+
.where(and(eq(apiKeys.keyHash, keyHash), isNull(apiKeys.revokedAt)))
|
|
67
|
+
.limit(1);
|
|
68
|
+
|
|
69
|
+
const key = rows[0];
|
|
70
|
+
if (!key) {
|
|
71
|
+
return c.json({ error: "Unauthorized" }, 401);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
if (key.expiresAt && key.expiresAt < new Date()) {
|
|
75
|
+
return c.json({ error: "API key expired" }, 401);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const keyContext: ApiKeyContext = {
|
|
79
|
+
id: key.id,
|
|
80
|
+
name: key.name,
|
|
81
|
+
scopes: key.scopes as string[],
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
c.set("apiKey", keyContext);
|
|
85
|
+
|
|
86
|
+
KEY_CACHE.set(keyHash, {
|
|
87
|
+
data: {
|
|
88
|
+
...keyContext,
|
|
89
|
+
expiresAt: key.expiresAt,
|
|
90
|
+
lastUsedAt: key.lastUsedAt,
|
|
91
|
+
},
|
|
92
|
+
cachedAt: Date.now(),
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
const shouldUpdateLastUsed =
|
|
96
|
+
!key.lastUsedAt ||
|
|
97
|
+
Date.now() - key.lastUsedAt.getTime() > LAST_USED_DEBOUNCE;
|
|
98
|
+
|
|
99
|
+
if (shouldUpdateLastUsed) {
|
|
100
|
+
db.update(apiKeys)
|
|
101
|
+
.set({ lastUsedAt: new Date() })
|
|
102
|
+
.where(eq(apiKeys.id, key.id))
|
|
103
|
+
.then(() => {})
|
|
104
|
+
.catch(() => {});
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
return next();
|
|
108
|
+
});
|
|
109
|
+
|
|
110
|
+
export function requireScope(scope: string) {
|
|
111
|
+
const required = SCOPE_HIERARCHY[scope] ?? 0;
|
|
112
|
+
|
|
113
|
+
return createMiddleware<AppEnv>(async (c, next) => {
|
|
114
|
+
const apiKey = c.get("apiKey");
|
|
115
|
+
if (!apiKey) {
|
|
116
|
+
return c.json({ error: "Unauthorized" }, 401);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
const maxScope = Math.max(
|
|
120
|
+
...apiKey.scopes.map((s: string) => SCOPE_HIERARCHY[s] ?? 0),
|
|
121
|
+
);
|
|
122
|
+
|
|
123
|
+
if (maxScope < required) {
|
|
124
|
+
return c.json({ error: "Forbidden: insufficient scope" }, 403);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
return next();
|
|
128
|
+
});
|
|
129
|
+
}
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { auditLogs } from "@hogsend/db";
|
|
2
|
+
import { createMiddleware } from "hono/factory";
|
|
3
|
+
import type { AppEnv } from "../app.js";
|
|
4
|
+
|
|
5
|
+
const MUTATION_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]);
|
|
6
|
+
|
|
7
|
+
function extractResource(path: string): {
|
|
8
|
+
resource: string;
|
|
9
|
+
resourceId: string | null;
|
|
10
|
+
} {
|
|
11
|
+
const parts = path
|
|
12
|
+
.replace(/^\/v1\/admin\//, "")
|
|
13
|
+
.split("/")
|
|
14
|
+
.filter(Boolean);
|
|
15
|
+
|
|
16
|
+
const resource = parts[0] ?? "unknown";
|
|
17
|
+
const resourceId = parts[1] ?? null;
|
|
18
|
+
return { resource, resourceId };
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export const auditMiddleware = createMiddleware<AppEnv>(async (c, next) => {
|
|
22
|
+
await next();
|
|
23
|
+
|
|
24
|
+
if (!MUTATION_METHODS.has(c.req.method)) return;
|
|
25
|
+
if (c.res.status >= 400) return;
|
|
26
|
+
|
|
27
|
+
const apiKey = c.get("apiKey");
|
|
28
|
+
const { db, logger } = c.get("container");
|
|
29
|
+
const { resource, resourceId } = extractResource(c.req.path);
|
|
30
|
+
|
|
31
|
+
db.insert(auditLogs)
|
|
32
|
+
.values({
|
|
33
|
+
actor: apiKey?.name ?? "unknown",
|
|
34
|
+
actorKeyId: apiKey?.id && apiKey.id !== "legacy" ? apiKey.id : null,
|
|
35
|
+
action: `${resource}.${c.req.method.toLowerCase()}`,
|
|
36
|
+
resource,
|
|
37
|
+
resourceId,
|
|
38
|
+
ipAddress:
|
|
39
|
+
c.req.header("x-forwarded-for") ?? c.req.header("x-real-ip") ?? null,
|
|
40
|
+
})
|
|
41
|
+
.then(() => {})
|
|
42
|
+
.catch((err: unknown) => {
|
|
43
|
+
logger.warn("Audit log write failed", {
|
|
44
|
+
error: err instanceof Error ? err.message : String(err),
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
});
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { Context } from "hono";
|
|
2
|
+
import { createMiddleware } from "hono/factory";
|
|
3
|
+
import type { AppEnv } from "../app.js";
|
|
4
|
+
|
|
5
|
+
async function resolveSession(c: Context<AppEnv>) {
|
|
6
|
+
const { auth } = c.get("container");
|
|
7
|
+
const session = await auth.api.getSession({ headers: c.req.raw.headers });
|
|
8
|
+
c.set("user", session?.user ?? null);
|
|
9
|
+
c.set("session", session?.session ?? null);
|
|
10
|
+
return session;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export const sessionMiddleware = createMiddleware<AppEnv>(async (c, next) => {
|
|
14
|
+
await resolveSession(c);
|
|
15
|
+
return next();
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
export const requireAuth = createMiddleware<AppEnv>(async (c, next) => {
|
|
19
|
+
const session = await resolveSession(c);
|
|
20
|
+
if (!session) {
|
|
21
|
+
return c.json({ error: "Unauthorized" }, 401);
|
|
22
|
+
}
|
|
23
|
+
return next();
|
|
24
|
+
});
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { ErrorHandler } from "hono";
|
|
2
|
+
import type { ContentfulStatusCode } from "hono/utils/http-status";
|
|
3
|
+
import type { AppEnv } from "../app.js";
|
|
4
|
+
|
|
5
|
+
export const errorHandler: ErrorHandler<AppEnv> = (err, c) => {
|
|
6
|
+
const logger = c.get("container").logger;
|
|
7
|
+
|
|
8
|
+
const status: ContentfulStatusCode =
|
|
9
|
+
"status" in err && typeof err.status === "number"
|
|
10
|
+
? (err.status as ContentfulStatusCode)
|
|
11
|
+
: 500;
|
|
12
|
+
const message = status === 500 ? "Internal Server Error" : err.message;
|
|
13
|
+
|
|
14
|
+
logger.error(err.message, {
|
|
15
|
+
stack: err.stack,
|
|
16
|
+
path: c.req.path,
|
|
17
|
+
method: c.req.method,
|
|
18
|
+
status,
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
return c.json({ error: message }, status);
|
|
22
|
+
};
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { createMiddleware } from "hono/factory";
|
|
2
|
+
import type { AppEnv } from "../app.js";
|
|
3
|
+
import { getRedis } from "../lib/redis.js";
|
|
4
|
+
|
|
5
|
+
const WINDOW_MS = 60_000;
|
|
6
|
+
const MAX_REQUESTS = 100;
|
|
7
|
+
|
|
8
|
+
const memoryStore = new Map<string, number[]>();
|
|
9
|
+
let cleanupCounter = 0;
|
|
10
|
+
const MAX_KEYS = 10_000;
|
|
11
|
+
|
|
12
|
+
export const rateLimit = createMiddleware<AppEnv>(async (c, next) => {
|
|
13
|
+
if (process.env.NODE_ENV === "test") return next();
|
|
14
|
+
|
|
15
|
+
const apiKey = c.get("apiKey");
|
|
16
|
+
const keyId = apiKey?.id ?? "anonymous";
|
|
17
|
+
const now = Date.now();
|
|
18
|
+
|
|
19
|
+
let count: number;
|
|
20
|
+
|
|
21
|
+
try {
|
|
22
|
+
const redis = getRedis();
|
|
23
|
+
if (redis) {
|
|
24
|
+
const windowKey = `ratelimit:${keyId}`;
|
|
25
|
+
const pipeline = redis.pipeline();
|
|
26
|
+
pipeline.zremrangebyscore(windowKey, 0, now - WINDOW_MS);
|
|
27
|
+
pipeline.zcard(windowKey);
|
|
28
|
+
pipeline.zadd(windowKey, now, `${now}:${Math.random()}`);
|
|
29
|
+
pipeline.expire(windowKey, Math.ceil(WINDOW_MS / 1000));
|
|
30
|
+
|
|
31
|
+
const results = await pipeline.exec();
|
|
32
|
+
count = (results?.[1]?.[1] as number) ?? 0;
|
|
33
|
+
} else {
|
|
34
|
+
throw new Error("No Redis");
|
|
35
|
+
}
|
|
36
|
+
} catch {
|
|
37
|
+
const entries = memoryStore.get(keyId) ?? [];
|
|
38
|
+
const cutoff = now - WINDOW_MS;
|
|
39
|
+
const valid = entries.filter((t) => t > cutoff);
|
|
40
|
+
valid.push(now);
|
|
41
|
+
memoryStore.set(keyId, valid);
|
|
42
|
+
count = valid.length - 1;
|
|
43
|
+
|
|
44
|
+
if (++cleanupCounter % 100 === 0 && memoryStore.size > MAX_KEYS) {
|
|
45
|
+
const sweepCutoff = now - WINDOW_MS;
|
|
46
|
+
for (const [key, entries] of memoryStore) {
|
|
47
|
+
const active = entries.filter((t) => t > sweepCutoff);
|
|
48
|
+
if (active.length === 0) memoryStore.delete(key);
|
|
49
|
+
else memoryStore.set(key, active);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
c.header("X-RateLimit-Limit", String(MAX_REQUESTS));
|
|
55
|
+
c.header(
|
|
56
|
+
"X-RateLimit-Remaining",
|
|
57
|
+
String(Math.max(0, MAX_REQUESTS - count - 1)),
|
|
58
|
+
);
|
|
59
|
+
|
|
60
|
+
if (count >= MAX_REQUESTS) {
|
|
61
|
+
return c.json({ error: "Rate limit exceeded" }, 429);
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return next();
|
|
65
|
+
});
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import type { MiddlewareHandler } from "hono";
|
|
2
|
+
import type { AppEnv } from "../app.js";
|
|
3
|
+
|
|
4
|
+
export const requestLogger: MiddlewareHandler<AppEnv> = async (c, next) => {
|
|
5
|
+
const start = Date.now();
|
|
6
|
+
|
|
7
|
+
await next();
|
|
8
|
+
|
|
9
|
+
const duration = Date.now() - start;
|
|
10
|
+
const logger = c.get("container").logger;
|
|
11
|
+
|
|
12
|
+
logger.http("request", {
|
|
13
|
+
method: c.req.method,
|
|
14
|
+
path: c.req.path,
|
|
15
|
+
status: c.res.status,
|
|
16
|
+
duration,
|
|
17
|
+
requestId: c.get("requestId"),
|
|
18
|
+
});
|
|
19
|
+
};
|