@hogsend/engine 0.40.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -8
- package/src/buckets/check-membership.ts +6 -1
- package/src/campaigns/cohort-sql.ts +198 -0
- package/src/campaigns/define-campaign.ts +207 -28
- package/src/campaigns/reconcile.ts +105 -11
- package/src/campaigns/steps.ts +48 -0
- package/src/connectors/define-action.ts +70 -0
- package/src/container.ts +124 -42
- package/src/destinations/presets/posthog.ts +7 -1
- package/src/index.ts +96 -2
- package/src/journeys/constants.ts +10 -0
- package/src/journeys/define-journey.ts +14 -405
- package/src/journeys/execute-journey-run.ts +641 -0
- package/src/journeys/graph/build-graph.ts +212 -56
- package/src/journeys/journey-boundary.ts +54 -0
- package/src/journeys/journey-context.ts +406 -59
- package/src/journeys/record-once.ts +110 -0
- package/src/lib/blueprint-lock.ts +21 -0
- package/src/lib/blueprints.ts +803 -0
- package/src/lib/campaign-send-key.ts +58 -0
- package/src/lib/connector-actions.ts +203 -19
- package/src/lib/contacts.ts +54 -0
- package/src/lib/email-service-types.ts +4 -1
- package/src/lib/email.ts +13 -2
- package/src/lib/event-names.ts +143 -0
- package/src/lib/feed.ts +77 -57
- package/src/lib/frequency-cap.ts +45 -15
- package/src/lib/hatchet-duration.ts +22 -0
- package/src/lib/html.ts +1 -0
- package/src/lib/ingestion.ts +145 -4
- package/src/lib/journey-suppress.ts +87 -0
- package/src/lib/links.ts +204 -18
- package/src/lib/outbound.ts +20 -0
- package/src/lib/preferences.ts +80 -1
- package/src/lib/tracked.ts +44 -29
- package/src/lib/tracking-event-names.ts +8 -0
- package/src/lib/tracking.ts +1 -5
- package/src/lib/webhook-signing.ts +6 -1
- package/src/lists/channels.ts +75 -0
- package/src/lists/define-list.ts +34 -0
- package/src/lists/registry.ts +29 -2
- package/src/mcp/authoring-guide.ts +160 -0
- package/src/mcp/blueprint-tools.ts +390 -0
- package/src/middleware/require-admin.ts +13 -0
- package/src/routes/admin/api-keys.ts +72 -0
- package/src/routes/admin/blueprints.ts +795 -0
- package/src/routes/admin/campaigns.ts +179 -3
- package/src/routes/admin/emails.ts +13 -0
- package/src/routes/admin/events.ts +55 -0
- package/src/routes/admin/index.ts +4 -0
- package/src/routes/admin/journeys.ts +4 -2
- package/src/routes/admin/links.ts +303 -36
- package/src/routes/admin/lists.ts +51 -0
- package/src/routes/campaigns/index.ts +44 -6
- package/src/routes/email/preferences.ts +50 -13
- package/src/routes/feed/recipient.ts +2 -43
- package/src/routes/index.ts +6 -0
- package/src/routes/lists/index.ts +138 -18
- package/src/routes/tracking/arrive.ts +260 -0
- package/src/routes/tracking/click-pipeline.ts +329 -0
- package/src/routes/tracking/click.ts +5 -270
- package/src/routes/tracking/index.ts +2 -0
- package/src/routes/tracking/vanity.ts +64 -0
- package/src/worker.ts +5 -0
- package/src/workflows/bucket-backfill.ts +4 -2
- package/src/workflows/bucket-reconcile.ts +11 -4
- package/src/workflows/check-alerts.ts +130 -1
- package/src/workflows/journey-blueprint-interpreter.ts +656 -0
- package/src/workflows/send-campaign.ts +574 -180
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { links, trackedLinks } from "@hogsend/db";
|
|
2
|
+
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
+
import { and, asc, eq } from "drizzle-orm";
|
|
4
|
+
import type { AppEnv } from "../../app.js";
|
|
5
|
+
import { canonicalTrackedRowFilter, normalizeSlug } from "../../lib/links.js";
|
|
6
|
+
import { clickSelection, handleTrackedClick } from "./click-pipeline.js";
|
|
7
|
+
|
|
8
|
+
// The vanity short path — `/l/:slug` layered over `/v1/t/c/:id`. Mounted at
|
|
9
|
+
// the APP ROOT (not under /v1/t) so the operator-facing URL stays short; the
|
|
10
|
+
// `/l` prefix collides with nothing (`/v1`, `/studio`, `/docs`, `/api/auth`,
|
|
11
|
+
// `/connectors`, webhooks).
|
|
12
|
+
const vanityRoute = createRoute({
|
|
13
|
+
method: "get",
|
|
14
|
+
path: "/l/:slug",
|
|
15
|
+
tags: ["Tracking"],
|
|
16
|
+
summary: "Vanity link redirect",
|
|
17
|
+
request: {
|
|
18
|
+
params: z.object({
|
|
19
|
+
slug: z.string(),
|
|
20
|
+
}),
|
|
21
|
+
},
|
|
22
|
+
responses: {
|
|
23
|
+
302: { description: "Redirect to original URL" },
|
|
24
|
+
},
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
export const vanityRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
28
|
+
vanityRoute,
|
|
29
|
+
async (c) => {
|
|
30
|
+
const { slug } = c.req.valid("param");
|
|
31
|
+
const { db, env } = c.get("container");
|
|
32
|
+
|
|
33
|
+
// Tolerate cased/typed-by-hand input (`/l/Black-Friday`): slugs are stored
|
|
34
|
+
// normalized, so normalize the inbound too. A malformed slug can't match
|
|
35
|
+
// anything — treat it like an unknown link (redirect home), mirroring the
|
|
36
|
+
// UUID route's missing-row behavior.
|
|
37
|
+
let normalized: string;
|
|
38
|
+
try {
|
|
39
|
+
normalized = normalizeSlug(slug);
|
|
40
|
+
} catch {
|
|
41
|
+
return c.redirect(env.API_PUBLIC_URL, 302);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
// Resolve the link's CANONICAL redirect row: a managed link has one
|
|
45
|
+
// tracked row minted alongside it, plus (later) a per-link QR scan row
|
|
46
|
+
// (`source: "qr"`) — the vanity path must never attribute to the QR row.
|
|
47
|
+
// Archived links keep resolving (same as the UUID route, which never
|
|
48
|
+
// checks archived_at) — a printed/shared slug should not 404 on archive.
|
|
49
|
+
const rows = await db
|
|
50
|
+
.select(clickSelection)
|
|
51
|
+
.from(links)
|
|
52
|
+
.innerJoin(trackedLinks, eq(trackedLinks.linkId, links.id))
|
|
53
|
+
.where(and(eq(links.slug, normalized), canonicalTrackedRowFilter()))
|
|
54
|
+
.orderBy(asc(trackedLinks.createdAt))
|
|
55
|
+
.limit(1);
|
|
56
|
+
|
|
57
|
+
const link = rows[0];
|
|
58
|
+
if (!link) {
|
|
59
|
+
return c.redirect(env.API_PUBLIC_URL, 302);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return handleTrackedClick(c, link);
|
|
63
|
+
},
|
|
64
|
+
);
|
package/src/worker.ts
CHANGED
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
} from "./workflows/deliver-webhook.js";
|
|
31
31
|
import { importContactsTask } from "./workflows/import-contacts.js";
|
|
32
32
|
import { importSuppressionsTask } from "./workflows/import-suppressions.js";
|
|
33
|
+
import { journeyBlueprintInterpreter } from "./workflows/journey-blueprint-interpreter.js";
|
|
33
34
|
import {
|
|
34
35
|
reapStuckCampaignsTask,
|
|
35
36
|
sendCampaignTask,
|
|
@@ -114,6 +115,10 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
|
|
|
114
115
|
sendFeedTask,
|
|
115
116
|
importContactsTask,
|
|
116
117
|
importSuppressionsTask,
|
|
118
|
+
// The generic blueprint interpreter — registered ONCE, always, like every
|
|
119
|
+
// other built-in. Blueprint content never changes worker registration:
|
|
120
|
+
// dispatch is a DB lookup at ingest (checkBlueprintTriggers, spec §5).
|
|
121
|
+
journeyBlueprintInterpreter,
|
|
117
122
|
sendCampaignTask,
|
|
118
123
|
reapStuckCampaignsTask,
|
|
119
124
|
deliverWebhookTask,
|
|
@@ -24,7 +24,7 @@ import {
|
|
|
24
24
|
import { getBucketRegistrySingleton } from "../buckets/registry-singleton.js";
|
|
25
25
|
import { getJourneyRegistrySingleton } from "../journeys/registry-singleton.js";
|
|
26
26
|
import { emitBucketTransition } from "../lib/bucket-emit.js";
|
|
27
|
-
import { contactKeySql } from "../lib/contacts.js";
|
|
27
|
+
import { contactKeySql, normalizeEmailOrNull } from "../lib/contacts.js";
|
|
28
28
|
import { hatchet } from "../lib/hatchet.js";
|
|
29
29
|
import type { Logger } from "../lib/logger.js";
|
|
30
30
|
import { createLogger } from "../lib/logger.js";
|
|
@@ -301,7 +301,9 @@ async function backfillJoins(opts: {
|
|
|
301
301
|
|
|
302
302
|
const rows = chunk.map((userId) => ({
|
|
303
303
|
userId,
|
|
304
|
-
|
|
304
|
+
// Normalized at the write site (audience-model.md wart #1) — belt and
|
|
305
|
+
// braces on top of the contacts row already being normalized.
|
|
306
|
+
userEmail: normalizeEmailOrNull(emailByUser.get(userId)),
|
|
305
307
|
bucketId: bucket.id,
|
|
306
308
|
status: "active" as const,
|
|
307
309
|
source: "backfill" as const,
|
|
@@ -41,8 +41,9 @@ import {
|
|
|
41
41
|
import { getBucketRegistrySingleton } from "../buckets/registry-singleton.js";
|
|
42
42
|
import { getJourneyRegistrySingleton } from "../journeys/registry-singleton.js";
|
|
43
43
|
import { emitBucketTransition } from "../lib/bucket-emit.js";
|
|
44
|
-
import { contactKeySql } from "../lib/contacts.js";
|
|
44
|
+
import { contactKeySql, normalizeEmailOrNull } from "../lib/contacts.js";
|
|
45
45
|
import { hatchet } from "../lib/hatchet.js";
|
|
46
|
+
import { toSleepDuration } from "../lib/hatchet-duration.js";
|
|
46
47
|
import type { Logger } from "../lib/logger.js";
|
|
47
48
|
import { createLogger } from "../lib/logger.js";
|
|
48
49
|
import { FIRST_TIME_FORMAT } from "./bucket-backfill.js";
|
|
@@ -220,8 +221,10 @@ export const bucketExpiryTask = hatchet.durableTask({
|
|
|
220
221
|
const registry = getBucketRegistrySingleton();
|
|
221
222
|
const journeyRegistry = getJourneyRegistrySingleton();
|
|
222
223
|
|
|
223
|
-
// Durable sleep to the deadline
|
|
224
|
-
|
|
224
|
+
// Durable sleep to the deadline — normalized to whole seconds: a raw ms
|
|
225
|
+
// number renders as a multi-unit duration string some hatchet-lite versions
|
|
226
|
+
// silently no-op (instant wake → premature expiry evaluation).
|
|
227
|
+
await ctx.sleepFor(toSleepDuration(input.msUntilExpiry));
|
|
225
228
|
|
|
226
229
|
const bucket = registry.get(input.bucketId);
|
|
227
230
|
if (!bucket?.criteria) {
|
|
@@ -1033,7 +1036,11 @@ async function reconcileJoinOne(opts: {
|
|
|
1033
1036
|
userId: string;
|
|
1034
1037
|
userEmail: string | null;
|
|
1035
1038
|
}): Promise<boolean> {
|
|
1036
|
-
const { db, logger, journeyRegistry, bucket, userId
|
|
1039
|
+
const { db, logger, journeyRegistry, bucket, userId } = opts;
|
|
1040
|
+
// Normalized at the write site (audience-model.md wart #1) — the contact row
|
|
1041
|
+
// should already be normalized, but the membership keyspace must never
|
|
1042
|
+
// depend on that.
|
|
1043
|
+
const userEmail = normalizeEmailOrNull(opts.userEmail);
|
|
1037
1044
|
|
|
1038
1045
|
// entryCount ordinal = 1 + ALL prior memberships (active + left). Shared with
|
|
1039
1046
|
// the real-time join path so the ordinal never drifts between the two writers.
|
|
@@ -4,13 +4,140 @@ import {
|
|
|
4
4
|
emailSends,
|
|
5
5
|
journeyStates,
|
|
6
6
|
} from "@hogsend/db";
|
|
7
|
-
import { and, eq, gte, sql } from "drizzle-orm";
|
|
7
|
+
import { and, eq, gte, isNull, sql } from "drizzle-orm";
|
|
8
8
|
import { checkAlertRules } from "../lib/alerting.js";
|
|
9
9
|
import { hatchet } from "../lib/hatchet.js";
|
|
10
10
|
import { createLogger, type Logger } from "../lib/logger.js";
|
|
11
11
|
|
|
12
12
|
const FAILURE_WINDOW_MINUTES = 60;
|
|
13
13
|
|
|
14
|
+
// A `waiting` row only counts as STRANDED once its deadline is more than this
|
|
15
|
+
// grace past due. The window must absorb the two legitimate reasons a deadline
|
|
16
|
+
// slips while the row is still (correctly) waiting: Hatchet wake latency after
|
|
17
|
+
// the durable sleep fires, and the `scheduleTimeout` headroom a saturated worker
|
|
18
|
+
// needs to replay-resume before the engine flips the row back to `active`. Set it
|
|
19
|
+
// too tight and a healthy-but-slow resume false-positives; an hour clears both.
|
|
20
|
+
const STRANDED_GRACE_MS = 60 * 60 * 1000;
|
|
21
|
+
|
|
22
|
+
// Cap the per-row detail attached to a single alert. Stranded rows should be
|
|
23
|
+
// rare, but a systemic outage could strand many at once — log the total count
|
|
24
|
+
// plus a bounded sample rather than an unbounded array.
|
|
25
|
+
const STRANDED_ALERT_SAMPLE_CAP = 50;
|
|
26
|
+
|
|
27
|
+
interface StrandedFinding {
|
|
28
|
+
stateId: string;
|
|
29
|
+
journeyId: string;
|
|
30
|
+
userId: string;
|
|
31
|
+
/** Which deadline is overdue: the `wait_deadline` column or a `digest:<label>`. */
|
|
32
|
+
deadlineSource: string;
|
|
33
|
+
deadline: string;
|
|
34
|
+
overdueMinutes: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** The `context.__digest__` sub-bag (flat `<label>:deadline` / `<label>:result`). */
|
|
38
|
+
function digestBag(context: unknown): Record<string, unknown> {
|
|
39
|
+
if (!context || typeof context !== "object") return {};
|
|
40
|
+
const bag = (context as Record<string, unknown>).__digest__;
|
|
41
|
+
return bag && typeof bag === "object" ? (bag as Record<string, unknown>) : {};
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function finding(
|
|
45
|
+
row: { id: string; journeyId: string; userId: string },
|
|
46
|
+
deadlineSource: string,
|
|
47
|
+
deadline: Date,
|
|
48
|
+
now: number,
|
|
49
|
+
): StrandedFinding {
|
|
50
|
+
return {
|
|
51
|
+
stateId: row.id,
|
|
52
|
+
journeyId: row.journeyId,
|
|
53
|
+
userId: row.userId,
|
|
54
|
+
deadlineSource,
|
|
55
|
+
deadline: deadline.toISOString(),
|
|
56
|
+
overdueMinutes: Math.round((now - deadline.getTime()) / 60000),
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Stranded-waiting detector. A journey run can die WITHOUT the engine's
|
|
61
|
+
// exit/resume catch firing (a lost `runs.cancel`, Hatchet data loss, a
|
|
62
|
+
// saturated-redeploy `scheduleTimeout`) — the `journey_states` row then stays
|
|
63
|
+
// `status='waiting'` forever, and the `already_active` enrollment guard silently
|
|
64
|
+
// absorbs EVERY future trigger event for that (user, journey) into a black hole.
|
|
65
|
+
// Digest journeys park in `waiting` for days by design, so exposure is maximal.
|
|
66
|
+
// v1 is DETECT + ALERT only: no status flips, no auto-repair. Exported as the
|
|
67
|
+
// test seam (mirrors how the reconcile workflows are exercised directly).
|
|
68
|
+
export async function surfaceStrandedWaiting(opts: {
|
|
69
|
+
db: Database;
|
|
70
|
+
logger: Logger;
|
|
71
|
+
}): Promise<void> {
|
|
72
|
+
const { db, logger } = opts;
|
|
73
|
+
|
|
74
|
+
try {
|
|
75
|
+
const rows = await db
|
|
76
|
+
.select({
|
|
77
|
+
id: journeyStates.id,
|
|
78
|
+
journeyId: journeyStates.journeyId,
|
|
79
|
+
userId: journeyStates.userId,
|
|
80
|
+
waitDeadline: journeyStates.waitDeadline,
|
|
81
|
+
updatedAt: journeyStates.updatedAt,
|
|
82
|
+
context: journeyStates.context,
|
|
83
|
+
})
|
|
84
|
+
.from(journeyStates)
|
|
85
|
+
.where(
|
|
86
|
+
and(
|
|
87
|
+
eq(journeyStates.status, "waiting"),
|
|
88
|
+
isNull(journeyStates.deletedAt),
|
|
89
|
+
// Only two stranding sources exist: the wait_deadline column and a
|
|
90
|
+
// digest deadline in the context bag. Plain ctx.sleep waits have
|
|
91
|
+
// neither — exclude them in SQL so the sweep doesn't haul every
|
|
92
|
+
// parked row's full context jsonb over the wire for nothing.
|
|
93
|
+
sql`(${journeyStates.waitDeadline} IS NOT NULL OR ${journeyStates.context} ? '__digest__')`,
|
|
94
|
+
),
|
|
95
|
+
);
|
|
96
|
+
|
|
97
|
+
const now = Date.now();
|
|
98
|
+
const cutoff = now - STRANDED_GRACE_MS;
|
|
99
|
+
const stranded: StrandedFinding[] = [];
|
|
100
|
+
|
|
101
|
+
for (const row of rows) {
|
|
102
|
+
// (a) the `wait_deadline` column — a `ctx.waitForEvent({ where })` re-arm loop
|
|
103
|
+
// persists an absolute deadline here; overdue past the grace = stranded.
|
|
104
|
+
if (row.waitDeadline && row.waitDeadline.getTime() < cutoff) {
|
|
105
|
+
stranded.push(finding(row, "wait_deadline", row.waitDeadline, now));
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
// (b) any digest deadline overdue past the grace WITHOUT a matching result.
|
|
109
|
+
// A recorded `<label>:result` means that digest flushed cleanly (the row may
|
|
110
|
+
// legitimately be parked on a LATER primitive), so only an UN-flushed overdue
|
|
111
|
+
// deadline is evidence of stranding. Parse the bag in JS — waiting rows are a
|
|
112
|
+
// small set, so no jsonb-path SQL is needed.
|
|
113
|
+
const bag = digestBag(row.context);
|
|
114
|
+
for (const [key, value] of Object.entries(bag)) {
|
|
115
|
+
if (!key.endsWith(":deadline") || typeof value !== "string") continue;
|
|
116
|
+
const label = key.slice(0, -":deadline".length);
|
|
117
|
+
if (Object.hasOwn(bag, `${label}:result`)) continue;
|
|
118
|
+
const deadline = new Date(value);
|
|
119
|
+
if (Number.isNaN(deadline.getTime())) continue;
|
|
120
|
+
if (deadline.getTime() < cutoff) {
|
|
121
|
+
stranded.push(finding(row, `digest:${label}`, deadline, now));
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (stranded.length > 0) {
|
|
127
|
+
logger.error("Stranded waiting journey states detected", {
|
|
128
|
+
count: stranded.length,
|
|
129
|
+
graceMinutes: STRANDED_GRACE_MS / 60000,
|
|
130
|
+
states: stranded.slice(0, STRANDED_ALERT_SAMPLE_CAP),
|
|
131
|
+
hint: "A waiting row whose deadline is long past-due means its Hatchet run died WITHOUT the engine's exit/resume catch (lost cancel, engine data loss, or a saturated-redeploy scheduleTimeout). The active-enrollment guard now absorbs every future trigger event for this user+journey into a black hole. Repair by cancelling the row via DELETE /v1/admin/journeys/:id/states/:stateId (or fail it) in the admin surface.",
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
} catch (err) {
|
|
135
|
+
logger.warn("Failed to check stranded waiting states", {
|
|
136
|
+
error: err instanceof Error ? err.message : String(err),
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
14
141
|
// Ruleless failure surfacing. The configured alert rules already cover failed
|
|
15
142
|
// journeys (journey_failure_spike) and failed sends (they drag delivery_issue's
|
|
16
143
|
// delivery rate down) — but a fresh install has NO alert_rules rows, so a
|
|
@@ -75,6 +202,8 @@ export const checkAlertsTask = hatchet.task({
|
|
|
75
202
|
|
|
76
203
|
await surfaceRecentFailures({ db, logger });
|
|
77
204
|
|
|
205
|
+
await surfaceStrandedWaiting({ db, logger });
|
|
206
|
+
|
|
78
207
|
await checkAlertRules({
|
|
79
208
|
db,
|
|
80
209
|
logger,
|