@hogsend/engine 0.39.0 → 0.41.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. package/package.json +9 -7
  2. package/src/buckets/check-membership.ts +6 -1
  3. package/src/campaigns/cohort-sql.ts +198 -0
  4. package/src/campaigns/define-campaign.ts +301 -0
  5. package/src/campaigns/reconcile.ts +391 -0
  6. package/src/campaigns/steps.ts +48 -0
  7. package/src/connectors/define-action.ts +70 -0
  8. package/src/container.ts +144 -42
  9. package/src/destinations/presets/posthog.ts +7 -1
  10. package/src/env.ts +14 -0
  11. package/src/index.ts +98 -3
  12. package/src/journeys/constants.ts +10 -0
  13. package/src/journeys/define-journey.ts +14 -405
  14. package/src/journeys/execute-journey-run.ts +608 -0
  15. package/src/journeys/graph/build-graph.ts +212 -56
  16. package/src/journeys/journey-boundary.ts +54 -0
  17. package/src/journeys/journey-context.ts +374 -59
  18. package/src/journeys/record-once.ts +110 -0
  19. package/src/lib/auth.ts +33 -1
  20. package/src/lib/blueprints.ts +576 -0
  21. package/src/lib/campaign-send-key.ts +58 -0
  22. package/src/lib/connector-actions.ts +203 -19
  23. package/src/lib/contacts.ts +54 -0
  24. package/src/lib/email-service-types.ts +4 -1
  25. package/src/lib/email.ts +13 -2
  26. package/src/lib/feed.ts +77 -57
  27. package/src/lib/frequency-cap.ts +45 -15
  28. package/src/lib/hatchet-duration.ts +22 -0
  29. package/src/lib/html.ts +1 -0
  30. package/src/lib/ingestion.ts +145 -4
  31. package/src/lib/journey-suppress.ts +87 -0
  32. package/src/lib/links.ts +204 -18
  33. package/src/lib/outbound.ts +20 -0
  34. package/src/lib/preferences.ts +80 -1
  35. package/src/lib/tracked.ts +44 -29
  36. package/src/lib/tracking-event-names.ts +8 -0
  37. package/src/lib/webhook-signing.ts +6 -1
  38. package/src/lists/channels.ts +75 -0
  39. package/src/lists/define-list.ts +34 -0
  40. package/src/lists/registry.ts +29 -2
  41. package/src/mcp/blueprint-tools.ts +460 -0
  42. package/src/routes/admin/blueprints.ts +711 -0
  43. package/src/routes/admin/campaigns.ts +341 -0
  44. package/src/routes/admin/emails.ts +13 -0
  45. package/src/routes/admin/index.ts +6 -0
  46. package/src/routes/admin/journeys.ts +4 -2
  47. package/src/routes/admin/links.ts +303 -36
  48. package/src/routes/admin/lists.ts +51 -0
  49. package/src/routes/campaigns/index.ts +297 -50
  50. package/src/routes/email/preferences.ts +50 -13
  51. package/src/routes/feed/recipient.ts +2 -43
  52. package/src/routes/index.ts +6 -0
  53. package/src/routes/lists/index.ts +138 -18
  54. package/src/routes/tracking/arrive.ts +260 -0
  55. package/src/routes/tracking/click-pipeline.ts +329 -0
  56. package/src/routes/tracking/click.ts +5 -270
  57. package/src/routes/tracking/index.ts +2 -0
  58. package/src/routes/tracking/vanity.ts +64 -0
  59. package/src/worker.ts +27 -0
  60. package/src/workflows/bucket-backfill.ts +4 -2
  61. package/src/workflows/bucket-reconcile.ts +11 -4
  62. package/src/workflows/check-alerts.ts +130 -1
  63. package/src/workflows/journey-blueprint-interpreter.ts +561 -0
  64. package/src/workflows/send-campaign.ts +673 -127
@@ -0,0 +1,64 @@
1
+ import { links, trackedLinks } from "@hogsend/db";
2
+ import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
3
+ import { and, asc, eq } from "drizzle-orm";
4
+ import type { AppEnv } from "../../app.js";
5
+ import { canonicalTrackedRowFilter, normalizeSlug } from "../../lib/links.js";
6
+ import { clickSelection, handleTrackedClick } from "./click-pipeline.js";
7
+
8
+ // The vanity short path — `/l/:slug` layered over `/v1/t/c/:id`. Mounted at
9
+ // the APP ROOT (not under /v1/t) so the operator-facing URL stays short; the
10
+ // `/l` prefix collides with nothing (`/v1`, `/studio`, `/docs`, `/api/auth`,
11
+ // `/connectors`, webhooks).
12
+ const vanityRoute = createRoute({
13
+ method: "get",
14
+ path: "/l/:slug",
15
+ tags: ["Tracking"],
16
+ summary: "Vanity link redirect",
17
+ request: {
18
+ params: z.object({
19
+ slug: z.string(),
20
+ }),
21
+ },
22
+ responses: {
23
+ 302: { description: "Redirect to original URL" },
24
+ },
25
+ });
26
+
27
+ export const vanityRouter = new OpenAPIHono<AppEnv>().openapi(
28
+ vanityRoute,
29
+ async (c) => {
30
+ const { slug } = c.req.valid("param");
31
+ const { db, env } = c.get("container");
32
+
33
+ // Tolerate cased/typed-by-hand input (`/l/Black-Friday`): slugs are stored
34
+ // normalized, so normalize the inbound too. A malformed slug can't match
35
+ // anything — treat it like an unknown link (redirect home), mirroring the
36
+ // UUID route's missing-row behavior.
37
+ let normalized: string;
38
+ try {
39
+ normalized = normalizeSlug(slug);
40
+ } catch {
41
+ return c.redirect(env.API_PUBLIC_URL, 302);
42
+ }
43
+
44
+ // Resolve the link's CANONICAL redirect row: a managed link has one
45
+ // tracked row minted alongside it, plus (later) a per-link QR scan row
46
+ // (`source: "qr"`) — the vanity path must never attribute to the QR row.
47
+ // Archived links keep resolving (same as the UUID route, which never
48
+ // checks archived_at) — a printed/shared slug should not 404 on archive.
49
+ const rows = await db
50
+ .select(clickSelection)
51
+ .from(links)
52
+ .innerJoin(trackedLinks, eq(trackedLinks.linkId, links.id))
53
+ .where(and(eq(links.slug, normalized), canonicalTrackedRowFilter()))
54
+ .orderBy(asc(trackedLinks.createdAt))
55
+ .limit(1);
56
+
57
+ const link = rows[0];
58
+ if (!link) {
59
+ return c.redirect(env.API_PUBLIC_URL, 302);
60
+ }
61
+
62
+ return handleTrackedClick(c, link);
63
+ },
64
+ );
package/src/worker.ts CHANGED
@@ -4,6 +4,8 @@ import {
4
4
  selectBucketReactionTasks,
5
5
  selectBucketTasks,
6
6
  } from "./buckets/registry.js";
7
+ import type { DefinedCampaign } from "./campaigns/define-campaign.js";
8
+ import { reconcileDefinedCampaigns } from "./campaigns/reconcile.js";
7
9
  import {
8
10
  type ConnectorRuntimeFactory,
9
11
  startConnectorRuntimes,
@@ -28,6 +30,7 @@ import {
28
30
  } from "./workflows/deliver-webhook.js";
29
31
  import { importContactsTask } from "./workflows/import-contacts.js";
30
32
  import { importSuppressionsTask } from "./workflows/import-suppressions.js";
33
+ import { journeyBlueprintInterpreter } from "./workflows/journey-blueprint-interpreter.js";
31
34
  import {
32
35
  reapStuckCampaignsTask,
33
36
  sendCampaignTask,
@@ -40,6 +43,11 @@ export interface CreateWorkerOptions {
40
43
  journeys: DefinedJourney[];
41
44
  /** Buckets whose fast-expiry timer tasks are registered. Defaults to none. */
42
45
  buckets?: DefinedBucket[];
46
+ /**
47
+ * Code-defined one-shot campaigns (broadcasts) reconciled at boot into
48
+ * scheduled `campaigns` rows. Defaults to `container.campaigns`.
49
+ */
50
+ campaigns?: DefinedCampaign[];
43
51
  /** Defaults to `container.env.ENABLED_JOURNEYS`. */
44
52
  enabledJourneys?: string;
45
53
  /** Defaults to `container.env.ENABLED_BUCKETS`. */
@@ -107,6 +115,10 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
107
115
  sendFeedTask,
108
116
  importContactsTask,
109
117
  importSuppressionsTask,
118
+ // The generic blueprint interpreter — registered ONCE, always, like every
119
+ // other built-in. Blueprint content never changes worker registration:
120
+ // dispatch is a DB lookup at ingest (checkBlueprintTriggers, spec §5).
121
+ journeyBlueprintInterpreter,
110
122
  sendCampaignTask,
111
123
  reapStuckCampaignsTask,
112
124
  deliverWebhookTask,
@@ -209,6 +221,21 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
209
221
  });
210
222
  });
211
223
 
224
+ // Reconcile code-defined campaigns into scheduled rows (create / sync /
225
+ // expire — see `reconcileDefinedCampaigns`). Same fire-and-forget stance
226
+ // as the bucket backfills above: best-effort, never blocks the listener.
227
+ const definedCampaigns = opts.campaigns ?? container.campaigns;
228
+ if (definedCampaigns.length > 0) {
229
+ reconcileDefinedCampaigns({
230
+ client: container,
231
+ campaigns: definedCampaigns,
232
+ }).catch((err) => {
233
+ container.logger.warn("Campaign reconcile (boot) failed", {
234
+ error: err instanceof Error ? err.message : String(err),
235
+ });
236
+ });
237
+ }
238
+
212
239
  await _worker.start();
213
240
  }
214
241
 
@@ -24,7 +24,7 @@ import {
24
24
  import { getBucketRegistrySingleton } from "../buckets/registry-singleton.js";
25
25
  import { getJourneyRegistrySingleton } from "../journeys/registry-singleton.js";
26
26
  import { emitBucketTransition } from "../lib/bucket-emit.js";
27
- import { contactKeySql } from "../lib/contacts.js";
27
+ import { contactKeySql, normalizeEmailOrNull } from "../lib/contacts.js";
28
28
  import { hatchet } from "../lib/hatchet.js";
29
29
  import type { Logger } from "../lib/logger.js";
30
30
  import { createLogger } from "../lib/logger.js";
@@ -301,7 +301,9 @@ async function backfillJoins(opts: {
301
301
 
302
302
  const rows = chunk.map((userId) => ({
303
303
  userId,
304
- userEmail: emailByUser.get(userId) ?? null,
304
+ // Normalized at the write site (audience-model.md wart #1) — belt and
305
+ // braces on top of the contacts row already being normalized.
306
+ userEmail: normalizeEmailOrNull(emailByUser.get(userId)),
305
307
  bucketId: bucket.id,
306
308
  status: "active" as const,
307
309
  source: "backfill" as const,
@@ -41,8 +41,9 @@ import {
41
41
  import { getBucketRegistrySingleton } from "../buckets/registry-singleton.js";
42
42
  import { getJourneyRegistrySingleton } from "../journeys/registry-singleton.js";
43
43
  import { emitBucketTransition } from "../lib/bucket-emit.js";
44
- import { contactKeySql } from "../lib/contacts.js";
44
+ import { contactKeySql, normalizeEmailOrNull } from "../lib/contacts.js";
45
45
  import { hatchet } from "../lib/hatchet.js";
46
+ import { toSleepDuration } from "../lib/hatchet-duration.js";
46
47
  import type { Logger } from "../lib/logger.js";
47
48
  import { createLogger } from "../lib/logger.js";
48
49
  import { FIRST_TIME_FORMAT } from "./bucket-backfill.js";
@@ -220,8 +221,10 @@ export const bucketExpiryTask = hatchet.durableTask({
220
221
  const registry = getBucketRegistrySingleton();
221
222
  const journeyRegistry = getJourneyRegistrySingleton();
222
223
 
223
- // Durable sleep to the deadline. Hatchet's sleepFor accepts a ms number.
224
- await ctx.sleepFor(input.msUntilExpiry);
224
+ // Durable sleep to the deadline — normalized to whole seconds: a raw ms
225
+ // number renders as a multi-unit duration string some hatchet-lite versions
226
+ // silently no-op (instant wake → premature expiry evaluation).
227
+ await ctx.sleepFor(toSleepDuration(input.msUntilExpiry));
225
228
 
226
229
  const bucket = registry.get(input.bucketId);
227
230
  if (!bucket?.criteria) {
@@ -1033,7 +1036,11 @@ async function reconcileJoinOne(opts: {
1033
1036
  userId: string;
1034
1037
  userEmail: string | null;
1035
1038
  }): Promise<boolean> {
1036
- const { db, logger, journeyRegistry, bucket, userId, userEmail } = opts;
1039
+ const { db, logger, journeyRegistry, bucket, userId } = opts;
1040
+ // Normalized at the write site (audience-model.md wart #1) — the contact row
1041
+ // should already be normalized, but the membership keyspace must never
1042
+ // depend on that.
1043
+ const userEmail = normalizeEmailOrNull(opts.userEmail);
1037
1044
 
1038
1045
  // entryCount ordinal = 1 + ALL prior memberships (active + left). Shared with
1039
1046
  // the real-time join path so the ordinal never drifts between the two writers.
@@ -4,13 +4,140 @@ import {
4
4
  emailSends,
5
5
  journeyStates,
6
6
  } from "@hogsend/db";
7
- import { and, eq, gte, sql } from "drizzle-orm";
7
+ import { and, eq, gte, isNull, sql } from "drizzle-orm";
8
8
  import { checkAlertRules } from "../lib/alerting.js";
9
9
  import { hatchet } from "../lib/hatchet.js";
10
10
  import { createLogger, type Logger } from "../lib/logger.js";
11
11
 
12
12
  const FAILURE_WINDOW_MINUTES = 60;
13
13
 
14
+ // A `waiting` row only counts as STRANDED once its deadline is more than this
15
+ // grace past due. The window must absorb the two legitimate reasons a deadline
16
+ // slips while the row is still (correctly) waiting: Hatchet wake latency after
17
+ // the durable sleep fires, and the `scheduleTimeout` headroom a saturated worker
18
+ // needs to replay-resume before the engine flips the row back to `active`. Set it
19
+ // too tight and a healthy-but-slow resume false-positives; an hour clears both.
20
+ const STRANDED_GRACE_MS = 60 * 60 * 1000;
21
+
22
+ // Cap the per-row detail attached to a single alert. Stranded rows should be
23
+ // rare, but a systemic outage could strand many at once — log the total count
24
+ // plus a bounded sample rather than an unbounded array.
25
+ const STRANDED_ALERT_SAMPLE_CAP = 50;
26
+
27
+ interface StrandedFinding {
28
+ stateId: string;
29
+ journeyId: string;
30
+ userId: string;
31
+ /** Which deadline is overdue: the `wait_deadline` column or a `digest:<label>`. */
32
+ deadlineSource: string;
33
+ deadline: string;
34
+ overdueMinutes: number;
35
+ }
36
+
37
+ /** The `context.__digest__` sub-bag (flat `<label>:deadline` / `<label>:result`). */
38
+ function digestBag(context: unknown): Record<string, unknown> {
39
+ if (!context || typeof context !== "object") return {};
40
+ const bag = (context as Record<string, unknown>).__digest__;
41
+ return bag && typeof bag === "object" ? (bag as Record<string, unknown>) : {};
42
+ }
43
+
44
+ function finding(
45
+ row: { id: string; journeyId: string; userId: string },
46
+ deadlineSource: string,
47
+ deadline: Date,
48
+ now: number,
49
+ ): StrandedFinding {
50
+ return {
51
+ stateId: row.id,
52
+ journeyId: row.journeyId,
53
+ userId: row.userId,
54
+ deadlineSource,
55
+ deadline: deadline.toISOString(),
56
+ overdueMinutes: Math.round((now - deadline.getTime()) / 60000),
57
+ };
58
+ }
59
+
60
+ // Stranded-waiting detector. A journey run can die WITHOUT the engine's
61
+ // exit/resume catch firing (a lost `runs.cancel`, Hatchet data loss, a
62
+ // saturated-redeploy `scheduleTimeout`) — the `journey_states` row then stays
63
+ // `status='waiting'` forever, and the `already_active` enrollment guard silently
64
+ // absorbs EVERY future trigger event for that (user, journey) into a black hole.
65
+ // Digest journeys park in `waiting` for days by design, so exposure is maximal.
66
+ // v1 is DETECT + ALERT only: no status flips, no auto-repair. Exported as the
67
+ // test seam (mirrors how the reconcile workflows are exercised directly).
68
+ export async function surfaceStrandedWaiting(opts: {
69
+ db: Database;
70
+ logger: Logger;
71
+ }): Promise<void> {
72
+ const { db, logger } = opts;
73
+
74
+ try {
75
+ const rows = await db
76
+ .select({
77
+ id: journeyStates.id,
78
+ journeyId: journeyStates.journeyId,
79
+ userId: journeyStates.userId,
80
+ waitDeadline: journeyStates.waitDeadline,
81
+ updatedAt: journeyStates.updatedAt,
82
+ context: journeyStates.context,
83
+ })
84
+ .from(journeyStates)
85
+ .where(
86
+ and(
87
+ eq(journeyStates.status, "waiting"),
88
+ isNull(journeyStates.deletedAt),
89
+ // Only two stranding sources exist: the wait_deadline column and a
90
+ // digest deadline in the context bag. Plain ctx.sleep waits have
91
+ // neither — exclude them in SQL so the sweep doesn't haul every
92
+ // parked row's full context jsonb over the wire for nothing.
93
+ sql`(${journeyStates.waitDeadline} IS NOT NULL OR ${journeyStates.context} ? '__digest__')`,
94
+ ),
95
+ );
96
+
97
+ const now = Date.now();
98
+ const cutoff = now - STRANDED_GRACE_MS;
99
+ const stranded: StrandedFinding[] = [];
100
+
101
+ for (const row of rows) {
102
+ // (a) the `wait_deadline` column — a `ctx.waitForEvent({ where })` re-arm loop
103
+ // persists an absolute deadline here; overdue past the grace = stranded.
104
+ if (row.waitDeadline && row.waitDeadline.getTime() < cutoff) {
105
+ stranded.push(finding(row, "wait_deadline", row.waitDeadline, now));
106
+ }
107
+
108
+ // (b) any digest deadline overdue past the grace WITHOUT a matching result.
109
+ // A recorded `<label>:result` means that digest flushed cleanly (the row may
110
+ // legitimately be parked on a LATER primitive), so only an UN-flushed overdue
111
+ // deadline is evidence of stranding. Parse the bag in JS — waiting rows are a
112
+ // small set, so no jsonb-path SQL is needed.
113
+ const bag = digestBag(row.context);
114
+ for (const [key, value] of Object.entries(bag)) {
115
+ if (!key.endsWith(":deadline") || typeof value !== "string") continue;
116
+ const label = key.slice(0, -":deadline".length);
117
+ if (Object.hasOwn(bag, `${label}:result`)) continue;
118
+ const deadline = new Date(value);
119
+ if (Number.isNaN(deadline.getTime())) continue;
120
+ if (deadline.getTime() < cutoff) {
121
+ stranded.push(finding(row, `digest:${label}`, deadline, now));
122
+ }
123
+ }
124
+ }
125
+
126
+ if (stranded.length > 0) {
127
+ logger.error("Stranded waiting journey states detected", {
128
+ count: stranded.length,
129
+ graceMinutes: STRANDED_GRACE_MS / 60000,
130
+ states: stranded.slice(0, STRANDED_ALERT_SAMPLE_CAP),
131
+ hint: "A waiting row whose deadline is long past-due means its Hatchet run died WITHOUT the engine's exit/resume catch (lost cancel, engine data loss, or a saturated-redeploy scheduleTimeout). The active-enrollment guard now absorbs every future trigger event for this user+journey into a black hole. Repair by cancelling the row via DELETE /v1/admin/journeys/:id/states/:stateId (or fail it) in the admin surface.",
132
+ });
133
+ }
134
+ } catch (err) {
135
+ logger.warn("Failed to check stranded waiting states", {
136
+ error: err instanceof Error ? err.message : String(err),
137
+ });
138
+ }
139
+ }
140
+
14
141
  // Ruleless failure surfacing. The configured alert rules already cover failed
15
142
  // journeys (journey_failure_spike) and failed sends (they drag delivery_issue's
16
143
  // delivery rate down) — but a fresh install has NO alert_rules rows, so a
@@ -75,6 +202,8 @@ export const checkAlertsTask = hatchet.task({
75
202
 
76
203
  await surfaceRecentFailures({ db, logger });
77
204
 
205
+ await surfaceStrandedWaiting({ db, logger });
206
+
78
207
  await checkAlertRules({
79
208
  db,
80
209
  logger,