@hogsend/engine 0.41.0 → 0.43.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/package.json +12 -9
  2. package/src/container.ts +192 -4
  3. package/src/env.ts +28 -0
  4. package/src/index.ts +53 -0
  5. package/src/journeys/execute-journey-run.ts +50 -17
  6. package/src/journeys/journey-boundary.ts +5 -1
  7. package/src/journeys/journey-context.ts +58 -0
  8. package/src/lib/blueprint-lock.ts +21 -0
  9. package/src/lib/blueprints.ts +273 -46
  10. package/src/lib/event-names.ts +143 -0
  11. package/src/lib/journey-suppress.ts +33 -2
  12. package/src/lib/outbound.ts +58 -0
  13. package/src/lib/phone.ts +19 -0
  14. package/src/lib/preferences.ts +32 -0
  15. package/src/lib/sms-frequency-cap.ts +47 -0
  16. package/src/lib/sms-inbound.ts +200 -0
  17. package/src/lib/sms-link-tracking.ts +160 -0
  18. package/src/lib/sms-mailer.ts +338 -0
  19. package/src/lib/sms-provider-registry.ts +35 -0
  20. package/src/lib/sms-providers-from-env.ts +100 -0
  21. package/src/lib/sms-service-types.ts +169 -0
  22. package/src/lib/sms-tracked.ts +590 -0
  23. package/src/lib/sms.ts +117 -0
  24. package/src/lib/tracking-event-names.ts +6 -0
  25. package/src/lib/tracking-events.ts +107 -1
  26. package/src/lib/tracking.ts +3 -6
  27. package/src/lib/webhook-signing.ts +17 -1
  28. package/src/lists/channels.ts +21 -0
  29. package/src/mcp/authoring-guide.ts +160 -0
  30. package/src/mcp/blueprint-tools.ts +38 -108
  31. package/src/middleware/require-admin.ts +13 -0
  32. package/src/routes/admin/api-keys.ts +72 -0
  33. package/src/routes/admin/blueprints.ts +94 -10
  34. package/src/routes/admin/events.ts +55 -0
  35. package/src/routes/email/unsubscribe.ts +1 -0
  36. package/src/routes/index.ts +6 -0
  37. package/src/routes/lists/index.ts +54 -7
  38. package/src/routes/tracking/click-pipeline.ts +83 -2
  39. package/src/routes/tracking/short.ts +61 -0
  40. package/src/routes/webhooks/index.ts +5 -1
  41. package/src/routes/webhooks/sms-provider.ts +110 -0
  42. package/src/workflows/import-contacts.ts +61 -4
  43. package/src/workflows/journey-blueprint-interpreter.ts +128 -33
@@ -0,0 +1,110 @@
1
+ import { WebhookHandshakeSignal } from "@hogsend/core";
2
+ import { createRoute, type OpenAPIHono, z } from "@hono/zod-openapi";
3
+ import type { Context } from "hono";
4
+ import type { AppEnv } from "../../app.js";
5
+ import { env } from "../../env.js";
6
+ import { headersToRecord } from "../../lib/headers.js";
7
+
8
+ /**
9
+ * Shared SMS-provider webhook dispatch for
10
+ * `POST /v1/webhooks/sms/:providerId`. Resolves the provider from the
11
+ * container's {@link SmsProviderRegistry} (404 on unknown id), reads the raw
12
+ * body as the EXACT received bytes, builds the canonical PUBLIC url (Twilio
13
+ * signs `API_PUBLIC_URL + path + query`, NOT `c.req.url` which has the wrong
14
+ * host behind a proxy), verifies + dispatches the normalized {@link SmsEvent},
15
+ * 200s a {@link WebhookHandshakeSignal} (intermediate status / unrecognized),
16
+ * and 401s a verification error.
17
+ *
18
+ * Responds with an empty TwiML document — Twilio's inbound MO webhook expects
19
+ * TwiML; status callbacks accept any 2xx.
20
+ */
21
+ export async function dispatchSmsProviderWebhook(
22
+ c: Context<AppEnv>,
23
+ providerId: string,
24
+ ) {
25
+ const { smsProviders, smsService, logger } = c.get("container");
26
+
27
+ const provider = smsProviders.get(providerId);
28
+ if (!provider) {
29
+ return c.json({ error: "Unknown SMS provider" }, 404);
30
+ }
31
+
32
+ const payload = await c.req.text();
33
+ const headers = headersToRecord(c.req.raw.headers);
34
+
35
+ // Canonical public URL the provider signed: API_PUBLIC_URL + path (+ query).
36
+ // Twilio's HMAC covers the URL string byte-for-byte, so normalize the one
37
+ // operator-controlled degree of freedom — a trailing slash on
38
+ // API_PUBLIC_URL would otherwise 401 EVERY callback and inbound STOP.
39
+ const requestUrl = new URL(c.req.url);
40
+ const base = env.API_PUBLIC_URL.replace(/\/+$/, "");
41
+ const url = `${base}${requestUrl.pathname}${requestUrl.search}`;
42
+
43
+ try {
44
+ const event = await provider.verifyWebhook({ payload, headers, url });
45
+ const result = await smsService.handleWebhook(event, providerId);
46
+ logger.info("SMS provider webhook processed", {
47
+ providerId,
48
+ type: event.type,
49
+ handled: result.handled,
50
+ });
51
+ return c.body(
52
+ '<?xml version="1.0" encoding="UTF-8"?><Response></Response>',
53
+ 200,
54
+ { "Content-Type": "text/xml" },
55
+ );
56
+ } catch (err) {
57
+ if (err instanceof WebhookHandshakeSignal) {
58
+ logger.info("SMS webhook handshake", { providerId, action: err.action });
59
+ return c.body(
60
+ '<?xml version="1.0" encoding="UTF-8"?><Response></Response>',
61
+ 200,
62
+ { "Content-Type": "text/xml" },
63
+ );
64
+ }
65
+ logger.warn("SMS provider webhook failed", {
66
+ providerId,
67
+ error: err instanceof Error ? err.message : String(err),
68
+ });
69
+ return c.json({ error: "Webhook verification failed" }, 401);
70
+ }
71
+ }
72
+
73
+ const smsProviderWebhookRoute = createRoute({
74
+ method: "post",
75
+ path: "/v1/webhooks/sms/{providerId}",
76
+ tags: ["Webhooks"],
77
+ summary: "SMS provider webhook receiver (status callbacks + inbound STOP)",
78
+ request: {
79
+ params: z.object({ providerId: z.string() }),
80
+ body: {
81
+ content: {
82
+ "application/x-www-form-urlencoded": {
83
+ schema: z.record(z.string(), z.unknown()),
84
+ },
85
+ },
86
+ },
87
+ },
88
+ responses: {
89
+ 200: { description: "Webhook processed (empty TwiML)" },
90
+ 401: {
91
+ content: {
92
+ "application/json": { schema: z.object({ error: z.string() }) },
93
+ },
94
+ description: "Missing or invalid webhook signature",
95
+ },
96
+ 404: {
97
+ content: {
98
+ "application/json": { schema: z.object({ error: z.string() }) },
99
+ },
100
+ description: "Unknown SMS provider",
101
+ },
102
+ },
103
+ });
104
+
105
+ export function registerSmsProviderRoutes(app: OpenAPIHono<AppEnv>) {
106
+ app.openapi(smsProviderWebhookRoute, (c) => {
107
+ const { providerId } = c.req.valid("param");
108
+ return dispatchSmsProviderWebhook(c, providerId);
109
+ });
110
+ }
@@ -1,17 +1,70 @@
1
- import { createDatabase, importJobs } from "@hogsend/db";
2
- import { eq } from "drizzle-orm";
1
+ import type { Database } from "@hogsend/db";
2
+ import { contacts, createDatabase, importJobs } from "@hogsend/db";
3
+ import { and, eq, isNull } from "drizzle-orm";
3
4
  import Papa from "papaparse";
4
5
  import { resolveOrCreateContact } from "../lib/contacts.js";
5
6
  import { hatchet } from "../lib/hatchet.js";
7
+ import { normalizePhone } from "../lib/phone.js";
6
8
 
7
9
  const BATCH_SIZE = 500;
8
10
 
9
11
  interface ImportRow {
10
12
  externalId?: string;
11
13
  email?: string;
14
+ /** Optional E.164 phone; attached to the resolved contact (SMS channel). */
15
+ phone?: string;
12
16
  properties?: Record<string, unknown>;
13
17
  }
14
18
 
19
+ /**
20
+ * True when `err` is the `contacts_phone_unique_idx` partial-unique violation
21
+ * (another live contact already owns the number). Walks the drizzle cause
22
+ * chain for PG 23505, mirroring `isSlugUniqueViolation` in lib/links.ts.
23
+ */
24
+ function isPhoneUniqueViolation(err: unknown): boolean {
25
+ let candidate: unknown = err;
26
+ while (candidate && typeof candidate === "object") {
27
+ const c = candidate as {
28
+ code?: string;
29
+ constraint_name?: string;
30
+ constraint?: string;
31
+ cause?: unknown;
32
+ };
33
+ if (c.code === "23505") {
34
+ const constraint = c.constraint_name ?? c.constraint ?? "";
35
+ return constraint.includes("contacts_phone_unique");
36
+ }
37
+ candidate = c.cause;
38
+ }
39
+ return false;
40
+ }
41
+
42
+ /**
43
+ * Best-effort attach a normalized phone to a resolved contact's `phone` column
44
+ * without minting an identity conflict. Only fills a NULL column, and swallows
45
+ * ONLY the partial-unique conflict (another live contact already owns that
46
+ * number) so one duplicate phone never fails the whole import row — any other
47
+ * DB error rethrows into the row's error accounting instead of silently
48
+ * dropping the phone (the task runs with retries: 0).
49
+ */
50
+ async function attachPhone(
51
+ db: Database,
52
+ contactId: string,
53
+ rawPhone: string | undefined,
54
+ ): Promise<void> {
55
+ const phone = normalizePhone(rawPhone);
56
+ if (!phone) return;
57
+ try {
58
+ await db
59
+ .update(contacts)
60
+ .set({ phone, updatedAt: new Date() })
61
+ .where(and(eq(contacts.id, contactId), isNull(contacts.phone)));
62
+ } catch (err) {
63
+ if (!isPhoneUniqueViolation(err)) throw err;
64
+ // Unique-index conflict (another contact owns this phone) — leave unset.
65
+ }
66
+ }
67
+
15
68
  export const importContactsTask = hatchet.task({
16
69
  name: "import-contacts",
17
70
  retries: 0,
@@ -77,7 +130,10 @@ export const importContactsTask = hatchet.task({
77
130
  userId: row.externalId,
78
131
  email: row.email,
79
132
  contactProperties: row.properties,
80
- }).then(() => ({ index: i + idx, ok: true }));
133
+ }).then(async (result) => {
134
+ await attachPhone(db, result.id, row.phone);
135
+ return { index: i + idx, ok: true };
136
+ });
81
137
  }),
82
138
  );
83
139
 
@@ -131,11 +187,12 @@ function parseCsv(data: string): ImportRow[] {
131
187
  }
132
188
 
133
189
  return result.data.map((row) => {
134
- const { externalId, email, ...rest } = row;
190
+ const { externalId, email, phone, ...rest } = row;
135
191
  const properties = Object.keys(rest).length > 0 ? rest : undefined;
136
192
  return {
137
193
  externalId: externalId || undefined,
138
194
  email: email || undefined,
195
+ phone: phone || undefined,
139
196
  properties: properties as Record<string, unknown> | undefined,
140
197
  };
141
198
  });
@@ -5,9 +5,11 @@ import {
5
5
  BlueprintUnreachableNodeError,
6
6
  type DurationObject,
7
7
  evaluateCondition,
8
+ isReservedEventName,
8
9
  type JourneyEdge,
9
10
  type JourneyNode,
10
11
  type PropertyCondition,
12
+ RESERVED_EVENT_NAMESPACES,
11
13
  serializeBlueprintError,
12
14
  validateBlueprintGraph,
13
15
  } from "@hogsend/core";
@@ -310,6 +312,17 @@ export async function walkBlueprintGraph(
310
312
  "event",
311
313
  node.meta?.event,
312
314
  );
315
+ // Save-time validation rejects reserved namespaces since the check
316
+ // was added, but the graph column is jsonb — a row saved BEFORE the
317
+ // rule (or written out-of-band) must still never forge an
318
+ // engine-emitted event through the ingest pipeline.
319
+ if (isReservedEventName(event)) {
320
+ throw new BlueprintNodeExecutionError(
321
+ blueprintId,
322
+ node.id,
323
+ `trigger node event "${event}" uses a reserved namespace (${RESERVED_EVENT_NAMESPACES.join("/")})`,
324
+ );
325
+ }
313
326
  await ctx.trigger({
314
327
  event,
315
328
  userId: user.id,
@@ -358,37 +371,12 @@ export async function walkBlueprintGraph(
358
371
  // code journey's run() returning normally.
359
372
  return null;
360
373
 
361
- case "end-exited": {
362
- // Terminal: flip the row terminal the same way checkExits does, then
363
- // abort the run via the SAME control-flow signal a mid-wait exit
364
- // uses — the lifecycle catch maps it to { status: "exited" } without
365
- // a "failed" write or a journey:failed push. Idempotent on replay
366
- // (the guarded update no-ops once terminal).
367
- const [exited] = await db
368
- .update(journeyStates)
369
- .set({
370
- status: "exited",
371
- exitedAt: new Date(),
372
- updatedAt: new Date(),
373
- })
374
- .where(
375
- and(
376
- eq(journeyStates.id, user.stateId),
377
- notInArray(journeyStates.status, [...TERMINAL_STATUSES]),
378
- ),
379
- )
380
- .returning({ id: journeyStates.id });
381
- if (exited) {
382
- logTransition({
383
- db,
384
- journeyStateId: user.stateId,
385
- from: null,
386
- to: "end-exited",
387
- action: "exited",
388
- });
389
- }
390
- throw new JourneyExitedError(user.stateId);
391
- }
374
+ case "end-exited":
375
+ // Terminal "exited": the SAME primitive a code journey (and promoted
376
+ // blueprint code) calls — a guarded flip to "exited" plus the
377
+ // JourneyExitedError the lifecycle maps to { status: "exited" } without
378
+ // a "failed" write or a journey:failed push. One mechanism, not two.
379
+ return ctx.exit();
392
380
 
393
381
  case "end-failed":
394
382
  // Terminal: matches a code journey's thrown error — the lifecycle
@@ -459,6 +447,75 @@ export async function walkBlueprintGraph(
459
447
  }
460
448
  }
461
449
 
450
+ /**
451
+ * A REPLAY (resume recovered by run id) that can no longer execute — the
452
+ * blueprint row vanished, or its stored graph now fails execution-time
453
+ * re-validation — must FAIL the recovered enrollment, not return `skipped`
454
+ * and leave a waiting/active row stranded non-terminal forever. Recovery is by
455
+ * the replay-stable Hatchet run id + journeyId, exactly as `executeJourneyRun`
456
+ * recovers an enrollment; a FRESH (never-enrolled) dispatch recovers nothing
457
+ * and the caller keeps returning `skipped` unchanged.
458
+ *
459
+ * Mirrors the failed path in `executeJourneyRun`: a guarded flip to "failed"
460
+ * (`notInArray(TERMINAL_STATUSES)`, so an already-terminal row is never
461
+ * clobbered) plus the same `journey:failed` side effects (structured
462
+ * errorMessage, failed transition, event push). Returns whether an existing
463
+ * enrollment was recovered for this run.
464
+ */
465
+ async function failRecoveredBlueprintEnrollment(opts: {
466
+ db: Database;
467
+ blueprintId: string;
468
+ workflowRunId: string;
469
+ userId: string;
470
+ message: string;
471
+ }): Promise<boolean> {
472
+ const { db, blueprintId, workflowRunId, userId, message } = opts;
473
+ if (!workflowRunId) return false;
474
+
475
+ const recovered = await db.query.journeyStates.findFirst({
476
+ where: and(
477
+ eq(journeyStates.hatchetRunId, workflowRunId),
478
+ eq(journeyStates.journeyId, blueprintId),
479
+ ),
480
+ });
481
+ if (!recovered) return false;
482
+
483
+ const [failed] = await db
484
+ .update(journeyStates)
485
+ .set({ status: "failed", errorMessage: message, updatedAt: new Date() })
486
+ .where(
487
+ and(
488
+ eq(journeyStates.id, recovered.id),
489
+ notInArray(journeyStates.status, [...TERMINAL_STATUSES]),
490
+ ),
491
+ )
492
+ .returning({ id: journeyStates.id });
493
+ // Already terminal — nothing was stranded; the recovery still happened.
494
+ if (!failed) return true;
495
+
496
+ logger.error("blueprint replay failed the recovered enrollment", {
497
+ blueprintId,
498
+ stateId: recovered.id,
499
+ userId,
500
+ error: message,
501
+ });
502
+ logTransition({
503
+ db,
504
+ journeyStateId: recovered.id,
505
+ from: recovered.currentNodeId ?? null,
506
+ to: null,
507
+ action: "failed",
508
+ detail: { error: message },
509
+ });
510
+ await hatchet.events.push("journey:failed", {
511
+ journeyId: blueprintId,
512
+ stateId: recovered.id,
513
+ userId,
514
+ error: message,
515
+ });
516
+ return true;
517
+ }
518
+
462
519
  /**
463
520
  * The event payload `checkBlueprintTriggers` pushes for each matching enabled
464
521
  * blueprint — one push per (blueprint, user) enrollment attempt.
@@ -498,6 +555,11 @@ export const journeyBlueprintInterpreter = hatchet.durableTask({
498
555
  fn: async (input: BlueprintRunPayload, hatchetCtx) => {
499
556
  const db = getDb();
500
557
  const blueprintId = input.blueprintId as string;
558
+ // Recovery anchor: a REPLAY resumes under the SAME run id, so a blueprint
559
+ // that has since vanished / gone invalid must fail its recovered
560
+ // enrollment rather than strand it (see failRecoveredBlueprintEnrollment).
561
+ const workflowRunId = hatchetCtx.workflowRunId();
562
+ const userId = input.userId as string;
501
563
 
502
564
  const row = await db.query.journeyBlueprints.findFirst({
503
565
  where: eq(journeyBlueprints.id, blueprintId),
@@ -506,7 +568,21 @@ export const journeyBlueprintInterpreter = hatchet.durableTask({
506
568
  logger.warn("blueprint run skipped: blueprint not found", {
507
569
  blueprintId,
508
570
  });
509
- return { status: "skipped", reason: "blueprint_not_found" };
571
+ const failed = await failRecoveredBlueprintEnrollment({
572
+ db,
573
+ blueprintId,
574
+ workflowRunId,
575
+ userId,
576
+ message: JSON.stringify(
577
+ serializeBlueprintError(
578
+ blueprintId,
579
+ new Error("blueprint row not found at execution time"),
580
+ ),
581
+ ),
582
+ });
583
+ return failed
584
+ ? { status: "failed", reason: "blueprint_not_found" }
585
+ : { status: "skipped", reason: "blueprint_not_found" };
510
586
  }
511
587
 
512
588
  // Defense-in-depth (spec §6): a saved blueprint already passed
@@ -518,7 +594,21 @@ export const journeyBlueprintInterpreter = hatchet.durableTask({
518
594
  "blueprint run skipped: stored graph failed execution-time validation",
519
595
  { blueprintId, issues: validated.issues },
520
596
  );
521
- return { status: "skipped", reason: "invalid_blueprint_graph" };
597
+ const failed = await failRecoveredBlueprintEnrollment({
598
+ db,
599
+ blueprintId,
600
+ workflowRunId,
601
+ userId,
602
+ message: JSON.stringify(
603
+ serializeBlueprintError(
604
+ blueprintId,
605
+ new Error("stored graph failed execution-time re-validation"),
606
+ ),
607
+ ),
608
+ });
609
+ return failed
610
+ ? { status: "failed", reason: "invalid_blueprint_graph" }
611
+ : { status: "skipped", reason: "invalid_blueprint_graph" };
522
612
  }
523
613
 
524
614
  // Version pin (spec §12): the version this run enrolled under, recorded
@@ -551,6 +641,11 @@ export const journeyBlueprintInterpreter = hatchet.durableTask({
551
641
  } satisfies EventPayloadInput,
552
642
  hatchetCtx,
553
643
  extraContext: { __blueprintVersion: enrolledVersion },
644
+ // Serialize this enrollment insert against a concurrent graph edit
645
+ // (updateBlueprint takes the SAME advisory lock around its in-flight
646
+ // count + update) so the run cannot become active between that count and
647
+ // the graph write — the desync window the in-flight gate exists to close.
648
+ serializeEnrollment: true,
554
649
  // Structured errorMessage (spec §6.1): { blueprintId, nodeId, message }
555
650
  // as JSON in the SAME journeyStates.errorMessage field code journeys
556
651
  // use, so Studio's error display needs no blueprint-specific branching.