@hogsend/engine 0.6.0 → 0.8.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hogsend/engine",
-  "version": "0.6.0",
+  "version": "0.8.0",
   "type": "module",
   "license": "MIT",
   "repository": {
@@ -35,13 +35,14 @@
     "papaparse": "^5.5.3",
     "picocolors": "^1.1.1",
     "resend": "^6.12.3",
+    "svix": "^1.95.1",
     "winston": "^3.19.0",
     "zod": "^4.4.3",
-    "@hogsend/core": "^0.6.0",
-    "@hogsend/db": "^0.6.0",
-    "@hogsend/email": "^0.6.0",
-    "@hogsend/plugin-posthog": "^0.6.0",
-    "@hogsend/plugin-resend": "^0.6.0"
+    "@hogsend/core": "^0.8.0",
+    "@hogsend/email": "^0.8.0",
+    "@hogsend/plugin-posthog": "^0.8.0",
+    "@hogsend/plugin-resend": "^0.8.0",
+    "@hogsend/db": "^0.8.0"
   },
   "devDependencies": {
     "@types/node": "^22.15.3",

package/src/app.ts

@@ -15,6 +15,7 @@ import { errorHandler } from "./middleware/error-handler.js";
 import { requestLogger } from "./middleware/request-logger.js";
 import { registerRoutes } from "./routes/index.js";
 import type { DefinedWebhookSource } from "./webhook-sources/define-webhook-source.js";
+import { presetsFromEnv } from "./webhook-sources/presets/index.js";
 
 type AuthSession = Awaited<ReturnType<Auth["api"]["getSession"]>>;
 
@@ -35,10 +36,32 @@ export interface CreateAppOptions {
   middleware?: MiddlewareHandler[];
   /** Webhook sources served at `/v1/webhooks/:sourceId`. */
   webhookSources?: DefinedWebhookSource[];
+  /**
+   * Auto-enable the shipped integration presets (Clerk, Supabase, Stripe,
+   * Segment) for every preset whose env secret is configured (gated further by
+   * `ENABLED_WEBHOOK_PRESETS`). Consumer-supplied `webhookSources` always win on
+   * an id collision. Set `false` to opt out entirely. Default `true`.
+   */
+  enablePresets?: boolean;
   /** Override the default error handler. */
   onError?: ErrorHandler<AppEnv>;
 }
 
+/**
+ * Merge env-enabled presets with the consumer's explicit sources. The
+ * consumer-supplied source WINS on an id collision (so a hand-tuned override of
+ * a preset replaces the shipped one rather than registering a duplicate route).
+ */
+function dedupeById(sources: DefinedWebhookSource[]): DefinedWebhookSource[] {
+  const byId = new Map<string, DefinedWebhookSource>();
+  for (const source of sources) {
+    // Last write wins; callers order presets BEFORE consumer sources so the
+    // consumer override lands last.
+    byId.set(source.meta.id, source);
+  }
+  return [...byId.values()];
+}
+
 export function createApp(
   container: HogsendClient,
   opts: CreateAppOptions = {},
@@ -96,7 +119,19 @@ export function createApp(
     return c.json({ needsSetup: existing.length === 0 });
   });
 
-  registerRoutes(app, { webhookSources: opts.webhookSources ?? [] });
+  // Merge env-enabled presets ahead of the consumer's explicit sources so a
+  // consumer override of a preset id wins (decision #13). `enablePresets`
+  // defaults true; setting only `STRIPE_WEBHOOK_SECRET` auto-mounts Stripe at
+  // `POST /v1/webhooks/stripe` and nothing else.
+  const enablePresets = opts.enablePresets ?? true;
+  const webhookSources = enablePresets
+    ? dedupeById([
+        ...presetsFromEnv(container.env),
+        ...(opts.webhookSources ?? []),
+      ])
+    : (opts.webhookSources ?? []);
+
+  registerRoutes(app, { webhookSources });
 
   // Serve the Studio SPA at /studio/* (static layer, no auth — the SPA gates
   // itself via /v1/auth/status + login; data endpoints stay behind requireAdmin).

package/src/buckets/check-membership.ts

@@ -51,7 +51,16 @@ export async function checkBucketMembership(opts: {
   userId: string;
   userEmail: string | null;
   event: string;
-  properties: Record<string, unknown>;
+  /**
+   * D2: the event payload — candidate-narrowing ONLY. It NO LONGER participates
+   * in property eval (the raw-payload overlay was the bucket-side conflation).
+   */
+  eventProperties: Record<string, unknown>;
+  /**
+   * D2: this-ingest contact-property patch, overlaid on the read contact row so
+   * the very first event after a property change evaluates correctly (risk 7).
+   */
+  contactProperties?: Record<string, unknown>;
   /** Optional override; defaults to the process bucket-registry singleton. */
   bucketRegistry?: ReturnType<typeof getBucketRegistrySingleton>;
 }): Promise<BucketTransition[]> {
@@ -63,7 +72,8 @@ export async function checkBucketMembership(opts: {
     userId,
     userEmail,
     event,
-    properties,
+    eventProperties,
+    contactProperties: contactPropertiesPatch,
   } = opts;
 
   // (1) Recursion guard — MUST be first. bucket:-prefixed events are transition
@@ -81,12 +91,18 @@ export async function checkBucketMembership(opts: {
 
   // (2) Candidate narrowing — the UNION of buckets referencing this event name
   // (eventIndex + the degenerate wildcard set) and buckets referencing any
-  // property present in this payload (propertyIndex). Section 6.2.
+  // property present in EITHER bag (propertyIndex): the eventProperties drive
+  // event-shaped criteria narrowing, the contactProperties patch surfaces a
+  // contact-property change so a property-criteria bucket is re-checked on the
+  // first event that mutates it. Section 6.2.
   const candidateMap = new Map<string, BucketMeta>();
   for (const bucket of bucketRegistry.getByReferencedEvent(event)) {
     candidateMap.set(bucket.id, bucket);
   }
-  for (const key of Object.keys(properties ?? {})) {
+  for (const key of [
+    ...Object.keys(eventProperties ?? {}),
+    ...Object.keys(contactPropertiesPatch ?? {}),
+  ]) {
     for (const bucket of bucketRegistry.getByReferencedProperty(key)) {
       candidateMap.set(bucket.id, bucket);
     }
@@ -109,19 +125,20 @@ export async function checkBucketMembership(opts: {
     return [];
   }
 
-  // (3) Property predicates evaluate against MERGED contact state, NOT the bare
-  // event payload (Section 6.1 rule #3). Read the EXISTING contacts row ONCE iff
-  // any surviving candidate references a property — pure event/count buckets skip
-  // the read entirely. We read the row that already exists (not the one
-  // upsertContact is concurrently writing) so we do not depend on the
-  // fire-and-forget upsert having run.
+  // (3) Property predicates evaluate against contact state ⊕ this-ingest
+  // contactProperties patch — NOT the raw event payload (Section 6.1 rule #3 /
+  // D2). Read the EXISTING contacts row ONCE iff any surviving candidate
+  // references a property — pure event/count buckets skip the read entirely.
+  // `ingestEvent` already awaited `resolveOrCreateContact` before us, so the row
+  // exists by the resolved key; the patch overlay still covers the read-after-
+  // write gap on a contact's very first event (risk 7).
   const needsContactState = candidates.some(
     (bucket) =>
       bucket.criteria != null &&
       collectPropertyNames(bucket.criteria).length > 0,
   );
 
-  let contactProperties: Record<string, unknown> = {};
+  let storedContactProps: Record<string, unknown> = {};
   let contactDeleted = false;
   if (needsContactState) {
     const [contact] = await db
@@ -133,7 +150,7 @@ export async function checkBucketMembership(opts: {
       .where(eq(contacts.externalId, userId))
       .limit(1);
     if (contact) {
-      contactProperties =
+      storedContactProps =
         (contact.properties as Record<string, unknown> | null) ?? {};
       contactDeleted = contact.deletedAt != null;
     }
@@ -144,10 +161,12 @@ export async function checkBucketMembership(opts: {
     return [];
   }
 
-  // event payload overlays cumulative contact state.
+  // this-ingest contactProperties patch overlays stored contact state. The raw
+  // event payload is REMOVED from property eval (D2 — bucket prop-criteria see
+  // contact state only).
   const journeyContext: Record<string, unknown> = {
-    ...contactProperties,
-    ...(properties ?? {}),
+    ...storedContactProps,
+    ...(contactPropertiesPatch ?? {}),
   };
 
   const transitions: BucketTransition[] = [];

package/src/container.ts

@@ -36,6 +36,8 @@ import { createLogger, type Logger } from "./lib/logger.js";
 import { createTrackedMailer } from "./lib/mailer.js";
 import { getPostHog } from "./lib/posthog.js";
 import { prepareTrackedHtml } from "./lib/tracking.js";
+import type { DefinedList } from "./lists/define-list.js";
+import { buildListRegistry, type ListRegistry } from "./lists/registry.js";
 
 export interface HogsendDefaults {
   /** Global fallback IANA timezone for scheduling. Defaults to "UTC". */
@@ -70,6 +72,14 @@ export interface HogsendClient {
    * Empty when no buckets are wired.
    */
   bucketRegistry: BucketRegistry;
+  /**
+   * The email-list registry (D3): code-defined subscription categories layered
+   * on `email_preferences.categories`, with the LOCKED polarity rule that is the
+   * single source of truth for the mailer's suppression check AND the preference
+   * center. Built and installed as the process singleton at client build (read
+   * elsewhere via `getListRegistry()`). Empty when no lists are wired.
+   */
+  listRegistry: ListRegistry;
   hatchet: HatchetClient;
   /**
    * The client repo's migration journal (`migrations/meta/_journal.json`),
@@ -91,6 +101,13 @@ export interface HogsendClientOptions {
   journeys?: DefinedJourney[];
   /** Buckets to register in the {@link BucketRegistry}. Defaults to none. */
   buckets?: DefinedBucket[];
+  /**
+   * Email lists (D3) to register in the {@link ListRegistry}. Each is a
+   * `defineList()` subscription category (id + name + `defaultOptIn`). The
+   * registry drives the mailer's list-aware suppression check and the
+   * preference center. Defaults to none (empty registry ⇒ legacy opt-in).
+   */
+  lists?: DefinedList[];
   /**
    * Email is a first-class channel. Its config is grouped here rather than
    * spread across top-level args — the engine owns the cohesive email pipeline
@@ -131,6 +148,11 @@ export interface HogsendClientOptions {
    * `env.ENABLED_BUCKETS`.
    */
   enabledBuckets?: string;
+  /**
+   * Comma-separated ids (or `*`) controlling which lists load. Defaults to
+   * `env.ENABLED_LISTS`.
+   */
+  enabledLists?: string;
   /**
    * The client repo's migration journal for the `schema.client` health block.
    * Defaults to `{ entries: [] }` (empty client track ⇒ trivially in sync).
@@ -237,6 +259,15 @@ export function createHogsendClient(
     bucket.membersIterator = accessor.membersIterator;
   }
 
+  // Build + install the list registry singleton (D3). Runs in BOTH the API and
+  // worker (both call createHogsendClient), so `getListRegistry()` resolves the
+  // wired lists in the mailer's suppression check and the preference center in
+  // either process. `buildListRegistry` installs the process singleton.
+  const listRegistry = buildListRegistry(
+    opts.lists ?? [],
+    opts.enabledLists ?? env.ENABLED_LISTS,
+  );
+
   const provider =
     opts.email?.provider ??
     createResendProvider({
@@ -293,6 +324,7 @@ export function createHogsendClient(
   // keep these at debug for non-boot contexts (tests, REPL, library use).
   logger.debug(`Journey registry loaded: ${registry.count()} journeys`);
   logger.debug(`Bucket registry loaded: ${bucketRegistry.count()} buckets`);
+  logger.debug(`List registry loaded: ${listRegistry.count()} lists`);
 
   return {
     env,
@@ -306,6 +338,7 @@ export function createHogsendClient(
     analytics,
     registry,
     bucketRegistry,
+    listRegistry,
     hatchet: opts.overrides?.hatchet ?? hatchet,
     clientJournal: opts.clientJournal ?? { entries: [] },
     defaults,

package/src/env.ts

@@ -65,8 +65,37 @@ export const env = createEnv({
     // Evaluated at worker boot — a toggle requires a worker restart; only the
     // bucket_configs DB override is hot.
     ENABLED_BUCKETS: z.string().default("*"),
+    // Email lists (D3): same `"*"`-or-csv contract as ENABLED_JOURNEYS /
+    // ENABLED_BUCKETS. Filters which `defineList()` lists are registered into the
+    // process ListRegistry (the suppression-polarity + preference-center source).
+    ENABLED_LISTS: z.string().default("*"),
     // Cadence for the engine-owned bucket reconcile cron (time-based leaves).
     BUCKET_RECONCILE_CRON: z.string().default("*/5 * * * *"),
+    // --- Outbound webhooks (Section 1.5/1.8) ---
+    // Cadence for the engine-owned outbound-delivery reaper cron (the retry
+    // scheduler + orphan-`sending` recovery). Declared for parity with
+    // BUCKET_RECONCILE_CRON; the delivery task also reads it raw off process.env.
+    OUTBOUND_WEBHOOK_REAPER_CRON: z.string().optional(),
+    // Delivery tunables — read raw off process.env inside the durable task;
+    // declared here so they are part of the validated env contract. All optional
+    // with task-internal defaults (MAX_ATTEMPTS 8, TIMEOUT 15s, BASE 5s,
+    // MAX_DELAY 6h, STUCK_AFTER 5min).
+    OUTBOUND_WEBHOOK_MAX_ATTEMPTS: z.coerce.number().optional(),
+    OUTBOUND_WEBHOOK_TIMEOUT_MS: z.coerce.number().optional(),
+    OUTBOUND_WEBHOOK_BASE_DELAY_MS: z.coerce.number().optional(),
+    OUTBOUND_WEBHOOK_MAX_DELAY_MS: z.coerce.number().optional(),
+    OUTBOUND_WEBHOOK_STUCK_AFTER_MS: z.coerce.number().optional(),
+    // --- Integration presets (Section 2.2) ---
+    // Signature-source secrets. The webhook route resolves a preset's secret via
+    // env[source.auth.envKey]; a signature source FAILS CLOSED when its secret is
+    // unset. Setting one auto-enables that preset at POST /v1/webhooks/<id>.
+    CLERK_WEBHOOK_SECRET: z.string().min(1).optional(),
+    SUPABASE_WEBHOOK_SECRET: z.string().min(1).optional(),
+    STRIPE_WEBHOOK_SECRET: z.string().min(1).optional(),
+    SEGMENT_WEBHOOK_SECRET: z.string().min(1).optional(),
+    // Preset enablement override: csv of preset ids, `"*"` (all with a secret),
+    // or `"none"`. Absent → auto-enable any preset whose secret is set.
+    ENABLED_WEBHOOK_PRESETS: z.string().optional(),
   },
   runtimeEnv: process.env,
   emptyStringAsUndefined: true,

package/src/index.ts

@@ -16,7 +16,6 @@ export type {
   PostHogService,
   SendResult,
   WebhookEvent,
-  WebhookEventType,
   WebhookHandlerMap,
 } from "@hogsend/core";
 // Core helpers used by content journeys (days/hours/minutes, condition + journey
@@ -150,6 +149,13 @@ export {
 // --- Logging ---
 export { createLogger, type Logger } from "./lib/logger.js";
 export { createTrackedMailer } from "./lib/mailer.js";
+// --- Outbound webhooks: emit spine (Section 1.4) ---
+export {
+  emitOutbound,
+  OUTBOUND_EVENTS,
+  type OutboundEventName,
+  type OutboundPayloads,
+} from "./lib/outbound.js";
 export { getPostHog } from "./lib/posthog.js";
 export { getRedisIfConnected } from "./lib/redis.js";
 export { type MountStudioResult, mountStudio } from "./lib/studio.js";
@@ -174,14 +180,48 @@ export {
 export {
   pushTrackingEvent,
   resolveEmailSendContext,
+  resolveEmailSendContextByResendId,
 } from "./lib/tracking-events.js";
+// --- Outbound webhooks: signing core (Section 1.2) ---
+export {
+  generateWebhookSecret,
+  type SignedWebhook,
+  signWebhook,
+  verifyWebhookSignature,
+  WEBHOOK_EVENT_TYPES,
+  type WebhookEventType,
+} from "./lib/webhook-signing.js";
+// --- Lists (D3) ---
+export {
+  type DefinedList,
+  defineList,
+  type ListMeta,
+} from "./lists/define-list.js";
+export { buildListRegistry, ListRegistry } from "./lists/registry.js";
+export {
+  getListRegistry,
+  resetListRegistry,
+  setListRegistry,
+} from "./lists/registry-singleton.js";
 // --- Webhook sources ---
 export {
   type DefinedWebhookSource,
   defineWebhookSource,
+  verifySignature,
+  type WebhookSourceAuth,
   type WebhookSourceCtx,
   type WebhookSourceMeta,
 } from "./webhook-sources/define-webhook-source.js";
+// --- Integration presets (Section 2.3/2.4) ---
+export {
+  clerkSource,
+  PRESET_SOURCES,
+  type PresetId,
+  presetsFromEnv,
+  segmentSource,
+  stripeSource,
+  supabaseSource,
+} from "./webhook-sources/presets/index.js";
 export {
   type CreateWorkerOptions,
   createWorker,
@@ -199,6 +239,12 @@ export {
   bucketReconcileTask,
 } from "./workflows/bucket-reconcile.js";
 export { checkAlertsTask } from "./workflows/check-alerts.js";
+// --- Outbound webhooks: durable delivery task + reaper (Section 1.5) ---
+export {
+  deliverWebhookTask,
+  reapDueWebhookDeliveriesTask,
+} from "./workflows/deliver-webhook.js";
 export { importContactsTask } from "./workflows/import-contacts.js";
+export { sendCampaignTask } from "./workflows/send-campaign.js";
 // --- Built-in Hatchet workflow tasks ---
 export { sendEmailTask } from "./workflows/send-email.js";

package/src/journeys/define-journey.ts

@@ -15,6 +15,7 @@ import {
 } from "../lib/enrollment-guards.js";
 import { hatchet } from "../lib/hatchet.js";
 import { createLogger } from "../lib/logger.js";
+import { emitOutbound } from "../lib/outbound.js";
 import { resolveTimezoneWithSource } from "../lib/timezone.js";
 import { getClientScheduleDefaults } from "./client-defaults-singleton.js";
 import { JOURNEY_EXECUTION_TIMEOUT } from "./constants.js";
@@ -190,12 +191,13 @@ export function defineJourney(options: {
       try {
         await options.run(user, ctx);
 
+        const completedAt = new Date();
         await db
           .update(journeyStates)
           .set({
             status: "completed",
-            completedAt: new Date(),
-            updatedAt: new Date(),
+            completedAt,
+            updatedAt: completedAt,
           })
           .where(eq(journeyStates.id, stateId));
 
@@ -205,6 +207,28 @@ export function defineJourney(options: {
           userId,
         });
 
+        // OUTBOUND `journey.completed` — fired alongside the internal
+        // `journey:completed` push. Runs in the WORKER (this durable task), so it
+        // uses the engine `db`/`hatchet`/`logger` singletons. `dedupeKey` =
+        // `journey.completed:<stateId>`: a Hatchet re-execution recomputes the
+        // identical key and the unique `(endpointId, dedupeKey)` index absorbs the
+        // duplicate (risk 3). `journey:failed` is NOT in the catalog → no emit.
+        void emitOutbound({
+          db,
+          hatchet,
+          logger,
+          event: "journey.completed",
+          dedupeKey: `journey.completed:${stateId}`,
+          payload: {
+            journeyId: meta.id,
+            journeyName: meta.name,
+            stateId,
+            userId,
+            userEmail,
+            completedAt: completedAt.toISOString(),
+          },
+        }).catch(logger.warn);
+
         return { stateId, status: "completed" };
       } catch (err) {
         // The journey reached a terminal state (exitOn / cancel) while suspended

package/src/journeys/journey-context.ts

@@ -298,6 +298,10 @@ export function createJourneyContext(
       userEmail: targetEmail,
       properties,
     }) {
+      // Keep the PUBLIC `TriggerOptions.properties` field name (decision #13 —
+      // renaming it would break consumer journeys + scaffold). Map it to the
+      // engine-internal `eventProperties` bag here; no `contactProperties` by
+      // default (a future `TriggerOptions.contactProperties` is deferred).
       await ingestEvent({
         db,
         registry,
@@ -307,7 +311,7 @@ export function createJourneyContext(
           event,
           userId: targetUserId,
           userEmail: targetEmail ?? userEmail,
-          properties: properties ?? {},
+          eventProperties: properties ?? {},
         },
       });
     },

package/src/lib/boot.ts

@@ -171,6 +171,6 @@ export function reportWorkerReady(info: WorkerReadyInfo): void {
     `  ${ok} ${tasks}`,
     "",
     `  ${dim("Listening — journeys fire as events arrive.")}`,
-    `  ${dim("Send one:")} ${color.cyan("POST /v1/ingest")} ${dim("· or Studio › Debug")}`,
+    `  ${dim("Send one:")} ${color.cyan("POST /v1/events")} ${dim("· or Studio › Debug")}`,
   ]);
 }

package/src/lib/bucket-emit.ts

@@ -6,6 +6,7 @@ import type { BucketLeaveReason } from "../buckets/bucket-reactions.js";
 import { syncBucketToPostHog } from "./bucket-posthog-sync.js";
 import { ingestEvent } from "./ingestion.js";
 import type { Logger } from "./logger.js";
+import { emitOutbound } from "./outbound.js";
 
 export type BucketTransitionKind = "entered" | "left" | "dwell";
 
@@ -123,7 +124,7 @@ export async function emitBucketTransition(opts: {
       event: eventName,
       userId,
       userEmail: userEmail ?? "",
-      properties,
+      eventProperties: properties,
       idempotencyKey,
     },
   });
@@ -141,9 +142,53 @@ export async function emitBucketTransition(opts: {
         event: genericEvent,
         userId,
         userEmail: userEmail ?? "",
-        properties,
+        eventProperties: properties,
         idempotencyKey: `bucket:${bucket.id}:${userId}:${kind}:${epoch}:generic`,
       },
     });
   }
+
+  // OUTBOUND `bucket.entered` / `bucket.left` — this is the SINGLE producer for
+  // all three transition origins (real-time / reconcile / fast-expiry), so the
+  // emit lives here once. GATED to enter/leave (dwell is a recurring membership
+  // tick, not in the catalog — same gate as the PostHog mirror above). The
+  // `dedupeKey` is the SAME deterministic `idempotencyKey` all three producers
+  // compute, so a Hatchet re-execution converges to ONE emission via the unique
+  // `(endpointId, dedupeKey)` index (risk 3). Fire-and-forget.
+  if (kind === "entered") {
+    void emitOutbound({
+      db,
+      hatchet,
+      logger,
+      event: "bucket.entered",
+      dedupeKey: idempotencyKey,
+      payload: {
+        bucketId: bucket.id,
+        bucketName: bucket.name,
+        userId,
+        userEmail,
+        transition: "entered",
+        entryCount: epoch,
+        source,
+      },
+    }).catch(logger.warn);
+  } else if (kind === "left") {
+    void emitOutbound({
+      db,
+      hatchet,
+      logger,
+      event: "bucket.left",
+      dedupeKey: idempotencyKey,
+      payload: {
+        bucketId: bucket.id,
+        bucketName: bucket.name,
+        userId,
+        userEmail,
+        transition: "left",
+        entryCount: epoch,
+        source,
+        ...(reason != null ? { reason } : {}),
+      },
+    }).catch(logger.warn);
+  }
 }