@hogsend/engine 0.1.1 → 0.2.0

package/src/container.ts

@@ -1,6 +1,6 @@
 import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
 import type { TimeZone } from "@hogsend/core";
-import type { JourneyRegistry } from "@hogsend/core/registry";
+import type { BucketRegistry, JourneyRegistry } from "@hogsend/core/registry";
 import type { SendWindow } from "@hogsend/core/schedule";
 import {
   createDatabase,
@@ -16,6 +16,8 @@ import {
   type EmailProvider,
 } from "@hogsend/plugin-resend";
 import type { Resend } from "resend";
+import type { DefinedBucket } from "./buckets/define-bucket.js";
+import { buildBucketRegistry } from "./buckets/registry.js";
 import { env } from "./env.js";
 import { setClientScheduleDefaults } from "./journeys/client-defaults-singleton.js";
 import type { DefinedJourney } from "./journeys/define-journey.js";
@@ -58,6 +60,13 @@ export interface HogsendClient {
   templates: TemplateRegistry;
   analytics?: PostHogService;
   registry: JourneyRegistry;
+  /**
+   * The bucket registry (id map + event/property inverted indexes for candidate
+   * narrowing). Built and installed as the process singleton at client build;
+   * the real-time ingest path reads it via `getBucketRegistrySingleton()`.
+   * Empty when no buckets are wired.
+   */
+  bucketRegistry: BucketRegistry;
   hatchet: HatchetClient;
   /**
    * The client repo's migration journal (`migrations/meta/_journal.json`),
@@ -77,6 +86,8 @@ export interface HogsendClient {
 export interface HogsendClientOptions {
   /** Journeys to register in the {@link JourneyRegistry}. Defaults to none. */
   journeys?: DefinedJourney[];
+  /** Buckets to register in the {@link BucketRegistry}. Defaults to none. */
+  buckets?: DefinedBucket[];
   /**
    * Email is a first-class channel. Its config is grouped here rather than
    * spread across top-level args — the engine owns the cohesive email pipeline
@@ -112,6 +123,11 @@ export interface HogsendClientOptions {
    * `env.ENABLED_JOURNEYS`.
    */
   enabledJourneys?: string;
+  /**
+   * Comma-separated ids (or `*`) controlling which buckets load. Defaults to
+   * `env.ENABLED_BUCKETS`.
+   */
+  enabledBuckets?: string;
   /**
    * The client repo's migration journal for the `schema.client` health block.
    * Defaults to `{ entries: [] }` (empty client track ⇒ trivially in sync).
@@ -182,6 +198,14 @@ export function createHogsendClient(
     opts.enabledJourneys ?? env.ENABLED_JOURNEYS,
   );
 
+  // Installs the bucket registry singleton in BOTH the API and worker processes
+  // (both call createHogsendClient); the real-time ingest path reads it via
+  // getBucketRegistrySingleton().
+  const bucketRegistry = buildBucketRegistry(
+    opts.buckets ?? [],
+    opts.enabledBuckets ?? env.ENABLED_BUCKETS,
+  );
+
   const provider =
     opts.email?.provider ??
     createResendProvider({
@@ -228,6 +252,7 @@ export function createHogsendClient(
   const analytics = opts.analytics ?? getPostHog();
 
   logger.info(`Journey registry loaded: ${registry.count()} journeys`);
+  logger.info(`Bucket registry loaded: ${bucketRegistry.count()} buckets`);
 
   return {
     env,
@@ -240,6 +265,7 @@ export function createHogsendClient(
     templates,
     analytics,
     registry,
+    bucketRegistry,
     hatchet: opts.overrides?.hatchet ?? hatchet,
     clientJournal: opts.clientJournal ?? { entries: [] },
     defaults,

package/src/env.ts

@@ -54,6 +54,12 @@ export const env = createEnv({
     ADMIN_API_KEY: z.string().min(1).optional(),
     API_PUBLIC_URL: z.string().url().default("http://localhost:3002"),
     ENABLED_JOURNEYS: z.string().default("*"),
+    // Buckets: same `"*"`-or-csv contract as ENABLED_JOURNEYS (Section 9.3).
+    // Evaluated at worker boot — a toggle requires a worker restart; only the
+    // bucket_configs DB override is hot.
+    ENABLED_BUCKETS: z.string().default("*"),
+    // Cadence for the engine-owned bucket reconcile cron (time-based leaves).
+    BUCKET_RECONCILE_CRON: z.string().default("*/5 * * * *"),
   },
   runtimeEnv: process.env,
   emptyStringAsUndefined: true,

package/src/index.ts

@@ -6,7 +6,10 @@
 // Core helpers used by content journeys (days/hours/minutes, condition + journey
 // types) so content can import everything from `@hogsend/engine`.
 export * from "@hogsend/core";
-export { JourneyRegistry } from "@hogsend/core/registry";
+export {
+  BucketRegistry,
+  JourneyRegistry,
+} from "@hogsend/core/registry";
 // --- Re-exports for content ---
 // Schema/version helpers used by the boot guard and the /v1/health route.
 export {
@@ -19,6 +22,25 @@ export {
 } from "@hogsend/db";
 // --- App / container / worker factories ---
 export { type AppEnv, type CreateAppOptions, createApp } from "./app.js";
+// --- Buckets ---
+export {
+  type BucketTransition,
+  type BucketTransitionKind,
+  checkBucketMembership,
+} from "./buckets/check-membership.js";
+export {
+  type DefinedBucket,
+  defineBucket,
+} from "./buckets/define-bucket.js";
+export {
+  buildBucketRegistry,
+  selectBucketTasks,
+} from "./buckets/registry.js";
+export {
+  getBucketRegistrySingleton,
+  resetBucketRegistry,
+  setBucketRegistry,
+} from "./buckets/registry-singleton.js";
 export {
   createHogsendClient,
   type HogsendClient,
@@ -50,6 +72,11 @@ export {
   type BatchedBackfillResult,
   runBatchedBackfill,
 } from "./lib/backfill.js";
+// --- Bucket transition emission (shared by real-time / cron / fast-expiry) ---
+export {
+  type BucketTransitionSource,
+  emitBucketTransition,
+} from "./lib/bucket-emit.js";
 // --- Infrastructure singletons ---
 export { getDb } from "./lib/db.js";
 // --- Email ---
@@ -121,6 +148,17 @@ export {
   createWorker,
   type Worker,
 } from "./worker.js";
+export {
+  type BucketBackfillInput,
+  bucketBackfillTask,
+  computeCriteriaHash,
+  enqueueBucketBackfills,
+} from "./workflows/bucket-backfill.js";
+export {
+  type BucketArmExpiryInput,
+  bucketExpiryTask,
+  bucketReconcileTask,
+} from "./workflows/bucket-reconcile.js";
 export { checkAlertsTask } from "./workflows/check-alerts.js";
 export { importContactsTask } from "./workflows/import-contacts.js";
 // --- Built-in Hatchet workflow tasks ---

package/src/lib/bucket-emit.ts

@@ -0,0 +1,107 @@
+import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
+import type { BucketMeta } from "@hogsend/core";
+import type { JourneyRegistry } from "@hogsend/core/registry";
+import type { Database } from "@hogsend/db";
+import { syncBucketToPostHog } from "./bucket-posthog-sync.js";
+import { ingestEvent } from "./ingestion.js";
+import type { Logger } from "./logger.js";
+
+export type BucketTransitionKind = "entered" | "left";
+
+/** Where a transition originated — carried on the emitted event properties. */
+export type BucketTransitionSource =
+  | "event"
+  | "reconcile"
+  | "backfill"
+  | "manual";
+
+/**
+ * Emit a bucket transition back through `ingestEvent` (the `ctx.trigger`
+ * precedent) — shared by ALL three producers (real-time `checkBucketMembership`,
+ * the reconcile cron, and the fast-expiry timer) so they compute byte-identical
+ * `idempotencyKey`s for the same transition and converge to ONE emission
+ * (Section 6.3 worked example).
+ *
+ * Persists to `userEvents`, pushes to Hatchet (routing to journeys), and runs
+ * `checkExits`. Emits the per-bucket ALIAS (`bucket:<kind>:<id>`) by default; the
+ * generic `bucket:<kind>` is emitted ONLY when a generic-bound journey actually
+ * exists (aliased-only default — Section 8.5). `epoch` is the winning membership
+ * row's `entryCount`, read off the single winning mutation by the caller.
+ */
+export async function emitBucketTransition(opts: {
+  db: Database;
+  /** The JOURNEY registry — forwarded into the recursive emit ingestEvent. */
+  registry: JourneyRegistry;
+  hatchet: HatchetClient;
+  logger: Logger;
+  kind: BucketTransitionKind;
+  bucket: BucketMeta;
+  userId: string;
+  userEmail: string | null;
+  epoch: number;
+  source?: BucketTransitionSource;
+}): Promise<void> {
+  const {
+    db,
+    registry,
+    hatchet,
+    logger,
+    kind,
+    bucket,
+    userId,
+    userEmail,
+    epoch,
+    source = "event",
+  } = opts;
+
+  const properties: Record<string, unknown> = {
+    bucketId: bucket.id,
+    bucketName: bucket.name,
+    userId,
+    transition: kind,
+    source,
+  };
+
+  // Optional PostHog person-property mirror (Section 12). Off by default; a
+  // no-op without POSTHOG_API_KEY. Wired here, the single transition path shared
+  // by all three producers (real-time / reconcile / fast-expiry), so the sync
+  // fires exactly once per emitted transition. Best-effort — it never blocks the
+  // event emit below.
+  syncBucketToPostHog({ logger, kind, bucket, userId });
+
+  // Per-bucket alias — the recommended, narrowly-routed binding. The
+  // deterministic idempotencyKey rides the userEvents dedup short-circuit as
+  // defense-in-depth (Section 6.3).
+  await ingestEvent({
+    db,
+    registry,
+    hatchet,
+    logger,
+    event: {
+      event: `bucket:${kind}:${bucket.id}`,
+      userId,
+      userEmail: userEmail ?? "",
+      properties,
+      idempotencyKey: `bucket:${bucket.id}:${userId}:${kind}:${epoch}`,
+    },
+  });
+
+  // Generic form — emitted ONLY if a journey actually binds to it, so the
+  // recursion-guarded generic event is not written for nothing (Section 8.5).
+  const genericEvent = `bucket:${kind}`;
+  if (registry.getByTriggerEvent(genericEvent).length > 0) {
+    await ingestEvent({
+      db,
+      registry,
+      hatchet,
+      logger,
+      event: {
+        event: genericEvent,
+        userId,
+        userEmail: userEmail ?? "",
+        properties,
+        idempotencyKey: `bucket:${bucket.id}:${userId}:${kind}:${epoch}:generic`,
+      },
+    });
+  }
+}

package/src/lib/bucket-posthog-sync.ts

@@ -0,0 +1,63 @@
+import type { BucketMeta } from "@hogsend/core";
+import type { Logger } from "./logger.js";
+import { getPostHog } from "./posthog.js";
+
+/**
+ * Optional PostHog person-property mirror for a bucket transition (Section 12).
+ *
+ * OFF BY DEFAULT — a no-op unless `meta.syncToPostHog === true`. Also a no-op
+ * without `POSTHOG_API_KEY` (`getPostHog()` returns undefined), so self-host
+ * setups that omit PostHog silently do nothing — documented, not broken.
+ *
+ * On JOIN it `$set`s a boolean person property `true`; on LEAVE it `$unset`s the
+ * same key. `$unset` (NOT `$set false`) is the recommended default: a cohort
+ * `key = true` excludes a false value, but a cohort `key is set` STILL matches a
+ * false value, so `$unset` is the only form where both cohort idioms behave
+ * correctly. The property key defaults to `hogsend_bucket_<id>`, overridable via
+ * `meta.postHogPropertyKey`.
+ *
+ * This reuses the existing `plugin-posthog` capture path (the same one
+ * `identify()` uses at journey-context.ts) — it adds no new integration surface
+ * and never pushes to any non-PostHog destination (the Section 2.4 anti-CDP
+ * invariant). Best-effort: a capture failure is logged and swallowed so a sync
+ * error never blocks a transition emit.
+ */
+export function syncBucketToPostHog(opts: {
+  logger: Logger;
+  kind: "entered" | "left";
+  bucket: BucketMeta;
+  userId: string;
+}): void {
+  const { logger, kind, bucket, userId } = opts;
+
+  if (!bucket.syncToPostHog) return;
+
+  const posthog = getPostHog();
+  if (!posthog) return; // no POSTHOG_API_KEY → silent no-op
+
+  const propertyKey =
+    bucket.postHogPropertyKey ?? `hogsend_bucket_${bucket.id}`;
+
+  try {
+    if (kind === "entered") {
+      // $set { key: true } — mirrors plugin-posthog identify() ($set path).
+      posthog.identify(userId, { [propertyKey]: true });
+    } else {
+      // $unset [key] — RECOMMENDED on leave (Section 12). The property is absent
+      // unless the user is currently a member, so both `key = true` and
+      // `key is set` cohorts behave correctly.
+      posthog.captureEvent({
+        distinctId: userId,
+        event: "$set",
+        properties: { $unset: [propertyKey] },
+      });
+    }
+  } catch (err) {
+    logger.warn("Bucket PostHog sync failed (best-effort)", {
+      bucketId: bucket.id,
+      userId,
+      kind,
+      error: err instanceof Error ? err.message : String(err),
+    });
+  }
+}

package/src/lib/ingestion.ts

@@ -3,6 +3,7 @@ import { evaluatePropertyConditions } from "@hogsend/core";
 import type { JourneyRegistry } from "@hogsend/core/registry";
 import { type Database, journeyStates, userEvents } from "@hogsend/db";
 import { and, eq, inArray, isNull } from "drizzle-orm";
+import { checkBucketMembership } from "../buckets/check-membership.js";
 import { upsertContact } from "./contacts.js";
 import type { Logger } from "./logger.js";
 
@@ -93,6 +94,30 @@ export async function ingestEvent(opts: {
     }),
   ]);
 
+  // Real-time bucket membership re-evaluation (Section 6.1). NOT part of the
+  // Promise.all above: its property eval reads MERGED contact state, and its
+  // bucket:entered/left emissions recurse back into ingestEvent (the recursion
+  // guard in checkBucketMembership bounds them). Best-effort: a bucket failure
+  // must not fail the ingest of the originating event.
+  try {
+    await checkBucketMembership({
+      db,
+      registry,
+      hatchet,
+      logger,
+      userId: event.userId,
+      userEmail: event.userEmail || null,
+      event: event.event,
+      properties: event.properties,
+    });
+  } catch (err) {
+    logger.warn("Bucket membership check failed", {
+      event: event.event,
+      userId: event.userId,
+      error: err instanceof Error ? err.message : String(err),
+    });
+  }
+
   logger.info("Event ingested", {
     event: event.event,
     userId: event.userId,