@hogsend/engine 0.1.1 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hogsend/engine",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "type": "module",
   "license": "MIT",
   "repository": {
@@ -35,8 +35,8 @@
     "resend": "^6.12.3",
     "winston": "^3.19.0",
     "zod": "^4.4.3",
-    "@hogsend/core": "^0.1.0",
-    "@hogsend/db": "^0.1.0",
+    "@hogsend/core": "^0.2.0",
+    "@hogsend/db": "^0.2.0",
     "@hogsend/email": "^0.1.0",
     "@hogsend/plugin-posthog": "^0.1.0",
     "@hogsend/plugin-resend": "^0.1.0"

package/src/buckets/check-membership.ts

@@ -0,0 +1,499 @@
+import type { HatchetClient } from "@hatchet-dev/typescript-sdk/v1/index.js";
+import {
+  type BucketMeta,
+  collectPropertyNames,
+  durationToMs,
+  evaluateCondition,
+} from "@hogsend/core";
+import type { JourneyRegistry } from "@hogsend/core/registry";
+import { bucketMemberships, contacts, type Database } from "@hogsend/db";
+import { and, eq, isNull, sql } from "drizzle-orm";
+import { emitBucketTransition } from "../lib/bucket-emit.js";
+import type { Logger } from "../lib/logger.js";
+import { getBucketRegistrySingleton } from "./registry-singleton.js";
+
+/** The reserved prefix every bucket transition event carries. */
+const BUCKET_EVENT_PREFIX = "bucket:";
+
+export type BucketTransitionKind = "entered" | "left";
+
+export interface BucketTransition {
+  bucketId: string;
+  transition: BucketTransitionKind;
+}
+
+/**
+ * Real-time bucket-membership re-evaluation, invoked from inside `ingestEvent`
+ * AFTER the `userEvents` insert / idempotency short-circuit (Section 6.1).
+ *
+ * For the ingested event it narrows the candidate buckets via the registry's
+ * event + property inverted indexes (Section 6.2), evaluates each candidate's
+ * criteria against MERGED contact state (Section 6.1 rule #3), diffs the result
+ * against the current `bucket_memberships` rows, and performs the atomic
+ * RETURNING-gated mutation (partial-unique INSERT for joins, compare-and-swap
+ * UPDATE for leaves — Section 6.3). On a real transition it emits
+ * `bucket:entered:<id>` / `bucket:left:<id>` back through `ingestEvent`, gated on
+ * the reentry policy and deferring leaves still inside `minDwell`.
+ *
+ * Returns the computed transition list so a unit test can assert enter/leave/no-op
+ * WITHOUT a live Hatchet (Section 14 — the testing seam). Production callers
+ * ignore the return value (the emission has already happened via recursion).
+ */
+export async function checkBucketMembership(opts: {
+  db: Database;
+  /** The JOURNEY registry — forwarded into the recursive emit ingestEvent. */
+  registry: JourneyRegistry;
+  hatchet: HatchetClient;
+  logger: Logger;
+  userId: string;
+  userEmail: string | null;
+  event: string;
+  properties: Record<string, unknown>;
+  /** Optional override; defaults to the process bucket-registry singleton. */
+  bucketRegistry?: ReturnType<typeof getBucketRegistrySingleton>;
+}): Promise<BucketTransition[]> {
+  const {
+    db,
+    registry,
+    hatchet,
+    logger,
+    userId,
+    userEmail,
+    event,
+    properties,
+  } = opts;
+
+  // (1) Recursion guard — MUST be first. bucket:-prefixed events are transition
+  // rows (still written to userEvents / pushed to Hatchet / run through
+  // checkExits) but MUST NOT trigger bucket re-evaluation, else the emit recurses
+  // forever. ingestEvent has no built-in re-entry guard, so this prefix check is
+  // the bound on recursion (Section 6.1 rule #1).
+  if (event.startsWith(BUCKET_EVENT_PREFIX)) {
+    return [];
+  }
+
+  // The bucket registry is resolved separately from the journey registry; the
+  // two are never conflated (Section 6.1 signature note).
+  const bucketRegistry = opts.bucketRegistry ?? getBucketRegistrySingleton();
+
+  // (2) Candidate narrowing — the UNION of buckets referencing this event name
+  // (eventIndex + the degenerate wildcard set) and buckets referencing any
+  // property present in this payload (propertyIndex). Section 6.2.
+  const candidateMap = new Map<string, BucketMeta>();
+  for (const bucket of bucketRegistry.getByReferencedEvent(event)) {
+    candidateMap.set(bucket.id, bucket);
+  }
+  for (const key of Object.keys(properties ?? {})) {
+    for (const bucket of bucketRegistry.getByReferencedProperty(key)) {
+      candidateMap.set(bucket.id, bucket);
+    }
+  }
+
+  if (candidateMap.size === 0) {
+    return [];
+  }
+
+  const candidates = Array.from(candidateMap.values()).filter(
+    // manual buckets are not criteria-driven; they never appear in the indexes,
+    // but guard defensively. enabled is the static load-time flag (the DB
+    // bucket_configs override is a later-phase concern, not read on this hot
+    // path — Section 6.2).
+    (bucket) =>
+      bucket.enabled && bucket.kind !== "manual" && bucket.criteria != null,
+  );
+
+  if (candidates.length === 0) {
+    return [];
+  }
+
+  // (3) Property predicates evaluate against MERGED contact state, NOT the bare
+  // event payload (Section 6.1 rule #3). Read the EXISTING contacts row ONCE iff
+  // any surviving candidate references a property — pure event/count buckets skip
+  // the read entirely. We read the row that already exists (not the one
+  // upsertContact is concurrently writing) so we do not depend on the
+  // fire-and-forget upsert having run.
+  const needsContactState = candidates.some(
+    (bucket) =>
+      bucket.criteria != null &&
+      collectPropertyNames(bucket.criteria).length > 0,
+  );
+
+  let contactProperties: Record<string, unknown> = {};
+  let contactDeleted = false;
+  if (needsContactState) {
+    const [contact] = await db
+      .select({
+        properties: contacts.properties,
+        deletedAt: contacts.deletedAt,
+      })
+      .from(contacts)
+      .where(eq(contacts.externalId, userId))
+      .limit(1);
+    if (contact) {
+      contactProperties =
+        (contact.properties as Record<string, unknown> | null) ?? {};
+      contactDeleted = contact.deletedAt != null;
+    }
+  }
+
+  // GDPR: never (re-)evaluate or emit for a soft-deleted contact (Section 8.6).
+  if (contactDeleted) {
+    return [];
+  }
+
+  // event payload overlays cumulative contact state.
+  const journeyContext: Record<string, unknown> = {
+    ...contactProperties,
+    ...(properties ?? {}),
+  };
+
+  const transitions: BucketTransition[] = [];
+
+  for (const bucket of candidates) {
+    if (!bucket.criteria) continue;
+
+    // wasMember — current active, non-deleted membership row (cheap pre-filter;
+    // the authoritative guard is the RETURNING-gated mutation below).
+    const active = await db.query.bucketMemberships.findFirst({
+      where: and(
+        eq(bucketMemberships.userId, userId),
+        eq(bucketMemberships.bucketId, bucket.id),
+        eq(bucketMemberships.status, "active"),
+        isNull(bucketMemberships.deletedAt),
+      ),
+    });
+    const wasMember = !!active;
+
+    // isMember — the criteria evaluation. event/count sub-conditions read
+    // userEvents (the just-stored row is visible on the same connection — the
+    // documented no-pooler assumption, Section 6.1 rule #2); property
+    // sub-conditions read the merged journeyContext.
+    const isMember = await evaluateCondition({
+      condition: bucket.criteria,
+      ctx: { db, userId, journeyContext },
+    });
+
+    if (!wasMember && isMember) {
+      const transition = await handleJoin({
+        db,
+        registry,
+        hatchet,
+        logger,
+        bucket,
+        userId,
+        userEmail,
+      });
+      if (transition) transitions.push(transition);
+    } else if (wasMember && isMember) {
+      // stable member → no transition, no emit. Cheap observability bump.
+      await db
+        .update(bucketMemberships)
+        .set({ lastEvaluatedAt: new Date() })
+        .where(eq(bucketMemberships.id, active.id));
+    } else if (wasMember && !isMember) {
+      const transition = await handleLeave({
+        db,
+        registry,
+        hatchet,
+        logger,
+        bucket,
+        active,
+        userId,
+        userEmail,
+      });
+      if (transition) transitions.push(transition);
+    }
+    // !wasMember && !isMember → nothing.
+  }
+
+  return transitions;
+}
+
+async function handleJoin(opts: {
+  db: Database;
+  registry: JourneyRegistry;
+  hatchet: HatchetClient;
+  logger: Logger;
+  bucket: BucketMeta;
+  userId: string;
+  userEmail: string | null;
+}): Promise<BucketTransition | null> {
+  const { db, registry, hatchet, logger, bucket, userId, userEmail } = opts;
+
+  // entryCount ordinal = 1 + count of ALL prior memberships (active + left) for
+  // this (user, bucket) (Section 6.3 / 8.2). priorCount also drives the reentry
+  // gate.
+  const [counted] = await db
+    .select({ priorCount: sql<number>`count(*)::int` })
+    .from(bucketMemberships)
+    .where(
+      and(
+        eq(bucketMemberships.userId, userId),
+        eq(bucketMemberships.bucketId, bucket.id),
+      ),
+    );
+  const priorCount = Number(counted?.priorCount ?? 0);
+  const epoch = priorCount + 1;
+
+  // INSERT a FRESH active row. ON CONFLICT DO NOTHING targets the partial active
+  // unique index (uq_user_bucket_active): a concurrent emitter that already
+  // inserted the active row makes THIS insert return zero rows → we do NOT emit
+  // (the loser mutates nothing — Section 6.3 governing rule).
+  const expiresAt = computeExpiresAt(bucket);
+  // Unconditional TTL deadline — set once on join, swept by the reconcile cron.
+  const maxDwellAt = bucket.maxDwell
+    ? new Date(Date.now() + durationToMs(bucket.maxDwell))
+    : null;
+  const inserted = await db
+    .insert(bucketMemberships)
+    .values({
+      userId,
+      userEmail,
+      bucketId: bucket.id,
+      status: "active",
+      source: "event",
+      entryCount: epoch,
+      expiresAt,
+      maxDwellAt,
+      lastEvaluatedAt: new Date(),
+    })
+    .onConflictDoNothing()
+    .returning({ id: bucketMemberships.id });
+
+  const insertedRow = inserted[0];
+  if (!insertedRow) {
+    // Lost the race; the winner emits. We did not change a row → no emit.
+    return null;
+  }
+
+  // Arm the per-user fast-expiry durable timer (Section 6.5) AFTER the active row
+  // is written. The cron remains the authoritative backstop, so a push failure
+  // is best-effort. We arm against the persisted expiresAt so the timer's CAS on
+  // wake matches the row (or no-ops if a later event re-armed the window).
+  if (bucket.fastExpiry && expiresAt) {
+    await armExpiryTimer({
+      hatchet,
+      logger,
+      bucket,
+      rowId: insertedRow.id,
+      userId,
+      userEmail,
+      expiresAt,
+    });
+  }
+
+  // The active row is always written (Studio size must reflect reality) and the
+  // epoch always advances via the real insert; only the bucket:entered emission
+  // is gated by the reentry policy (Section 6.3).
+  if (shouldEmitJoin(bucket, priorCount)) {
+    await emitBucketTransition({
+      db,
+      registry,
+      hatchet,
+      logger,
+      kind: "entered",
+      bucket,
+      userId,
+      userEmail,
+      epoch,
+      source: "event",
+    });
+  } else {
+    logger.info("Bucket join emit suppressed by reentry policy", {
+      bucketId: bucket.id,
+      userId,
+      reentry: bucket.reentry ?? "unlimited",
+    });
+  }
+
+  return { bucketId: bucket.id, transition: "entered" };
+}
+
+async function handleLeave(opts: {
+  db: Database;
+  registry: JourneyRegistry;
+  hatchet: HatchetClient;
+  logger: Logger;
+  bucket: BucketMeta;
+  active: typeof bucketMemberships.$inferSelect;
+  userId: string;
+  userEmail: string | null;
+}): Promise<BucketTransition | null> {
+  const { db, registry, hatchet, logger, bucket, active, userId, userEmail } =
+    opts;
+
+  // minDwell DEFERS (never silently drops) the leave (Section 6.3). We set a
+  // pending-leave deadline on expiresAt = enteredAt + minDwell so the reconcile
+  // cron / fastExpiry timer re-checks after the dwell window and emits the leave
+  // via the CAS path. We do NOT emit now.
+  if (withinMinDwell(active, bucket)) {
+    const deadline = new Date(
+      active.enteredAt.getTime() +
+        durationToMs(bucket.minDwell as NonNullable<BucketMeta["minDwell"]>),
+    );
+    await db
+      .update(bucketMemberships)
+      .set({ expiresAt: deadline, lastEvaluatedAt: new Date() })
+      .where(
+        and(
+          eq(bucketMemberships.id, active.id),
+          eq(bucketMemberships.status, "active"),
+        ),
+      );
+    logger.info("Bucket leave deferred by minDwell", {
+      bucketId: bucket.id,
+      userId,
+      deferUntil: deadline.toISOString(),
+    });
+    return null;
+  }
+
+  // Compare-and-swap: only the emitter whose UPDATE actually flips the active row
+  // emits. A concurrent emitter that already flipped it matches zero rows → no
+  // emit (Section 6.3).
+  const left = await db
+    .update(bucketMemberships)
+    .set({
+      status: "left",
+      leftAt: new Date(),
+      lastEvaluatedAt: new Date(),
+      updatedAt: new Date(),
+    })
+    .where(
+      and(
+        eq(bucketMemberships.id, active.id),
+        eq(bucketMemberships.status, "active"),
+      ),
+    )
+    .returning({
+      id: bucketMemberships.id,
+      entryCount: bucketMemberships.entryCount,
+    });
+
+  const flipped = left[0];
+  if (!flipped) {
+    return null;
+  }
+
+  await emitBucketTransition({
+    db,
+    registry,
+    hatchet,
+    logger,
+    kind: "left",
+    bucket,
+    userId,
+    userEmail,
+    epoch: flipped.entryCount,
+    source: "event",
+  });
+
+  return { bucketId: bucket.id, transition: "left" };
+}
+
+/**
+ * The `reentry` emit gate, consulted on the JOIN transition only (Section 6.3).
+ * Suppressing the emit still wrote the active row and advanced the epoch — only
+ * the `bucket:entered` ingestEvent recursion is skipped.
+ */
+function shouldEmitJoin(bucket: BucketMeta, priorCount: number): boolean {
+  // First-ever join always emits.
+  if (priorCount === 0) return true;
+  switch (bucket.reentry ?? "unlimited") {
+    case "unlimited":
+      return true;
+    case "once":
+      // Any prior membership → suppress (mirrors checkEntryLimit "once").
+      return false;
+    case "once_per_period":
+      // Conservative without re-reading prior rows here: a per-period emit gate
+      // needs the most-recent prior transition timestamp. The active row was
+      // just inserted; the prior-row timestamps are not loaded on this path, so
+      // we treat once_per_period as "emit" once the active insert won and let
+      // the journey-side entryLimit/entryPeriod enforce cooldown (the documented
+      // two-layer gating, Section 13). A precise prior-transition lookup is a
+      // later-phase refinement.
+      return true;
+    default:
+      return true;
+  }
+}
+
+/** True while the active membership is still inside its minDwell window. */
+function withinMinDwell(
+  active: typeof bucketMemberships.$inferSelect,
+  bucket: BucketMeta,
+): boolean {
+  if (!bucket.minDwell) return false;
+  const elapsed = Date.now() - active.enteredAt.getTime();
+  return elapsed < durationToMs(bucket.minDwell);
+}
+
+/**
+ * Arm the shared per-user fast-expiry durable timer by pushing a
+ * `bucket:arm-expiry` event (Section 6.5). The single shared `bucketExpiryTask`
+ * durableTask (workflows/bucket-reconcile.ts) routes on `onEvents:
+ * ["bucket:arm-expiry"]`, durably sleeps to the deadline, then leaves via a CAS
+ * keyed on the ARMED `expiresAt`. The `bucket:`-prefixed event is recursion-guarded
+ * by `checkBucketMembership`, so arming does NOT re-enter bucket evaluation.
+ * Best-effort: the cron is the authoritative backstop, so a push failure is logged
+ * and swallowed.
+ */
+async function armExpiryTimer(opts: {
+  hatchet: HatchetClient;
+  logger: Logger;
+  bucket: BucketMeta;
+  rowId: string;
+  userId: string;
+  userEmail: string | null;
+  expiresAt: Date;
+}): Promise<void> {
+  const { hatchet, logger, bucket, rowId, userId, userEmail, expiresAt } = opts;
+  const msUntilExpiry = Math.max(0, expiresAt.getTime() - Date.now());
+  try {
+    await hatchet.events.push("bucket:arm-expiry", {
+      rowId,
+      bucketId: bucket.id,
+      userId,
+      userEmail,
+      armedExpiresAt: expiresAt.toISOString(),
+      msUntilExpiry,
+    });
+  } catch (err) {
+    logger.warn("Bucket fast-expiry arm failed (cron backstop covers it)", {
+      bucketId: bucket.id,
+      userId,
+      error: err instanceof Error ? err.message : String(err),
+    });
+  }
+}
+
+/**
+ * The persisted membership-expiry / fastExpiry arming epoch. Non-time-based,
+ * non-fastExpiry buckets have no deadline (returns null). Time-based / fastExpiry
+ * buckets that carry a single `within` window get `now + within`; the reconcile
+ * cron (Phase 2) and fastExpiry timer (Phase 3) own the actual leave.
+ */
+function computeExpiresAt(bucket: BucketMeta): Date | null {
+  if (!bucket.criteria) return null;
+  if (!bucket.timeBased && !bucket.fastExpiry) return null;
+  const within = firstWithin(bucket.criteria);
+  if (!within) return null;
+  return new Date(Date.now() + durationToMs(within));
+}
+
+/** Find the first EventCondition.within in a criteria tree (depth-first). */
+function firstWithin(
+  criteria: NonNullable<BucketMeta["criteria"]>,
+): NonNullable<BucketMeta["minDwell"]> | null {
+  if (criteria.type === "event" && criteria.within) {
+    return criteria.within;
+  }
+  if (criteria.type === "composite") {
+    for (const child of criteria.conditions) {
+      const found = firstWithin(child);
+      if (found) return found;
+    }
+  }
+  return null;
+}

package/src/buckets/define-bucket.ts

@@ -0,0 +1,29 @@
+import type { BucketMeta } from "@hogsend/core/types";
+import type { hatchet } from "../lib/hatchet.js";
+
+export interface DefinedBucket {
+  meta: BucketMeta;
+  /**
+   * The only task a bucket ever holds is the opt-in per-user fast-expiry timer,
+   * which is a DURABLE task (it `ctx.sleepFor`s — Section 6.5), so the type MUST
+   * be the durableTask return type, mirroring
+   * `DefinedJourney.task = ReturnType<typeof hatchet.durableTask>`
+   * (define-journey.ts:34) — NOT `hatchet.task`. The common case is
+   * declarative-only (no task), like webhookSources; the engine-wide
+   * `bucketReconcileTask` handles time-based leaves regardless.
+   */
+  task?: ReturnType<typeof hatchet.durableTask>;
+}
+
+export function defineBucket(options: { meta: BucketMeta }): DefinedBucket {
+  // bucketMetaSchema.parse happens at BucketRegistry.register (the journey
+  // precedent). defineBucket stays a PURE passthrough — identical in shape to
+  // defineWebhookSource (define-webhook-source.ts:30-34) — and does NOT branch
+  // on meta or construct any task. This keeps the three primitives consistent
+  // and avoids building a Hatchet durableTask at module-load before validation
+  // has run. The fast-expiry durableTask is synthesized later, at worker build,
+  // by selectBucketTasks(buckets, enabled) reading meta.fastExpiry (Section 9.4)
+  // — that is the single place a bucket's task is constructed, AFTER the
+  // registry has validated the meta.
+  return { meta: options.meta };
+}

package/src/buckets/registry-singleton.ts

@@ -0,0 +1,21 @@
+import type { BucketRegistry } from "@hogsend/core/registry";
+
+let _registry: BucketRegistry | undefined;
+
+export function setBucketRegistry(registry: BucketRegistry): void {
+  _registry = registry;
+}
+
+export function getBucketRegistrySingleton(): BucketRegistry {
+  if (!_registry) {
+    throw new Error(
+      "Bucket registry not initialized. Call setBucketRegistry() at startup.",
+    );
+  }
+  return _registry;
+}
+
+/** Reset the singleton — only for test cleanup. */
+export function resetBucketRegistry(): void {
+  _registry = undefined;
+}

package/src/buckets/registry.ts

@@ -0,0 +1,62 @@
+import { BucketRegistry } from "@hogsend/core/registry";
+import { parseEnabledFilter } from "../journeys/registry.js";
+import { bucketExpiryTask } from "../workflows/bucket-reconcile.js";
+import type { DefinedBucket } from "./define-bucket.js";
+import { setBucketRegistry } from "./registry-singleton.js";
+
+/**
+ * Build a {@link BucketRegistry} from an injected array of buckets, applying the
+ * enabled filter, and install it as the process singleton (so the real-time
+ * ingest path and the reconcile cron can resolve it). Returns the registry.
+ *
+ * `parseEnabledFilter` (journeys/registry.ts) is reused as-is — `ENABLED_BUCKETS`
+ * honours the same `"*"`-or-csv contract as `ENABLED_JOURNEYS` (Section 9.3).
+ * `BucketRegistry.register()` runs `bucketMetaSchema.parse()` internally, so no
+ * separate validation step is needed here.
+ */
+export function buildBucketRegistry(
+  buckets: DefinedBucket[],
+  enabledFilter?: string,
+): BucketRegistry {
+  const registry = new BucketRegistry();
+  const enabled = parseEnabledFilter(enabledFilter);
+
+  for (const bucket of buckets) {
+    if (enabled === "*" || enabled.has(bucket.meta.id)) {
+      registry.register(bucket.meta);
+    }
+  }
+
+  setBucketRegistry(registry);
+  return registry;
+}
+
+/**
+ * Select the Hatchet durable tasks for the enabled buckets. This is the SINGLE
+ * place a bucket's per-user fast-expiry timer task is constructed (Section
+ * 4.3/9.4) — task construction happens at worker build, AFTER the registry has
+ * validated every meta, never at module-load.
+ *
+ * Only `meta.fastExpiry` buckets contribute a task; the engine-wide
+ * `bucketReconcileTask` (registered separately in `baseWorkflows`) owns
+ * time-based leaves regardless. The fast-expiry timer is a single shared
+ * `durableTask` keyed on `bucket:arm-expiry` (per-bucket arming is by event
+ * payload, not per-bucket task instances), so it is registered once if ANY
+ * enabled bucket opts in.
+ */
+export function selectBucketTasks(
+  buckets: DefinedBucket[],
+  enabledFilter?: string,
+): NonNullable<DefinedBucket["task"]>[] {
+  const enabled = parseEnabledFilter(enabledFilter);
+  const hasFastExpiry = buckets.some(
+    (b) =>
+      (enabled === "*" || enabled.has(b.meta.id)) && b.meta.fastExpiry === true,
+  );
+  if (!hasFastExpiry) return [];
+
+  // The single shared `bucket:arm-expiry` durableTask, registered ONCE because
+  // any enabled bucket opts in (per-bucket arming is by event payload). Cast to
+  // the DefinedBucket task shape — both are `hatchet.durableTask` returns.
+  return [bucketExpiryTask as NonNullable<DefinedBucket["task"]>];
+}