@hogsend/engine 0.2.0 → 0.3.0

package/src/workflows/bucket-backfill.ts

@@ -3,7 +3,6 @@ import type { JsonObject } from "@hatchet-dev/typescript-sdk/v1/types.js";
 import {
   type BucketMeta,
   type ConditionEval,
-  type DurationObject,
   durationToMs,
   evaluateCondition,
 } from "@hogsend/core";
@@ -16,7 +15,12 @@ import {
   importJobs,
   userEvents,
 } from "@hogsend/db";
-import { and, eq, gte, inArray, isNull, sql } from "drizzle-orm";
+import { and, eq, gt, gte, inArray, isNull, sql } from "drizzle-orm";
+import {
+  computeExpiresAt,
+  computeMaxDwellAt,
+  matchesEventCount,
+} from "../buckets/membership-epoch.js";
 import { getBucketRegistrySingleton } from "../buckets/registry-singleton.js";
 import { getJourneyRegistrySingleton } from "../journeys/registry-singleton.js";
 import { emitBucketTransition } from "../lib/bucket-emit.js";
@@ -28,7 +32,7 @@ import { createLogger } from "../lib/logger.js";
 const BATCH_SIZE = 500;
 
 /** import_jobs.format discriminator for the reused status record (Section 6.6). */
-const FIRST_TIME_FORMAT = "bucket-backfill";
+export const FIRST_TIME_FORMAT = "bucket-backfill";
 const REEVAL_FORMAT = "bucket-reeval";
 
 /**
@@ -199,6 +203,18 @@ async function backfillJoins(opts: {
     .set({ totalRows: matcherIds.length, updatedAt: new Date() })
     .where(eq(importJobs.id, jobId));
 
+  // Unconditional max-dwell TTL deadline, stamped once at insert (mirrors the
+  // live join, check-membership.ts). null when the bucket has no maxDwell; the
+  // TTL sweep (reconcileBucketTtlLeaves) filters isNotNull(maxDwellAt), so an
+  // unset value would never be force-left.
+  const maxDwellAt = computeMaxDwellAt(bucket);
+
+  // Fix C (DEFERRED): backfilled fastExpiry rows are NOT armed with a
+  // bucket:arm-expiry durable timer here — they are picked up by the next cron
+  // sweep instead (reconcileBucketLeaves / reconcileBucketTtlLeaves are the
+  // authoritative backstop). Conscious choice (cron cadence, default 5m), not an
+  // omission: arming at backfill would fan out one durable task per inserted row.
+
   let inserted = 0;
   for (let i = 0; i < matcherIds.length; i += BATCH_SIZE) {
     const chunk = matcherIds.slice(i, i + BATCH_SIZE);
@@ -214,14 +230,38 @@ async function backfillJoins(opts: {
       chunkContacts.map((c) => [c.externalId, c.email]),
     );
 
+    // Fix A: entryCount = 1 + prior memberships for each (user, bucket), the
+    // same monotonic ordinal the live join computes (check-membership.ts). On a
+    // FIRST-TIME backfill priorCount is 0 → entryCount 1 (unchanged); on a
+    // REEVAL re-join of a user with historical "left" rows it advances the
+    // epoch correctly. ONE batched GROUP BY per chunk (never per-user — the set-
+    // based path must not reintroduce the O(P) serial-query trap).
+    const priorCounts = await db
+      .select({
+        userId: bucketMemberships.userId,
+        cnt: sql<number>`count(*)::int`,
+      })
+      .from(bucketMemberships)
+      .where(
+        and(
+          eq(bucketMemberships.bucketId, bucket.id),
+          inArray(bucketMemberships.userId, chunk),
+        ),
+      )
+      .groupBy(bucketMemberships.userId);
+    const priorByUser = new Map(
+      priorCounts.map((r) => [r.userId, Number(r.cnt)]),
+    );
+
     const rows = chunk.map((userId) => ({
       userId,
       userEmail: emailByUser.get(userId) ?? null,
       bucketId: bucket.id,
       status: "active" as const,
       source: "backfill" as const,
-      entryCount: 1,
-      expiresAt: computeBackfillExpiresAt(bucket),
+      entryCount: 1 + (priorByUser.get(userId) ?? 0),
+      expiresAt: computeExpiresAt(bucket),
+      maxDwellAt,
       lastEvaluatedAt: new Date(),
     }));
 
@@ -343,8 +383,22 @@ async function selectEventMatchers(
     : null;
 
   // count gte N / exists → SELECT user_id ... GROUP BY HAVING. not_exists
-  // (absence) → live contacts with NO such event in the window (anti-join).
+  // (absence) → live contacts who EVER fired the event but have NONE in the
+  // window (lapsed-only). A bare windowed `not_exists within W` is treated as
+  // LAPSED-ONLY (never-active EXCLUDED) in BOTH this backfill and the cron
+  // (bucket-reconcile.ts reconcileBucketJoins, the everFired floor), so the two
+  // writers agree: brand-new never-active signups are NOT materialized for an
+  // absence-within-window bucket — only users who once did X and then stopped.
   if (criteria.check === "not_exists") {
+    // everFired floor: contacts who fired the event AT LEAST ONCE (no window),
+    // mirroring the cron's `ever_fired` semi-join. Excludes never-active
+    // contacts so the two writers select the same lapsed-only cohort.
+    const everFired = db
+      .selectDistinct({ userId: userEvents.userId })
+      .from(userEvents)
+      .where(eq(userEvents.event, criteria.eventName))
+      .as("ever_fired");
+
     const present = db
       .select({ userId: userEvents.userId })
       .from(userEvents)
@@ -360,115 +414,94 @@ async function selectEventMatchers(
     const rows = await db
       .select({ userId: contacts.externalId })
       .from(contacts)
+      .innerJoin(everFired, eq(everFired.userId, contacts.externalId))
       .leftJoin(present, eq(present.userId, contacts.externalId))
       .where(and(isNull(contacts.deletedAt), isNull(present.userId)));
     return rows.map((r) => r.userId);
   }
 
-  // exists / count: group counts then filter by the operator.
+  // exists / count: group counts then filter by the operator. Fix B: innerJoin
+  // live contacts (GDPR — only materialize memberships for non-deleted contacts
+  // that actually exist), mirroring selectEventLeavers in bucket-reconcile.ts.
+  // The not_exists branch above already filters contacts.deletedAt; without this
+  // join the positive-event path could materialize active rows for soft-deleted
+  // or orphan-event userIds, diverging from the live/reconcile paths.
   const rows = await db
     .select({
       userId: userEvents.userId,
       cnt: sql<number>`count(*)::int`,
     })
     .from(userEvents)
+    .innerJoin(contacts, eq(contacts.externalId, userEvents.userId))
     .where(
       and(
         eq(userEvents.event, criteria.eventName),
+        isNull(contacts.deletedAt),
         cutoff ? gte(userEvents.occurredAt, cutoff) : undefined,
       ),
     )
     .groupBy(userEvents.userId);
 
   return rows
-    .filter((r) => matchesCount(criteria, Number(r.cnt)))
+    .filter((r) => matchesEventCount(criteria, Number(r.cnt)))
     .map((r) => r.userId);
 }
 
-/** True when a windowed count satisfies the (exists/count) criterion. */
-function matchesCount(
-  criteria: Extract<ConditionEval, { type: "event" }>,
-  count: number,
-): boolean {
-  switch (criteria.check) {
-    case "exists":
-      return count > 0;
-    case "count": {
-      if (!criteria.operator || criteria.value === undefined) return count > 0;
-      switch (criteria.operator) {
-        case "gt":
-          return count > criteria.value;
-        case "gte":
-          return count >= criteria.value;
-        case "lt":
-          return count < criteria.value;
-        case "lte":
-          return count <= criteria.value;
-        case "eq":
-          return count === criteria.value;
-        default:
-          return false;
-      }
-    }
-    default:
-      return false;
-  }
-}
-
 /**
  * Composite/multi-condition fallback (the documented O(P) exception, Section 6.6):
- * a chunked per-contact `evaluateCondition` loop over live contacts. Property
+ * a per-contact `evaluateCondition` loop over live contacts. Property
  * sub-conditions evaluate against the contact's merged properties.
+ *
+ * KEYSET PAGINATION by `contacts.externalId` in BATCH_SIZE pages (mirrors
+ * reconcileBucketJoins' `externalId asc` paging): each page selects
+ * `WHERE externalId > :cursor ORDER BY externalId ASC LIMIT BATCH_SIZE`,
+ * evaluates the criteria per contact, then advances the cursor to the last
+ * externalId of the page — repeating until a short page ends the scan. The whole
+ * contacts table is never held in memory at once.
  */
 async function selectCompositeMatchers(
   db: Database,
   criteria: ConditionEval,
 ): Promise<string[]> {
-  const liveContacts = await db
-    .select({
-      externalId: contacts.externalId,
-      properties: contacts.properties,
-    })
-    .from(contacts)
-    .where(isNull(contacts.deletedAt));
-
   const matchers: string[] = [];
-  for (const contact of liveContacts) {
-    const isMember = await evaluateCondition({
-      condition: criteria,
-      ctx: {
-        db,
-        userId: contact.externalId,
-        journeyContext:
-          (contact.properties as Record<string, unknown> | null) ?? {},
-      },
-    });
-    if (isMember) matchers.push(contact.externalId);
-  }
-  return matchers;
-}
+  let cursor: string | null = null;
 
-/** now + within for time-based / fastExpiry buckets; null otherwise. */
-function computeBackfillExpiresAt(bucket: BucketMeta): Date | null {
-  if (!bucket.criteria) return null;
-  if (!bucket.timeBased && !bucket.fastExpiry) return null;
-  const within = firstWithin(bucket.criteria);
-  if (!within) return null;
-  return new Date(Date.now() + durationToMs(within));
-}
+  for (;;) {
+    const page = await db
+      .select({
+        externalId: contacts.externalId,
+        properties: contacts.properties,
+      })
+      .from(contacts)
+      .where(
+        and(
+          isNull(contacts.deletedAt),
+          cursor != null ? gt(contacts.externalId, cursor) : undefined,
+        ),
+      )
+      .orderBy(sql`${contacts.externalId} asc`)
+      .limit(BATCH_SIZE);
 
-/** Find the first EventCondition.within in a criteria tree (depth-first). */
-function firstWithin(criteria: ConditionEval): DurationObject | null {
-  if (criteria.type === "event" && criteria.within) {
-    return criteria.within;
-  }
-  if (criteria.type === "composite") {
-    for (const child of criteria.conditions) {
-      const found = firstWithin(child);
-      if (found) return found;
+    for (const contact of page) {
+      const isMember = await evaluateCondition({
+        condition: criteria,
+        ctx: {
+          db,
+          userId: contact.externalId,
+          journeyContext:
+            (contact.properties as Record<string, unknown> | null) ?? {},
+        },
+      });
+      if (isMember) matchers.push(contact.externalId);
     }
+
+    // A short page (fewer than a full batch) means the scan is exhausted.
+    if (page.length < BATCH_SIZE) break;
+    cursor = page[page.length - 1]?.externalId ?? null;
+    if (cursor == null) break;
   }
-  return null;
+
+  return matchers;
 }
 
 /**
@@ -535,7 +568,11 @@ export async function enqueueBucketBackfills(opts: {
 
       if (!job) continue;
 
-      await bucketBackfillTask.run({
+      // runNoWait (fire-and-forget): this is called from worker boot BEFORE the
+      // listener starts, so awaiting the run would deadlock (the run needs the
+      // listener that `_worker.start()` brings up). The triggered run queues and
+      // executes once listening; the task itself persists the criteriaHash.
+      await bucketBackfillTask.runNoWait({
         jobId: job.id,
         bucketId: bucket.id,
         mode,