@hogsend/engine 0.4.0 → 0.6.0

package/src/workflows/bucket-backfill.ts

@@ -15,7 +15,7 @@ import {
   importJobs,
   userEvents,
 } from "@hogsend/db";
-import { and, eq, gt, gte, inArray, isNull, sql } from "drizzle-orm";
+import { and, eq, gt, gte, inArray, isNull, max, sql } from "drizzle-orm";
 import {
   computeExpiresAt,
   computeMaxDwellAt,
@@ -209,6 +209,18 @@ async function backfillJoins(opts: {
   // unset value would never be force-left.
   const maxDwellAt = computeMaxDwellAt(bucket);
 
+  // Historical dwell anchor (Section 6.3 / LOCKED DECISION 1). For a
+  // windowed/event criterion the anchor is `max(occurredAt)` of the qualifying
+  // event = "when they became dormant" (e.g. went-dormant = the last
+  // `app_opened`). The dwell gate reads `coalesce(dwellAnchorAt, enteredAt)`, so
+  // backfilled members start the dwell clock at their real historical instant
+  // rather than the deploy-time `enteredAt`. Shapes with no cheap per-matcher
+  // timestamp leave the anchor NULL (fall back to enteredAt). The live join path
+  // (handleJoin) never sets dwellAnchorAt, so post-deploy joins clock from their
+  // real enteredAt. Computed batched per chunk (one GROUP BY max(occurredAt),
+  // mirroring the priorCounts GROUP BY) — never per-user serial queries.
+  const anchorEvent = resolveDwellAnchorEvent(criteria);
+
   // Fix C (DEFERRED): backfilled fastExpiry rows are NOT armed with a
   // bucket:arm-expiry durable timer here — they are picked up by the next cron
   // sweep instead (reconcileBucketLeaves / reconcileBucketTtlLeaves are the
@@ -253,6 +265,35 @@ async function backfillJoins(opts: {
       priorCounts.map((r) => [r.userId, Number(r.cnt)]),
     );
 
+    // Batched dwell-anchor derivation (LOCKED DECISION 1): one GROUP BY
+    // max(occurredAt) over the qualifying event for THIS chunk, mirroring the
+    // priorCounts GROUP BY above (never per-user serial queries). Only computed
+    // when the criteria shape exposes a cheap per-matcher anchor event; an empty
+    // map leaves dwellAnchorAt NULL → the dwell gate falls back to enteredAt.
+    let anchorByUser = new Map<string, Date>();
+    if (anchorEvent != null) {
+      const anchors = await db
+        .select({
+          userId: userEvents.userId,
+          lastAt: max(userEvents.occurredAt),
+        })
+        .from(userEvents)
+        .where(
+          and(
+            eq(userEvents.event, anchorEvent),
+            inArray(userEvents.userId, chunk),
+          ),
+        )
+        .groupBy(userEvents.userId);
+      anchorByUser = new Map(
+        anchors
+          .filter(
+            (r): r is { userId: string; lastAt: Date } => r.lastAt != null,
+          )
+          .map((r) => [r.userId, r.lastAt]),
+      );
+    }
+
     const rows = chunk.map((userId) => ({
       userId,
       userEmail: emailByUser.get(userId) ?? null,
@@ -262,6 +303,8 @@ async function backfillJoins(opts: {
       entryCount: 1 + (priorByUser.get(userId) ?? 0),
       expiresAt: computeExpiresAt(bucket),
       maxDwellAt,
+      // Historical dwell anchor where derivable; NULL otherwise (→ enteredAt).
+      dwellAnchorAt: anchorByUser.get(userId) ?? null,
       lastEvaluatedAt: new Date(),
     }));
 
@@ -365,6 +408,7 @@ async function reevalLeaves(opts: {
         userEmail: row.userEmail,
         epoch: row.entryCount,
         source: "backfill",
+        reason: "criteria",
       });
     }
     leftCount += flipped.length;
@@ -504,6 +548,51 @@ async function selectCompositeMatchers(
   return matchers;
 }
 
+/**
+ * Resolve the event whose `max(occurredAt)` is the historical dwell anchor for a
+ * backfilled member (LOCKED DECISION 1 / Section 6.3) — "when they became
+ * dormant". Returns an event name only for the windowed/event shapes that expose
+ * a cheap per-matcher timestamp; `null` for everything else (the anchor stays
+ * NULL and the dwell gate falls back to `enteredAt`):
+ *
+ *   - a single windowed `event` criterion → its `eventName` (the last qualifying
+ *     occurrence is the window boundary, e.g. the last `app_opened`).
+ *   - the lapsed-active composite `all(event(X).exists(),
+ *     event(X).within(W).not_exists())` → event X (the flagship went-dormant
+ *     shape; the last X is when they lapsed).
+ *
+ * Other shapes (property/count composites, OR-of-absence, multi-event) have no
+ * single cheap per-matcher timestamp, so they keep a NULL anchor.
+ */
+function resolveDwellAnchorEvent(criteria: ConditionEval): string | null {
+  if (criteria.type === "event") {
+    return criteria.within != null ? criteria.eventName : null;
+  }
+  // Lapsed-active composite — two legs on the SAME event X: an unwindowed
+  // exists() anchor and a windowed not_exists() leg. Mirrors
+  // isLapsedActiveComposite in bucket-reconcile.ts.
+  if (
+    criteria.type === "composite" &&
+    criteria.operator === "and" &&
+    criteria.conditions.length === 2
+  ) {
+    const existsLeg = criteria.conditions.find(
+      (c) => c.type === "event" && c.check === "exists" && c.within == null,
+    );
+    const notExistsLeg = criteria.conditions.find(
+      (c) => c.type === "event" && c.check === "not_exists" && c.within != null,
+    );
+    if (
+      existsLeg?.type === "event" &&
+      notExistsLeg?.type === "event" &&
+      existsLeg.eventName === notExistsLeg.eventName
+    ) {
+      return notExistsLeg.eventName;
+    }
+  }
+  return null;
+}
+
 /**
  * Upsert the bucket's current criteria fingerprint onto `bucket_configs` (Section
  * 6.6 B). Mirrors the admin enable/disable onConflictDoUpdate target.

package/src/workflows/bucket-reconcile.ts

@@ -7,6 +7,7 @@ import {
   durationToMs,
   evaluateCondition,
 } from "@hogsend/core";
+import type { JourneyMeta } from "@hogsend/core/types";
 import {
   bucketConfigs,
   bucketMemberships,
@@ -27,6 +28,7 @@ import {
   or,
   sql,
 } from "drizzle-orm";
+import type { BucketLeaveReason } from "../buckets/bucket-reactions.js";
 import { shouldEmitJoin } from "../buckets/check-membership.js";
 import {
   BUCKET_EVENT_PREFIX,
@@ -97,12 +99,21 @@ export const bucketReconcileTask = hatchet.task({
       // kind:"manual" buckets are NEVER auto-recomputed (early-continue).
       if (bucket.kind === "manual" || !bucket.criteria) continue;
 
-      // Process a bucket here iff a clock can flip its membership: a TIME-BASED
-      // criteria window (criteria-driven leaves/joins) OR an unconditional
-      // `maxDwell` TTL (membership-age-driven leaves). timeBased is honoured
-      // explicitly OR inferred from a `within` window.
+      // Process a bucket here iff a clock can flip its membership OR fire a
+      // membership-age dwell: a TIME-BASED criteria window (criteria-driven
+      // leaves/joins), an unconditional `maxDwell` TTL (membership-age-driven
+      // leaves), OR a `dwell` reaction (membership-age-driven fire). timeBased is
+      // honoured explicitly OR inferred from a `within` window. The dwell-only
+      // bucket falls through and runs ONLY the dwell pass (the criteria pass is
+      // behind `if (timeBased)`, the TTL pass behind `if (bucket.maxDwell)`).
       const timeBased = isTimeBased(bucket);
-      if (!timeBased && !bucket.maxDwell) continue;
+      const dwellReactions = journeyRegistry
+        .getAll()
+        .filter(
+          (j) => j.sourceBucketId === bucket.id && j.reactionKind === "dwell",
+        );
+      const hasDwell = dwellReactions.length > 0;
+      if (!timeBased && !bucket.maxDwell && !hasDwell) continue;
 
       try {
         if (timeBased) {
@@ -145,6 +156,21 @@ export const bucketReconcileTask = hatchet.task({
             bucket,
           });
         }
+
+        // Dwell pass — runs AFTER the TTL pass (ordering is load-bearing: a
+        // member force-left by maxDwell earlier this iteration is status='left'
+        // here, so the dwell scan's status='active' filter excludes it). Fires
+        // `bucket:dwell:<id>:<label>` over the continuously-dwelling active
+        // population at cron resolution (Section 6.4–6.6).
+        if (hasDwell) {
+          reconciled += await reconcileBucketDwell({
+            db,
+            logger,
+            journeyRegistry,
+            bucket,
+            dwellReactions,
+          });
+        }
       } catch (err) {
         logger.error("Bucket reconcile failed", {
           bucketId: bucket.id,
@@ -255,6 +281,8 @@ export const bucketExpiryTask = hatchet.durableTask({
       userEmail: input.userEmail,
       epoch: flipped.entryCount,
       source: "reconcile",
+      // Fast-expiry is a criteria re-confirm leave (Section 6.7).
+      reason: "criteria",
     });
 
     return { status: "left", rowId: flipped.id };
@@ -287,6 +315,7 @@ async function reconcileBucketLeaves(opts: {
       journeyRegistry,
       bucket,
       userIds: leaverIds,
+      reason: "criteria",
     });
   }
 
@@ -421,7 +450,14 @@ async function reconcileCompositeLeaves(opts: {
   }
 
   if (leaverIds.length === 0) return 0;
-  return bulkLeave({ db, logger, journeyRegistry, bucket, userIds: leaverIds });
+  return bulkLeave({
+    db,
+    logger,
+    journeyRegistry,
+    bucket,
+    userIds: leaverIds,
+    reason: "criteria",
+  });
 }
 
 /**
@@ -462,6 +498,7 @@ async function reconcileBucketTtlLeaves(opts: {
     journeyRegistry,
     bucket,
     userIds: expired.map((r) => r.userId),
+    reason: "maxDwell",
   });
 }
 
@@ -478,8 +515,10 @@ async function bulkLeave(opts: {
   journeyRegistry: ReturnType<typeof getJourneyRegistrySingleton>;
   bucket: BucketMeta;
   userIds: string[];
+  /** Why these members leave — TTL passes "maxDwell", criteria passes "criteria". */
+  reason: BucketLeaveReason;
 }): Promise<number> {
-  const { db, logger, journeyRegistry, bucket, userIds } = opts;
+  const { db, logger, journeyRegistry, bucket, userIds, reason } = opts;
 
   const dwellMs = bucket.minDwell ? durationToMs(bucket.minDwell) : 0;
   const dwellCutoff = dwellMs > 0 ? new Date(Date.now() - dwellMs) : null;
@@ -524,12 +563,171 @@ async function bulkLeave(opts: {
       userEmail: row.userEmail,
       epoch: row.entryCount,
       source: "reconcile",
+      reason,
     });
   }
 
   return flipped.length;
 }
 
+/**
+ * Dwell pass for one bucket (Section 6.4–6.6). Fires `bucket:dwell:<id>:<label>`
+ * over the EXISTING continuously-dwelling active population at cron resolution —
+ * its unique value over `on("enter") + ctx.sleep`. Idempotent across sweeps,
+ * interoperable with maxDwell/fastExpiry, and routed through
+ * `emitBucketTransition` (NOT a raw push) for `userEvents`/exitOn/history/analytics
+ * parity (Section 6.1):
+ *
+ *  - PUSH FIRST (at-least-once; the deterministic idempotencyKey + the userEvents
+ *    dedup absorb a same-sweep retry), THEN stamp `dwellState` (the inter-sweep
+ *    "already fired this membership" gate). The stamp's `status='active'` clause
+ *    makes the leave/fastExpiry interop correct (a row flipped to `left` between
+ *    SELECT and UPDATE no-ops).
+ *  - The dwell clock is `coalesce(dwellAnchorAt, enteredAt)` — backfilled members
+ *    use their derived historical anchor (LOCKED DECISION 1), live joins use
+ *    enteredAt (anchor NULL).
+ *  - Candidates are ordered `lastEvaluatedAt asc nulls first` (oldest-served-first)
+ *    so a busy `every` bucket cannot starve members past BATCH_SIZE; the stamp
+ *    bumps `lastEvaluatedAt`, advancing the cursor. Hitting BATCH_SIZE is logged
+ *    once per sweep (visibility, not silent).
+ *  - First-deploy quiet window: reuse `firstTimeBackfillIncomplete` so the
+ *    pre-existing/backfilled population is not blasted before the first-time
+ *    backfill has settled.
+ *
+ * `every` is fires-at-most-once-per-sweep, coalescing (one catch-up fire after a
+ * multi-interval outage); `dwellCount` is the deterministic interval ordinal
+ * `floor((sweepInstant - anchor) / offsetMs)` (gap-stable, NOT a fire count). For
+ * `after` the ordinal is always 1 (one-shot).
+ */
+async function reconcileBucketDwell(opts: {
+  db: Database;
+  logger: Logger;
+  journeyRegistry: ReturnType<typeof getJourneyRegistrySingleton>;
+  bucket: BucketMeta;
+  dwellReactions: JourneyMeta[];
+}): Promise<number> {
+  const { db, logger, journeyRegistry, bucket, dwellReactions } = opts;
+
+  // First-deploy quiet window: do not blast the pre-existing/backfilled
+  // population before the first-time backfill has settled (reuse the guard).
+  if (await firstTimeBackfillIncomplete(db, bucket)) return 0;
+
+  // Captured once per invocation and reused for the ordinal. The ordinal is
+  // floor((sweepInstant - anchor) / offsetMs), so it is grid-quantized: a Hatchet
+  // retry (a fresh fn() invocation, seconds–minutes later) lands in the SAME
+  // interval window and recomputes the SAME ordinal → SAME idempotencyKey →
+  // absorbed by the userEvents dedup. `after` is always ordinal 1 (fully stable).
+  // Residual edge: an `every` retry that straddles an interval boundary yields a
+  // new ordinal and thus one extra dwell fire — bounded to a single duplicate by
+  // the key, and only possible for sub-retry-window intervals. Documented, not
+  // load-bearing for the common (hours/days) intervals.
+  const sweepInstant = Date.now();
+  let fired = 0;
+
+  for (const reaction of dwellReactions) {
+    const schedule = reaction.dwellSchedule;
+    if (!schedule) continue;
+    const { label, after, every } = schedule;
+    const offsetMs = after ?? every;
+    if (offsetMs == null) continue;
+    const cutoff = new Date(sweepInstant - offsetMs);
+
+    // Continuous-member gate. coalesce(dwellAnchorAt, enteredAt) is the dwell
+    // clock. Oldest-served-first (Section 6.5).
+    const candidates = await db
+      .select({
+        id: bucketMemberships.id,
+        userId: bucketMemberships.userId,
+        userEmail: bucketMemberships.userEmail,
+        entryCount: bucketMemberships.entryCount,
+        anchor: sql<Date>`coalesce(${bucketMemberships.dwellAnchorAt}, ${bucketMemberships.enteredAt})`,
+        dwellState: bucketMemberships.dwellState,
+      })
+      .from(bucketMemberships)
+      .innerJoin(contacts, eq(contacts.externalId, bucketMemberships.userId))
+      .where(
+        and(
+          eq(bucketMemberships.bucketId, bucket.id),
+          eq(bucketMemberships.status, "active"),
+          isNull(bucketMemberships.deletedAt),
+          isNull(contacts.deletedAt),
+          // Fold the comparison into the fragment with an explicit cast: a JS
+          // Date passed to lte() against a raw sql`coalesce(...)` fragment has no
+          // column type to drive param encoding, so the pg driver throws on the
+          // Date (and the per-bucket try/catch would silently swallow it → 0
+          // dwell fires). Binding the ISO string + ::timestamptz is well-typed.
+          sql`coalesce(${bucketMemberships.dwellAnchorAt}, ${bucketMemberships.enteredAt}) <= ${cutoff.toISOString()}::timestamptz`,
+        ),
+      )
+      .orderBy(sql`${bucketMemberships.lastEvaluatedAt} asc nulls first`)
+      .limit(BATCH_SIZE);
+
+    if (candidates.length >= BATCH_SIZE) {
+      logger.warn("Bucket dwell pass bounded to BATCH_SIZE/tick", {
+        bucketId: bucket.id,
+        label,
+        batchSize: BATCH_SIZE,
+      });
+    }
+
+    for (const m of candidates) {
+      const state = (m.dwellState ?? {}) as Record<string, string>;
+      const lastFired = state[label] ? Date.parse(state[label]) : null;
+      const anchorMs = new Date(m.anchor).getTime();
+
+      if (after != null) {
+        // one-shot: already fired for this membership → skip.
+        if (lastFired != null) continue;
+      } else {
+        // every: not yet due since the last fire (or the anchor).
+        const since = lastFired ?? anchorMs;
+        if (sweepInstant - since < offsetMs) continue;
+      }
+
+      // Deterministic per (membership, sweepInstant) so a retry recomputes it.
+      const ordinal =
+        after != null ? 1 : Math.floor((sweepInstant - anchorMs) / offsetMs);
+
+      // PUSH FIRST (at-least-once; idempotencyKey + userEvents dedup absorb
+      // retries), THEN stamp. emitBucketTransition handles the
+      // userEvents/exitOn/analytics parity.
+      await emitBucketTransition({
+        db,
+        registry: journeyRegistry,
+        hatchet,
+        logger,
+        kind: "dwell",
+        bucket,
+        userId: m.userId,
+        userEmail: m.userEmail,
+        epoch: m.entryCount,
+        source: "reconcile",
+        dwellLabel: label,
+        dwellOrdinal: ordinal,
+      });
+
+      // Stamp the membership (inter-sweep gate). status='active' clause = leave
+      // interop (a row flipped to 'left' between SELECT and UPDATE no-ops).
+      await db
+        .update(bucketMemberships)
+        .set({
+          dwellState: sql`jsonb_set(coalesce(${bucketMemberships.dwellState}, '{}'::jsonb), ${`{${label}}`}, ${`"${new Date(sweepInstant).toISOString()}"`}::jsonb)`,
+          lastEvaluatedAt: new Date(),
+          updatedAt: new Date(),
+        })
+        .where(
+          and(
+            eq(bucketMemberships.id, m.id),
+            eq(bucketMemberships.status, "active"),
+          ),
+        );
+      fired += 1;
+    }
+  }
+
+  return fired;
+}
+
 /**
  * reconcileJoins (absence buckets): materialize NEW members the real-time path
  * cannot see — a user who STOPS doing X fires no event, so only the clock can

package/src/workflows/check-alerts.ts

@@ -16,7 +16,6 @@ export const checkAlertsTask = hatchet.task({
     await checkAlertRules({
       db,
       logger,
-      resendApiKey: process.env.RESEND_API_KEY,
     });
 
     return { checked: true };

package/src/workflows/send-email.ts

@@ -1,22 +1,16 @@
 import { NonRetryableError } from "@hatchet-dev/typescript-sdk/v1/index.js";
-import { createResendClient } from "@hogsend/plugin-resend";
+import { EmailSendError } from "@hogsend/email";
+import { getEmailService } from "../lib/email.js";
 import { hatchet } from "../lib/hatchet.js";
 
-const resend = createResendClient({
-  apiKey: process.env.RESEND_API_KEY ?? "",
-});
-
-const NON_RETRYABLE_CODES = new Set([
-  "validation_error",
-  "missing_required_field",
-  "invalid_api_key",
-  "not_found",
-  "restricted_api_key",
-]);
-
 export const sendEmailTask = hatchet.task({
   name: "send-email",
-  retries: 3,
+  // The EmailProvider owns transient-failure backoff internally (classified
+  // exponential retry in its `send`), and permanent failures fail fast below via
+  // NonRetryableError — so Hatchet's retry is just ONE durability re-attempt for a
+  // worker crash/timeout, not a second transient-retry loop layered on the
+  // provider's. (Previously 3, which multiplied with the provider's own retries.)
+  retries: 1,
   executionTimeout: "30s",
   backoff: { factor: 2, maxSeconds: 30 },
   fn: async (input: {
@@ -28,27 +22,35 @@ export const sendEmailTask = hatchet.task({
     tags?: Array<{ name: string; value: string }>;
     headers?: Record<string, string>;
   }) => {
-    const { data, error } = await resend.emails.send({
-      from:
-        input.from ??
-        process.env.RESEND_FROM_EMAIL ??
-        "Hogsend <noreply@hogsend.com>",
-      to: input.to,
-      subject: input.subject,
-      html: input.html,
-      replyTo: input.replyTo,
-      tags: input.tags,
-      headers: input.headers,
-    });
+    // Deliver through the injected, provider-backed mailer (set by
+    // createHogsendClient → setEmailService). `sendRaw` calls the swappable
+    // EmailProvider's `send`, resolving the default `from` from the mailer
+    // config — so a swapped provider is honored and this task no longer
+    // constructs a raw Resend client of its own. The provider already retries
+    // transient failures internally and surfaces a classified EmailSendError;
+    // map a non-retryable one to Hatchet's NonRetryableError so the task's own
+    // retry/backoff doesn't re-attempt a permanent failure.
+    const emailService = getEmailService();
 
-    if (error) {
-      const name = (error as { name?: string }).name ?? "";
-      if (NON_RETRYABLE_CODES.has(name)) {
-        throw new NonRetryableError(`${name}: ${error.message}`);
+    try {
+      // `from` is optional: when absent the mailer's `resolveFrom` falls back to
+      // its configured defaultFrom (env.RESEND_FROM_EMAIL).
+      const result = await emailService.sendRaw({
+        from: input.from,
+        to: input.to,
+        subject: input.subject,
+        html: input.html,
+        replyTo: input.replyTo,
+        tags: input.tags,
+        headers: input.headers,
+      });
+
+      return { emailId: result.id };
+    } catch (error) {
+      if (error instanceof EmailSendError && !error.retryable) {
+        throw new NonRetryableError(error.message);
       }
-      throw new Error(`Failed to send email: ${error.message}`);
+      throw error;
     }
-
-    return { emailId: data?.id ?? "" };
   },
 });