@hogsend/engine 0.6.0 → 0.7.0

package/src/workflows/bucket-reconcile.ts

@@ -41,6 +41,7 @@ import {
 import { getBucketRegistrySingleton } from "../buckets/registry-singleton.js";
 import { getJourneyRegistrySingleton } from "../journeys/registry-singleton.js";
 import { emitBucketTransition } from "../lib/bucket-emit.js";
+import { contactKeySql } from "../lib/contacts.js";
 import { hatchet } from "../lib/hatchet.js";
 import type { Logger } from "../lib/logger.js";
 import { createLogger } from "../lib/logger.js";
@@ -841,18 +842,27 @@ async function reconcileBucketJoins(opts: {
     ? selectPresentInAllWindows(db, absenceLegs)
     : null;
 
+  // The membership/event tables key on the RESOLVED string key (external_id ??
+  // anonymous_id ?? contact.id), NOT necessarily external_id — email-only /
+  // anonymous contacts have a NULL external_id and are keyed on their uuid /
+  // anonymous_id. Joining on contacts.externalId would force external_id NOT NULL
+  // for every candidate (the coalesce would collapse to external_id) and silently
+  // drop exactly the dormant email-only contacts this cron exists to reconcile.
+  // Join on the SAME coalesce expression so the projected key matches the join.
+  const contactKey = contactKeySql();
+
   const baseQuery = db
     .select({
-      userId: contacts.externalId,
+      userId: contactKey,
       email: contacts.email,
     })
     .from(contacts)
-    .innerJoin(everFired, eq(everFired.userId, contacts.externalId))
-    .leftJoin(activeMembers, eq(activeMembers.userId, contacts.externalId));
+    .innerJoin(everFired, eq(everFired.userId, contactKey))
+    .leftJoin(activeMembers, eq(activeMembers.userId, contactKey));
 
   const candidates = await (presentInAll
     ? baseQuery
-        .leftJoin(presentInAll, eq(presentInAll.userId, contacts.externalId))
+        .leftJoin(presentInAll, eq(presentInAll.userId, contactKey))
         .where(
           and(
             isNull(contacts.deletedAt),
@@ -864,7 +874,12 @@ async function reconcileBucketJoins(opts: {
         and(isNull(contacts.deletedAt), isNull(activeMembers.userId)),
       )
   )
-    .orderBy(sql`${contacts.externalId} asc`)
+    // Deterministic scan order for the bounded re-run (no keyset cursor; the
+    // scan advances as reconciled matchers become active members and drop out).
+    // Order by contacts.id (the non-null unique PK) so the scan is null-safe and
+    // stable even for null-external_id contacts now that the join is on the
+    // coalesce key.
+    .orderBy(sql`${contacts.id} asc`)
     .limit(BATCH_SIZE);
 
   // SET-BASED / EXACT shapes (Fix #3) — every candidate row is a true matcher,

package/src/workflows/import-contacts.ts

@@ -1,11 +1,17 @@
 import { createDatabase, importJobs } from "@hogsend/db";
 import { eq } from "drizzle-orm";
 import Papa from "papaparse";
-import { upsertContact } from "../lib/contacts.js";
+import { resolveOrCreateContact } from "../lib/contacts.js";
 import { hatchet } from "../lib/hatchet.js";
 
 const BATCH_SIZE = 500;
 
+interface ImportRow {
+  externalId?: string;
+  email?: string;
+  properties?: Record<string, unknown>;
+}
+
 export const importContactsTask = hatchet.task({
   name: "import-contacts",
   retries: 0,
@@ -20,11 +26,7 @@ export const importContactsTask = hatchet.task({
       .set({ status: "processing", updatedAt: new Date() })
       .where(eq(importJobs.id, input.jobId));
 
-    let rows: Array<{
-      externalId: string;
-      email?: string;
-      properties?: Record<string, unknown>;
-    }>;
+    let rows: ImportRow[];
 
     try {
       if (input.format === "json") {
@@ -61,14 +63,22 @@ export const importContactsTask = hatchet.task({
     for (let i = 0; i < rows.length; i += BATCH_SIZE) {
       const batch = rows.slice(i, i + BATCH_SIZE);
       const results = await Promise.allSettled(
-        batch.map((row, idx) =>
-          upsertContact({
+        batch.map((row, idx) => {
+          // Accept email-only rows (D1): require AT LEAST one identity key,
+          // not externalId specifically. The resolver upserts/merges on
+          // whichever keys are present.
+          if (!row.externalId && !row.email) {
+            return Promise.reject(
+              new Error("Row has neither externalId nor email"),
+            );
+          }
+          return resolveOrCreateContact({
             db,
-            externalId: row.externalId,
+            userId: row.externalId,
             email: row.email,
-            properties: row.properties,
-          }).then(() => ({ index: i + idx, ok: true })),
-        ),
+            contactProperties: row.properties,
+          }).then(() => ({ index: i + idx, ok: true }));
+        }),
       );
 
       results.forEach((result, batchIdx) => {
@@ -108,25 +118,23 @@ export const importContactsTask = hatchet.task({
   },
 });
 
-function parseCsv(data: string): Array<{
-  externalId: string;
-  email?: string;
-  properties?: Record<string, unknown>;
-}> {
+function parseCsv(data: string): ImportRow[] {
   const result = Papa.parse<Record<string, string>>(data, {
     header: true,
     skipEmptyLines: true,
   });
 
-  if (!result.meta.fields?.includes("externalId")) {
-    throw new Error("CSV must have an externalId column");
+  const fields = result.meta.fields ?? [];
+  // Accept email-only imports (D1): require AT LEAST one identity column.
+  if (!fields.includes("externalId") && !fields.includes("email")) {
+    throw new Error("CSV must have an externalId or email column");
   }
 
   return result.data.map((row) => {
     const { externalId, email, ...rest } = row;
     const properties = Object.keys(rest).length > 0 ? rest : undefined;
     return {
-      externalId: externalId ?? "",
+      externalId: externalId || undefined,
       email: email || undefined,
       properties: properties as Record<string, unknown> | undefined,
     };