@hogsend/engine 0.56.0 → 0.57.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,448 @@
1
+ import type { JsonObject } from "@hatchet-dev/typescript-sdk/v1/types.js";
2
+ import { createDatabase, type Database, importJobs } from "@hogsend/db";
3
+ import { and, eq, inArray, sql } from "drizzle-orm";
4
+ import { ALIAS_REASON_BACKFILL } from "../lib/contacts.js";
5
+ import { hatchet } from "../lib/hatchet.js";
6
+ import type { Logger } from "../lib/logger.js";
7
+ import { createLogger } from "../lib/logger.js";
8
+
9
+ /** `import_jobs.format` discriminator for the reused status record. */
10
+ export const IDENTITY_ALIAS_BACKFILL_FORMAT = "identity-alias-backfill";
11
+
12
+ /** Contacts per keyset batch — the `bucket-backfill.ts` precedent. */
13
+ const BATCH_SIZE = 500;
14
+
15
+ /**
16
+ * PRD 02 T3 — fill `contact_aliases` with a row per identity key of every LIVE
17
+ * contact, so the identity table can become the source of truth `findByKey`
18
+ * reads first. Keyset-paginated on the `contacts` PK; one classify SELECT plus
19
+ * one guarded INSERT per batch; no long-held transaction, no table lock.
20
+ *
21
+ * Invariants, each pinned by a test:
22
+ * - IDEMPOTENT — `ON CONFLICT (alias_kind, alias_value) DO NOTHING` against
23
+ * the plain unique index; a re-run (or a resume from zero) inserts nothing.
24
+ * - NEVER STEALS — a `(kind, value)` already claimed by a DIFFERENT contact is
25
+ * skipped and counted (`conflicting`), never repointed.
26
+ * - Soft-deleted contacts are EXCLUDED (`deleted_at IS NULL`): their keys are
27
+ * either a merged loser's (already aliased to the survivor by
28
+ * `recordMergeAliases`) or an erased person's (must not be resurrected).
29
+ * - Emails land NORMALIZED (`lower(trim(email))`) so the alias row matches the
30
+ * value the resolver's probes always compare against.
31
+ * - The row-uuid pseudo-key (`('external', contacts.id)`) is deliberately NOT
32
+ * backfilled — on a deployment using uuids as external ids it would contend
33
+ * with real keys, a history-theft-shaped outcome for a defensive row. The
34
+ * uuid probe stays last in `findByKey` instead.
35
+ */
36
+ export interface IdentityAliasBackfillInput extends JsonObject {
37
+ jobId: string;
38
+ /** Classify + count only; write nothing. Also reports the mixed-case email
39
+ * count (the datum behind PRD 02 T6's per-deployment decision). */
40
+ dryRun?: boolean;
41
+ }
42
+
43
+ // Extends JsonObject: this is a Hatchet task return value and must be
44
+ // JSON-serializable (the same constraint every workflow output carries).
45
+ export interface IdentityAliasBackfillResult extends JsonObject {
46
+ status: "completed" | "failed";
47
+ dryRun: boolean;
48
+ /** Live contacts scanned. */
49
+ scanned: number;
50
+ /** Alias rows written (projected, in a dry run). */
51
+ inserted: number;
52
+ /** Pairs already aliased to the SAME contact (benign; dominated by the
53
+ * dual-write once it has been live for a while). */
54
+ present: number;
55
+ /** Pairs whose `(kind, value)` is claimed by a DIFFERENT contact — the
56
+ * divergence metric. Never repointed; surfaced via the parity endpoint. */
57
+ conflicting: number;
58
+ /** Live contacts whose stored email differs from `lower(trim(email))` —
59
+ * re-measure per deployment before trusting the read flip for email (T6). */
60
+ mixedCaseEmails: number;
61
+ reason?: string;
62
+ }
63
+
64
+ /** The four identity-column → alias-kind projections of one contacts batch.
65
+ * Fragment shared by the classify SELECT and the INSERT so the two can never
66
+ * disagree about what a "pair" is. */
67
+ function pairsCte(afterId: string | null) {
68
+ return sql`
69
+ batch AS (
70
+ SELECT id, external_id, email, anonymous_id, discord_id
71
+ FROM contacts
72
+ WHERE deleted_at IS NULL
73
+ AND (${afterId}::uuid IS NULL OR id > ${afterId}::uuid)
74
+ ORDER BY id
75
+ LIMIT ${BATCH_SIZE}
76
+ ),
77
+ pairs AS (
78
+ SELECT id, 'external'::text AS kind, external_id AS value
79
+ FROM batch WHERE external_id IS NOT NULL
80
+ UNION ALL
81
+ SELECT id, 'email', lower(trim(email))
82
+ FROM batch WHERE email IS NOT NULL AND trim(email) <> ''
83
+ UNION ALL
84
+ SELECT id, 'anonymous', anonymous_id
85
+ FROM batch WHERE anonymous_id IS NOT NULL
86
+ UNION ALL
87
+ SELECT id, 'discord', discord_id
88
+ FROM batch WHERE discord_id IS NOT NULL
89
+ )`;
90
+ }
91
+
92
+ interface BatchClassification {
93
+ scanned: number;
94
+ lastId: string | null;
95
+ toInsert: number;
96
+ present: number;
97
+ conflicting: number;
98
+ }
99
+
100
+ /** One round trip: how many contacts this batch scans, where the cursor lands,
101
+ * and how each candidate pair classifies against the existing alias rows. */
102
+ async function classifyBatch(
103
+ db: Database,
104
+ afterId: string | null,
105
+ ): Promise<BatchClassification> {
106
+ const rows = await db.execute<{
107
+ scanned: number;
108
+ last_id: string | null;
109
+ to_insert: number;
110
+ present: number;
111
+ conflicting: number;
112
+ }>(sql`
113
+ WITH ${pairsCte(afterId)},
114
+ classified AS (
115
+ SELECT p.id, a.contact_id AS owner
116
+ FROM pairs p
117
+ LEFT JOIN contact_aliases a
118
+ ON a.alias_kind = p.kind AND a.alias_value = p.value
119
+ )
120
+ SELECT (SELECT count(*) FROM batch)::int AS scanned,
121
+ -- no max(uuid) aggregate exists; take the keyset tail explicitly
122
+ (SELECT id FROM batch ORDER BY id DESC LIMIT 1)::text AS last_id,
123
+ count(*) FILTER (WHERE c.owner IS NULL)::int AS to_insert,
124
+ count(*) FILTER (WHERE c.owner = c.id)::int AS present,
125
+ count(*) FILTER (WHERE c.owner IS NOT NULL
126
+ AND c.owner <> c.id)::int AS conflicting
127
+ FROM classified c
128
+ `);
129
+ const row = rows[0];
130
+ return {
131
+ scanned: Number(row?.scanned ?? 0),
132
+ lastId: row?.last_id ?? null,
133
+ toInsert: Number(row?.to_insert ?? 0),
134
+ present: Number(row?.present ?? 0),
135
+ conflicting: Number(row?.conflicting ?? 0),
136
+ };
137
+ }
138
+
139
+ /** True when the error chain carries a Postgres FK violation (23503). Drizzle
140
+ * wraps the driver error, so walk `cause` (the 23505 lesson from the partial-
141
+ * index onConflict work applies to every SQLSTATE). */
142
+ function isFkViolation(err: unknown): boolean {
143
+ let cursor: unknown = err;
144
+ for (let i = 0; i < 5 && cursor; i++) {
145
+ if (
146
+ typeof cursor === "object" &&
147
+ "code" in cursor &&
148
+ (cursor as { code?: unknown }).code === "23503"
149
+ ) {
150
+ return true;
151
+ }
152
+ cursor = (cursor as { cause?: unknown }).cause;
153
+ }
154
+ return false;
155
+ }
156
+
157
+ /**
158
+ * The write half — the PRD's verbatim statement. Returns rows inserted.
159
+ *
160
+ * Retries (bounded) on an FK violation: a contact HARD-deleted between this
161
+ * statement's snapshot and its inserts (a GDPR purge script, a test cleanup)
162
+ * makes the alias row's `contact_id` FK fail even though the batch CTE read the
163
+ * row as live. Each retry re-reads the batch under a fresh snapshot — the
164
+ * deleted contact is gone from it — so a racing purge costs a retry, not the
165
+ * whole job. Any other error, or a third consecutive FK loss, propagates.
166
+ */
167
+ async function insertBatch(db: Database, afterId: string | null) {
168
+ for (let attempt = 0; ; attempt++) {
169
+ try {
170
+ const rows = await db.execute<{ contact_id: string }>(sql`
171
+ WITH ${pairsCte(afterId)}
172
+ INSERT INTO contact_aliases
173
+ (contact_id, alias_kind, alias_value, from_contact_id, reason,
174
+ created_at, updated_at)
175
+ SELECT id, kind, value, NULL, ${ALIAS_REASON_BACKFILL}, now(), now()
176
+ FROM pairs
177
+ ON CONFLICT (alias_kind, alias_value) DO NOTHING
178
+ RETURNING contact_id
179
+ `);
180
+ return rows.length;
181
+ } catch (err) {
182
+ if (attempt >= 2 || !isFkViolation(err)) throw err;
183
+ }
184
+ }
185
+ }
186
+
187
+ /**
188
+ * The task body, exported directly so tests and operator scripts can run it
189
+ * without a Hatchet engine. `jobId` is optional: when present, progress is
190
+ * recorded on the `import_jobs` row (`processedRows` = contacts scanned,
191
+ * `failedRows` = conflicting pairs — the divergence metric, NOT an error
192
+ * count; the full breakdown is in the return value and the completion log).
193
+ */
194
+ export async function runIdentityAliasBackfill(opts: {
195
+ db: Database;
196
+ logger: Logger;
197
+ jobId?: string;
198
+ dryRun?: boolean;
199
+ }): Promise<IdentityAliasBackfillResult> {
200
+ const { db, logger, jobId } = opts;
201
+ const dryRun = opts.dryRun === true;
202
+
203
+ const markJob = async (
204
+ patch: Partial<typeof importJobs.$inferInsert>,
205
+ ): Promise<void> => {
206
+ if (!jobId) return;
207
+ await db
208
+ .update(importJobs)
209
+ .set({ ...patch, updatedAt: new Date() })
210
+ .where(eq(importJobs.id, jobId));
211
+ };
212
+
213
+ try {
214
+ const [totals] = await db.execute<{ live: number; mixed: number }>(sql`
215
+ SELECT count(*)::int AS live,
216
+ count(*) FILTER (
217
+ WHERE email IS NOT NULL
218
+ AND email IS DISTINCT FROM lower(trim(email))
219
+ )::int AS mixed
220
+ FROM contacts
221
+ WHERE deleted_at IS NULL
222
+ `);
223
+ const mixedCaseEmails = Number(totals?.mixed ?? 0);
224
+ await markJob({
225
+ status: "processing",
226
+ totalRows: Number(totals?.live ?? 0),
227
+ });
228
+
229
+ let cursor: string | null = null;
230
+ let scanned = 0;
231
+ let inserted = 0;
232
+ let present = 0;
233
+ let conflicting = 0;
234
+
235
+ for (;;) {
236
+ const batch = await classifyBatch(db, cursor);
237
+ if (batch.scanned === 0) break;
238
+
239
+ if (!dryRun) {
240
+ // The classify → insert pair is two statements, so a resolve committing
241
+ // between them can shrink the insert below `toInsert` — the INSERT's
242
+ // own ON CONFLICT arbiter is the correctness guard; the counts are
243
+ // observability. Count what actually landed.
244
+ inserted += await insertBatch(db, cursor);
245
+ } else {
246
+ inserted += batch.toInsert;
247
+ }
248
+ scanned += batch.scanned;
249
+ present += batch.present;
250
+ conflicting += batch.conflicting;
251
+ cursor = batch.lastId;
252
+
253
+ await markJob({ processedRows: scanned, failedRows: conflicting });
254
+ if (batch.scanned < BATCH_SIZE) break;
255
+ }
256
+
257
+ if (!dryRun) {
258
+ // ERASURE-RACE SWEEP. A contact erased WHILE this job runs can have its
259
+ // keys re-inserted from a batch statement's stale snapshot (the batch
260
+ // read the contact as live; the erasure hook's delete committed first;
261
+ // this insert committed second). That would resurrect an erased person's
262
+ // identity keys — the exact leak PRD 02 T1 exists to close. This final
263
+ // statement runs AFTER every insert has committed, so its snapshot sees
264
+ // any erasure that raced a batch: whichever of the two commits second
265
+ // now cleans up. Scoped to backfill-authored rows — merge-trail aliases
266
+ // pointing at a soft-deleted survivor are `recordMergeAliases`' business,
267
+ // not this job's.
268
+ const swept = await db.execute<{ id: string }>(sql`
269
+ DELETE FROM contact_aliases a
270
+ USING contacts c
271
+ WHERE a.contact_id = c.id
272
+ AND c.deleted_at IS NOT NULL
273
+ AND a.reason = ${ALIAS_REASON_BACKFILL}
274
+ RETURNING a.id
275
+ `);
276
+ if (swept.length > 0) {
277
+ logger.warn("Identity alias backfill swept erased-contact rows", {
278
+ swept: swept.length,
279
+ });
280
+ // Clamp: the sweep can also collect a PREVIOUS run's raced rows,
281
+ // which this run never counted as inserts.
282
+ inserted = Math.max(0, inserted - swept.length);
283
+ }
284
+ }
285
+
286
+ await markJob({
287
+ status: "completed",
288
+ processedRows: scanned,
289
+ failedRows: conflicting,
290
+ });
291
+ logger.info("Identity alias backfill complete", {
292
+ dryRun,
293
+ scanned,
294
+ inserted,
295
+ present,
296
+ conflicting,
297
+ mixedCaseEmails,
298
+ });
299
+ return {
300
+ status: "completed",
301
+ dryRun,
302
+ scanned,
303
+ inserted,
304
+ present,
305
+ conflicting,
306
+ mixedCaseEmails,
307
+ };
308
+ } catch (err) {
309
+ const message = err instanceof Error ? err.message : String(err);
310
+ await markJob({
311
+ status: "failed",
312
+ errors: [{ row: 0, error: message }],
313
+ }).catch(() => undefined);
314
+ logger.error("Identity alias backfill failed", { dryRun, message });
315
+ return {
316
+ status: "failed",
317
+ dryRun,
318
+ scanned: 0,
319
+ inserted: 0,
320
+ present: 0,
321
+ conflicting: 0,
322
+ mixedCaseEmails: 0,
323
+ reason: message,
324
+ };
325
+ }
326
+ }
327
+
328
+ export const identityAliasBackfillTask = hatchet.task({
329
+ name: "identity-alias-backfill",
330
+ retries: 0,
331
+ executionTimeout: "600s",
332
+ fn: async (input: IdentityAliasBackfillInput) => {
333
+ const { db } = createDatabase({ url: process.env.DATABASE_URL ?? "" });
334
+ const logger = createLogger(process.env.LOG_LEVEL ?? "info");
335
+ return runIdentityAliasBackfill({
336
+ db,
337
+ logger,
338
+ jobId: input.jobId,
339
+ dryRun: input.dryRun === true,
340
+ });
341
+ },
342
+ });
343
+
344
+ /**
345
+ * Worker-boot enqueue (mirrors `enqueueBucketBackfills`): fire the backfill
346
+ * once per deployment without an operator having to remember it — the engine
347
+ * is a published dependency, and a manual step every consumer must run before
348
+ * the read flip is how deployments strand. Skips when ANY non-failed job of
349
+ * this format exists (pending/processing/completed); a failed run re-enqueues
350
+ * on the next boot, and `POST /v1/admin/identity/alias-backfill` forces a
351
+ * re-run past a stale record. Re-running is always safe (ON CONFLICT DO
352
+ * NOTHING), just not free — hence the skip. Best-effort: a failure to enqueue
353
+ * must never crash worker boot.
354
+ */
355
+ export async function enqueueIdentityAliasBackfill(opts: {
356
+ db: Database;
357
+ logger: Logger;
358
+ }): Promise<void> {
359
+ const { db, logger } = opts;
360
+ try {
361
+ const existing = await db
362
+ .select({ id: importJobs.id })
363
+ .from(importJobs)
364
+ .where(
365
+ and(
366
+ eq(importJobs.format, IDENTITY_ALIAS_BACKFILL_FORMAT),
367
+ inArray(importJobs.status, ["pending", "processing", "completed"]),
368
+ ),
369
+ )
370
+ .limit(1);
371
+ if (existing[0]) return;
372
+
373
+ const [job] = await db
374
+ .insert(importJobs)
375
+ .values({
376
+ fileName: IDENTITY_ALIAS_BACKFILL_FORMAT,
377
+ format: IDENTITY_ALIAS_BACKFILL_FORMAT,
378
+ status: "pending",
379
+ })
380
+ .returning({ id: importJobs.id });
381
+ if (!job) return;
382
+
383
+ // runNoWait: called from worker boot BEFORE the listener starts — awaiting
384
+ // the run would deadlock (same reasoning as enqueueBucketBackfills).
385
+ await identityAliasBackfillTask.runNoWait({ jobId: job.id, dryRun: false });
386
+ logger.info("Identity alias backfill enqueued", { jobId: job.id });
387
+ } catch (err) {
388
+ logger.warn("Identity alias backfill enqueue failed", {
389
+ error: err instanceof Error ? err.message : String(err),
390
+ });
391
+ }
392
+ }
393
+
394
+ // ---------------------------------------------------------------------------
395
+ // PRD 02 T4 — read-only parity verifier
396
+ // ---------------------------------------------------------------------------
397
+
398
+ export interface AliasParityRow {
399
+ kind: string;
400
+ /** Alias and column resolve to DIFFERENT live contacts. MUST be 0 before
401
+ * trusting alias-first resolution; a non-zero value is a data bug to fix on
402
+ * its own, never to flip over. */
403
+ diverged: number;
404
+ /** Alias target is dead/missing but a live column owner exists — expected,
405
+ * handled by `findByKey`'s live-target fall-through. */
406
+ aliasDead: number;
407
+ /** Alias exists with no column owner — the merge/promote population, and
408
+ * (after PRD 03) the point of the table. */
409
+ aliasOnly: number;
410
+ }
411
+
412
+ /** "Would alias-first return a different contact than column-first?", per kind,
413
+ * over every alias row. Pure read; no writes, no locks beyond the reads. */
414
+ export async function identityAliasParity(
415
+ db: Database,
416
+ ): Promise<AliasParityRow[]> {
417
+ const rows = await db.execute<{
418
+ alias_kind: string;
419
+ diverged: number;
420
+ alias_dead: number;
421
+ alias_only: number;
422
+ }>(sql`
423
+ SELECT a.alias_kind,
424
+ count(*) FILTER (WHERE ac.id IS NOT NULL AND cc.id IS NOT NULL
425
+ AND ac.id <> cc.id)::int AS diverged,
426
+ count(*) FILTER (WHERE ac.id IS NULL
427
+ AND cc.id IS NOT NULL)::int AS alias_dead,
428
+ count(*) FILTER (WHERE ac.id IS NOT NULL
429
+ AND cc.id IS NULL)::int AS alias_only
430
+ FROM contact_aliases a
431
+ LEFT JOIN contacts ac
432
+ ON ac.id = a.contact_id AND ac.deleted_at IS NULL
433
+ LEFT JOIN contacts cc
434
+ ON cc.deleted_at IS NULL AND (
435
+ (a.alias_kind = 'external' AND cc.external_id = a.alias_value)
436
+ OR (a.alias_kind = 'email' AND lower(trim(cc.email)) = a.alias_value)
437
+ OR (a.alias_kind = 'anonymous' AND cc.anonymous_id = a.alias_value)
438
+ OR (a.alias_kind = 'discord' AND cc.discord_id = a.alias_value))
439
+ GROUP BY 1
440
+ ORDER BY 1
441
+ `);
442
+ return rows.map((r) => ({
443
+ kind: r.alias_kind,
444
+ diverged: Number(r.diverged),
445
+ aliasDead: Number(r.alias_dead),
446
+ aliasOnly: Number(r.alias_only),
447
+ }));
448
+ }
@@ -130,6 +130,16 @@ export const importContactsTask = hatchet.task({
130
130
  userId: row.externalId,
131
131
  email: row.email,
132
132
  contactProperties: row.properties,
133
+ // PRD 06 T4 (L5 row 21): an operator import asserts identity by
134
+ // externalId/email only (the key guard above rejects rows with
135
+ // neither; `phone` attaches OUTSIDE the resolver, below) — so
136
+ // create-on-miss, no clamp, and the narrow `trustedKinds` is the
137
+ // honest statement. Enforced by T5.
138
+ policy: {
139
+ create: "on-miss",
140
+ allowMerge: "any",
141
+ trustedKinds: ["external", "email"],
142
+ },
133
143
  }).then(async (result) => {
134
144
  await attachPhone(db, result.id, row.phone);
135
145
  return { index: i + idx, ok: true };
@@ -19,7 +19,7 @@ import {
19
19
  waveConditionSql,
20
20
  } from "../campaigns/cohort-sql.js";
21
21
  import { campaignSendKey } from "../lib/campaign-send-key.js";
22
- import { normalizeEmail } from "../lib/contacts.js";
22
+ import { contactKeySql, normalizeEmail } from "../lib/contacts.js";
23
23
  import { getDb } from "../lib/db.js";
24
24
  import { getEmailService } from "../lib/email.js";
25
25
  import { hatchet } from "../lib/hatchet.js";
@@ -973,18 +973,36 @@ async function* resolveBucketRecipients(
973
973
  ];
974
974
  if (cursor) conditions.push(gt(bucketMemberships.id, cursor));
975
975
 
976
- return db
977
- .select({
978
- id: bucketMemberships.id,
979
- userId: bucketMemberships.userId,
980
- membershipEmail: bucketMemberships.userEmail,
981
- contactEmail: contacts.email,
982
- })
983
- .from(bucketMemberships)
984
- .innerJoin(contacts, eq(contacts.externalId, bucketMemberships.userId))
985
- .where(and(...conditions))
986
- .orderBy(bucketMemberships.id)
987
- .limit(CHUNK_SIZE);
976
+ return (
977
+ db
978
+ .select({
979
+ id: bucketMemberships.id,
980
+ userId: bucketMemberships.userId,
981
+ membershipEmail: bucketMemberships.userEmail,
982
+ contactEmail: contacts.email,
983
+ })
984
+ .from(bucketMemberships)
985
+ // Join on the CANONICAL key, not `external_id` alone.
986
+ // `bucket_memberships.user_id` holds `coalesce(external_id,
987
+ // anonymous_id, id)`, so a contact with a NULL `external_id` — an
988
+ // email-only subscriber keyed on its uuid, or an anonymous visitor
989
+ // keyed on its `anonymous_id` — matched nothing here and was silently
990
+ // dropped from the audience. They were genuine active members of the
991
+ // bucket; they simply never received the campaign, and nothing
992
+ // surfaced it: a send report showing 40 of 40 delivered looks complete
993
+ // whether or not the audience was built correctly.
994
+ //
995
+ // Suppression is unaffected by widening this join. Every gate lives in
996
+ // the same `conditions` array applied to this query and is keyed on
997
+ // `recipientEmail` (`coalesce(membership email, contact email)`, lowered
998
+ // and trimmed), not on the join column — so a newly-visible member is
999
+ // filtered by the identical unsubscribed/suppressed `NOT EXISTS` as
1000
+ // everyone else, and the mailer re-checks preferences per send.
1001
+ .innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
1002
+ .where(and(...conditions))
1003
+ .orderBy(bucketMemberships.id)
1004
+ .limit(CHUNK_SIZE)
1005
+ );
988
1006
  },
989
1007
  cursorOf: (row) => row.id,
990
1008
  map: (row) => {