@hogsend/engine 0.58.0 → 0.59.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,771 @@
1
+ import type { JsonObject } from "@hatchet-dev/typescript-sdk/v1/types.js";
2
+ import {
3
+ contacts,
4
+ createDatabase,
5
+ type Database,
6
+ importJobs,
7
+ } from "@hogsend/db";
8
+ import { and, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm";
9
+ import { contactKeySql } from "../lib/contacts.js";
10
+ import { hatchet } from "../lib/hatchet.js";
11
+ import type { Logger } from "../lib/logger.js";
12
+ import { createLogger } from "../lib/logger.js";
13
+
14
+ /**
15
+ * PRD 04 T5 — fill the five history tables' `contact_id` bookkeeping column for
16
+ * rows that predate the dual-write (and for rows a best-effort dual-write
17
+ * missed, D6).
18
+ *
19
+ * NEVER a migration (`docs/UPGRADING.md:82-87`): this is a chunked, paced,
20
+ * resumable Hatchet task, modelled on `identity-alias-backfill.ts` — an
21
+ * `import_jobs` progress row, boot-enqueued, forceable from admin.
22
+ *
23
+ * Shape, per D3/D4:
24
+ * - PASS 1 iterates **contacts**, not events. Per live contact it derives the
25
+ * canonical key (`contactKeySql()` = `external_id ?? anonymous_id ?? id`) and
26
+ * issues one bounded `UPDATE … WHERE id IN (SELECT id … WHERE user_id = $key
27
+ * AND contact_id IS NULL LIMIT $n)` per table, looped to zero. Every table
28
+ * has a leading `user_id` index, so no statement seq-scans `user_events`.
29
+ * - PASS 2 fills rows keyed on a STALE key that only `contact_aliases` knows,
30
+ * restricted to kinds `external`/`anonymous` (the only kinds a canonical key
31
+ * can ever be). A value owned by more than one live contact across those two
32
+ * kinds is SKIPPED and logged — never guessed (D4).
33
+ *
34
+ * Invariants, each pinned by a test:
35
+ * - **Only `contact_id` is ever written.** No inserts, no deletes, no other
36
+ * column touched (not even `updated_at` — the statements set one column).
37
+ * - **`contact_id IS NULL` guards every UPDATE**, so a re-run, a resume, or the
38
+ * periodic re-sweep (D6) updates zero rows and costs only the probes.
39
+ * - **A row whose key owns NO live contact stays NULL. Forever** (D5). The
40
+ * completion criterion is not "zero NULLs"; T6's probe is what judges it.
41
+ * - **Not a repair tool.** A row already carrying a WRONG non-NULL value is
42
+ * left alone by design (risk 3) — the fix for that is a targeted corrective
43
+ * job, not this sweep.
44
+ */
45
+
46
+ /** `import_jobs.format` discriminator for the reused status record. */
47
+ export const CONTACT_ID_BACKFILL_FORMAT = "identity-contact-id-backfill";
48
+
49
+ /** The five string-keyed history tables carrying the column (D1). */
50
+ const TABLES = [
51
+ "user_events",
52
+ "journey_states",
53
+ "bucket_memberships",
54
+ "email_sends",
55
+ "email_preferences",
56
+ ] as const;
57
+
58
+ export type ContactIdBackfillTable = (typeof TABLES)[number];
59
+
60
+ /** Per-table row counts. A mapped type (not an interface) so it carries the
61
+ * implicit index signature the Hatchet `JsonObject` return contract needs. */
62
+ export type ContactIdBackfillCounts = Record<ContactIdBackfillTable, number>;
63
+
64
+ /** Live contacts per keyset chunk — the `bucket-backfill.ts` precedent. */
65
+ const DEFAULT_CONTACTS_PER_CHUNK = 500;
66
+ /** Rows per UPDATE statement. The sizing RULE (D3) is what matters: keep any
67
+ * single statement under ~1s and ~10,000 row locks. This default is a guess
68
+ * about someone else's data; measure and override per deployment. */
69
+ const DEFAULT_ROWS_PER_STATEMENT = 5_000;
70
+ /** Pause after a statement that actually wrote, so autovacuum keeps up with the
71
+ * dead tuples a non-HOT (indexed column) update leaves behind (D3). */
72
+ const DEFAULT_PAUSE_MS = 25;
73
+ /** Runaway guard: a per-(contact, table) loop that never drains is a bug, not a
74
+ * big table. 5,000 iterations × the default cap is 25M rows for ONE key. */
75
+ const MAX_STATEMENTS_PER_LOOP = 5_000;
76
+ /** D6 — how stale the newest COMPLETED sweep may be before boot re-enqueues. */
77
+ const DEFAULT_RESWEEP_HOURS = 24;
78
+ /** Ambiguous alias values logged per run (the rest are counted only). */
79
+ const AMBIGUOUS_SAMPLE = 20;
80
+
81
+ export interface ContactIdBackfillInput extends JsonObject {
82
+ /** `import_jobs` row to record progress on. Optional: the task body runs
83
+ * standalone (tests, operator scripts) without one. */
84
+ jobId?: string;
85
+ /** Live contacts read per keyset chunk. Default 500. */
86
+ contactsPerChunk?: number;
87
+ /** Rows per bounded UPDATE. Default 5,000. */
88
+ rowsPerStatement?: number;
89
+ /** Milliseconds to pause after a statement that wrote. Default 25. */
90
+ pauseMs?: number;
91
+ }
92
+
93
+ // Extends JsonObject: this is a Hatchet task return value and must be
94
+ // JSON-serializable (the constraint every workflow output carries).
95
+ export interface ContactIdBackfillResult extends JsonObject {
96
+ status: "completed" | "failed";
97
+ /** Live contacts visited by pass 1. */
98
+ contactsScanned: number;
99
+ /** Rows stamped by pass 1, per table (the canonical-key population). */
100
+ canonical: ContactIdBackfillCounts;
101
+ /** Rows stamped by pass 2, per table (the stale-alias population). */
102
+ alias: ContactIdBackfillCounts;
103
+ /** `canonical + alias` across all five tables — the number a re-run must
104
+ * report as 0 (the idempotence assertion). */
105
+ updated: number;
106
+ /** Alias VALUES owned by more than one live contact across the two permitted
107
+ * kinds. Skipped, never guessed (D4); a non-zero count wants a human. */
108
+ ambiguousAliases: number;
109
+ /** UPDATE statements issued. Observability for the pacing rule. */
110
+ statements: number;
111
+ reason?: string;
112
+ }
113
+
114
+ const zeroCounts = (): ContactIdBackfillCounts => ({
115
+ user_events: 0,
116
+ journey_states: 0,
117
+ bucket_memberships: 0,
118
+ email_sends: 0,
119
+ email_preferences: 0,
120
+ });
121
+
122
+ const sleep = (ms: number) =>
123
+ new Promise<void>((resolve) => setTimeout(resolve, ms));
124
+
125
+ /** Positive-integer option with a documented default; a nonsense value falls
126
+ * back rather than producing an unbounded or zero-progress statement. */
127
+ function positiveInt(value: number | undefined, fallback: number): number {
128
+ if (value === undefined) return fallback;
129
+ return Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
130
+ }
131
+
132
+ /**
133
+ * PASS 1, one table, one contact. The PRD's verbatim statement (D3), looped
134
+ * until it affects zero rows. Returns rows stamped and statements issued.
135
+ *
136
+ * The `IN (SELECT … LIMIT n)` shape (rather than a bare `WHERE user_id = …`) is
137
+ * what bounds the row-lock count: the fat tail of this system is a bot anon id
138
+ * with tens of thousands of events under ONE key, and an unbounded statement
139
+ * there is the lock/WAL spike D3 exists to avoid.
140
+ */
141
+ async function fillCanonicalKey(opts: {
142
+ db: Database;
143
+ table: ContactIdBackfillTable;
144
+ key: string;
145
+ contactId: string;
146
+ rowsPerStatement: number;
147
+ pauseMs: number;
148
+ }): Promise<{ updated: number; statements: number }> {
149
+ const { db, table, key, contactId, rowsPerStatement, pauseMs } = opts;
150
+ const name = sql.identifier(table);
151
+ let updated = 0;
152
+ let statements = 0;
153
+
154
+ for (;;) {
155
+ const rows = await db.execute<{ id: string }>(sql`
156
+ UPDATE ${name} SET contact_id = ${contactId}::uuid
157
+ WHERE id IN (
158
+ SELECT id FROM ${name}
159
+ WHERE user_id = ${key} AND contact_id IS NULL
160
+ LIMIT ${rowsPerStatement}
161
+ )
162
+ RETURNING id
163
+ `);
164
+ statements += 1;
165
+ if (rows.length === 0) return { updated, statements };
166
+ updated += rows.length;
167
+
168
+ if (statements >= MAX_STATEMENTS_PER_LOOP) {
169
+ throw new Error(
170
+ `contact-id backfill: ${table} did not drain for key after ` +
171
+ `${statements} statements (${updated} rows) — refusing to loop`,
172
+ );
173
+ }
174
+ // Only a statement that WROTE created dead tuples, so only that one earns
175
+ // the pause. Without this the steady-state re-sweep (D6) would idle for
176
+ // 5 × pauseMs per contact while updating nothing.
177
+ if (pauseMs > 0) await sleep(pauseMs);
178
+ }
179
+ }
180
+
181
+ /**
182
+ * The set of alias values that resolve UNAMBIGUOUSLY to one live contact, under
183
+ * the two kinds a canonical key can be. `uniqueIndex(alias_kind, alias_value)`
184
+ * is per-KIND, so one string can legally sit under both `external` and
185
+ * `anonymous`; when those disagree on the contact this CTE drops the value (the
186
+ * `HAVING count(DISTINCT …) = 1` clause) and `countAmbiguousAliases` reports it.
187
+ *
188
+ * `array_agg(…)[1]` rather than `min()`: `min(uuid)` only exists from PG 14, and
189
+ * the aggregate choice is irrelevant when the HAVING proves there is exactly one
190
+ * distinct value.
191
+ */
192
+ const resolvedAliasesCte = sql`
193
+ resolved AS (
194
+ SELECT a.alias_value AS alias_key,
195
+ (array_agg(a.contact_id ORDER BY a.contact_id))[1] AS contact_id
196
+ FROM contact_aliases a
197
+ JOIN contacts c ON c.id = a.contact_id AND c.deleted_at IS NULL
198
+ WHERE a.alias_kind IN ('external', 'anonymous')
199
+ GROUP BY a.alias_value
200
+ HAVING count(DISTINCT a.contact_id) = 1
201
+ )`;
202
+
203
+ /** D4's skip rule, measured: alias values whose two permitted kinds point at
204
+ * DIFFERENT live contacts. Never resolved; reported so a human can. */
205
+ async function countAmbiguousAliases(
206
+ db: Database,
207
+ ): Promise<{ total: number; sample: string[] }> {
208
+ const rows = await db.execute<{ alias_value: string }>(sql`
209
+ SELECT a.alias_value
210
+ FROM contact_aliases a
211
+ JOIN contacts c ON c.id = a.contact_id AND c.deleted_at IS NULL
212
+ WHERE a.alias_kind IN ('external', 'anonymous')
213
+ GROUP BY a.alias_value
214
+ HAVING count(DISTINCT a.contact_id) > 1
215
+ `);
216
+ return {
217
+ total: rows.length,
218
+ sample: rows.slice(0, AMBIGUOUS_SAMPLE).map((r) => r.alias_value),
219
+ };
220
+ }
221
+
222
+ /**
223
+ * PASS 2's work list: alias values that resolve to exactly one live contact AND
224
+ * are NOT already the canonical key of some live contact — i.e. precisely the
225
+ * stale/second-device population pass 1 cannot reach.
226
+ *
227
+ * The `NOT EXISTS` exclusion is not just an optimisation: an alias value that IS
228
+ * a live contact's canonical key was already fully drained by pass 1 (every row
229
+ * under that key is now non-NULL), so re-visiting it can only issue zero-row
230
+ * probes. Excluding it keeps pass 2 proportional to the merge/promote
231
+ * population rather than to the whole alias table (which, post-PRD-02, holds
232
+ * roughly one row per identity column per live contact — C2).
233
+ *
234
+ * Keyset-paginated on `alias_key` so the work list itself is bounded too.
235
+ */
236
+ async function selectStaleAliasKeys(opts: {
237
+ db: Database;
238
+ afterKey: string | null;
239
+ limit: number;
240
+ }): Promise<Array<{ key: string; contactId: string }>> {
241
+ const { db, afterKey, limit } = opts;
242
+ const rows = await db.execute<{ alias_key: string; contact_id: string }>(sql`
243
+ WITH ${resolvedAliasesCte}
244
+ SELECT r.alias_key, r.contact_id::text AS contact_id
245
+ FROM resolved r
246
+ WHERE (${afterKey}::text IS NULL OR r.alias_key > ${afterKey}::text)
247
+ AND NOT EXISTS (
248
+ SELECT 1 FROM contacts
249
+ WHERE contacts.deleted_at IS NULL
250
+ AND ${contactKeySql()} = r.alias_key
251
+ )
252
+ ORDER BY r.alias_key
253
+ LIMIT ${limit}
254
+ `);
255
+ return rows.map((r) => ({ key: r.alias_key, contactId: r.contact_id }));
256
+ }
257
+
258
+ /**
259
+ * The task body, exported directly so tests and operator scripts can run it
260
+ * without a Hatchet engine. `jobId` is optional; when present, progress lands on
261
+ * the `import_jobs` row (`totalRows` = live contacts, `processedRows` = contacts
262
+ * done, `failedRows` = ambiguous alias values skipped — a divergence metric, NOT
263
+ * an error count).
264
+ *
265
+ * RESUME is free rather than bookkept: an interrupted run is re-driven from the
266
+ * top, re-reads live contacts ordered by `id`, and every UPDATE's
267
+ * `contact_id IS NULL` guard makes an already-done contact a single zero-row
268
+ * probe per table. No cursor is persisted because none is needed.
269
+ */
270
+ export async function runContactIdBackfill(opts: {
271
+ db: Database;
272
+ logger: Logger;
273
+ jobId?: string;
274
+ contactsPerChunk?: number;
275
+ rowsPerStatement?: number;
276
+ pauseMs?: number;
277
+ }): Promise<ContactIdBackfillResult> {
278
+ const { db, logger, jobId } = opts;
279
+ const contactsPerChunk = positiveInt(
280
+ opts.contactsPerChunk,
281
+ DEFAULT_CONTACTS_PER_CHUNK,
282
+ );
283
+ const rowsPerStatement = positiveInt(
284
+ opts.rowsPerStatement,
285
+ DEFAULT_ROWS_PER_STATEMENT,
286
+ );
287
+ const pauseMs =
288
+ opts.pauseMs === undefined
289
+ ? DEFAULT_PAUSE_MS
290
+ : Number.isFinite(opts.pauseMs) && opts.pauseMs >= 0
291
+ ? Math.floor(opts.pauseMs)
292
+ : DEFAULT_PAUSE_MS;
293
+
294
+ const canonical = zeroCounts();
295
+ const alias = zeroCounts();
296
+ let contactsScanned = 0;
297
+ let statements = 0;
298
+ let ambiguousAliases = 0;
299
+
300
+ const markJob = async (
301
+ patch: Partial<typeof importJobs.$inferInsert>,
302
+ ): Promise<void> => {
303
+ if (!jobId) return;
304
+ await db
305
+ .update(importJobs)
306
+ .set({ ...patch, updatedAt: new Date() })
307
+ .where(eq(importJobs.id, jobId));
308
+ };
309
+
310
+ try {
311
+ const [totals] = await db.execute<{ live: number }>(sql`
312
+ SELECT count(*)::int AS live FROM contacts WHERE deleted_at IS NULL
313
+ `);
314
+ await markJob({
315
+ status: "processing",
316
+ totalRows: Number(totals?.live ?? 0),
317
+ });
318
+
319
+ // ---- PASS 1: the canonical key of every live contact (D3) --------------
320
+ const resolvedKey = contactKeySql();
321
+ let cursor: string | null = null;
322
+ for (;;) {
323
+ const chunk = await db
324
+ .select({ id: contacts.id, key: resolvedKey })
325
+ .from(contacts)
326
+ .where(
327
+ and(
328
+ isNull(contacts.deletedAt),
329
+ cursor ? gt(contacts.id, cursor) : undefined,
330
+ ),
331
+ )
332
+ .orderBy(contacts.id)
333
+ .limit(contactsPerChunk);
334
+ if (chunk.length === 0) break;
335
+
336
+ for (const row of chunk) {
337
+ for (const table of TABLES) {
338
+ const filled = await fillCanonicalKey({
339
+ db,
340
+ table,
341
+ key: row.key,
342
+ contactId: row.id,
343
+ rowsPerStatement,
344
+ pauseMs,
345
+ });
346
+ canonical[table] += filled.updated;
347
+ statements += filled.statements;
348
+ }
349
+ }
350
+
351
+ contactsScanned += chunk.length;
352
+ cursor = chunk[chunk.length - 1]?.id ?? null;
353
+ await markJob({ processedRows: contactsScanned });
354
+ if (chunk.length < contactsPerChunk) break;
355
+ }
356
+
357
+ // ---- PASS 2: stale keys only `contact_aliases` knows (D4) --------------
358
+ const ambiguous = await countAmbiguousAliases(db);
359
+ ambiguousAliases = ambiguous.total;
360
+ if (ambiguousAliases > 0) {
361
+ logger.warn("Contact-id backfill skipped ambiguous alias values", {
362
+ count: ambiguousAliases,
363
+ sample: ambiguous.sample,
364
+ });
365
+ }
366
+
367
+ let aliasCursor: string | null = null;
368
+ for (;;) {
369
+ const staleKeys = await selectStaleAliasKeys({
370
+ db,
371
+ afterKey: aliasCursor,
372
+ limit: contactsPerChunk,
373
+ });
374
+ if (staleKeys.length === 0) break;
375
+
376
+ for (const entry of staleKeys) {
377
+ for (const table of TABLES) {
378
+ const filled = await fillCanonicalKey({
379
+ db,
380
+ table,
381
+ key: entry.key,
382
+ contactId: entry.contactId,
383
+ rowsPerStatement,
384
+ pauseMs,
385
+ });
386
+ alias[table] += filled.updated;
387
+ statements += filled.statements;
388
+ }
389
+ }
390
+
391
+ aliasCursor = staleKeys[staleKeys.length - 1]?.key ?? null;
392
+ if (staleKeys.length < contactsPerChunk) break;
393
+ }
394
+
395
+ const updated =
396
+ TABLES.reduce((sum, t) => sum + canonical[t], 0) +
397
+ TABLES.reduce((sum, t) => sum + alias[t], 0);
398
+
399
+ await markJob({
400
+ status: "completed",
401
+ processedRows: contactsScanned,
402
+ failedRows: ambiguousAliases,
403
+ });
404
+ logger.info("Contact-id backfill complete", {
405
+ contactsScanned,
406
+ canonical,
407
+ alias,
408
+ updated,
409
+ ambiguousAliases,
410
+ statements,
411
+ });
412
+ return {
413
+ status: "completed",
414
+ contactsScanned,
415
+ canonical,
416
+ alias,
417
+ updated,
418
+ ambiguousAliases,
419
+ statements,
420
+ };
421
+ } catch (err) {
422
+ const message = err instanceof Error ? err.message : String(err);
423
+ await markJob({
424
+ status: "failed",
425
+ errors: [{ row: 0, error: message }],
426
+ }).catch(() => undefined);
427
+ logger.error("Contact-id backfill failed", { message });
428
+ return {
429
+ status: "failed",
430
+ contactsScanned,
431
+ canonical,
432
+ alias,
433
+ updated:
434
+ TABLES.reduce((sum, t) => sum + canonical[t], 0) +
435
+ TABLES.reduce((sum, t) => sum + alias[t], 0),
436
+ ambiguousAliases,
437
+ statements,
438
+ reason: message,
439
+ };
440
+ }
441
+ }
442
+
443
+ // ---------------------------------------------------------------------------
444
+ // T6 — the invariant probe
445
+ // ---------------------------------------------------------------------------
446
+
447
+ /** One table's verdict. `missing` and `mismatched` are failures; `orphaned` is
448
+ * information (D5) and is expected to be non-zero forever. */
449
+ export interface ContactIdVerifyCounts {
450
+ /** A live contact owns the row's `user_id` — canonically OR by alias — but
451
+ * `contact_id` is NULL. After a completed sweep this is a HOLE: either the
452
+ * backfill missed the row or a dual-write site dropped it (D6). MUST be 0. */
453
+ missing: number;
454
+ /** `contact_id` points at a contact that owns the row's `user_id` NEITHER
455
+ * canonically NOR by alias (including a pointer at no contact at all). The
456
+ * corruption detector, and the one count an FK could never give: an FK proves
457
+ * the uuid EXISTS, this proves it is the RIGHT uuid. MUST be 0. */
458
+ mismatched: number;
459
+ /** `contact_id` is NULL and NO live contact owns the key either way — a
460
+ * refused anonymous ingest, a keyless raw send. Expected, permitted, reported.
461
+ * NEVER a failure (D5). */
462
+ orphaned: number;
463
+ }
464
+
465
+ /** The probe's verdict for the whole database (or, when scoped, for the keys it
466
+ * was pointed at). */
467
+ export interface ContactIdVerifyResult {
468
+ tables: Record<ContactIdBackfillTable, ContactIdVerifyCounts>;
469
+ /** The five tables summed — what the alert line and the gate read. */
470
+ totals: ContactIdVerifyCounts;
471
+ /**
472
+ * **THE PRD 05 ENTRY GATE.** True iff EVERY table reports `missing === 0` AND
473
+ * `mismatched === 0` — i.e. every row a live contact owns is stamped, and
474
+ * every stamp points at a contact that really owns that row's key. `orphaned`
475
+ * is deliberately NOT part of the gate (D5 makes it legitimately non-zero
476
+ * forever). PRD 05 flips reads from `user_id` onto `contact_id`; flipping
477
+ * while this is false means silently losing history (`missing`) or attributing
478
+ * it to the wrong person (`mismatched`). Judge it AFTER a completed sweep — a
479
+ * false reading with no sweep on record is just a pending backfill.
480
+ */
481
+ flipReady: boolean;
482
+ }
483
+
484
+ /**
485
+ * "Contact `c` owns key `k`" — the **ALIAS-AWARE** ownership definition
486
+ * (T6's "T4 review correction", LOCKED):
487
+ *
488
+ * `coalesce(c.external_id, c.anonymous_id, c.id::text) = k` (canonical)
489
+ * **OR** a live alias row `(contact_id = c.id, alias_value = k,
490
+ * alias_kind IN ('external','anonymous'))` (alias)
491
+ *
492
+ * The bare canonical coalesce is WRONG post-PRD-03 and is not a nicety: a
493
+ * second-device anonymous id lives ONLY in `contact_aliases`, the dual-write
494
+ * deliberately resolves it (C1), and a coalesce-only `mismatched` probe would
495
+ * flag every such CORRECTLY-stamped row as corruption — a false alarm that would
496
+ * block the gate forever on any deployment where anyone has ever used two
497
+ * devices. Kinds are restricted to `external`/`anonymous` for the same reason
498
+ * pass 2 restricts them (D4): those are the only kinds a canonical key can ever
499
+ * be, so an `email`/`discord` alias is NOT ownership.
500
+ *
501
+ * Three properties worth naming:
502
+ *
503
+ * - `missing` and `orphaned` PARTITION the NULL-stamped rows: every row with
504
+ * `contact_id IS NULL` is one or the other, never both, never neither. So
505
+ * `missing + orphaned = count(*) WHERE contact_id IS NULL`, which is what
506
+ * makes "zero NULLs" the wrong completion criterion and this the right one.
507
+ * - `mismatched` does NOT require the pointed-at contact to be LIVE. A
508
+ * soft-deleted contact whose key still matches is consistent data (someone
509
+ * was deleted), not corruption; demanding liveness would fail the gate on
510
+ * every GDPR delete. The merge-regression case risk 5 worries about is still
511
+ * caught, because a merge REWRITES the row's `user_id` to the survivor's key
512
+ * — so a row stranded on a soft-deleted loser mismatches on the KEY. A
513
+ * pointer at a contact row that does not exist at all is counted (that is the
514
+ * FK's job, and this probe is meant to be strictly stronger than an FK).
515
+ * - **`user_id IS NULL` rows are NOT counted as mismatched, and the reasoning
516
+ * is load-bearing.** D7 makes a raw/keyless send stamp `contact_id = NULL`
517
+ * (`lib/tracked.ts` resolves `sendContactId` only `if (options.userId)`), so
518
+ * the instinct — "a keyless row carrying a `contact_id` must be a new writer
519
+ * violating D7" — looks safe. It is FALSE: the admin resend path
520
+ * (`routes/admin/bulk.ts`, D8 row 11) inserts its new `email_sends` row with
521
+ * `contactId: email.contactId` copied off the source row and does NOT copy
522
+ * `userId` (the pre-existing gap T4d files as do-not-fix). So
523
+ * `user_id IS NULL AND contact_id IS NOT NULL` is reachable TODAY, on a
524
+ * committed path, carrying a CORRECT contact_id. Counting it would pin
525
+ * `mismatched > 0` on any deployment that has ever resent a bounced email —
526
+ * the same permanent false alarm the alias correction exists to prevent. The
527
+ * cost, stated plainly: a future writer resolving a keyless send by recipient
528
+ * ADDRESS would be indistinguishable from a resend, so this probe cannot
529
+ * catch that particular D7 violation. `email_sends` is the only one of the
530
+ * five tables where the case exists at all — the other four are
531
+ * `user_id NOT NULL`.
532
+ *
533
+ * Two set-based statements per table, no per-row work: one over the NULL-stamped
534
+ * rows (which `missing`/`orphaned` partition) and one over the stamped rows
535
+ * (whose `contact_id IS NOT NULL` predicate is exactly D2's partial index). The
536
+ * canonical leg is the same `contactKeySql()` coalesce the backfill keys on; the
537
+ * alias leg probes `contact_aliases_kind_value_idx` (an `IN` on the leading
538
+ * `alias_kind` column keeps the index usable).
539
+ */
540
+ async function verifyTable(
541
+ db: Database,
542
+ table: ContactIdBackfillTable,
543
+ userIds: string[] | undefined,
544
+ ): Promise<ContactIdVerifyCounts> {
545
+ const name = sql.identifier(table);
546
+ const scope = scopeToKeys(userIds);
547
+
548
+ const [nulls] = await db.execute<{ missing: number; orphaned: number }>(sql`
549
+ SELECT count(*) FILTER (WHERE s.owned)::int AS missing,
550
+ count(*) FILTER (WHERE NOT s.owned)::int AS orphaned
551
+ FROM (
552
+ SELECT (
553
+ EXISTS (
554
+ SELECT 1 FROM contacts
555
+ WHERE contacts.deleted_at IS NULL
556
+ AND ${contactKeySql()} = t.user_id
557
+ )
558
+ OR EXISTS (
559
+ SELECT 1 FROM contact_aliases a
560
+ JOIN contacts ON contacts.id = a.contact_id
561
+ AND contacts.deleted_at IS NULL
562
+ WHERE a.alias_value = t.user_id
563
+ AND a.alias_kind IN ('external', 'anonymous')
564
+ )
565
+ ) AS owned
566
+ FROM ${name} t
567
+ WHERE t.contact_id IS NULL${scope}
568
+ ) s
569
+ `);
570
+
571
+ const [stamped] = await db.execute<{ mismatched: number }>(sql`
572
+ SELECT count(*)::int AS mismatched
573
+ FROM ${name} t
574
+ LEFT JOIN contacts ON contacts.id = t.contact_id
575
+ WHERE t.contact_id IS NOT NULL
576
+ AND t.user_id IS NOT NULL${scope}
577
+ AND ${contactKeySql()} IS DISTINCT FROM t.user_id
578
+ AND NOT EXISTS (
579
+ SELECT 1 FROM contact_aliases a
580
+ WHERE a.contact_id = contacts.id
581
+ AND a.alias_value = t.user_id
582
+ AND a.alias_kind IN ('external', 'anonymous')
583
+ )
584
+ `);
585
+
586
+ return {
587
+ missing: Number(nulls?.missing ?? 0),
588
+ mismatched: Number(stamped?.mismatched ?? 0),
589
+ orphaned: Number(nulls?.orphaned ?? 0),
590
+ };
591
+ }
592
+
593
+ /** The optional key restriction, appended to both statements' WHERE. `undefined`
594
+ * = the whole table (the gate); an EMPTY list means "these zero keys", which is
595
+ * `false`, not "everything". */
596
+ function scopeToKeys(userIds: string[] | undefined) {
597
+ if (userIds === undefined) return sql.empty();
598
+ if (userIds.length === 0) return sql` AND false`;
599
+ return sql` AND t.user_id IN (${sql.join(
600
+ userIds.map((value) => sql`${value}`),
601
+ sql`, `,
602
+ )})`;
603
+ }
604
+
605
+ /**
606
+ * PRD 04 T6 — the invariant probe, and the thing this PRD ships INSTEAD of a
607
+ * foreign key (D1): an FK proves the uuid exists, this proves it is the right
608
+ * uuid. It is also the only control that can catch a dual-write writing the
609
+ * WRONG contact (risk 3) — the backfill cannot, because it only fills NULLs and
610
+ * will happily leave a wrong non-NULL value in place. **The backfill is not a
611
+ * repair tool**; if `mismatched > 0` the fix is a targeted corrective job.
612
+ *
613
+ * Read-only. Five tables × two set-based statements, run SEQUENTIALLY rather
614
+ * than fanned out: these are full-table probes by nature, and five concurrent
615
+ * scans of the two largest tables in the system is not what an operator wants
616
+ * from an on-demand admin endpoint. Nothing here is on a hot path.
617
+ *
618
+ * `userIds` restricts every count to rows carrying one of those exact `user_id`
619
+ * values. **The gate is the UNSCOPED call** — that is what the admin route runs
620
+ * and what `flipReady` means. The scope exists so a caller can ask the question
621
+ * about a known set of keys; in practice that caller is the test suite, which
622
+ * shares one Postgres with concurrently-running files and would otherwise be
623
+ * reduced to asserting deltas on counts other writers move underneath it.
624
+ */
625
+ export async function verifyContactIdBackfill(opts: {
626
+ db: Database;
627
+ userIds?: string[];
628
+ }): Promise<ContactIdVerifyResult> {
629
+ const tables = {} as Record<ContactIdBackfillTable, ContactIdVerifyCounts>;
630
+ const totals: ContactIdVerifyCounts = {
631
+ missing: 0,
632
+ mismatched: 0,
633
+ orphaned: 0,
634
+ };
635
+
636
+ for (const table of TABLES) {
637
+ const counts = await verifyTable(opts.db, table, opts.userIds);
638
+ tables[table] = counts;
639
+ totals.missing += counts.missing;
640
+ totals.mismatched += counts.mismatched;
641
+ totals.orphaned += counts.orphaned;
642
+ }
643
+
644
+ return {
645
+ tables,
646
+ totals,
647
+ flipReady: totals.missing === 0 && totals.mismatched === 0,
648
+ };
649
+ }
650
+
651
+ export const contactIdBackfillTask = hatchet.task({
652
+ name: "identity-contact-id-backfill",
653
+ retries: 0,
654
+ // Longer than the alias backfill's 600s: this one walks every live contact
655
+ // against the two largest tables in the system. A run that outlives the
656
+ // timeout is not lost — the next re-sweep resumes it for free.
657
+ executionTimeout: "3600s",
658
+ fn: async (input: ContactIdBackfillInput) => {
659
+ const { db } = createDatabase({ url: process.env.DATABASE_URL ?? "" });
660
+ const logger = createLogger(process.env.LOG_LEVEL ?? "info");
661
+ return runContactIdBackfill({
662
+ db,
663
+ logger,
664
+ jobId: input.jobId,
665
+ contactsPerChunk: input.contactsPerChunk,
666
+ rowsPerStatement: input.rowsPerStatement,
667
+ pauseMs: input.pauseMs,
668
+ });
669
+ },
670
+ });
671
+
672
+ /**
673
+ * D6 — how stale the newest COMPLETED sweep may be before boot runs another.
674
+ * Read raw off `process.env` (the `OUTBOUND_WEBHOOK_*` tunable stance) and
675
+ * validated here: a missing, non-numeric or non-positive value falls back to 24
676
+ * hours rather than disabling the sweep or hammering it.
677
+ */
678
+ export function contactIdResweepIntervalMs(logger?: Logger): number {
679
+ const raw = process.env.CONTACT_ID_BACKFILL_RESWEEP_HOURS;
680
+ if (raw === undefined || raw.trim() === "") {
681
+ return DEFAULT_RESWEEP_HOURS * 60 * 60 * 1000;
682
+ }
683
+ const hours = Number(raw);
684
+ if (!Number.isFinite(hours) || hours <= 0) {
685
+ logger?.warn("Ignoring invalid CONTACT_ID_BACKFILL_RESWEEP_HOURS", {
686
+ value: raw,
687
+ fallbackHours: DEFAULT_RESWEEP_HOURS,
688
+ });
689
+ return DEFAULT_RESWEEP_HOURS * 60 * 60 * 1000;
690
+ }
691
+ return hours * 60 * 60 * 1000;
692
+ }
693
+
694
+ /**
695
+ * Worker-boot enqueue. Unlike the alias backfill's once-per-deployment gate,
696
+ * this is a **periodic reconcile sweep** (D6, revised): the dual-write is
697
+ * best-effort, so "T5 will fill it later" only holds while a later T5 exists.
698
+ * A run therefore starts when EITHER
699
+ * - no completed job of this format exists yet, OR
700
+ * - the newest completed one is older than `CONTACT_ID_BACKFILL_RESWEEP_HOURS`
701
+ * (default 24h).
702
+ * A `pending`/`processing` row always wins — two sweeps must never stack. (A
703
+ * run killed hard enough to never mark its row terminal therefore parks the
704
+ * boot sweep until an operator marks it failed; `POST
705
+ * /v1/admin/maintenance/backfill-contact-id` bypasses this guard entirely and
706
+ * is the escape hatch.)
707
+ *
708
+ * Re-running is cheap by construction: every UPDATE is guarded by
709
+ * `contact_id IS NULL`, so a steady-state sweep is five zero-row probes per
710
+ * contact and writes nothing. Boot-triggered rather than operator-triggered
711
+ * deliberately — the engine is a published dependency, and a manual step every
712
+ * consumer must remember is how deployments strand at PRD 05. Best-effort: a
713
+ * failure to enqueue must never crash worker boot.
714
+ */
715
+ export async function enqueueContactIdBackfill(opts: {
716
+ db: Database;
717
+ logger: Logger;
718
+ }): Promise<void> {
719
+ const { db, logger } = opts;
720
+ try {
721
+ const inflight = await db
722
+ .select({ id: importJobs.id })
723
+ .from(importJobs)
724
+ .where(
725
+ and(
726
+ eq(importJobs.format, CONTACT_ID_BACKFILL_FORMAT),
727
+ inArray(importJobs.status, ["pending", "processing"]),
728
+ ),
729
+ )
730
+ .limit(1);
731
+ if (inflight[0]) return;
732
+
733
+ const [latest] = await db
734
+ .select({ id: importJobs.id, updatedAt: importJobs.updatedAt })
735
+ .from(importJobs)
736
+ .where(
737
+ and(
738
+ eq(importJobs.format, CONTACT_ID_BACKFILL_FORMAT),
739
+ eq(importJobs.status, "completed"),
740
+ ),
741
+ )
742
+ .orderBy(desc(importJobs.updatedAt))
743
+ .limit(1);
744
+
745
+ const intervalMs = contactIdResweepIntervalMs(logger);
746
+ if (latest && Date.now() - latest.updatedAt.getTime() < intervalMs) return;
747
+
748
+ const [job] = await db
749
+ .insert(importJobs)
750
+ .values({
751
+ fileName: CONTACT_ID_BACKFILL_FORMAT,
752
+ format: CONTACT_ID_BACKFILL_FORMAT,
753
+ status: "pending",
754
+ })
755
+ .returning({ id: importJobs.id });
756
+ if (!job) return;
757
+
758
+ // runNoWait: called from worker boot BEFORE the listener starts — awaiting
759
+ // the run would deadlock (the `enqueueBucketBackfills` reasoning).
760
+ await contactIdBackfillTask.runNoWait({ jobId: job.id });
761
+ logger.info("Contact-id backfill enqueued", {
762
+ jobId: job.id,
763
+ previousSweepAt: latest?.updatedAt.toISOString() ?? null,
764
+ resweepIntervalMs: intervalMs,
765
+ });
766
+ } catch (err) {
767
+ logger.warn("Contact-id backfill enqueue failed", {
768
+ error: err instanceof Error ? err.message : String(err),
769
+ });
770
+ }
771
+ }