@hogsend/engine 0.58.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -11
- package/src/buckets/check-membership.ts +6 -0
- package/src/index.ts +16 -0
- package/src/journeys/execute-journey-run.ts +83 -1
- package/src/lib/contacts.ts +114 -0
- package/src/lib/ingestion.ts +9 -0
- package/src/lib/preferences.ts +39 -2
- package/src/lib/tracked.ts +31 -0
- package/src/routes/admin/bulk.ts +4 -0
- package/src/routes/admin/index.ts +2 -0
- package/src/routes/admin/maintenance.ts +210 -0
- package/src/routes/admin/preferences.ts +9 -1
- package/src/routes/lists/index.ts +5 -0
- package/src/worker.ts +20 -0
- package/src/workflows/backfill-contact-id.ts +771 -0
- package/src/workflows/bucket-backfill.ts +9 -1
- package/src/workflows/bucket-reconcile.ts +4 -0
|
@@ -0,0 +1,771 @@
|
|
|
1
|
+
import type { JsonObject } from "@hatchet-dev/typescript-sdk/v1/types.js";
|
|
2
|
+
import {
|
|
3
|
+
contacts,
|
|
4
|
+
createDatabase,
|
|
5
|
+
type Database,
|
|
6
|
+
importJobs,
|
|
7
|
+
} from "@hogsend/db";
|
|
8
|
+
import { and, desc, eq, gt, inArray, isNull, sql } from "drizzle-orm";
|
|
9
|
+
import { contactKeySql } from "../lib/contacts.js";
|
|
10
|
+
import { hatchet } from "../lib/hatchet.js";
|
|
11
|
+
import type { Logger } from "../lib/logger.js";
|
|
12
|
+
import { createLogger } from "../lib/logger.js";
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* PRD 04 T5 — fill the five history tables' `contact_id` bookkeeping column for
|
|
16
|
+
* rows that predate the dual-write (and for rows a best-effort dual-write
|
|
17
|
+
* missed, D6).
|
|
18
|
+
*
|
|
19
|
+
* NEVER a migration (`docs/UPGRADING.md:82-87`): this is a chunked, paced,
|
|
20
|
+
* resumable Hatchet task, modelled on `identity-alias-backfill.ts` — an
|
|
21
|
+
* `import_jobs` progress row, boot-enqueued, forceable from admin.
|
|
22
|
+
*
|
|
23
|
+
* Shape, per D3/D4:
|
|
24
|
+
* - PASS 1 iterates **contacts**, not events. Per live contact it derives the
|
|
25
|
+
* canonical key (`contactKeySql()` = `external_id ?? anonymous_id ?? id`) and
|
|
26
|
+
* issues one bounded `UPDATE … WHERE id IN (SELECT id … WHERE user_id = $key
|
|
27
|
+
* AND contact_id IS NULL LIMIT $n)` per table, looped to zero. Every table
|
|
28
|
+
* has a leading `user_id` index, so no statement seq-scans `user_events`.
|
|
29
|
+
* - PASS 2 fills rows keyed on a STALE key that only `contact_aliases` knows,
|
|
30
|
+
* restricted to kinds `external`/`anonymous` (the only kinds a canonical key
|
|
31
|
+
* can ever be). A value owned by more than one live contact across those two
|
|
32
|
+
* kinds is SKIPPED and logged — never guessed (D4).
|
|
33
|
+
*
|
|
34
|
+
* Invariants, each pinned by a test:
|
|
35
|
+
* - **Only `contact_id` is ever written.** No inserts, no deletes, no other
|
|
36
|
+
* column touched (not even `updated_at` — the statements set one column).
|
|
37
|
+
* - **`contact_id IS NULL` guards every UPDATE**, so a re-run, a resume, or the
|
|
38
|
+
* periodic re-sweep (D6) updates zero rows and costs only the probes.
|
|
39
|
+
* - **A row whose key owns NO live contact stays NULL. Forever** (D5). The
|
|
40
|
+
* completion criterion is not "zero NULLs"; T6's probe is what judges it.
|
|
41
|
+
* - **Not a repair tool.** A row already carrying a WRONG non-NULL value is
|
|
42
|
+
* left alone by design (risk 3) — the fix for that is a targeted corrective
|
|
43
|
+
* job, not this sweep.
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
/** `import_jobs.format` discriminator for the reused status record. */
|
|
47
|
+
export const CONTACT_ID_BACKFILL_FORMAT = "identity-contact-id-backfill";
|
|
48
|
+
|
|
49
|
+
/** The five string-keyed history tables carrying the column (D1). */
|
|
50
|
+
const TABLES = [
|
|
51
|
+
"user_events",
|
|
52
|
+
"journey_states",
|
|
53
|
+
"bucket_memberships",
|
|
54
|
+
"email_sends",
|
|
55
|
+
"email_preferences",
|
|
56
|
+
] as const;
|
|
57
|
+
|
|
58
|
+
export type ContactIdBackfillTable = (typeof TABLES)[number];
|
|
59
|
+
|
|
60
|
+
/** Per-table row counts. A mapped type (not an interface) so it carries the
|
|
61
|
+
* implicit index signature the Hatchet `JsonObject` return contract needs. */
|
|
62
|
+
export type ContactIdBackfillCounts = Record<ContactIdBackfillTable, number>;
|
|
63
|
+
|
|
64
|
+
/** Live contacts per keyset chunk — the `bucket-backfill.ts` precedent. */
|
|
65
|
+
const DEFAULT_CONTACTS_PER_CHUNK = 500;
|
|
66
|
+
/** Rows per UPDATE statement. The sizing RULE (D3) is what matters: keep any
|
|
67
|
+
* single statement under ~1s and ~10,000 row locks. This default is a guess
|
|
68
|
+
* about someone else's data; measure and override per deployment. */
|
|
69
|
+
const DEFAULT_ROWS_PER_STATEMENT = 5_000;
|
|
70
|
+
/** Pause after a statement that actually wrote, so autovacuum keeps up with the
|
|
71
|
+
* dead tuples a non-HOT (indexed column) update leaves behind (D3). */
|
|
72
|
+
const DEFAULT_PAUSE_MS = 25;
|
|
73
|
+
/** Runaway guard: a per-(contact, table) loop that never drains is a bug, not a
|
|
74
|
+
* big table. 5,000 iterations × the default cap is 25M rows for ONE key. */
|
|
75
|
+
const MAX_STATEMENTS_PER_LOOP = 5_000;
|
|
76
|
+
/** D6 — how stale the newest COMPLETED sweep may be before boot re-enqueues. */
|
|
77
|
+
const DEFAULT_RESWEEP_HOURS = 24;
|
|
78
|
+
/** Ambiguous alias values logged per run (the rest are counted only). */
|
|
79
|
+
const AMBIGUOUS_SAMPLE = 20;
|
|
80
|
+
|
|
81
|
+
export interface ContactIdBackfillInput extends JsonObject {
|
|
82
|
+
/** `import_jobs` row to record progress on. Optional: the task body runs
|
|
83
|
+
* standalone (tests, operator scripts) without one. */
|
|
84
|
+
jobId?: string;
|
|
85
|
+
/** Live contacts read per keyset chunk. Default 500. */
|
|
86
|
+
contactsPerChunk?: number;
|
|
87
|
+
/** Rows per bounded UPDATE. Default 5,000. */
|
|
88
|
+
rowsPerStatement?: number;
|
|
89
|
+
/** Milliseconds to pause after a statement that wrote. Default 25. */
|
|
90
|
+
pauseMs?: number;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Extends JsonObject: this is a Hatchet task return value and must be
|
|
94
|
+
// JSON-serializable (the constraint every workflow output carries).
|
|
95
|
+
export interface ContactIdBackfillResult extends JsonObject {
|
|
96
|
+
status: "completed" | "failed";
|
|
97
|
+
/** Live contacts visited by pass 1. */
|
|
98
|
+
contactsScanned: number;
|
|
99
|
+
/** Rows stamped by pass 1, per table (the canonical-key population). */
|
|
100
|
+
canonical: ContactIdBackfillCounts;
|
|
101
|
+
/** Rows stamped by pass 2, per table (the stale-alias population). */
|
|
102
|
+
alias: ContactIdBackfillCounts;
|
|
103
|
+
/** `canonical + alias` across all five tables — the number a re-run must
|
|
104
|
+
* report as 0 (the idempotence assertion). */
|
|
105
|
+
updated: number;
|
|
106
|
+
/** Alias VALUES owned by more than one live contact across the two permitted
|
|
107
|
+
* kinds. Skipped, never guessed (D4); a non-zero count wants a human. */
|
|
108
|
+
ambiguousAliases: number;
|
|
109
|
+
/** UPDATE statements issued. Observability for the pacing rule. */
|
|
110
|
+
statements: number;
|
|
111
|
+
reason?: string;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const zeroCounts = (): ContactIdBackfillCounts => ({
|
|
115
|
+
user_events: 0,
|
|
116
|
+
journey_states: 0,
|
|
117
|
+
bucket_memberships: 0,
|
|
118
|
+
email_sends: 0,
|
|
119
|
+
email_preferences: 0,
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
const sleep = (ms: number) =>
|
|
123
|
+
new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
124
|
+
|
|
125
|
+
/** Positive-integer option with a documented default; a nonsense value falls
|
|
126
|
+
* back rather than producing an unbounded or zero-progress statement. */
|
|
127
|
+
function positiveInt(value: number | undefined, fallback: number): number {
|
|
128
|
+
if (value === undefined) return fallback;
|
|
129
|
+
return Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* PASS 1, one table, one contact. The PRD's verbatim statement (D3), looped
|
|
134
|
+
* until it affects zero rows. Returns rows stamped and statements issued.
|
|
135
|
+
*
|
|
136
|
+
* The `IN (SELECT … LIMIT n)` shape (rather than a bare `WHERE user_id = …`) is
|
|
137
|
+
* what bounds the row-lock count: the fat tail of this system is a bot anon id
|
|
138
|
+
* with tens of thousands of events under ONE key, and an unbounded statement
|
|
139
|
+
* there is the lock/WAL spike D3 exists to avoid.
|
|
140
|
+
*/
|
|
141
|
+
async function fillCanonicalKey(opts: {
|
|
142
|
+
db: Database;
|
|
143
|
+
table: ContactIdBackfillTable;
|
|
144
|
+
key: string;
|
|
145
|
+
contactId: string;
|
|
146
|
+
rowsPerStatement: number;
|
|
147
|
+
pauseMs: number;
|
|
148
|
+
}): Promise<{ updated: number; statements: number }> {
|
|
149
|
+
const { db, table, key, contactId, rowsPerStatement, pauseMs } = opts;
|
|
150
|
+
const name = sql.identifier(table);
|
|
151
|
+
let updated = 0;
|
|
152
|
+
let statements = 0;
|
|
153
|
+
|
|
154
|
+
for (;;) {
|
|
155
|
+
const rows = await db.execute<{ id: string }>(sql`
|
|
156
|
+
UPDATE ${name} SET contact_id = ${contactId}::uuid
|
|
157
|
+
WHERE id IN (
|
|
158
|
+
SELECT id FROM ${name}
|
|
159
|
+
WHERE user_id = ${key} AND contact_id IS NULL
|
|
160
|
+
LIMIT ${rowsPerStatement}
|
|
161
|
+
)
|
|
162
|
+
RETURNING id
|
|
163
|
+
`);
|
|
164
|
+
statements += 1;
|
|
165
|
+
if (rows.length === 0) return { updated, statements };
|
|
166
|
+
updated += rows.length;
|
|
167
|
+
|
|
168
|
+
if (statements >= MAX_STATEMENTS_PER_LOOP) {
|
|
169
|
+
throw new Error(
|
|
170
|
+
`contact-id backfill: ${table} did not drain for key after ` +
|
|
171
|
+
`${statements} statements (${updated} rows) — refusing to loop`,
|
|
172
|
+
);
|
|
173
|
+
}
|
|
174
|
+
// Only a statement that WROTE created dead tuples, so only that one earns
|
|
175
|
+
// the pause. Without this the steady-state re-sweep (D6) would idle for
|
|
176
|
+
// 5 × pauseMs per contact while updating nothing.
|
|
177
|
+
if (pauseMs > 0) await sleep(pauseMs);
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* The set of alias values that resolve UNAMBIGUOUSLY to one live contact, under
|
|
183
|
+
* the two kinds a canonical key can be. `uniqueIndex(alias_kind, alias_value)`
|
|
184
|
+
* is per-KIND, so one string can legally sit under both `external` and
|
|
185
|
+
* `anonymous`; when those disagree on the contact this CTE drops the value (the
|
|
186
|
+
* `HAVING count(DISTINCT …) = 1` clause) and `countAmbiguousAliases` reports it.
|
|
187
|
+
*
|
|
188
|
+
* `array_agg(…)[1]` rather than `min()`: `min(uuid)` only exists from PG 14, and
|
|
189
|
+
* the aggregate choice is irrelevant when the HAVING proves there is exactly one
|
|
190
|
+
* distinct value.
|
|
191
|
+
*/
|
|
192
|
+
const resolvedAliasesCte = sql`
|
|
193
|
+
resolved AS (
|
|
194
|
+
SELECT a.alias_value AS alias_key,
|
|
195
|
+
(array_agg(a.contact_id ORDER BY a.contact_id))[1] AS contact_id
|
|
196
|
+
FROM contact_aliases a
|
|
197
|
+
JOIN contacts c ON c.id = a.contact_id AND c.deleted_at IS NULL
|
|
198
|
+
WHERE a.alias_kind IN ('external', 'anonymous')
|
|
199
|
+
GROUP BY a.alias_value
|
|
200
|
+
HAVING count(DISTINCT a.contact_id) = 1
|
|
201
|
+
)`;
|
|
202
|
+
|
|
203
|
+
/** D4's skip rule, measured: alias values whose two permitted kinds point at
|
|
204
|
+
* DIFFERENT live contacts. Never resolved; reported so a human can. */
|
|
205
|
+
async function countAmbiguousAliases(
|
|
206
|
+
db: Database,
|
|
207
|
+
): Promise<{ total: number; sample: string[] }> {
|
|
208
|
+
const rows = await db.execute<{ alias_value: string }>(sql`
|
|
209
|
+
SELECT a.alias_value
|
|
210
|
+
FROM contact_aliases a
|
|
211
|
+
JOIN contacts c ON c.id = a.contact_id AND c.deleted_at IS NULL
|
|
212
|
+
WHERE a.alias_kind IN ('external', 'anonymous')
|
|
213
|
+
GROUP BY a.alias_value
|
|
214
|
+
HAVING count(DISTINCT a.contact_id) > 1
|
|
215
|
+
`);
|
|
216
|
+
return {
|
|
217
|
+
total: rows.length,
|
|
218
|
+
sample: rows.slice(0, AMBIGUOUS_SAMPLE).map((r) => r.alias_value),
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* PASS 2's work list: alias values that resolve to exactly one live contact AND
|
|
224
|
+
* are NOT already the canonical key of some live contact — i.e. precisely the
|
|
225
|
+
* stale/second-device population pass 1 cannot reach.
|
|
226
|
+
*
|
|
227
|
+
* The `NOT EXISTS` exclusion is not just an optimisation: an alias value that IS
|
|
228
|
+
* a live contact's canonical key was already fully drained by pass 1 (every row
|
|
229
|
+
* under that key is now non-NULL), so re-visiting it can only issue zero-row
|
|
230
|
+
* probes. Excluding it keeps pass 2 proportional to the merge/promote
|
|
231
|
+
* population rather than to the whole alias table (which, post-PRD-02, holds
|
|
232
|
+
* roughly one row per identity column per live contact — C2).
|
|
233
|
+
*
|
|
234
|
+
* Keyset-paginated on `alias_key` so the work list itself is bounded too.
|
|
235
|
+
*/
|
|
236
|
+
async function selectStaleAliasKeys(opts: {
|
|
237
|
+
db: Database;
|
|
238
|
+
afterKey: string | null;
|
|
239
|
+
limit: number;
|
|
240
|
+
}): Promise<Array<{ key: string; contactId: string }>> {
|
|
241
|
+
const { db, afterKey, limit } = opts;
|
|
242
|
+
const rows = await db.execute<{ alias_key: string; contact_id: string }>(sql`
|
|
243
|
+
WITH ${resolvedAliasesCte}
|
|
244
|
+
SELECT r.alias_key, r.contact_id::text AS contact_id
|
|
245
|
+
FROM resolved r
|
|
246
|
+
WHERE (${afterKey}::text IS NULL OR r.alias_key > ${afterKey}::text)
|
|
247
|
+
AND NOT EXISTS (
|
|
248
|
+
SELECT 1 FROM contacts
|
|
249
|
+
WHERE contacts.deleted_at IS NULL
|
|
250
|
+
AND ${contactKeySql()} = r.alias_key
|
|
251
|
+
)
|
|
252
|
+
ORDER BY r.alias_key
|
|
253
|
+
LIMIT ${limit}
|
|
254
|
+
`);
|
|
255
|
+
return rows.map((r) => ({ key: r.alias_key, contactId: r.contact_id }));
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/**
|
|
259
|
+
* The task body, exported directly so tests and operator scripts can run it
|
|
260
|
+
* without a Hatchet engine. `jobId` is optional; when present, progress lands on
|
|
261
|
+
* the `import_jobs` row (`totalRows` = live contacts, `processedRows` = contacts
|
|
262
|
+
* done, `failedRows` = ambiguous alias values skipped — a divergence metric, NOT
|
|
263
|
+
* an error count).
|
|
264
|
+
*
|
|
265
|
+
* RESUME is free rather than bookkept: an interrupted run is re-driven from the
|
|
266
|
+
* top, re-reads live contacts ordered by `id`, and every UPDATE's
|
|
267
|
+
* `contact_id IS NULL` guard makes an already-done contact a single zero-row
|
|
268
|
+
* probe per table. No cursor is persisted because none is needed.
|
|
269
|
+
*/
|
|
270
|
+
export async function runContactIdBackfill(opts: {
|
|
271
|
+
db: Database;
|
|
272
|
+
logger: Logger;
|
|
273
|
+
jobId?: string;
|
|
274
|
+
contactsPerChunk?: number;
|
|
275
|
+
rowsPerStatement?: number;
|
|
276
|
+
pauseMs?: number;
|
|
277
|
+
}): Promise<ContactIdBackfillResult> {
|
|
278
|
+
const { db, logger, jobId } = opts;
|
|
279
|
+
const contactsPerChunk = positiveInt(
|
|
280
|
+
opts.contactsPerChunk,
|
|
281
|
+
DEFAULT_CONTACTS_PER_CHUNK,
|
|
282
|
+
);
|
|
283
|
+
const rowsPerStatement = positiveInt(
|
|
284
|
+
opts.rowsPerStatement,
|
|
285
|
+
DEFAULT_ROWS_PER_STATEMENT,
|
|
286
|
+
);
|
|
287
|
+
const pauseMs =
|
|
288
|
+
opts.pauseMs === undefined
|
|
289
|
+
? DEFAULT_PAUSE_MS
|
|
290
|
+
: Number.isFinite(opts.pauseMs) && opts.pauseMs >= 0
|
|
291
|
+
? Math.floor(opts.pauseMs)
|
|
292
|
+
: DEFAULT_PAUSE_MS;
|
|
293
|
+
|
|
294
|
+
const canonical = zeroCounts();
|
|
295
|
+
const alias = zeroCounts();
|
|
296
|
+
let contactsScanned = 0;
|
|
297
|
+
let statements = 0;
|
|
298
|
+
let ambiguousAliases = 0;
|
|
299
|
+
|
|
300
|
+
const markJob = async (
|
|
301
|
+
patch: Partial<typeof importJobs.$inferInsert>,
|
|
302
|
+
): Promise<void> => {
|
|
303
|
+
if (!jobId) return;
|
|
304
|
+
await db
|
|
305
|
+
.update(importJobs)
|
|
306
|
+
.set({ ...patch, updatedAt: new Date() })
|
|
307
|
+
.where(eq(importJobs.id, jobId));
|
|
308
|
+
};
|
|
309
|
+
|
|
310
|
+
try {
|
|
311
|
+
const [totals] = await db.execute<{ live: number }>(sql`
|
|
312
|
+
SELECT count(*)::int AS live FROM contacts WHERE deleted_at IS NULL
|
|
313
|
+
`);
|
|
314
|
+
await markJob({
|
|
315
|
+
status: "processing",
|
|
316
|
+
totalRows: Number(totals?.live ?? 0),
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
// ---- PASS 1: the canonical key of every live contact (D3) --------------
|
|
320
|
+
const resolvedKey = contactKeySql();
|
|
321
|
+
let cursor: string | null = null;
|
|
322
|
+
for (;;) {
|
|
323
|
+
const chunk = await db
|
|
324
|
+
.select({ id: contacts.id, key: resolvedKey })
|
|
325
|
+
.from(contacts)
|
|
326
|
+
.where(
|
|
327
|
+
and(
|
|
328
|
+
isNull(contacts.deletedAt),
|
|
329
|
+
cursor ? gt(contacts.id, cursor) : undefined,
|
|
330
|
+
),
|
|
331
|
+
)
|
|
332
|
+
.orderBy(contacts.id)
|
|
333
|
+
.limit(contactsPerChunk);
|
|
334
|
+
if (chunk.length === 0) break;
|
|
335
|
+
|
|
336
|
+
for (const row of chunk) {
|
|
337
|
+
for (const table of TABLES) {
|
|
338
|
+
const filled = await fillCanonicalKey({
|
|
339
|
+
db,
|
|
340
|
+
table,
|
|
341
|
+
key: row.key,
|
|
342
|
+
contactId: row.id,
|
|
343
|
+
rowsPerStatement,
|
|
344
|
+
pauseMs,
|
|
345
|
+
});
|
|
346
|
+
canonical[table] += filled.updated;
|
|
347
|
+
statements += filled.statements;
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
contactsScanned += chunk.length;
|
|
352
|
+
cursor = chunk[chunk.length - 1]?.id ?? null;
|
|
353
|
+
await markJob({ processedRows: contactsScanned });
|
|
354
|
+
if (chunk.length < contactsPerChunk) break;
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
// ---- PASS 2: stale keys only `contact_aliases` knows (D4) --------------
|
|
358
|
+
const ambiguous = await countAmbiguousAliases(db);
|
|
359
|
+
ambiguousAliases = ambiguous.total;
|
|
360
|
+
if (ambiguousAliases > 0) {
|
|
361
|
+
logger.warn("Contact-id backfill skipped ambiguous alias values", {
|
|
362
|
+
count: ambiguousAliases,
|
|
363
|
+
sample: ambiguous.sample,
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
let aliasCursor: string | null = null;
|
|
368
|
+
for (;;) {
|
|
369
|
+
const staleKeys = await selectStaleAliasKeys({
|
|
370
|
+
db,
|
|
371
|
+
afterKey: aliasCursor,
|
|
372
|
+
limit: contactsPerChunk,
|
|
373
|
+
});
|
|
374
|
+
if (staleKeys.length === 0) break;
|
|
375
|
+
|
|
376
|
+
for (const entry of staleKeys) {
|
|
377
|
+
for (const table of TABLES) {
|
|
378
|
+
const filled = await fillCanonicalKey({
|
|
379
|
+
db,
|
|
380
|
+
table,
|
|
381
|
+
key: entry.key,
|
|
382
|
+
contactId: entry.contactId,
|
|
383
|
+
rowsPerStatement,
|
|
384
|
+
pauseMs,
|
|
385
|
+
});
|
|
386
|
+
alias[table] += filled.updated;
|
|
387
|
+
statements += filled.statements;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
aliasCursor = staleKeys[staleKeys.length - 1]?.key ?? null;
|
|
392
|
+
if (staleKeys.length < contactsPerChunk) break;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
const updated =
|
|
396
|
+
TABLES.reduce((sum, t) => sum + canonical[t], 0) +
|
|
397
|
+
TABLES.reduce((sum, t) => sum + alias[t], 0);
|
|
398
|
+
|
|
399
|
+
await markJob({
|
|
400
|
+
status: "completed",
|
|
401
|
+
processedRows: contactsScanned,
|
|
402
|
+
failedRows: ambiguousAliases,
|
|
403
|
+
});
|
|
404
|
+
logger.info("Contact-id backfill complete", {
|
|
405
|
+
contactsScanned,
|
|
406
|
+
canonical,
|
|
407
|
+
alias,
|
|
408
|
+
updated,
|
|
409
|
+
ambiguousAliases,
|
|
410
|
+
statements,
|
|
411
|
+
});
|
|
412
|
+
return {
|
|
413
|
+
status: "completed",
|
|
414
|
+
contactsScanned,
|
|
415
|
+
canonical,
|
|
416
|
+
alias,
|
|
417
|
+
updated,
|
|
418
|
+
ambiguousAliases,
|
|
419
|
+
statements,
|
|
420
|
+
};
|
|
421
|
+
} catch (err) {
|
|
422
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
423
|
+
await markJob({
|
|
424
|
+
status: "failed",
|
|
425
|
+
errors: [{ row: 0, error: message }],
|
|
426
|
+
}).catch(() => undefined);
|
|
427
|
+
logger.error("Contact-id backfill failed", { message });
|
|
428
|
+
return {
|
|
429
|
+
status: "failed",
|
|
430
|
+
contactsScanned,
|
|
431
|
+
canonical,
|
|
432
|
+
alias,
|
|
433
|
+
updated:
|
|
434
|
+
TABLES.reduce((sum, t) => sum + canonical[t], 0) +
|
|
435
|
+
TABLES.reduce((sum, t) => sum + alias[t], 0),
|
|
436
|
+
ambiguousAliases,
|
|
437
|
+
statements,
|
|
438
|
+
reason: message,
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
// ---------------------------------------------------------------------------
|
|
444
|
+
// T6 — the invariant probe
|
|
445
|
+
// ---------------------------------------------------------------------------
|
|
446
|
+
|
|
447
|
+
/** One table's verdict. `missing` and `mismatched` are failures; `orphaned` is
|
|
448
|
+
* information (D5) and is expected to be non-zero forever. */
|
|
449
|
+
export interface ContactIdVerifyCounts {
|
|
450
|
+
/** A live contact owns the row's `user_id` — canonically OR by alias — but
|
|
451
|
+
* `contact_id` is NULL. After a completed sweep this is a HOLE: either the
|
|
452
|
+
* backfill missed the row or a dual-write site dropped it (D6). MUST be 0. */
|
|
453
|
+
missing: number;
|
|
454
|
+
/** `contact_id` points at a contact that owns the row's `user_id` NEITHER
|
|
455
|
+
* canonically NOR by alias (including a pointer at no contact at all). The
|
|
456
|
+
* corruption detector, and the one count an FK could never give: an FK proves
|
|
457
|
+
* the uuid EXISTS, this proves it is the RIGHT uuid. MUST be 0. */
|
|
458
|
+
mismatched: number;
|
|
459
|
+
/** `contact_id` is NULL and NO live contact owns the key either way — a
|
|
460
|
+
* refused anonymous ingest, a keyless raw send. Expected, permitted, reported.
|
|
461
|
+
* NEVER a failure (D5). */
|
|
462
|
+
orphaned: number;
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
/** The probe's verdict for the whole database (or, when scoped, for the keys it
|
|
466
|
+
* was pointed at). */
|
|
467
|
+
export interface ContactIdVerifyResult {
|
|
468
|
+
tables: Record<ContactIdBackfillTable, ContactIdVerifyCounts>;
|
|
469
|
+
/** The five tables summed — what the alert line and the gate read. */
|
|
470
|
+
totals: ContactIdVerifyCounts;
|
|
471
|
+
/**
|
|
472
|
+
* **THE PRD 05 ENTRY GATE.** True iff EVERY table reports `missing === 0` AND
|
|
473
|
+
* `mismatched === 0` — i.e. every row a live contact owns is stamped, and
|
|
474
|
+
* every stamp points at a contact that really owns that row's key. `orphaned`
|
|
475
|
+
* is deliberately NOT part of the gate (D5 makes it legitimately non-zero
|
|
476
|
+
* forever). PRD 05 flips reads from `user_id` onto `contact_id`; flipping
|
|
477
|
+
* while this is false means silently losing history (`missing`) or attributing
|
|
478
|
+
* it to the wrong person (`mismatched`). Judge it AFTER a completed sweep — a
|
|
479
|
+
* false reading with no sweep on record is just a pending backfill.
|
|
480
|
+
*/
|
|
481
|
+
flipReady: boolean;
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
/**
|
|
485
|
+
* "Contact `c` owns key `k`" — the **ALIAS-AWARE** ownership definition
|
|
486
|
+
* (T6's "T4 review correction", LOCKED):
|
|
487
|
+
*
|
|
488
|
+
* `coalesce(c.external_id, c.anonymous_id, c.id::text) = k` (canonical)
|
|
489
|
+
* **OR** a live alias row `(contact_id = c.id, alias_value = k,
|
|
490
|
+
* alias_kind IN ('external','anonymous'))` (alias)
|
|
491
|
+
*
|
|
492
|
+
* The bare canonical coalesce is WRONG post-PRD-03 and is not a nicety: a
|
|
493
|
+
* second-device anonymous id lives ONLY in `contact_aliases`, the dual-write
|
|
494
|
+
* deliberately resolves it (C1), and a coalesce-only `mismatched` probe would
|
|
495
|
+
* flag every such CORRECTLY-stamped row as corruption — a false alarm that would
|
|
496
|
+
* block the gate forever on any deployment where anyone has ever used two
|
|
497
|
+
* devices. Kinds are restricted to `external`/`anonymous` for the same reason
|
|
498
|
+
* pass 2 restricts them (D4): those are the only kinds a canonical key can ever
|
|
499
|
+
* be, so an `email`/`discord` alias is NOT ownership.
|
|
500
|
+
*
|
|
501
|
+
* Three properties worth naming:
|
|
502
|
+
*
|
|
503
|
+
* - `missing` and `orphaned` PARTITION the NULL-stamped rows: every row with
|
|
504
|
+
* `contact_id IS NULL` is one or the other, never both, never neither. So
|
|
505
|
+
* `missing + orphaned = count(*) WHERE contact_id IS NULL`, which is what
|
|
506
|
+
* makes "zero NULLs" the wrong completion criterion and this the right one.
|
|
507
|
+
* - `mismatched` does NOT require the pointed-at contact to be LIVE. A
|
|
508
|
+
* soft-deleted contact whose key still matches is consistent data (someone
|
|
509
|
+
* was deleted), not corruption; demanding liveness would fail the gate on
|
|
510
|
+
* every GDPR delete. The merge-regression case risk 5 worries about is still
|
|
511
|
+
* caught, because a merge REWRITES the row's `user_id` to the survivor's key
|
|
512
|
+
* — so a row stranded on a soft-deleted loser mismatches on the KEY. A
|
|
513
|
+
* pointer at a contact row that does not exist at all is counted (that is the
|
|
514
|
+
* FK's job, and this probe is meant to be strictly stronger than an FK).
|
|
515
|
+
* - **`user_id IS NULL` rows are NOT counted as mismatched, and the reasoning
|
|
516
|
+
* is load-bearing.** D7 makes a raw/keyless send stamp `contact_id = NULL`
|
|
517
|
+
* (`lib/tracked.ts` resolves `sendContactId` only `if (options.userId)`), so
|
|
518
|
+
* the instinct — "a keyless row carrying a `contact_id` must be a new writer
|
|
519
|
+
* violating D7" — looks safe. It is FALSE: the admin resend path
|
|
520
|
+
* (`routes/admin/bulk.ts`, D8 row 11) inserts its new `email_sends` row with
|
|
521
|
+
* `contactId: email.contactId` copied off the source row and does NOT copy
|
|
522
|
+
* `userId` (the pre-existing gap T4d files as do-not-fix). So
|
|
523
|
+
* `user_id IS NULL AND contact_id IS NOT NULL` is reachable TODAY, on a
|
|
524
|
+
* committed path, carrying a CORRECT contact_id. Counting it would pin
|
|
525
|
+
* `mismatched > 0` on any deployment that has ever resent a bounced email —
|
|
526
|
+
* the same permanent false alarm the alias correction exists to prevent. The
|
|
527
|
+
* cost, stated plainly: a future writer resolving a keyless send by recipient
|
|
528
|
+
* ADDRESS would be indistinguishable from a resend, so this probe cannot
|
|
529
|
+
* catch that particular D7 violation. `email_sends` is the only one of the
|
|
530
|
+
* five tables where the case exists at all — the other four are
|
|
531
|
+
* `user_id NOT NULL`.
|
|
532
|
+
*
|
|
533
|
+
* Two set-based statements per table, no per-row work: one over the NULL-stamped
|
|
534
|
+
* rows (which `missing`/`orphaned` partition) and one over the stamped rows
|
|
535
|
+
* (whose `contact_id IS NOT NULL` predicate is exactly D2's partial index). The
|
|
536
|
+
* canonical leg is the same `contactKeySql()` coalesce the backfill keys on; the
|
|
537
|
+
* alias leg probes `contact_aliases_kind_value_idx` (an `IN` on the leading
|
|
538
|
+
* `alias_kind` column keeps the index usable).
|
|
539
|
+
*/
|
|
540
|
+
async function verifyTable(
|
|
541
|
+
db: Database,
|
|
542
|
+
table: ContactIdBackfillTable,
|
|
543
|
+
userIds: string[] | undefined,
|
|
544
|
+
): Promise<ContactIdVerifyCounts> {
|
|
545
|
+
const name = sql.identifier(table);
|
|
546
|
+
const scope = scopeToKeys(userIds);
|
|
547
|
+
|
|
548
|
+
const [nulls] = await db.execute<{ missing: number; orphaned: number }>(sql`
|
|
549
|
+
SELECT count(*) FILTER (WHERE s.owned)::int AS missing,
|
|
550
|
+
count(*) FILTER (WHERE NOT s.owned)::int AS orphaned
|
|
551
|
+
FROM (
|
|
552
|
+
SELECT (
|
|
553
|
+
EXISTS (
|
|
554
|
+
SELECT 1 FROM contacts
|
|
555
|
+
WHERE contacts.deleted_at IS NULL
|
|
556
|
+
AND ${contactKeySql()} = t.user_id
|
|
557
|
+
)
|
|
558
|
+
OR EXISTS (
|
|
559
|
+
SELECT 1 FROM contact_aliases a
|
|
560
|
+
JOIN contacts ON contacts.id = a.contact_id
|
|
561
|
+
AND contacts.deleted_at IS NULL
|
|
562
|
+
WHERE a.alias_value = t.user_id
|
|
563
|
+
AND a.alias_kind IN ('external', 'anonymous')
|
|
564
|
+
)
|
|
565
|
+
) AS owned
|
|
566
|
+
FROM ${name} t
|
|
567
|
+
WHERE t.contact_id IS NULL${scope}
|
|
568
|
+
) s
|
|
569
|
+
`);
|
|
570
|
+
|
|
571
|
+
const [stamped] = await db.execute<{ mismatched: number }>(sql`
|
|
572
|
+
SELECT count(*)::int AS mismatched
|
|
573
|
+
FROM ${name} t
|
|
574
|
+
LEFT JOIN contacts ON contacts.id = t.contact_id
|
|
575
|
+
WHERE t.contact_id IS NOT NULL
|
|
576
|
+
AND t.user_id IS NOT NULL${scope}
|
|
577
|
+
AND ${contactKeySql()} IS DISTINCT FROM t.user_id
|
|
578
|
+
AND NOT EXISTS (
|
|
579
|
+
SELECT 1 FROM contact_aliases a
|
|
580
|
+
WHERE a.contact_id = contacts.id
|
|
581
|
+
AND a.alias_value = t.user_id
|
|
582
|
+
AND a.alias_kind IN ('external', 'anonymous')
|
|
583
|
+
)
|
|
584
|
+
`);
|
|
585
|
+
|
|
586
|
+
return {
|
|
587
|
+
missing: Number(nulls?.missing ?? 0),
|
|
588
|
+
mismatched: Number(stamped?.mismatched ?? 0),
|
|
589
|
+
orphaned: Number(nulls?.orphaned ?? 0),
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
/** The optional key restriction, appended to both statements' WHERE. `undefined`
|
|
594
|
+
* = the whole table (the gate); an EMPTY list means "these zero keys", which is
|
|
595
|
+
* `false`, not "everything". */
|
|
596
|
+
function scopeToKeys(userIds: string[] | undefined) {
|
|
597
|
+
if (userIds === undefined) return sql.empty();
|
|
598
|
+
if (userIds.length === 0) return sql` AND false`;
|
|
599
|
+
return sql` AND t.user_id IN (${sql.join(
|
|
600
|
+
userIds.map((value) => sql`${value}`),
|
|
601
|
+
sql`, `,
|
|
602
|
+
)})`;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
/**
|
|
606
|
+
* PRD 04 T6 — the invariant probe, and the thing this PRD ships INSTEAD of a
|
|
607
|
+
* foreign key (D1): an FK proves the uuid exists, this proves it is the right
|
|
608
|
+
* uuid. It is also the only control that can catch a dual-write writing the
|
|
609
|
+
* WRONG contact (risk 3) — the backfill cannot, because it only fills NULLs and
|
|
610
|
+
* will happily leave a wrong non-NULL value in place. **The backfill is not a
|
|
611
|
+
* repair tool**; if `mismatched > 0` the fix is a targeted corrective job.
|
|
612
|
+
*
|
|
613
|
+
* Read-only. Five tables × two set-based statements, run SEQUENTIALLY rather
|
|
614
|
+
* than fanned out: these are full-table probes by nature, and five concurrent
|
|
615
|
+
* scans of the two largest tables in the system is not what an operator wants
|
|
616
|
+
* from an on-demand admin endpoint. Nothing here is on a hot path.
|
|
617
|
+
*
|
|
618
|
+
* `userIds` restricts every count to rows carrying one of those exact `user_id`
|
|
619
|
+
* values. **The gate is the UNSCOPED call** — that is what the admin route runs
|
|
620
|
+
* and what `flipReady` means. The scope exists so a caller can ask the question
|
|
621
|
+
* about a known set of keys; in practice that caller is the test suite, which
|
|
622
|
+
* shares one Postgres with concurrently-running files and would otherwise be
|
|
623
|
+
* reduced to asserting deltas on counts other writers move underneath it.
|
|
624
|
+
*/
|
|
625
|
+
export async function verifyContactIdBackfill(opts: {
|
|
626
|
+
db: Database;
|
|
627
|
+
userIds?: string[];
|
|
628
|
+
}): Promise<ContactIdVerifyResult> {
|
|
629
|
+
const tables = {} as Record<ContactIdBackfillTable, ContactIdVerifyCounts>;
|
|
630
|
+
const totals: ContactIdVerifyCounts = {
|
|
631
|
+
missing: 0,
|
|
632
|
+
mismatched: 0,
|
|
633
|
+
orphaned: 0,
|
|
634
|
+
};
|
|
635
|
+
|
|
636
|
+
for (const table of TABLES) {
|
|
637
|
+
const counts = await verifyTable(opts.db, table, opts.userIds);
|
|
638
|
+
tables[table] = counts;
|
|
639
|
+
totals.missing += counts.missing;
|
|
640
|
+
totals.mismatched += counts.mismatched;
|
|
641
|
+
totals.orphaned += counts.orphaned;
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
return {
|
|
645
|
+
tables,
|
|
646
|
+
totals,
|
|
647
|
+
flipReady: totals.missing === 0 && totals.mismatched === 0,
|
|
648
|
+
};
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
export const contactIdBackfillTask = hatchet.task({
|
|
652
|
+
name: "identity-contact-id-backfill",
|
|
653
|
+
retries: 0,
|
|
654
|
+
// Longer than the alias backfill's 600s: this one walks every live contact
|
|
655
|
+
// against the two largest tables in the system. A run that outlives the
|
|
656
|
+
// timeout is not lost — the next re-sweep resumes it for free.
|
|
657
|
+
executionTimeout: "3600s",
|
|
658
|
+
fn: async (input: ContactIdBackfillInput) => {
|
|
659
|
+
const { db } = createDatabase({ url: process.env.DATABASE_URL ?? "" });
|
|
660
|
+
const logger = createLogger(process.env.LOG_LEVEL ?? "info");
|
|
661
|
+
return runContactIdBackfill({
|
|
662
|
+
db,
|
|
663
|
+
logger,
|
|
664
|
+
jobId: input.jobId,
|
|
665
|
+
contactsPerChunk: input.contactsPerChunk,
|
|
666
|
+
rowsPerStatement: input.rowsPerStatement,
|
|
667
|
+
pauseMs: input.pauseMs,
|
|
668
|
+
});
|
|
669
|
+
},
|
|
670
|
+
});
|
|
671
|
+
|
|
672
|
+
/**
|
|
673
|
+
* D6 — how stale the newest COMPLETED sweep may be before boot runs another.
|
|
674
|
+
* Read raw off `process.env` (the `OUTBOUND_WEBHOOK_*` tunable stance) and
|
|
675
|
+
* validated here: a missing, non-numeric or non-positive value falls back to 24
|
|
676
|
+
* hours rather than disabling the sweep or hammering it.
|
|
677
|
+
*/
|
|
678
|
+
export function contactIdResweepIntervalMs(logger?: Logger): number {
|
|
679
|
+
const raw = process.env.CONTACT_ID_BACKFILL_RESWEEP_HOURS;
|
|
680
|
+
if (raw === undefined || raw.trim() === "") {
|
|
681
|
+
return DEFAULT_RESWEEP_HOURS * 60 * 60 * 1000;
|
|
682
|
+
}
|
|
683
|
+
const hours = Number(raw);
|
|
684
|
+
if (!Number.isFinite(hours) || hours <= 0) {
|
|
685
|
+
logger?.warn("Ignoring invalid CONTACT_ID_BACKFILL_RESWEEP_HOURS", {
|
|
686
|
+
value: raw,
|
|
687
|
+
fallbackHours: DEFAULT_RESWEEP_HOURS,
|
|
688
|
+
});
|
|
689
|
+
return DEFAULT_RESWEEP_HOURS * 60 * 60 * 1000;
|
|
690
|
+
}
|
|
691
|
+
return hours * 60 * 60 * 1000;
|
|
692
|
+
}
|
|
693
|
+
|
|
694
|
+
/**
|
|
695
|
+
* Worker-boot enqueue. Unlike the alias backfill's once-per-deployment gate,
|
|
696
|
+
* this is a **periodic reconcile sweep** (D6, revised): the dual-write is
|
|
697
|
+
* best-effort, so "T5 will fill it later" only holds while a later T5 exists.
|
|
698
|
+
* A run therefore starts when EITHER
|
|
699
|
+
* - no completed job of this format exists yet, OR
|
|
700
|
+
* - the newest completed one is older than `CONTACT_ID_BACKFILL_RESWEEP_HOURS`
|
|
701
|
+
* (default 24h).
|
|
702
|
+
* A `pending`/`processing` row always wins — two sweeps must never stack. (A
|
|
703
|
+
* run killed hard enough to never mark its row terminal therefore parks the
|
|
704
|
+
* boot sweep until an operator marks it failed; `POST
|
|
705
|
+
* /v1/admin/maintenance/backfill-contact-id` bypasses this guard entirely and
|
|
706
|
+
* is the escape hatch.)
|
|
707
|
+
*
|
|
708
|
+
* Re-running is cheap by construction: every UPDATE is guarded by
|
|
709
|
+
* `contact_id IS NULL`, so a steady-state sweep is five zero-row probes per
|
|
710
|
+
* contact and writes nothing. Boot-triggered rather than operator-triggered
|
|
711
|
+
* deliberately — the engine is a published dependency, and a manual step every
|
|
712
|
+
* consumer must remember is how deployments strand at PRD 05. Best-effort: a
|
|
713
|
+
* failure to enqueue must never crash worker boot.
|
|
714
|
+
*/
|
|
715
|
+
export async function enqueueContactIdBackfill(opts: {
|
|
716
|
+
db: Database;
|
|
717
|
+
logger: Logger;
|
|
718
|
+
}): Promise<void> {
|
|
719
|
+
const { db, logger } = opts;
|
|
720
|
+
try {
|
|
721
|
+
const inflight = await db
|
|
722
|
+
.select({ id: importJobs.id })
|
|
723
|
+
.from(importJobs)
|
|
724
|
+
.where(
|
|
725
|
+
and(
|
|
726
|
+
eq(importJobs.format, CONTACT_ID_BACKFILL_FORMAT),
|
|
727
|
+
inArray(importJobs.status, ["pending", "processing"]),
|
|
728
|
+
),
|
|
729
|
+
)
|
|
730
|
+
.limit(1);
|
|
731
|
+
if (inflight[0]) return;
|
|
732
|
+
|
|
733
|
+
const [latest] = await db
|
|
734
|
+
.select({ id: importJobs.id, updatedAt: importJobs.updatedAt })
|
|
735
|
+
.from(importJobs)
|
|
736
|
+
.where(
|
|
737
|
+
and(
|
|
738
|
+
eq(importJobs.format, CONTACT_ID_BACKFILL_FORMAT),
|
|
739
|
+
eq(importJobs.status, "completed"),
|
|
740
|
+
),
|
|
741
|
+
)
|
|
742
|
+
.orderBy(desc(importJobs.updatedAt))
|
|
743
|
+
.limit(1);
|
|
744
|
+
|
|
745
|
+
const intervalMs = contactIdResweepIntervalMs(logger);
|
|
746
|
+
if (latest && Date.now() - latest.updatedAt.getTime() < intervalMs) return;
|
|
747
|
+
|
|
748
|
+
const [job] = await db
|
|
749
|
+
.insert(importJobs)
|
|
750
|
+
.values({
|
|
751
|
+
fileName: CONTACT_ID_BACKFILL_FORMAT,
|
|
752
|
+
format: CONTACT_ID_BACKFILL_FORMAT,
|
|
753
|
+
status: "pending",
|
|
754
|
+
})
|
|
755
|
+
.returning({ id: importJobs.id });
|
|
756
|
+
if (!job) return;
|
|
757
|
+
|
|
758
|
+
// runNoWait: called from worker boot BEFORE the listener starts — awaiting
|
|
759
|
+
// the run would deadlock (the `enqueueBucketBackfills` reasoning).
|
|
760
|
+
await contactIdBackfillTask.runNoWait({ jobId: job.id });
|
|
761
|
+
logger.info("Contact-id backfill enqueued", {
|
|
762
|
+
jobId: job.id,
|
|
763
|
+
previousSweepAt: latest?.updatedAt.toISOString() ?? null,
|
|
764
|
+
resweepIntervalMs: intervalMs,
|
|
765
|
+
});
|
|
766
|
+
} catch (err) {
|
|
767
|
+
logger.warn("Contact-id backfill enqueue failed", {
|
|
768
|
+
error: err instanceof Error ? err.message : String(err),
|
|
769
|
+
});
|
|
770
|
+
}
|
|
771
|
+
}
|