@hogsend/engine 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -11
- package/src/buckets/bucket-access.ts +19 -10
- package/src/buckets/check-membership.ts +61 -8
- package/src/buckets/membership-epoch.ts +11 -1
- package/src/campaigns/cohort-sql.ts +3 -0
- package/src/index.ts +26 -6
- package/src/journeys/execute-journey-run.ts +180 -29
- package/src/journeys/journey-context.ts +43 -7
- package/src/lib/agent/tools.ts +29 -4
- package/src/lib/attribution-backfill.ts +27 -4
- package/src/lib/attribution.ts +22 -3
- package/src/lib/connector-actions.ts +4 -1
- package/src/lib/contacts.ts +518 -148
- package/src/lib/conversion-dispatch.ts +16 -2
- package/src/lib/enrollment-guards.ts +19 -12
- package/src/lib/feed.ts +5 -5
- package/src/lib/flags.ts +26 -9
- package/src/lib/ingestion.ts +23 -2
- package/src/lib/journey-lift.ts +17 -5
- package/src/lib/preferences.ts +116 -30
- package/src/lib/recipient-preferences.ts +16 -2
- package/src/lib/revenue.ts +20 -5
- package/src/lib/sms-tracked.ts +18 -1
- package/src/lib/tracked.ts +50 -8
- package/src/lib/tracking-events.ts +28 -0
- package/src/lib/unique-violation.ts +44 -0
- package/src/routes/admin/buckets.ts +20 -2
- package/src/routes/admin/bulk.ts +18 -2
- package/src/routes/admin/contacts.ts +4 -3
- package/src/routes/admin/conversions.ts +20 -4
- package/src/routes/admin/emails.ts +17 -2
- package/src/routes/admin/events.ts +32 -41
- package/src/routes/admin/funnels.ts +6 -1
- package/src/routes/admin/groups.ts +4 -0
- package/src/routes/admin/impact.ts +30 -10
- package/src/routes/admin/index.ts +2 -0
- package/src/routes/admin/journey-impact.ts +18 -9
- package/src/routes/admin/journeys.ts +8 -1
- package/src/routes/admin/maintenance.ts +216 -0
- package/src/routes/admin/preferences.ts +85 -38
- package/src/routes/admin/reporting.ts +25 -5
- package/src/routes/admin/targeting.ts +1 -0
- package/src/routes/admin/timeline.ts +16 -10
- package/src/routes/email/preferences.ts +18 -1
- package/src/routes/lists/index.ts +11 -6
- package/src/worker.ts +20 -0
- package/src/workflows/backfill-contact-id.ts +951 -0
- package/src/workflows/bucket-backfill.ts +145 -59
- package/src/workflows/bucket-reconcile.ts +154 -50
- package/src/workflows/check-alerts.ts +4 -0
- package/src/workflows/impact-digest.ts +10 -3
- package/src/workflows/journey-blueprint-interpreter.ts +8 -1
- package/src/workflows/send-campaign.ts +18 -11
|
@@ -0,0 +1,951 @@
|
|
|
1
|
+
import type { JsonObject } from "@hatchet-dev/typescript-sdk/v1/types.js";
|
|
2
|
+
import {
|
|
3
|
+
contacts,
|
|
4
|
+
createDatabase,
|
|
5
|
+
type Database,
|
|
6
|
+
importJobs,
|
|
7
|
+
} from "@hogsend/db";
|
|
8
|
+
import { and, desc, eq, gt, inArray, isNull, type SQL, sql } from "drizzle-orm";
|
|
9
|
+
import { contactKeySql } from "../lib/contacts.js";
|
|
10
|
+
import { hatchet } from "../lib/hatchet.js";
|
|
11
|
+
import type { Logger } from "../lib/logger.js";
|
|
12
|
+
import { createLogger } from "../lib/logger.js";
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* PRD 04 T5 — fill the five history tables' `contact_id` bookkeeping column for
|
|
16
|
+
* rows that predate the dual-write (and for rows a best-effort dual-write
|
|
17
|
+
* missed, D6).
|
|
18
|
+
*
|
|
19
|
+
* NEVER a migration (`docs/UPGRADING.md:82-87`): this is a chunked, paced,
|
|
20
|
+
* resumable Hatchet task, modelled on `identity-alias-backfill.ts` — an
|
|
21
|
+
* `import_jobs` progress row, boot-enqueued, forceable from admin.
|
|
22
|
+
*
|
|
23
|
+
* Shape, per D3/D4:
|
|
24
|
+
* - PASS 1 iterates **contacts**, not events. Per live contact it derives the
|
|
25
|
+
* canonical key (`contactKeySql()` = `external_id ?? anonymous_id ?? id`) and
|
|
26
|
+
* issues one bounded `UPDATE … WHERE id IN (SELECT id … WHERE user_id = $key
|
|
27
|
+
* AND contact_id IS NULL LIMIT $n)` per table, looped to zero. Every table
|
|
28
|
+
* has a leading `user_id` index, so no statement seq-scans `user_events`.
|
|
29
|
+
* - PASS 2 fills rows keyed on a STALE key that only `contact_aliases` knows,
|
|
30
|
+
* restricted to kinds `external`/`anonymous` (the only kinds a canonical key
|
|
31
|
+
* can ever be). A value owned by more than one live contact across those two
|
|
32
|
+
* kinds is SKIPPED and logged — never guessed (D4).
|
|
33
|
+
*
|
|
34
|
+
* Invariants, each pinned by a test:
|
|
35
|
+
* - **Only `contact_id` is ever written.** No inserts, no deletes, no other
|
|
36
|
+
* column touched (not even `updated_at` — the statements set one column).
|
|
37
|
+
* - **`contact_id IS NULL` guards every UPDATE**, so a re-run, a resume, or the
|
|
38
|
+
* periodic re-sweep (D6) updates zero rows and costs only the probes.
|
|
39
|
+
* - **A row whose key owns NO live contact stays NULL. Forever** (D5). The
|
|
40
|
+
* completion criterion is not "zero NULLs"; T6's probe is what judges it.
|
|
41
|
+
* - **Not a repair tool.** A row already carrying a WRONG non-NULL value is
|
|
42
|
+
* left alone by design (risk 3) — the fix for that is a targeted corrective
|
|
43
|
+
* job, not this sweep.
|
|
44
|
+
*/
|
|
45
|
+
|
|
46
|
+
/** `import_jobs.format` discriminator for the reused status record. */
|
|
47
|
+
export const CONTACT_ID_BACKFILL_FORMAT = "identity-contact-id-backfill";
|
|
48
|
+
|
|
49
|
+
/** The five string-keyed history tables carrying the column (D1). */
|
|
50
|
+
const TABLES = [
|
|
51
|
+
"user_events",
|
|
52
|
+
"journey_states",
|
|
53
|
+
"bucket_memberships",
|
|
54
|
+
"email_sends",
|
|
55
|
+
"email_preferences",
|
|
56
|
+
] as const;
|
|
57
|
+
|
|
58
|
+
export type ContactIdBackfillTable = (typeof TABLES)[number];
|
|
59
|
+
|
|
60
|
+
/** Per-table row counts. A mapped type (not an interface) so it carries the
|
|
61
|
+
* implicit index signature the Hatchet `JsonObject` return contract needs. */
|
|
62
|
+
export type ContactIdBackfillCounts = Record<ContactIdBackfillTable, number>;
|
|
63
|
+
|
|
64
|
+
/** Live contacts per keyset chunk — the `bucket-backfill.ts` precedent. */
|
|
65
|
+
const DEFAULT_CONTACTS_PER_CHUNK = 500;
|
|
66
|
+
/** Rows per UPDATE statement. The sizing RULE (D3) is what matters: keep any
|
|
67
|
+
* single statement under ~1s and ~10,000 row locks. This default is a guess
|
|
68
|
+
* about someone else's data; measure and override per deployment. */
|
|
69
|
+
const DEFAULT_ROWS_PER_STATEMENT = 5_000;
|
|
70
|
+
/** Pause after a statement that actually wrote, so autovacuum keeps up with the
|
|
71
|
+
* dead tuples a non-HOT (indexed column) update leaves behind (D3). */
|
|
72
|
+
const DEFAULT_PAUSE_MS = 25;
|
|
73
|
+
/** Runaway guard: a per-(contact, table) loop that never drains is a bug, not a
|
|
74
|
+
* big table. 5,000 iterations × the default cap is 25M rows for ONE key. */
|
|
75
|
+
const MAX_STATEMENTS_PER_LOOP = 5_000;
|
|
76
|
+
/** D6 — how stale the newest COMPLETED sweep may be before boot re-enqueues. */
|
|
77
|
+
const DEFAULT_RESWEEP_HOURS = 24;
|
|
78
|
+
/** Ambiguous alias values logged per run (the rest are counted only). */
|
|
79
|
+
const AMBIGUOUS_SAMPLE = 20;
|
|
80
|
+
|
|
81
|
+
export interface ContactIdBackfillInput extends JsonObject {
|
|
82
|
+
/** `import_jobs` row to record progress on. Optional: the task body runs
|
|
83
|
+
* standalone (tests, operator scripts) without one. */
|
|
84
|
+
jobId?: string;
|
|
85
|
+
/** Live contacts read per keyset chunk. Default 500. */
|
|
86
|
+
contactsPerChunk?: number;
|
|
87
|
+
/** Rows per bounded UPDATE. Default 5,000. */
|
|
88
|
+
rowsPerStatement?: number;
|
|
89
|
+
/** Milliseconds to pause after a statement that wrote. Default 25. */
|
|
90
|
+
pauseMs?: number;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// Extends JsonObject: this is a Hatchet task return value and must be
|
|
94
|
+
// JSON-serializable (the constraint every workflow output carries).
|
|
95
|
+
export interface ContactIdBackfillResult extends JsonObject {
|
|
96
|
+
status: "completed" | "failed";
|
|
97
|
+
/** Live contacts visited by pass 1. */
|
|
98
|
+
contactsScanned: number;
|
|
99
|
+
/** Rows stamped by pass 1, per table (the canonical-key population). */
|
|
100
|
+
canonical: ContactIdBackfillCounts;
|
|
101
|
+
/** Rows stamped by pass 2, per table (the stale-alias population). */
|
|
102
|
+
alias: ContactIdBackfillCounts;
|
|
103
|
+
/** `canonical + alias` across all five tables — the number a re-run must
|
|
104
|
+
* report as 0 (the idempotence assertion). */
|
|
105
|
+
updated: number;
|
|
106
|
+
/** Alias VALUES owned by more than one live contact across the two permitted
|
|
107
|
+
* kinds. Skipped, never guessed (D4); a non-zero count wants a human. */
|
|
108
|
+
ambiguousAliases: number;
|
|
109
|
+
/** PRD 05 T3 — stale `email_preferences` rows whose opt-out state was folded
|
|
110
|
+
* into the contact's already-stamped row for the same address, and which were
|
|
111
|
+
* then deleted. The only rows this job ever removes. */
|
|
112
|
+
preferencesFolded: number;
|
|
113
|
+
/** UPDATE statements issued. Observability for the pacing rule. */
|
|
114
|
+
statements: number;
|
|
115
|
+
reason?: string;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const zeroCounts = (): ContactIdBackfillCounts => ({
|
|
119
|
+
user_events: 0,
|
|
120
|
+
journey_states: 0,
|
|
121
|
+
bucket_memberships: 0,
|
|
122
|
+
email_sends: 0,
|
|
123
|
+
email_preferences: 0,
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
const sleep = (ms: number) =>
|
|
127
|
+
new Promise<void>((resolve) => setTimeout(resolve, ms));
|
|
128
|
+
|
|
129
|
+
/** Positive-integer option with a documented default; a nonsense value falls
|
|
130
|
+
* back rather than producing an unbounded or zero-progress statement. */
|
|
131
|
+
function positiveInt(value: number | undefined, fallback: number): number {
|
|
132
|
+
if (value === undefined) return fallback;
|
|
133
|
+
return Number.isFinite(value) && value >= 1 ? Math.floor(value) : fallback;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** The three tables migration 0071 gives a CONTACT-scoped partial unique index
|
|
137
|
+
* (PRD 05 T3). Everything below that special-cases a table keys off this set. */
|
|
138
|
+
const CONTACT_SCOPED_TABLES = new Set<ContactIdBackfillTable>([
|
|
139
|
+
"journey_states",
|
|
140
|
+
"bucket_memberships",
|
|
141
|
+
"email_preferences",
|
|
142
|
+
]);
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* PRD 05 T3 — "the owning contact ALREADY holds a row that this row's stamp
|
|
146
|
+
* would duplicate", correlated to the row `t` and to an owner-id expression.
|
|
147
|
+
*
|
|
148
|
+
* The sweep is a FOURTH writer to the three contact-scoped unique indexes, and
|
|
149
|
+
* the only one whose rows start OUTSIDE them: `contact_id IS NULL` is outside
|
|
150
|
+
* every one of the three predicates, so it is the STAMP that moves a row in.
|
|
151
|
+
* When the contact already owns a stamped twin (same live journey, same live
|
|
152
|
+
* bucket, same address — exactly the population adoption creates by stamping
|
|
153
|
+
* `contact_id` without rewriting `user_id`) the stamp raises 23505, and this
|
|
154
|
+
* job's blanket catch would turn ONE such row into a failed job that abandons
|
|
155
|
+
* the whole sweep — which the boot / 24h re-enqueue then re-hits forever.
|
|
156
|
+
*
|
|
157
|
+
* Each arm reproduces its index's WHERE clause EXACTLY. Keep in sync with:
|
|
158
|
+
* uq_contact_journey_active packages/db/src/schema/journey-states.ts
|
|
159
|
+
* uq_contact_bucket_active .../bucket-memberships.ts
|
|
160
|
+
* email_preferences_contact_email_idx .../email-preferences.ts
|
|
161
|
+
*/
|
|
162
|
+
function contactScopedTwinExists(
|
|
163
|
+
table: ContactIdBackfillTable,
|
|
164
|
+
owner: SQL,
|
|
165
|
+
): SQL {
|
|
166
|
+
switch (table) {
|
|
167
|
+
case "journey_states":
|
|
168
|
+
return sql`t.status IN ('active', 'waiting') AND EXISTS (
|
|
169
|
+
SELECT 1 FROM journey_states x
|
|
170
|
+
WHERE x.contact_id = ${owner}
|
|
171
|
+
AND x.journey_id = t.journey_id
|
|
172
|
+
AND x.status IN ('active', 'waiting'))`;
|
|
173
|
+
case "bucket_memberships":
|
|
174
|
+
return sql`t.status = 'active' AND t.deleted_at IS NULL AND EXISTS (
|
|
175
|
+
SELECT 1 FROM bucket_memberships x
|
|
176
|
+
WHERE x.contact_id = ${owner}
|
|
177
|
+
AND x.bucket_id = t.bucket_id
|
|
178
|
+
AND x.status = 'active'
|
|
179
|
+
AND x.deleted_at IS NULL)`;
|
|
180
|
+
case "email_preferences":
|
|
181
|
+
return sql`EXISTS (
|
|
182
|
+
SELECT 1 FROM email_preferences x
|
|
183
|
+
WHERE x.contact_id = ${owner} AND x.email = t.email)`;
|
|
184
|
+
default:
|
|
185
|
+
return sql`false`;
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* PRD 05 T3 — the ONE fold this sweep performs. Everywhere else it only ever
|
|
191
|
+
* fills a NULL column; `email_preferences` earns the exception because a stale
|
|
192
|
+
* row left NULL is an OPT-OUT the flipped (contact-scoped) read will never see.
|
|
193
|
+
*
|
|
194
|
+
* Per stale-keyed NULL row whose contact already holds a row for the same
|
|
195
|
+
* address: fold the stale state into the TWIN under the merge path's
|
|
196
|
+
* `foldEmailPreferences` rule — OR the opt-outs (`unsubscribed_all`,
|
|
197
|
+
* `suppressed`), union the categories with the twin winning a conflict EXCEPT
|
|
198
|
+
* that FALSE always wins (an opt-out is never lost), which also means a grant
|
|
199
|
+
* the twin never heard of (a stale `sms: true` consent) survives — then DELETE
|
|
200
|
+
* the stale row so nothing is left outside the index. Never the other
|
|
201
|
+
* direction: the twin is the row the flipped read resolves.
|
|
202
|
+
*
|
|
203
|
+
* Postgres runs every data-modifying CTE exactly once and to completion whether
|
|
204
|
+
* or not the primary query reads it, so the fold lands before the delete; the
|
|
205
|
+
* two touch disjoint rows (the twin is stamped, the stale row is not).
|
|
206
|
+
*/
|
|
207
|
+
async function foldStalePreferences(opts: {
|
|
208
|
+
db: Database;
|
|
209
|
+
key: string;
|
|
210
|
+
contactId: string;
|
|
211
|
+
}): Promise<number> {
|
|
212
|
+
const { db, key, contactId } = opts;
|
|
213
|
+
const rows = await db.execute<{ id: string }>(sql`
|
|
214
|
+
WITH stale AS (
|
|
215
|
+
SELECT p.id, p.email, p.unsubscribed_all, p.suppressed, p.suppressed_at,
|
|
216
|
+
p.bounce_count, p.last_bounce_at, p.categories
|
|
217
|
+
FROM email_preferences p
|
|
218
|
+
WHERE p.user_id = ${key}
|
|
219
|
+
AND p.contact_id IS NULL
|
|
220
|
+
AND EXISTS (
|
|
221
|
+
SELECT 1 FROM email_preferences t
|
|
222
|
+
WHERE t.contact_id = ${contactId}::uuid AND t.email = p.email
|
|
223
|
+
)
|
|
224
|
+
), folded AS (
|
|
225
|
+
UPDATE email_preferences t
|
|
226
|
+
SET unsubscribed_all = t.unsubscribed_all OR s.unsubscribed_all,
|
|
227
|
+
suppressed = t.suppressed OR s.suppressed,
|
|
228
|
+
suppressed_at = least(t.suppressed_at, s.suppressed_at),
|
|
229
|
+
bounce_count = greatest(t.bounce_count, s.bounce_count),
|
|
230
|
+
last_bounce_at = least(t.last_bounce_at, s.last_bounce_at),
|
|
231
|
+
categories = coalesce(s.categories, '{}'::jsonb)
|
|
232
|
+
|| coalesce(t.categories, '{}'::jsonb)
|
|
233
|
+
|| coalesce((
|
|
234
|
+
SELECT jsonb_object_agg(k.key, 'false'::jsonb)
|
|
235
|
+
FROM jsonb_each(coalesce(s.categories, '{}'::jsonb)) k
|
|
236
|
+
WHERE k.value = 'false'::jsonb
|
|
237
|
+
), '{}'::jsonb),
|
|
238
|
+
updated_at = now()
|
|
239
|
+
FROM stale s
|
|
240
|
+
WHERE t.contact_id = ${contactId}::uuid AND t.email = s.email
|
|
241
|
+
)
|
|
242
|
+
DELETE FROM email_preferences WHERE id IN (SELECT id FROM stale)
|
|
243
|
+
RETURNING id
|
|
244
|
+
`);
|
|
245
|
+
return rows.length;
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* PASS 1, one table, one contact. The PRD's verbatim statement (D3), looped
|
|
250
|
+
* until it affects zero rows. Returns rows stamped and statements issued.
|
|
251
|
+
*
|
|
252
|
+
* The `IN (SELECT … LIMIT n)` shape (rather than a bare `WHERE user_id = …`) is
|
|
253
|
+
* what bounds the row-lock count: the fat tail of this system is a bot anon id
|
|
254
|
+
* with tens of thousands of events under ONE key, and an unbounded statement
|
|
255
|
+
* there is the lock/WAL spike D3 exists to avoid.
|
|
256
|
+
*
|
|
257
|
+
* PRD 05 T3: on the three contact-scoped tables the inner SELECT also carries
|
|
258
|
+
* {@link contactScopedTwinExists} as a NOT guard, so a row whose stamp would
|
|
259
|
+
* collide is never selected — it is SKIPPED, not stamped and not repaired. The
|
|
260
|
+
* sweep does not get to cancel a live enrollment or membership to make room;
|
|
261
|
+
* `verifyContactIdBackfill` reports what it skipped as `duplicates` (not
|
|
262
|
+
* `missing`) so the gate can still open and the triage list stays visible.
|
|
263
|
+
* Skipping in the SELECT (rather than swallowing a 23505) also keeps the loop
|
|
264
|
+
* terminating: a skipped row simply stops being a candidate.
|
|
265
|
+
*/
|
|
266
|
+
async function fillCanonicalKey(opts: {
|
|
267
|
+
db: Database;
|
|
268
|
+
table: ContactIdBackfillTable;
|
|
269
|
+
key: string;
|
|
270
|
+
contactId: string;
|
|
271
|
+
rowsPerStatement: number;
|
|
272
|
+
pauseMs: number;
|
|
273
|
+
}): Promise<{ updated: number; statements: number; folded: number }> {
|
|
274
|
+
const { db, table, key, contactId, rowsPerStatement, pauseMs } = opts;
|
|
275
|
+
const name = sql.identifier(table);
|
|
276
|
+
let updated = 0;
|
|
277
|
+
let statements = 0;
|
|
278
|
+
let folded = 0;
|
|
279
|
+
|
|
280
|
+
// Opt-outs first: a folded row is deleted, so it never reaches the guard.
|
|
281
|
+
if (table === "email_preferences") {
|
|
282
|
+
folded = await foldStalePreferences({ db, key, contactId });
|
|
283
|
+
statements += 1;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
const guard = CONTACT_SCOPED_TABLES.has(table)
|
|
287
|
+
? sql` AND NOT (${contactScopedTwinExists(table, sql`${contactId}::uuid`)})`
|
|
288
|
+
: sql.empty();
|
|
289
|
+
|
|
290
|
+
for (;;) {
|
|
291
|
+
const rows = await db.execute<{ id: string }>(sql`
|
|
292
|
+
UPDATE ${name} SET contact_id = ${contactId}::uuid
|
|
293
|
+
WHERE id IN (
|
|
294
|
+
SELECT t.id FROM ${name} t
|
|
295
|
+
WHERE t.user_id = ${key} AND t.contact_id IS NULL${guard}
|
|
296
|
+
LIMIT ${rowsPerStatement}
|
|
297
|
+
)
|
|
298
|
+
RETURNING id
|
|
299
|
+
`);
|
|
300
|
+
statements += 1;
|
|
301
|
+
if (rows.length === 0) return { updated, statements, folded };
|
|
302
|
+
updated += rows.length;
|
|
303
|
+
|
|
304
|
+
if (statements >= MAX_STATEMENTS_PER_LOOP) {
|
|
305
|
+
throw new Error(
|
|
306
|
+
`contact-id backfill: ${table} did not drain for key after ` +
|
|
307
|
+
`${statements} statements (${updated} rows) — refusing to loop`,
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
// Only a statement that WROTE created dead tuples, so only that one earns
|
|
311
|
+
// the pause. Without this the steady-state re-sweep (D6) would idle for
|
|
312
|
+
// 5 × pauseMs per contact while updating nothing.
|
|
313
|
+
if (pauseMs > 0) await sleep(pauseMs);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
/**
|
|
318
|
+
* The set of alias values that resolve UNAMBIGUOUSLY to one live contact, under
|
|
319
|
+
* the two kinds a canonical key can be. `uniqueIndex(alias_kind, alias_value)`
|
|
320
|
+
* is per-KIND, so one string can legally sit under both `external` and
|
|
321
|
+
* `anonymous`; when those disagree on the contact this CTE drops the value (the
|
|
322
|
+
* `HAVING count(DISTINCT …) = 1` clause) and `countAmbiguousAliases` reports it.
|
|
323
|
+
*
|
|
324
|
+
* `array_agg(…)[1]` rather than `min()`: `min(uuid)` only exists from PG 14, and
|
|
325
|
+
* the aggregate choice is irrelevant when the HAVING proves there is exactly one
|
|
326
|
+
* distinct value.
|
|
327
|
+
*/
|
|
328
|
+
const resolvedAliasesCte = sql`
|
|
329
|
+
resolved AS (
|
|
330
|
+
SELECT a.alias_value AS alias_key,
|
|
331
|
+
(array_agg(a.contact_id ORDER BY a.contact_id))[1] AS contact_id
|
|
332
|
+
FROM contact_aliases a
|
|
333
|
+
JOIN contacts c ON c.id = a.contact_id AND c.deleted_at IS NULL
|
|
334
|
+
WHERE a.alias_kind IN ('external', 'anonymous')
|
|
335
|
+
GROUP BY a.alias_value
|
|
336
|
+
HAVING count(DISTINCT a.contact_id) = 1
|
|
337
|
+
)`;
|
|
338
|
+
|
|
339
|
+
/** D4's skip rule, measured: alias values whose two permitted kinds point at
|
|
340
|
+
* DIFFERENT live contacts. Never resolved; reported so a human can. */
|
|
341
|
+
async function countAmbiguousAliases(
|
|
342
|
+
db: Database,
|
|
343
|
+
): Promise<{ total: number; sample: string[] }> {
|
|
344
|
+
const rows = await db.execute<{ alias_value: string }>(sql`
|
|
345
|
+
SELECT a.alias_value
|
|
346
|
+
FROM contact_aliases a
|
|
347
|
+
JOIN contacts c ON c.id = a.contact_id AND c.deleted_at IS NULL
|
|
348
|
+
WHERE a.alias_kind IN ('external', 'anonymous')
|
|
349
|
+
GROUP BY a.alias_value
|
|
350
|
+
HAVING count(DISTINCT a.contact_id) > 1
|
|
351
|
+
`);
|
|
352
|
+
return {
|
|
353
|
+
total: rows.length,
|
|
354
|
+
sample: rows.slice(0, AMBIGUOUS_SAMPLE).map((r) => r.alias_value),
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* PASS 2's work list: alias values that resolve to exactly one live contact AND
|
|
360
|
+
* are NOT already the canonical key of some live contact — i.e. precisely the
|
|
361
|
+
* stale/second-device population pass 1 cannot reach.
|
|
362
|
+
*
|
|
363
|
+
* The `NOT EXISTS` exclusion is not just an optimisation: an alias value that IS
|
|
364
|
+
* a live contact's canonical key was already fully drained by pass 1 (every row
|
|
365
|
+
* under that key is now non-NULL), so re-visiting it can only issue zero-row
|
|
366
|
+
* probes. Excluding it keeps pass 2 proportional to the merge/promote
|
|
367
|
+
* population rather than to the whole alias table (which, post-PRD-02, holds
|
|
368
|
+
* roughly one row per identity column per live contact — C2).
|
|
369
|
+
*
|
|
370
|
+
* Keyset-paginated on `alias_key` so the work list itself is bounded too.
|
|
371
|
+
*/
|
|
372
|
+
async function selectStaleAliasKeys(opts: {
|
|
373
|
+
db: Database;
|
|
374
|
+
afterKey: string | null;
|
|
375
|
+
limit: number;
|
|
376
|
+
}): Promise<Array<{ key: string; contactId: string }>> {
|
|
377
|
+
const { db, afterKey, limit } = opts;
|
|
378
|
+
const rows = await db.execute<{ alias_key: string; contact_id: string }>(sql`
|
|
379
|
+
WITH ${resolvedAliasesCte}
|
|
380
|
+
SELECT r.alias_key, r.contact_id::text AS contact_id
|
|
381
|
+
FROM resolved r
|
|
382
|
+
WHERE (${afterKey}::text IS NULL OR r.alias_key > ${afterKey}::text)
|
|
383
|
+
AND NOT EXISTS (
|
|
384
|
+
SELECT 1 FROM contacts
|
|
385
|
+
WHERE contacts.deleted_at IS NULL
|
|
386
|
+
AND ${contactKeySql()} = r.alias_key
|
|
387
|
+
)
|
|
388
|
+
ORDER BY r.alias_key
|
|
389
|
+
LIMIT ${limit}
|
|
390
|
+
`);
|
|
391
|
+
return rows.map((r) => ({ key: r.alias_key, contactId: r.contact_id }));
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/**
|
|
395
|
+
* The task body, exported directly so tests and operator scripts can run it
|
|
396
|
+
* without a Hatchet engine. `jobId` is optional; when present, progress lands on
|
|
397
|
+
* the `import_jobs` row (`totalRows` = live contacts, `processedRows` = contacts
|
|
398
|
+
* done, `failedRows` = ambiguous alias values skipped — a divergence metric, NOT
|
|
399
|
+
* an error count).
|
|
400
|
+
*
|
|
401
|
+
* RESUME is free rather than bookkept: an interrupted run is re-driven from the
|
|
402
|
+
* top, re-reads live contacts ordered by `id`, and every UPDATE's
|
|
403
|
+
* `contact_id IS NULL` guard makes an already-done contact a single zero-row
|
|
404
|
+
* probe per table. No cursor is persisted because none is needed.
|
|
405
|
+
*/
|
|
406
|
+
export async function runContactIdBackfill(opts: {
|
|
407
|
+
db: Database;
|
|
408
|
+
logger: Logger;
|
|
409
|
+
jobId?: string;
|
|
410
|
+
contactsPerChunk?: number;
|
|
411
|
+
rowsPerStatement?: number;
|
|
412
|
+
pauseMs?: number;
|
|
413
|
+
}): Promise<ContactIdBackfillResult> {
|
|
414
|
+
const { db, logger, jobId } = opts;
|
|
415
|
+
const contactsPerChunk = positiveInt(
|
|
416
|
+
opts.contactsPerChunk,
|
|
417
|
+
DEFAULT_CONTACTS_PER_CHUNK,
|
|
418
|
+
);
|
|
419
|
+
const rowsPerStatement = positiveInt(
|
|
420
|
+
opts.rowsPerStatement,
|
|
421
|
+
DEFAULT_ROWS_PER_STATEMENT,
|
|
422
|
+
);
|
|
423
|
+
const pauseMs =
|
|
424
|
+
opts.pauseMs === undefined
|
|
425
|
+
? DEFAULT_PAUSE_MS
|
|
426
|
+
: Number.isFinite(opts.pauseMs) && opts.pauseMs >= 0
|
|
427
|
+
? Math.floor(opts.pauseMs)
|
|
428
|
+
: DEFAULT_PAUSE_MS;
|
|
429
|
+
|
|
430
|
+
const canonical = zeroCounts();
|
|
431
|
+
const alias = zeroCounts();
|
|
432
|
+
let contactsScanned = 0;
|
|
433
|
+
let statements = 0;
|
|
434
|
+
let ambiguousAliases = 0;
|
|
435
|
+
let preferencesFolded = 0;
|
|
436
|
+
|
|
437
|
+
const markJob = async (
|
|
438
|
+
patch: Partial<typeof importJobs.$inferInsert>,
|
|
439
|
+
): Promise<void> => {
|
|
440
|
+
if (!jobId) return;
|
|
441
|
+
await db
|
|
442
|
+
.update(importJobs)
|
|
443
|
+
.set({ ...patch, updatedAt: new Date() })
|
|
444
|
+
.where(eq(importJobs.id, jobId));
|
|
445
|
+
};
|
|
446
|
+
|
|
447
|
+
try {
|
|
448
|
+
const [totals] = await db.execute<{ live: number }>(sql`
|
|
449
|
+
SELECT count(*)::int AS live FROM contacts WHERE deleted_at IS NULL
|
|
450
|
+
`);
|
|
451
|
+
await markJob({
|
|
452
|
+
status: "processing",
|
|
453
|
+
totalRows: Number(totals?.live ?? 0),
|
|
454
|
+
});
|
|
455
|
+
|
|
456
|
+
// ---- PASS 1: the canonical key of every live contact (D3) --------------
|
|
457
|
+
const resolvedKey = contactKeySql();
|
|
458
|
+
let cursor: string | null = null;
|
|
459
|
+
for (;;) {
|
|
460
|
+
const chunk = await db
|
|
461
|
+
.select({ id: contacts.id, key: resolvedKey })
|
|
462
|
+
.from(contacts)
|
|
463
|
+
.where(
|
|
464
|
+
and(
|
|
465
|
+
isNull(contacts.deletedAt),
|
|
466
|
+
cursor ? gt(contacts.id, cursor) : undefined,
|
|
467
|
+
),
|
|
468
|
+
)
|
|
469
|
+
.orderBy(contacts.id)
|
|
470
|
+
.limit(contactsPerChunk);
|
|
471
|
+
if (chunk.length === 0) break;
|
|
472
|
+
|
|
473
|
+
for (const row of chunk) {
|
|
474
|
+
for (const table of TABLES) {
|
|
475
|
+
const filled = await fillCanonicalKey({
|
|
476
|
+
db,
|
|
477
|
+
table,
|
|
478
|
+
key: row.key,
|
|
479
|
+
contactId: row.id,
|
|
480
|
+
rowsPerStatement,
|
|
481
|
+
pauseMs,
|
|
482
|
+
});
|
|
483
|
+
canonical[table] += filled.updated;
|
|
484
|
+
statements += filled.statements;
|
|
485
|
+
preferencesFolded += filled.folded;
|
|
486
|
+
}
|
|
487
|
+
}
|
|
488
|
+
|
|
489
|
+
contactsScanned += chunk.length;
|
|
490
|
+
cursor = chunk[chunk.length - 1]?.id ?? null;
|
|
491
|
+
await markJob({ processedRows: contactsScanned });
|
|
492
|
+
if (chunk.length < contactsPerChunk) break;
|
|
493
|
+
}
|
|
494
|
+
|
|
495
|
+
// ---- PASS 2: stale keys only `contact_aliases` knows (D4) --------------
|
|
496
|
+
const ambiguous = await countAmbiguousAliases(db);
|
|
497
|
+
ambiguousAliases = ambiguous.total;
|
|
498
|
+
if (ambiguousAliases > 0) {
|
|
499
|
+
logger.warn("Contact-id backfill skipped ambiguous alias values", {
|
|
500
|
+
count: ambiguousAliases,
|
|
501
|
+
sample: ambiguous.sample,
|
|
502
|
+
});
|
|
503
|
+
}
|
|
504
|
+
|
|
505
|
+
let aliasCursor: string | null = null;
|
|
506
|
+
for (;;) {
|
|
507
|
+
const staleKeys = await selectStaleAliasKeys({
|
|
508
|
+
db,
|
|
509
|
+
afterKey: aliasCursor,
|
|
510
|
+
limit: contactsPerChunk,
|
|
511
|
+
});
|
|
512
|
+
if (staleKeys.length === 0) break;
|
|
513
|
+
|
|
514
|
+
for (const entry of staleKeys) {
|
|
515
|
+
for (const table of TABLES) {
|
|
516
|
+
const filled = await fillCanonicalKey({
|
|
517
|
+
db,
|
|
518
|
+
table,
|
|
519
|
+
key: entry.key,
|
|
520
|
+
contactId: entry.contactId,
|
|
521
|
+
rowsPerStatement,
|
|
522
|
+
pauseMs,
|
|
523
|
+
});
|
|
524
|
+
alias[table] += filled.updated;
|
|
525
|
+
statements += filled.statements;
|
|
526
|
+
preferencesFolded += filled.folded;
|
|
527
|
+
}
|
|
528
|
+
}
|
|
529
|
+
|
|
530
|
+
aliasCursor = staleKeys[staleKeys.length - 1]?.key ?? null;
|
|
531
|
+
if (staleKeys.length < contactsPerChunk) break;
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
const updated =
|
|
535
|
+
TABLES.reduce((sum, t) => sum + canonical[t], 0) +
|
|
536
|
+
TABLES.reduce((sum, t) => sum + alias[t], 0);
|
|
537
|
+
|
|
538
|
+
await markJob({
|
|
539
|
+
status: "completed",
|
|
540
|
+
processedRows: contactsScanned,
|
|
541
|
+
failedRows: ambiguousAliases,
|
|
542
|
+
});
|
|
543
|
+
logger.info("Contact-id backfill complete", {
|
|
544
|
+
contactsScanned,
|
|
545
|
+
canonical,
|
|
546
|
+
alias,
|
|
547
|
+
updated,
|
|
548
|
+
ambiguousAliases,
|
|
549
|
+
preferencesFolded,
|
|
550
|
+
statements,
|
|
551
|
+
});
|
|
552
|
+
return {
|
|
553
|
+
status: "completed",
|
|
554
|
+
contactsScanned,
|
|
555
|
+
canonical,
|
|
556
|
+
alias,
|
|
557
|
+
updated,
|
|
558
|
+
ambiguousAliases,
|
|
559
|
+
preferencesFolded,
|
|
560
|
+
statements,
|
|
561
|
+
};
|
|
562
|
+
} catch (err) {
|
|
563
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
564
|
+
await markJob({
|
|
565
|
+
status: "failed",
|
|
566
|
+
errors: [{ row: 0, error: message }],
|
|
567
|
+
}).catch(() => undefined);
|
|
568
|
+
logger.error("Contact-id backfill failed", { message });
|
|
569
|
+
return {
|
|
570
|
+
status: "failed",
|
|
571
|
+
contactsScanned,
|
|
572
|
+
canonical,
|
|
573
|
+
alias,
|
|
574
|
+
updated:
|
|
575
|
+
TABLES.reduce((sum, t) => sum + canonical[t], 0) +
|
|
576
|
+
TABLES.reduce((sum, t) => sum + alias[t], 0),
|
|
577
|
+
ambiguousAliases,
|
|
578
|
+
preferencesFolded,
|
|
579
|
+
statements,
|
|
580
|
+
reason: message,
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
|
|
585
|
+
// ---------------------------------------------------------------------------
|
|
586
|
+
// T6 — the invariant probe
|
|
587
|
+
// ---------------------------------------------------------------------------
|
|
588
|
+
|
|
589
|
+
/** One table's verdict. `missing` and `mismatched` are failures; `orphaned`
|
|
590
|
+
* (D5) and `duplicates` (PRD 05 T3) are information and may be non-zero. */
|
|
591
|
+
export interface ContactIdVerifyCounts {
|
|
592
|
+
/** A live contact owns the row's `user_id` — canonically OR by alias — but
|
|
593
|
+
* `contact_id` is NULL and the stamp WOULD land. After a completed sweep this
|
|
594
|
+
* is a HOLE: either the backfill missed the row or a dual-write site dropped
|
|
595
|
+
* it (D6). MUST be 0. */
|
|
596
|
+
missing: number;
|
|
597
|
+
/**
|
|
598
|
+
* PRD 05 T3 — a live contact owns the row's `user_id`, `contact_id` is NULL,
|
|
599
|
+
* and stamping it would violate one of the contact-scoped partial unique
|
|
600
|
+
* indexes because the contact ALREADY holds a row for that live journey / live
|
|
601
|
+
* bucket / address. The sweep skips these on purpose (cancelling someone's
|
|
602
|
+
* live enrollment is not a background job's call), so counting them as
|
|
603
|
+
* `missing` would pin the gate shut forever on a population no sweep can
|
|
604
|
+
* drain. Reported separately so operators can triage it; NOT part of the gate.
|
|
605
|
+
*/
|
|
606
|
+
duplicates: number;
|
|
607
|
+
/** `contact_id` points at a contact that owns the row's `user_id` NEITHER
|
|
608
|
+
* canonically NOR by alias (including a pointer at no contact at all). The
|
|
609
|
+
* corruption detector, and the one count an FK could never give: an FK proves
|
|
610
|
+
* the uuid EXISTS, this proves it is the RIGHT uuid. MUST be 0. */
|
|
611
|
+
mismatched: number;
|
|
612
|
+
/** `contact_id` is NULL and NO live contact owns the key either way — a
|
|
613
|
+
* refused anonymous ingest, a keyless raw send. Expected, permitted, reported.
|
|
614
|
+
* NEVER a failure (D5). */
|
|
615
|
+
orphaned: number;
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
/** The probe's verdict for the whole database (or, when scoped, for the keys it
|
|
619
|
+
* was pointed at). */
|
|
620
|
+
export interface ContactIdVerifyResult {
|
|
621
|
+
tables: Record<ContactIdBackfillTable, ContactIdVerifyCounts>;
|
|
622
|
+
/** The five tables summed — what the alert line and the gate read. */
|
|
623
|
+
totals: ContactIdVerifyCounts;
|
|
624
|
+
/**
|
|
625
|
+
* **THE PRD 05 ENTRY GATE.** True iff EVERY table reports `missing === 0` AND
|
|
626
|
+
* `mismatched === 0` — i.e. every row a live contact owns is stamped, and
|
|
627
|
+
* every stamp points at a contact that really owns that row's key. `orphaned`
|
|
628
|
+
* and `duplicates` are deliberately NOT part of the gate (D5 makes the first
|
|
629
|
+
* legitimately non-zero forever; the second is a population the sweep skips on
|
|
630
|
+
* purpose — see {@link ContactIdVerifyCounts.duplicates} — so gating on it
|
|
631
|
+
* would shut the door on a number no sweep can drain). PRD 05 flips reads from `user_id` onto `contact_id`; flipping
|
|
632
|
+
* while this is false means silently losing history (`missing`) or attributing
|
|
633
|
+
* it to the wrong person (`mismatched`). Judge it AFTER a completed sweep — a
|
|
634
|
+
* false reading with no sweep on record is just a pending backfill.
|
|
635
|
+
*/
|
|
636
|
+
flipReady: boolean;
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
/**
|
|
640
|
+
* "Contact `c` owns key `k`" — the **ALIAS-AWARE** ownership definition
|
|
641
|
+
* (T6's "T4 review correction", LOCKED):
|
|
642
|
+
*
|
|
643
|
+
* `coalesce(c.external_id, c.anonymous_id, c.id::text) = k` (canonical)
|
|
644
|
+
* **OR** a live alias row `(contact_id = c.id, alias_value = k,
|
|
645
|
+
* alias_kind IN ('external','anonymous'))` (alias)
|
|
646
|
+
*
|
|
647
|
+
* The bare canonical coalesce is WRONG post-PRD-03 and is not a nicety: a
|
|
648
|
+
* second-device anonymous id lives ONLY in `contact_aliases`, the dual-write
|
|
649
|
+
* deliberately resolves it (C1), and a coalesce-only `mismatched` probe would
|
|
650
|
+
* flag every such CORRECTLY-stamped row as corruption — a false alarm that would
|
|
651
|
+
* block the gate forever on any deployment where anyone has ever used two
|
|
652
|
+
* devices. Kinds are restricted to `external`/`anonymous` for the same reason
|
|
653
|
+
* pass 2 restricts them (D4): those are the only kinds a canonical key can ever
|
|
654
|
+
* be, so an `email`/`discord` alias is NOT ownership.
|
|
655
|
+
*
|
|
656
|
+
* Three properties worth naming:
|
|
657
|
+
*
|
|
658
|
+
* - `missing`, `duplicates` and `orphaned` PARTITION the NULL-stamped rows:
|
|
659
|
+
* every row with `contact_id IS NULL` is exactly one of the three, so
|
|
660
|
+
* `missing + duplicates + orphaned = count(*) WHERE contact_id IS NULL` —
|
|
661
|
+
* which is what makes "zero NULLs" the wrong completion criterion and this
|
|
662
|
+
* the right one. `duplicates` (PRD 05 T3) splits off the owned rows the sweep
|
|
663
|
+
* deliberately leaves NULL because stamping them would violate a
|
|
664
|
+
* contact-scoped unique index.
|
|
665
|
+
* - `mismatched` does NOT require the pointed-at contact to be LIVE. A
|
|
666
|
+
* soft-deleted contact whose key still matches is consistent data (someone
|
|
667
|
+
* was deleted), not corruption; demanding liveness would fail the gate on
|
|
668
|
+
* every GDPR delete. The merge-regression case risk 5 worries about is still
|
|
669
|
+
* caught, because a merge REWRITES the row's `user_id` to the survivor's key
|
|
670
|
+
* — so a row stranded on a soft-deleted loser mismatches on the KEY. A
|
|
671
|
+
* pointer at a contact row that does not exist at all is counted (that is the
|
|
672
|
+
* FK's job, and this probe is meant to be strictly stronger than an FK).
|
|
673
|
+
* - **`user_id IS NULL` rows are NOT counted as mismatched, and the reasoning
|
|
674
|
+
* is load-bearing.** D7 makes a raw/keyless send stamp `contact_id = NULL`
|
|
675
|
+
* (`lib/tracked.ts` resolves `sendContactId` only `if (options.userId)`), so
|
|
676
|
+
* the instinct — "a keyless row carrying a `contact_id` must be a new writer
|
|
677
|
+
* violating D7" — looks safe. It is FALSE: the admin resend path
|
|
678
|
+
* (`routes/admin/bulk.ts`, D8 row 11) inserts its new `email_sends` row with
|
|
679
|
+
* `contactId: email.contactId` copied off the source row and does NOT copy
|
|
680
|
+
* `userId` (the pre-existing gap T4d files as do-not-fix). So
|
|
681
|
+
* `user_id IS NULL AND contact_id IS NOT NULL` is reachable TODAY, on a
|
|
682
|
+
* committed path, carrying a CORRECT contact_id. Counting it would pin
|
|
683
|
+
* `mismatched > 0` on any deployment that has ever resent a bounced email —
|
|
684
|
+
* the same permanent false alarm the alias correction exists to prevent. The
|
|
685
|
+
* cost, stated plainly: a future writer resolving a keyless send by recipient
|
|
686
|
+
* ADDRESS would be indistinguishable from a resend, so this probe cannot
|
|
687
|
+
* catch that particular D7 violation. `email_sends` is the only one of the
|
|
688
|
+
* five tables where the case exists at all — the other four are
|
|
689
|
+
* `user_id NOT NULL`.
|
|
690
|
+
*
|
|
691
|
+
* Two set-based statements per table, no per-row work: one over the NULL-stamped
|
|
692
|
+
* rows (which `missing`/`orphaned` partition) and one over the stamped rows
|
|
693
|
+
* (whose `contact_id IS NOT NULL` predicate is exactly D2's partial index). The
|
|
694
|
+
* canonical leg is the same `contactKeySql()` coalesce the backfill keys on; the
|
|
695
|
+
* alias leg probes `contact_aliases_kind_value_idx` (an `IN` on the leading
|
|
696
|
+
* `alias_kind` column keeps the index usable).
|
|
697
|
+
*/
|
|
698
|
+
async function verifyTable(
|
|
699
|
+
db: Database,
|
|
700
|
+
table: ContactIdBackfillTable,
|
|
701
|
+
userIds: string[] | undefined,
|
|
702
|
+
): Promise<ContactIdVerifyCounts> {
|
|
703
|
+
const name = sql.identifier(table);
|
|
704
|
+
const scope = scopeToKeys(userIds);
|
|
705
|
+
|
|
706
|
+
// PRD 05 T3 — the owner is now RESOLVED (a LATERAL) rather than merely
|
|
707
|
+
// asserted (two EXISTS), because `duplicates` needs the owner's id to ask
|
|
708
|
+
// whether that contact already holds the row this stamp would duplicate. Same
|
|
709
|
+
// ownership definition, same two legs, same cost profile (the canonical leg is
|
|
710
|
+
// a `coalesce(...)` comparison either way, so neither shape is index-driven).
|
|
711
|
+
const collides = CONTACT_SCOPED_TABLES.has(table)
|
|
712
|
+
? contactScopedTwinExists(table, sql`owner.id`)
|
|
713
|
+
: sql`false`;
|
|
714
|
+
|
|
715
|
+
const [nulls] = await db.execute<{
|
|
716
|
+
missing: number;
|
|
717
|
+
duplicates: number;
|
|
718
|
+
orphaned: number;
|
|
719
|
+
}>(sql`
|
|
720
|
+
SELECT count(*) FILTER (WHERE s.owner IS NOT NULL AND NOT s.collides)::int
|
|
721
|
+
AS missing,
|
|
722
|
+
count(*) FILTER (WHERE s.owner IS NOT NULL AND s.collides)::int
|
|
723
|
+
AS duplicates,
|
|
724
|
+
count(*) FILTER (WHERE s.owner IS NULL)::int
|
|
725
|
+
AS orphaned
|
|
726
|
+
FROM (
|
|
727
|
+
SELECT owner.id AS owner, (${collides}) AS collides
|
|
728
|
+
FROM ${name} t
|
|
729
|
+
LEFT JOIN LATERAL (
|
|
730
|
+
SELECT contacts.id
|
|
731
|
+
FROM contacts
|
|
732
|
+
WHERE contacts.deleted_at IS NULL
|
|
733
|
+
AND (
|
|
734
|
+
${contactKeySql()} = t.user_id
|
|
735
|
+
OR EXISTS (
|
|
736
|
+
SELECT 1 FROM contact_aliases a
|
|
737
|
+
WHERE a.contact_id = contacts.id
|
|
738
|
+
AND a.alias_value = t.user_id
|
|
739
|
+
AND a.alias_kind IN ('external', 'anonymous')
|
|
740
|
+
)
|
|
741
|
+
)
|
|
742
|
+
LIMIT 1
|
|
743
|
+
) owner ON true
|
|
744
|
+
WHERE t.contact_id IS NULL${scope}
|
|
745
|
+
) s
|
|
746
|
+
`);
|
|
747
|
+
|
|
748
|
+
const [stamped] = await db.execute<{ mismatched: number }>(sql`
|
|
749
|
+
SELECT count(*)::int AS mismatched
|
|
750
|
+
FROM ${name} t
|
|
751
|
+
LEFT JOIN contacts ON contacts.id = t.contact_id
|
|
752
|
+
WHERE t.contact_id IS NOT NULL
|
|
753
|
+
AND t.user_id IS NOT NULL${scope}
|
|
754
|
+
AND ${contactKeySql()} IS DISTINCT FROM t.user_id
|
|
755
|
+
AND NOT EXISTS (
|
|
756
|
+
SELECT 1 FROM contact_aliases a
|
|
757
|
+
WHERE a.contact_id = contacts.id
|
|
758
|
+
AND a.alias_value = t.user_id
|
|
759
|
+
AND a.alias_kind IN ('external', 'anonymous')
|
|
760
|
+
)
|
|
761
|
+
`);
|
|
762
|
+
|
|
763
|
+
return {
|
|
764
|
+
missing: Number(nulls?.missing ?? 0),
|
|
765
|
+
duplicates: Number(nulls?.duplicates ?? 0),
|
|
766
|
+
mismatched: Number(stamped?.mismatched ?? 0),
|
|
767
|
+
orphaned: Number(nulls?.orphaned ?? 0),
|
|
768
|
+
};
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
/** The optional key restriction, appended to both statements' WHERE. `undefined`
|
|
772
|
+
* = the whole table (the gate); an EMPTY list means "these zero keys", which is
|
|
773
|
+
* `false`, not "everything". */
|
|
774
|
+
function scopeToKeys(userIds: string[] | undefined) {
|
|
775
|
+
if (userIds === undefined) return sql.empty();
|
|
776
|
+
if (userIds.length === 0) return sql` AND false`;
|
|
777
|
+
return sql` AND t.user_id IN (${sql.join(
|
|
778
|
+
userIds.map((value) => sql`${value}`),
|
|
779
|
+
sql`, `,
|
|
780
|
+
)})`;
|
|
781
|
+
}
|
|
782
|
+
|
|
783
|
+
/**
|
|
784
|
+
* PRD 04 T6 — the invariant probe, and the thing this PRD ships INSTEAD of a
|
|
785
|
+
* foreign key (D1): an FK proves the uuid exists, this proves it is the right
|
|
786
|
+
* uuid. It is also the only control that can catch a dual-write writing the
|
|
787
|
+
* WRONG contact (risk 3) — the backfill cannot, because it only fills NULLs and
|
|
788
|
+
* will happily leave a wrong non-NULL value in place. **The backfill is not a
|
|
789
|
+
* repair tool**; if `mismatched > 0` the fix is a targeted corrective job.
|
|
790
|
+
*
|
|
791
|
+
* Read-only. Five tables × two set-based statements, run SEQUENTIALLY rather
|
|
792
|
+
* than fanned out: these are full-table probes by nature, and five concurrent
|
|
793
|
+
* scans of the two largest tables in the system is not what an operator wants
|
|
794
|
+
* from an on-demand admin endpoint. Nothing here is on a hot path.
|
|
795
|
+
*
|
|
796
|
+
* `userIds` restricts every count to rows carrying one of those exact `user_id`
|
|
797
|
+
* values. **The gate is the UNSCOPED call** — that is what the admin route runs
|
|
798
|
+
* and what `flipReady` means. The scope exists so a caller can ask the question
|
|
799
|
+
* about a known set of keys; in practice that caller is the test suite, which
|
|
800
|
+
* shares one Postgres with concurrently-running files and would otherwise be
|
|
801
|
+
* reduced to asserting deltas on counts other writers move underneath it.
|
|
802
|
+
*/
|
|
803
|
+
export async function verifyContactIdBackfill(opts: {
|
|
804
|
+
db: Database;
|
|
805
|
+
userIds?: string[];
|
|
806
|
+
}): Promise<ContactIdVerifyResult> {
|
|
807
|
+
const tables = {} as Record<ContactIdBackfillTable, ContactIdVerifyCounts>;
|
|
808
|
+
const totals: ContactIdVerifyCounts = {
|
|
809
|
+
missing: 0,
|
|
810
|
+
duplicates: 0,
|
|
811
|
+
mismatched: 0,
|
|
812
|
+
orphaned: 0,
|
|
813
|
+
};
|
|
814
|
+
|
|
815
|
+
for (const table of TABLES) {
|
|
816
|
+
const counts = await verifyTable(opts.db, table, opts.userIds);
|
|
817
|
+
tables[table] = counts;
|
|
818
|
+
totals.missing += counts.missing;
|
|
819
|
+
totals.duplicates += counts.duplicates;
|
|
820
|
+
totals.mismatched += counts.mismatched;
|
|
821
|
+
totals.orphaned += counts.orphaned;
|
|
822
|
+
}
|
|
823
|
+
|
|
824
|
+
return {
|
|
825
|
+
tables,
|
|
826
|
+
totals,
|
|
827
|
+
flipReady: totals.missing === 0 && totals.mismatched === 0,
|
|
828
|
+
};
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
export const contactIdBackfillTask = hatchet.task({
|
|
832
|
+
name: "identity-contact-id-backfill",
|
|
833
|
+
retries: 0,
|
|
834
|
+
// Longer than the alias backfill's 600s: this one walks every live contact
|
|
835
|
+
// against the two largest tables in the system. A run that outlives the
|
|
836
|
+
// timeout is not lost — the next re-sweep resumes it for free.
|
|
837
|
+
executionTimeout: "3600s",
|
|
838
|
+
fn: async (input: ContactIdBackfillInput) => {
|
|
839
|
+
const { db } = createDatabase({ url: process.env.DATABASE_URL ?? "" });
|
|
840
|
+
const logger = createLogger(process.env.LOG_LEVEL ?? "info");
|
|
841
|
+
return runContactIdBackfill({
|
|
842
|
+
db,
|
|
843
|
+
logger,
|
|
844
|
+
jobId: input.jobId,
|
|
845
|
+
contactsPerChunk: input.contactsPerChunk,
|
|
846
|
+
rowsPerStatement: input.rowsPerStatement,
|
|
847
|
+
pauseMs: input.pauseMs,
|
|
848
|
+
});
|
|
849
|
+
},
|
|
850
|
+
});
|
|
851
|
+
|
|
852
|
+
/**
|
|
853
|
+
* D6 — how stale the newest COMPLETED sweep may be before boot runs another.
|
|
854
|
+
* Read raw off `process.env` (the `OUTBOUND_WEBHOOK_*` tunable stance) and
|
|
855
|
+
* validated here: a missing, non-numeric or non-positive value falls back to 24
|
|
856
|
+
* hours rather than disabling the sweep or hammering it.
|
|
857
|
+
*/
|
|
858
|
+
export function contactIdResweepIntervalMs(logger?: Logger): number {
|
|
859
|
+
const raw = process.env.CONTACT_ID_BACKFILL_RESWEEP_HOURS;
|
|
860
|
+
if (raw === undefined || raw.trim() === "") {
|
|
861
|
+
return DEFAULT_RESWEEP_HOURS * 60 * 60 * 1000;
|
|
862
|
+
}
|
|
863
|
+
const hours = Number(raw);
|
|
864
|
+
if (!Number.isFinite(hours) || hours <= 0) {
|
|
865
|
+
logger?.warn("Ignoring invalid CONTACT_ID_BACKFILL_RESWEEP_HOURS", {
|
|
866
|
+
value: raw,
|
|
867
|
+
fallbackHours: DEFAULT_RESWEEP_HOURS,
|
|
868
|
+
});
|
|
869
|
+
return DEFAULT_RESWEEP_HOURS * 60 * 60 * 1000;
|
|
870
|
+
}
|
|
871
|
+
return hours * 60 * 60 * 1000;
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
/**
|
|
875
|
+
* Worker-boot enqueue. Unlike the alias backfill's once-per-deployment gate,
|
|
876
|
+
* this is a **periodic reconcile sweep** (D6, revised): the dual-write is
|
|
877
|
+
* best-effort, so "T5 will fill it later" only holds while a later T5 exists.
|
|
878
|
+
* A run therefore starts when EITHER
|
|
879
|
+
* - no completed job of this format exists yet, OR
|
|
880
|
+
* - the newest completed one is older than `CONTACT_ID_BACKFILL_RESWEEP_HOURS`
|
|
881
|
+
* (default 24h).
|
|
882
|
+
* A `pending`/`processing` row always wins — two sweeps must never stack. (A
|
|
883
|
+
* run killed hard enough to never mark its row terminal therefore parks the
|
|
884
|
+
* boot sweep until an operator marks it failed; `POST
|
|
885
|
+
* /v1/admin/maintenance/backfill-contact-id` bypasses this guard entirely and
|
|
886
|
+
* is the escape hatch.)
|
|
887
|
+
*
|
|
888
|
+
* Re-running is cheap by construction: every UPDATE is guarded by
|
|
889
|
+
* `contact_id IS NULL`, so a steady-state sweep is five zero-row probes per
|
|
890
|
+
* contact and writes nothing. Boot-triggered rather than operator-triggered
|
|
891
|
+
* deliberately — the engine is a published dependency, and a manual step every
|
|
892
|
+
* consumer must remember is how deployments strand at PRD 05. Best-effort: a
|
|
893
|
+
* failure to enqueue must never crash worker boot.
|
|
894
|
+
*/
|
|
895
|
+
export async function enqueueContactIdBackfill(opts: {
|
|
896
|
+
db: Database;
|
|
897
|
+
logger: Logger;
|
|
898
|
+
}): Promise<void> {
|
|
899
|
+
const { db, logger } = opts;
|
|
900
|
+
try {
|
|
901
|
+
const inflight = await db
|
|
902
|
+
.select({ id: importJobs.id })
|
|
903
|
+
.from(importJobs)
|
|
904
|
+
.where(
|
|
905
|
+
and(
|
|
906
|
+
eq(importJobs.format, CONTACT_ID_BACKFILL_FORMAT),
|
|
907
|
+
inArray(importJobs.status, ["pending", "processing"]),
|
|
908
|
+
),
|
|
909
|
+
)
|
|
910
|
+
.limit(1);
|
|
911
|
+
if (inflight[0]) return;
|
|
912
|
+
|
|
913
|
+
const [latest] = await db
|
|
914
|
+
.select({ id: importJobs.id, updatedAt: importJobs.updatedAt })
|
|
915
|
+
.from(importJobs)
|
|
916
|
+
.where(
|
|
917
|
+
and(
|
|
918
|
+
eq(importJobs.format, CONTACT_ID_BACKFILL_FORMAT),
|
|
919
|
+
eq(importJobs.status, "completed"),
|
|
920
|
+
),
|
|
921
|
+
)
|
|
922
|
+
.orderBy(desc(importJobs.updatedAt))
|
|
923
|
+
.limit(1);
|
|
924
|
+
|
|
925
|
+
const intervalMs = contactIdResweepIntervalMs(logger);
|
|
926
|
+
if (latest && Date.now() - latest.updatedAt.getTime() < intervalMs) return;
|
|
927
|
+
|
|
928
|
+
const [job] = await db
|
|
929
|
+
.insert(importJobs)
|
|
930
|
+
.values({
|
|
931
|
+
fileName: CONTACT_ID_BACKFILL_FORMAT,
|
|
932
|
+
format: CONTACT_ID_BACKFILL_FORMAT,
|
|
933
|
+
status: "pending",
|
|
934
|
+
})
|
|
935
|
+
.returning({ id: importJobs.id });
|
|
936
|
+
if (!job) return;
|
|
937
|
+
|
|
938
|
+
// runNoWait: called from worker boot BEFORE the listener starts — awaiting
|
|
939
|
+
// the run would deadlock (the `enqueueBucketBackfills` reasoning).
|
|
940
|
+
await contactIdBackfillTask.runNoWait({ jobId: job.id });
|
|
941
|
+
logger.info("Contact-id backfill enqueued", {
|
|
942
|
+
jobId: job.id,
|
|
943
|
+
previousSweepAt: latest?.updatedAt.toISOString() ?? null,
|
|
944
|
+
resweepIntervalMs: intervalMs,
|
|
945
|
+
});
|
|
946
|
+
} catch (err) {
|
|
947
|
+
logger.warn("Contact-id backfill enqueue failed", {
|
|
948
|
+
error: err instanceof Error ? err.message : String(err),
|
|
949
|
+
});
|
|
950
|
+
}
|
|
951
|
+
}
|