@hogsend/engine 0.56.0 → 0.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -11
- package/src/buckets/bucket-access.ts +5 -10
- package/src/buckets/check-membership.ts +78 -6
- package/src/index.ts +31 -1
- package/src/journeys/execute-journey-run.ts +56 -1
- package/src/journeys/journey-context.ts +59 -0
- package/src/lib/bucket-emit.test.ts +83 -2
- package/src/lib/bucket-emit.ts +16 -0
- package/src/lib/contacts.ts +1031 -141
- package/src/lib/crm-ingest.ts +9 -0
- package/src/lib/feed.ts +64 -3
- package/src/lib/identity-service.ts +15 -2
- package/src/lib/ingestion.ts +180 -14
- package/src/lib/tracking-events.ts +33 -6
- package/src/routes/admin/agent.ts +26 -0
- package/src/routes/admin/contacts.ts +65 -5
- package/src/routes/admin/identity.ts +197 -0
- package/src/routes/admin/index.ts +2 -0
- package/src/routes/contacts/index.ts +50 -7
- package/src/routes/events/index.ts +94 -6
- package/src/routes/feed/index.ts +36 -0
- package/src/routes/feed/recipient.ts +44 -2
- package/src/routes/lists/index.ts +21 -0
- package/src/routes/tracking/arrive.ts +42 -3
- package/src/worker.ts +18 -0
- package/src/workflows/bucket-backfill.ts +1 -1
- package/src/workflows/bucket-reconcile.ts +129 -4
- package/src/workflows/identity-alias-backfill.ts +448 -0
- package/src/workflows/import-contacts.ts +10 -0
- package/src/workflows/send-campaign.ts +31 -13
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
import type { JsonObject } from "@hatchet-dev/typescript-sdk/v1/types.js";
|
|
2
|
+
import { createDatabase, type Database, importJobs } from "@hogsend/db";
|
|
3
|
+
import { and, eq, inArray, sql } from "drizzle-orm";
|
|
4
|
+
import { ALIAS_REASON_BACKFILL } from "../lib/contacts.js";
|
|
5
|
+
import { hatchet } from "../lib/hatchet.js";
|
|
6
|
+
import type { Logger } from "../lib/logger.js";
|
|
7
|
+
import { createLogger } from "../lib/logger.js";
|
|
8
|
+
|
|
9
|
+
/** `import_jobs.format` discriminator for the reused status record. */
|
|
10
|
+
export const IDENTITY_ALIAS_BACKFILL_FORMAT = "identity-alias-backfill";
|
|
11
|
+
|
|
12
|
+
/** Contacts per keyset batch — the `bucket-backfill.ts` precedent. */
|
|
13
|
+
const BATCH_SIZE = 500;
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* PRD 02 T3 — fill `contact_aliases` with a row per identity key of every LIVE
|
|
17
|
+
* contact, so the identity table can become the source of truth `findByKey`
|
|
18
|
+
* reads first. Keyset-paginated on the `contacts` PK; one classify SELECT plus
|
|
19
|
+
* one guarded INSERT per batch; no long-held transaction, no table lock.
|
|
20
|
+
*
|
|
21
|
+
* Invariants, each pinned by a test:
|
|
22
|
+
* - IDEMPOTENT — `ON CONFLICT (alias_kind, alias_value) DO NOTHING` against
|
|
23
|
+
* the plain unique index; a re-run (or a resume from zero) inserts nothing.
|
|
24
|
+
* - NEVER STEALS — a `(kind, value)` already claimed by a DIFFERENT contact is
|
|
25
|
+
* skipped and counted (`conflicting`), never repointed.
|
|
26
|
+
* - Soft-deleted contacts are EXCLUDED (`deleted_at IS NULL`): their keys are
|
|
27
|
+
* either a merged loser's (already aliased to the survivor by
|
|
28
|
+
* `recordMergeAliases`) or an erased person's (must not be resurrected).
|
|
29
|
+
* - Emails land NORMALIZED (`lower(trim(email))`) so the alias row matches the
|
|
30
|
+
* value the resolver's probes always compare against.
|
|
31
|
+
* - The row-uuid pseudo-key (`('external', contacts.id)`) is deliberately NOT
|
|
32
|
+
* backfilled — on a deployment using uuids as external ids it would contend
|
|
33
|
+
* with real keys, a history-theft-shaped outcome for a defensive row. The
|
|
34
|
+
* uuid probe stays last in `findByKey` instead.
|
|
35
|
+
*/
|
|
36
|
+
export interface IdentityAliasBackfillInput extends JsonObject {
|
|
37
|
+
jobId: string;
|
|
38
|
+
/** Classify + count only; write nothing. Also reports the mixed-case email
|
|
39
|
+
* count (the datum behind PRD 02 T6's per-deployment decision). */
|
|
40
|
+
dryRun?: boolean;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Extends JsonObject: this is a Hatchet task return value and must be
|
|
44
|
+
// JSON-serializable (the same constraint every workflow output carries).
|
|
45
|
+
export interface IdentityAliasBackfillResult extends JsonObject {
|
|
46
|
+
status: "completed" | "failed";
|
|
47
|
+
dryRun: boolean;
|
|
48
|
+
/** Live contacts scanned. */
|
|
49
|
+
scanned: number;
|
|
50
|
+
/** Alias rows written (projected, in a dry run). */
|
|
51
|
+
inserted: number;
|
|
52
|
+
/** Pairs already aliased to the SAME contact (benign; dominated by the
|
|
53
|
+
* dual-write once it has been live for a while). */
|
|
54
|
+
present: number;
|
|
55
|
+
/** Pairs whose `(kind, value)` is claimed by a DIFFERENT contact — the
|
|
56
|
+
* divergence metric. Never repointed; surfaced via the parity endpoint. */
|
|
57
|
+
conflicting: number;
|
|
58
|
+
/** Live contacts whose stored email differs from `lower(trim(email))` —
|
|
59
|
+
* re-measure per deployment before trusting the read flip for email (T6). */
|
|
60
|
+
mixedCaseEmails: number;
|
|
61
|
+
reason?: string;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** The four identity-column → alias-kind projections of one contacts batch.
|
|
65
|
+
* Fragment shared by the classify SELECT and the INSERT so the two can never
|
|
66
|
+
* disagree about what a "pair" is. */
|
|
67
|
+
function pairsCte(afterId: string | null) {
|
|
68
|
+
return sql`
|
|
69
|
+
batch AS (
|
|
70
|
+
SELECT id, external_id, email, anonymous_id, discord_id
|
|
71
|
+
FROM contacts
|
|
72
|
+
WHERE deleted_at IS NULL
|
|
73
|
+
AND (${afterId}::uuid IS NULL OR id > ${afterId}::uuid)
|
|
74
|
+
ORDER BY id
|
|
75
|
+
LIMIT ${BATCH_SIZE}
|
|
76
|
+
),
|
|
77
|
+
pairs AS (
|
|
78
|
+
SELECT id, 'external'::text AS kind, external_id AS value
|
|
79
|
+
FROM batch WHERE external_id IS NOT NULL
|
|
80
|
+
UNION ALL
|
|
81
|
+
SELECT id, 'email', lower(trim(email))
|
|
82
|
+
FROM batch WHERE email IS NOT NULL AND trim(email) <> ''
|
|
83
|
+
UNION ALL
|
|
84
|
+
SELECT id, 'anonymous', anonymous_id
|
|
85
|
+
FROM batch WHERE anonymous_id IS NOT NULL
|
|
86
|
+
UNION ALL
|
|
87
|
+
SELECT id, 'discord', discord_id
|
|
88
|
+
FROM batch WHERE discord_id IS NOT NULL
|
|
89
|
+
)`;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
interface BatchClassification {
|
|
93
|
+
scanned: number;
|
|
94
|
+
lastId: string | null;
|
|
95
|
+
toInsert: number;
|
|
96
|
+
present: number;
|
|
97
|
+
conflicting: number;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** One round trip: how many contacts this batch scans, where the cursor lands,
|
|
101
|
+
* and how each candidate pair classifies against the existing alias rows. */
|
|
102
|
+
async function classifyBatch(
|
|
103
|
+
db: Database,
|
|
104
|
+
afterId: string | null,
|
|
105
|
+
): Promise<BatchClassification> {
|
|
106
|
+
const rows = await db.execute<{
|
|
107
|
+
scanned: number;
|
|
108
|
+
last_id: string | null;
|
|
109
|
+
to_insert: number;
|
|
110
|
+
present: number;
|
|
111
|
+
conflicting: number;
|
|
112
|
+
}>(sql`
|
|
113
|
+
WITH ${pairsCte(afterId)},
|
|
114
|
+
classified AS (
|
|
115
|
+
SELECT p.id, a.contact_id AS owner
|
|
116
|
+
FROM pairs p
|
|
117
|
+
LEFT JOIN contact_aliases a
|
|
118
|
+
ON a.alias_kind = p.kind AND a.alias_value = p.value
|
|
119
|
+
)
|
|
120
|
+
SELECT (SELECT count(*) FROM batch)::int AS scanned,
|
|
121
|
+
-- no max(uuid) aggregate exists; take the keyset tail explicitly
|
|
122
|
+
(SELECT id FROM batch ORDER BY id DESC LIMIT 1)::text AS last_id,
|
|
123
|
+
count(*) FILTER (WHERE c.owner IS NULL)::int AS to_insert,
|
|
124
|
+
count(*) FILTER (WHERE c.owner = c.id)::int AS present,
|
|
125
|
+
count(*) FILTER (WHERE c.owner IS NOT NULL
|
|
126
|
+
AND c.owner <> c.id)::int AS conflicting
|
|
127
|
+
FROM classified c
|
|
128
|
+
`);
|
|
129
|
+
const row = rows[0];
|
|
130
|
+
return {
|
|
131
|
+
scanned: Number(row?.scanned ?? 0),
|
|
132
|
+
lastId: row?.last_id ?? null,
|
|
133
|
+
toInsert: Number(row?.to_insert ?? 0),
|
|
134
|
+
present: Number(row?.present ?? 0),
|
|
135
|
+
conflicting: Number(row?.conflicting ?? 0),
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** True when the error chain carries a Postgres FK violation (23503). Drizzle
|
|
140
|
+
* wraps the driver error, so walk `cause` (the 23505 lesson from the partial-
|
|
141
|
+
* index onConflict work applies to every SQLSTATE). */
|
|
142
|
+
function isFkViolation(err: unknown): boolean {
|
|
143
|
+
let cursor: unknown = err;
|
|
144
|
+
for (let i = 0; i < 5 && cursor; i++) {
|
|
145
|
+
if (
|
|
146
|
+
typeof cursor === "object" &&
|
|
147
|
+
"code" in cursor &&
|
|
148
|
+
(cursor as { code?: unknown }).code === "23503"
|
|
149
|
+
) {
|
|
150
|
+
return true;
|
|
151
|
+
}
|
|
152
|
+
cursor = (cursor as { cause?: unknown }).cause;
|
|
153
|
+
}
|
|
154
|
+
return false;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* The write half — the PRD's verbatim statement. Returns rows inserted.
|
|
159
|
+
*
|
|
160
|
+
* Retries (bounded) on an FK violation: a contact HARD-deleted between this
|
|
161
|
+
* statement's snapshot and its inserts (a GDPR purge script, a test cleanup)
|
|
162
|
+
* makes the alias row's `contact_id` FK fail even though the batch CTE read the
|
|
163
|
+
* row as live. Each retry re-reads the batch under a fresh snapshot — the
|
|
164
|
+
* deleted contact is gone from it — so a racing purge costs a retry, not the
|
|
165
|
+
* whole job. Any other error, or a third consecutive FK loss, propagates.
|
|
166
|
+
*/
|
|
167
|
+
async function insertBatch(db: Database, afterId: string | null) {
|
|
168
|
+
for (let attempt = 0; ; attempt++) {
|
|
169
|
+
try {
|
|
170
|
+
const rows = await db.execute<{ contact_id: string }>(sql`
|
|
171
|
+
WITH ${pairsCte(afterId)}
|
|
172
|
+
INSERT INTO contact_aliases
|
|
173
|
+
(contact_id, alias_kind, alias_value, from_contact_id, reason,
|
|
174
|
+
created_at, updated_at)
|
|
175
|
+
SELECT id, kind, value, NULL, ${ALIAS_REASON_BACKFILL}, now(), now()
|
|
176
|
+
FROM pairs
|
|
177
|
+
ON CONFLICT (alias_kind, alias_value) DO NOTHING
|
|
178
|
+
RETURNING contact_id
|
|
179
|
+
`);
|
|
180
|
+
return rows.length;
|
|
181
|
+
} catch (err) {
|
|
182
|
+
if (attempt >= 2 || !isFkViolation(err)) throw err;
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* The task body, exported directly so tests and operator scripts can run it
|
|
189
|
+
* without a Hatchet engine. `jobId` is optional: when present, progress is
|
|
190
|
+
* recorded on the `import_jobs` row (`processedRows` = contacts scanned,
|
|
191
|
+
* `failedRows` = conflicting pairs — the divergence metric, NOT an error
|
|
192
|
+
* count; the full breakdown is in the return value and the completion log).
|
|
193
|
+
*/
|
|
194
|
+
export async function runIdentityAliasBackfill(opts: {
|
|
195
|
+
db: Database;
|
|
196
|
+
logger: Logger;
|
|
197
|
+
jobId?: string;
|
|
198
|
+
dryRun?: boolean;
|
|
199
|
+
}): Promise<IdentityAliasBackfillResult> {
|
|
200
|
+
const { db, logger, jobId } = opts;
|
|
201
|
+
const dryRun = opts.dryRun === true;
|
|
202
|
+
|
|
203
|
+
const markJob = async (
|
|
204
|
+
patch: Partial<typeof importJobs.$inferInsert>,
|
|
205
|
+
): Promise<void> => {
|
|
206
|
+
if (!jobId) return;
|
|
207
|
+
await db
|
|
208
|
+
.update(importJobs)
|
|
209
|
+
.set({ ...patch, updatedAt: new Date() })
|
|
210
|
+
.where(eq(importJobs.id, jobId));
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
try {
|
|
214
|
+
const [totals] = await db.execute<{ live: number; mixed: number }>(sql`
|
|
215
|
+
SELECT count(*)::int AS live,
|
|
216
|
+
count(*) FILTER (
|
|
217
|
+
WHERE email IS NOT NULL
|
|
218
|
+
AND email IS DISTINCT FROM lower(trim(email))
|
|
219
|
+
)::int AS mixed
|
|
220
|
+
FROM contacts
|
|
221
|
+
WHERE deleted_at IS NULL
|
|
222
|
+
`);
|
|
223
|
+
const mixedCaseEmails = Number(totals?.mixed ?? 0);
|
|
224
|
+
await markJob({
|
|
225
|
+
status: "processing",
|
|
226
|
+
totalRows: Number(totals?.live ?? 0),
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
let cursor: string | null = null;
|
|
230
|
+
let scanned = 0;
|
|
231
|
+
let inserted = 0;
|
|
232
|
+
let present = 0;
|
|
233
|
+
let conflicting = 0;
|
|
234
|
+
|
|
235
|
+
for (;;) {
|
|
236
|
+
const batch = await classifyBatch(db, cursor);
|
|
237
|
+
if (batch.scanned === 0) break;
|
|
238
|
+
|
|
239
|
+
if (!dryRun) {
|
|
240
|
+
// The classify → insert pair is two statements, so a resolve committing
|
|
241
|
+
// between them can shrink the insert below `toInsert` — the INSERT's
|
|
242
|
+
// own ON CONFLICT arbiter is the correctness guard; the counts are
|
|
243
|
+
// observability. Count what actually landed.
|
|
244
|
+
inserted += await insertBatch(db, cursor);
|
|
245
|
+
} else {
|
|
246
|
+
inserted += batch.toInsert;
|
|
247
|
+
}
|
|
248
|
+
scanned += batch.scanned;
|
|
249
|
+
present += batch.present;
|
|
250
|
+
conflicting += batch.conflicting;
|
|
251
|
+
cursor = batch.lastId;
|
|
252
|
+
|
|
253
|
+
await markJob({ processedRows: scanned, failedRows: conflicting });
|
|
254
|
+
if (batch.scanned < BATCH_SIZE) break;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
if (!dryRun) {
|
|
258
|
+
// ERASURE-RACE SWEEP. A contact erased WHILE this job runs can have its
|
|
259
|
+
// keys re-inserted from a batch statement's stale snapshot (the batch
|
|
260
|
+
// read the contact as live; the erasure hook's delete committed first;
|
|
261
|
+
// this insert committed second). That would resurrect an erased person's
|
|
262
|
+
// identity keys — the exact leak PRD 02 T1 exists to close. This final
|
|
263
|
+
// statement runs AFTER every insert has committed, so its snapshot sees
|
|
264
|
+
// any erasure that raced a batch: whichever of the two commits second
|
|
265
|
+
// now cleans up. Scoped to backfill-authored rows — merge-trail aliases
|
|
266
|
+
// pointing at a soft-deleted survivor are `recordMergeAliases`' business,
|
|
267
|
+
// not this job's.
|
|
268
|
+
const swept = await db.execute<{ id: string }>(sql`
|
|
269
|
+
DELETE FROM contact_aliases a
|
|
270
|
+
USING contacts c
|
|
271
|
+
WHERE a.contact_id = c.id
|
|
272
|
+
AND c.deleted_at IS NOT NULL
|
|
273
|
+
AND a.reason = ${ALIAS_REASON_BACKFILL}
|
|
274
|
+
RETURNING a.id
|
|
275
|
+
`);
|
|
276
|
+
if (swept.length > 0) {
|
|
277
|
+
logger.warn("Identity alias backfill swept erased-contact rows", {
|
|
278
|
+
swept: swept.length,
|
|
279
|
+
});
|
|
280
|
+
// Clamp: the sweep can also collect a PREVIOUS run's raced rows,
|
|
281
|
+
// which this run never counted as inserts.
|
|
282
|
+
inserted = Math.max(0, inserted - swept.length);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
await markJob({
|
|
287
|
+
status: "completed",
|
|
288
|
+
processedRows: scanned,
|
|
289
|
+
failedRows: conflicting,
|
|
290
|
+
});
|
|
291
|
+
logger.info("Identity alias backfill complete", {
|
|
292
|
+
dryRun,
|
|
293
|
+
scanned,
|
|
294
|
+
inserted,
|
|
295
|
+
present,
|
|
296
|
+
conflicting,
|
|
297
|
+
mixedCaseEmails,
|
|
298
|
+
});
|
|
299
|
+
return {
|
|
300
|
+
status: "completed",
|
|
301
|
+
dryRun,
|
|
302
|
+
scanned,
|
|
303
|
+
inserted,
|
|
304
|
+
present,
|
|
305
|
+
conflicting,
|
|
306
|
+
mixedCaseEmails,
|
|
307
|
+
};
|
|
308
|
+
} catch (err) {
|
|
309
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
310
|
+
await markJob({
|
|
311
|
+
status: "failed",
|
|
312
|
+
errors: [{ row: 0, error: message }],
|
|
313
|
+
}).catch(() => undefined);
|
|
314
|
+
logger.error("Identity alias backfill failed", { dryRun, message });
|
|
315
|
+
return {
|
|
316
|
+
status: "failed",
|
|
317
|
+
dryRun,
|
|
318
|
+
scanned: 0,
|
|
319
|
+
inserted: 0,
|
|
320
|
+
present: 0,
|
|
321
|
+
conflicting: 0,
|
|
322
|
+
mixedCaseEmails: 0,
|
|
323
|
+
reason: message,
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
export const identityAliasBackfillTask = hatchet.task({
|
|
329
|
+
name: "identity-alias-backfill",
|
|
330
|
+
retries: 0,
|
|
331
|
+
executionTimeout: "600s",
|
|
332
|
+
fn: async (input: IdentityAliasBackfillInput) => {
|
|
333
|
+
const { db } = createDatabase({ url: process.env.DATABASE_URL ?? "" });
|
|
334
|
+
const logger = createLogger(process.env.LOG_LEVEL ?? "info");
|
|
335
|
+
return runIdentityAliasBackfill({
|
|
336
|
+
db,
|
|
337
|
+
logger,
|
|
338
|
+
jobId: input.jobId,
|
|
339
|
+
dryRun: input.dryRun === true,
|
|
340
|
+
});
|
|
341
|
+
},
|
|
342
|
+
});
|
|
343
|
+
|
|
344
|
+
/**
|
|
345
|
+
* Worker-boot enqueue (mirrors `enqueueBucketBackfills`): fire the backfill
|
|
346
|
+
* once per deployment without an operator having to remember it — the engine
|
|
347
|
+
* is a published dependency, and a manual step every consumer must run before
|
|
348
|
+
* the read flip is how deployments strand. Skips when ANY non-failed job of
|
|
349
|
+
* this format exists (pending/processing/completed); a failed run re-enqueues
|
|
350
|
+
* on the next boot, and `POST /v1/admin/identity/alias-backfill` forces a
|
|
351
|
+
* re-run past a stale record. Re-running is always safe (ON CONFLICT DO
|
|
352
|
+
* NOTHING), just not free — hence the skip. Best-effort: a failure to enqueue
|
|
353
|
+
* must never crash worker boot.
|
|
354
|
+
*/
|
|
355
|
+
export async function enqueueIdentityAliasBackfill(opts: {
|
|
356
|
+
db: Database;
|
|
357
|
+
logger: Logger;
|
|
358
|
+
}): Promise<void> {
|
|
359
|
+
const { db, logger } = opts;
|
|
360
|
+
try {
|
|
361
|
+
const existing = await db
|
|
362
|
+
.select({ id: importJobs.id })
|
|
363
|
+
.from(importJobs)
|
|
364
|
+
.where(
|
|
365
|
+
and(
|
|
366
|
+
eq(importJobs.format, IDENTITY_ALIAS_BACKFILL_FORMAT),
|
|
367
|
+
inArray(importJobs.status, ["pending", "processing", "completed"]),
|
|
368
|
+
),
|
|
369
|
+
)
|
|
370
|
+
.limit(1);
|
|
371
|
+
if (existing[0]) return;
|
|
372
|
+
|
|
373
|
+
const [job] = await db
|
|
374
|
+
.insert(importJobs)
|
|
375
|
+
.values({
|
|
376
|
+
fileName: IDENTITY_ALIAS_BACKFILL_FORMAT,
|
|
377
|
+
format: IDENTITY_ALIAS_BACKFILL_FORMAT,
|
|
378
|
+
status: "pending",
|
|
379
|
+
})
|
|
380
|
+
.returning({ id: importJobs.id });
|
|
381
|
+
if (!job) return;
|
|
382
|
+
|
|
383
|
+
// runNoWait: called from worker boot BEFORE the listener starts — awaiting
|
|
384
|
+
// the run would deadlock (same reasoning as enqueueBucketBackfills).
|
|
385
|
+
await identityAliasBackfillTask.runNoWait({ jobId: job.id, dryRun: false });
|
|
386
|
+
logger.info("Identity alias backfill enqueued", { jobId: job.id });
|
|
387
|
+
} catch (err) {
|
|
388
|
+
logger.warn("Identity alias backfill enqueue failed", {
|
|
389
|
+
error: err instanceof Error ? err.message : String(err),
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
// ---------------------------------------------------------------------------
|
|
395
|
+
// PRD 02 T4 — read-only parity verifier
|
|
396
|
+
// ---------------------------------------------------------------------------
|
|
397
|
+
|
|
398
|
+
export interface AliasParityRow {
|
|
399
|
+
kind: string;
|
|
400
|
+
/** Alias and column resolve to DIFFERENT live contacts. MUST be 0 before
|
|
401
|
+
* trusting alias-first resolution; a non-zero value is a data bug to fix on
|
|
402
|
+
* its own, never to flip over. */
|
|
403
|
+
diverged: number;
|
|
404
|
+
/** Alias target is dead/missing but a live column owner exists — expected,
|
|
405
|
+
* handled by `findByKey`'s live-target fall-through. */
|
|
406
|
+
aliasDead: number;
|
|
407
|
+
/** Alias exists with no column owner — the merge/promote population, and
|
|
408
|
+
* (after PRD 03) the point of the table. */
|
|
409
|
+
aliasOnly: number;
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
/** "Would alias-first return a different contact than column-first?", per kind,
|
|
413
|
+
* over every alias row. Pure read; no writes, no locks beyond the reads. */
|
|
414
|
+
export async function identityAliasParity(
|
|
415
|
+
db: Database,
|
|
416
|
+
): Promise<AliasParityRow[]> {
|
|
417
|
+
const rows = await db.execute<{
|
|
418
|
+
alias_kind: string;
|
|
419
|
+
diverged: number;
|
|
420
|
+
alias_dead: number;
|
|
421
|
+
alias_only: number;
|
|
422
|
+
}>(sql`
|
|
423
|
+
SELECT a.alias_kind,
|
|
424
|
+
count(*) FILTER (WHERE ac.id IS NOT NULL AND cc.id IS NOT NULL
|
|
425
|
+
AND ac.id <> cc.id)::int AS diverged,
|
|
426
|
+
count(*) FILTER (WHERE ac.id IS NULL
|
|
427
|
+
AND cc.id IS NOT NULL)::int AS alias_dead,
|
|
428
|
+
count(*) FILTER (WHERE ac.id IS NOT NULL
|
|
429
|
+
AND cc.id IS NULL)::int AS alias_only
|
|
430
|
+
FROM contact_aliases a
|
|
431
|
+
LEFT JOIN contacts ac
|
|
432
|
+
ON ac.id = a.contact_id AND ac.deleted_at IS NULL
|
|
433
|
+
LEFT JOIN contacts cc
|
|
434
|
+
ON cc.deleted_at IS NULL AND (
|
|
435
|
+
(a.alias_kind = 'external' AND cc.external_id = a.alias_value)
|
|
436
|
+
OR (a.alias_kind = 'email' AND lower(trim(cc.email)) = a.alias_value)
|
|
437
|
+
OR (a.alias_kind = 'anonymous' AND cc.anonymous_id = a.alias_value)
|
|
438
|
+
OR (a.alias_kind = 'discord' AND cc.discord_id = a.alias_value))
|
|
439
|
+
GROUP BY 1
|
|
440
|
+
ORDER BY 1
|
|
441
|
+
`);
|
|
442
|
+
return rows.map((r) => ({
|
|
443
|
+
kind: r.alias_kind,
|
|
444
|
+
diverged: Number(r.diverged),
|
|
445
|
+
aliasDead: Number(r.alias_dead),
|
|
446
|
+
aliasOnly: Number(r.alias_only),
|
|
447
|
+
}));
|
|
448
|
+
}
|
|
@@ -130,6 +130,16 @@ export const importContactsTask = hatchet.task({
|
|
|
130
130
|
userId: row.externalId,
|
|
131
131
|
email: row.email,
|
|
132
132
|
contactProperties: row.properties,
|
|
133
|
+
// PRD 06 T4 (L5 row 21): an operator import asserts identity by
|
|
134
|
+
// externalId/email only (the key guard above rejects rows with
|
|
135
|
+
// neither; `phone` attaches OUTSIDE the resolver, below) — so
|
|
136
|
+
// create-on-miss, no clamp, and the narrow `trustedKinds` is the
|
|
137
|
+
// honest statement. Enforced by T5.
|
|
138
|
+
policy: {
|
|
139
|
+
create: "on-miss",
|
|
140
|
+
allowMerge: "any",
|
|
141
|
+
trustedKinds: ["external", "email"],
|
|
142
|
+
},
|
|
133
143
|
}).then(async (result) => {
|
|
134
144
|
await attachPhone(db, result.id, row.phone);
|
|
135
145
|
return { index: i + idx, ok: true };
|
|
@@ -19,7 +19,7 @@ import {
|
|
|
19
19
|
waveConditionSql,
|
|
20
20
|
} from "../campaigns/cohort-sql.js";
|
|
21
21
|
import { campaignSendKey } from "../lib/campaign-send-key.js";
|
|
22
|
-
import { normalizeEmail } from "../lib/contacts.js";
|
|
22
|
+
import { contactKeySql, normalizeEmail } from "../lib/contacts.js";
|
|
23
23
|
import { getDb } from "../lib/db.js";
|
|
24
24
|
import { getEmailService } from "../lib/email.js";
|
|
25
25
|
import { hatchet } from "../lib/hatchet.js";
|
|
@@ -973,18 +973,36 @@ async function* resolveBucketRecipients(
|
|
|
973
973
|
];
|
|
974
974
|
if (cursor) conditions.push(gt(bucketMemberships.id, cursor));
|
|
975
975
|
|
|
976
|
-
return
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
976
|
+
return (
|
|
977
|
+
db
|
|
978
|
+
.select({
|
|
979
|
+
id: bucketMemberships.id,
|
|
980
|
+
userId: bucketMemberships.userId,
|
|
981
|
+
membershipEmail: bucketMemberships.userEmail,
|
|
982
|
+
contactEmail: contacts.email,
|
|
983
|
+
})
|
|
984
|
+
.from(bucketMemberships)
|
|
985
|
+
// Join on the CANONICAL key, not `external_id` alone.
|
|
986
|
+
// `bucket_memberships.user_id` holds `coalesce(external_id,
|
|
987
|
+
// anonymous_id, id)`, so a contact with a NULL `external_id` — an
|
|
988
|
+
// email-only subscriber keyed on its uuid, or an anonymous visitor
|
|
989
|
+
// keyed on its `anonymous_id` — matched nothing here and was silently
|
|
990
|
+
// dropped from the audience. They were genuine active members of the
|
|
991
|
+
// bucket; they simply never received the campaign, and nothing
|
|
992
|
+
// surfaced it: a send report showing 40 of 40 delivered looks complete
|
|
993
|
+
// whether or not the audience was built correctly.
|
|
994
|
+
//
|
|
995
|
+
// Suppression is unaffected by widening this join. Every gate lives in
|
|
996
|
+
// the same `conditions` array applied to this query and is keyed on
|
|
997
|
+
// `recipientEmail` (`coalesce(membership email, contact email)`, lowered
|
|
998
|
+
// and trimmed), not on the join column — so a newly-visible member is
|
|
999
|
+
// filtered by the identical unsubscribed/suppressed `NOT EXISTS` as
|
|
1000
|
+
// everyone else, and the mailer re-checks preferences per send.
|
|
1001
|
+
.innerJoin(contacts, eq(contactKeySql(), bucketMemberships.userId))
|
|
1002
|
+
.where(and(...conditions))
|
|
1003
|
+
.orderBy(bucketMemberships.id)
|
|
1004
|
+
.limit(CHUNK_SIZE)
|
|
1005
|
+
);
|
|
988
1006
|
},
|
|
989
1007
|
cursorOf: (row) => row.id,
|
|
990
1008
|
map: (row) => {
|