@hogsend/engine 0.58.0 → 0.60.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -11
- package/src/buckets/bucket-access.ts +19 -10
- package/src/buckets/check-membership.ts +61 -8
- package/src/buckets/membership-epoch.ts +11 -1
- package/src/campaigns/cohort-sql.ts +3 -0
- package/src/index.ts +26 -6
- package/src/journeys/execute-journey-run.ts +180 -29
- package/src/journeys/journey-context.ts +43 -7
- package/src/lib/agent/tools.ts +29 -4
- package/src/lib/attribution-backfill.ts +27 -4
- package/src/lib/attribution.ts +22 -3
- package/src/lib/connector-actions.ts +4 -1
- package/src/lib/contacts.ts +518 -148
- package/src/lib/conversion-dispatch.ts +16 -2
- package/src/lib/enrollment-guards.ts +19 -12
- package/src/lib/feed.ts +5 -5
- package/src/lib/flags.ts +26 -9
- package/src/lib/ingestion.ts +23 -2
- package/src/lib/journey-lift.ts +17 -5
- package/src/lib/preferences.ts +116 -30
- package/src/lib/recipient-preferences.ts +16 -2
- package/src/lib/revenue.ts +20 -5
- package/src/lib/sms-tracked.ts +18 -1
- package/src/lib/tracked.ts +50 -8
- package/src/lib/tracking-events.ts +28 -0
- package/src/lib/unique-violation.ts +44 -0
- package/src/routes/admin/buckets.ts +20 -2
- package/src/routes/admin/bulk.ts +18 -2
- package/src/routes/admin/contacts.ts +4 -3
- package/src/routes/admin/conversions.ts +20 -4
- package/src/routes/admin/emails.ts +17 -2
- package/src/routes/admin/events.ts +32 -41
- package/src/routes/admin/funnels.ts +6 -1
- package/src/routes/admin/groups.ts +4 -0
- package/src/routes/admin/impact.ts +30 -10
- package/src/routes/admin/index.ts +2 -0
- package/src/routes/admin/journey-impact.ts +18 -9
- package/src/routes/admin/journeys.ts +8 -1
- package/src/routes/admin/maintenance.ts +216 -0
- package/src/routes/admin/preferences.ts +85 -38
- package/src/routes/admin/reporting.ts +25 -5
- package/src/routes/admin/targeting.ts +1 -0
- package/src/routes/admin/timeline.ts +16 -10
- package/src/routes/email/preferences.ts +18 -1
- package/src/routes/lists/index.ts +11 -6
- package/src/worker.ts +20 -0
- package/src/workflows/backfill-contact-id.ts +951 -0
- package/src/workflows/bucket-backfill.ts +145 -59
- package/src/workflows/bucket-reconcile.ts +154 -50
- package/src/workflows/check-alerts.ts +4 -0
- package/src/workflows/impact-digest.ts +10 -3
- package/src/workflows/journey-blueprint-interpreter.ts +8 -1
- package/src/workflows/send-campaign.ts +18 -11
|
@@ -176,6 +176,26 @@ export const impactOverviewRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
176
176
|
const since = new Date(Date.now() - days * 24 * 60 * 60 * 1000);
|
|
177
177
|
const sinceTs = sql`${since.toISOString()}::timestamptz`;
|
|
178
178
|
|
|
179
|
+
// WHO a journey_states row is about — `bySubject` written in SQL. The
|
|
180
|
+
// owning contact when the `contact_id` FK is stamped, else the mutable
|
|
181
|
+
// text key (a contactless enrollment is a permanent supported state: the
|
|
182
|
+
// engine refuses to mint a contact on observation). Either/or per row,
|
|
183
|
+
// never both. A bare `count(distinct js.contact_id)` would SILENTLY DROP
|
|
184
|
+
// every contactless enrollment, because `count(distinct …)` skips NULLs —
|
|
185
|
+
// and an under-counted denominator reads as "quiet journey", not as a bug.
|
|
186
|
+
const jsSubject = sql`coalesce(js.contact_id::text, js.user_id)`;
|
|
187
|
+
// The conversion cohort joins on the contact FK, not the text key.
|
|
188
|
+
// `conversions.contact_id` is NOT NULL, so a conversion always belongs to
|
|
189
|
+
// a resolved contact and this needs no text-key fallback: when
|
|
190
|
+
// `js.contact_id` is NULL the subject has no contact, therefore no
|
|
191
|
+
// conversion, and the NULL comparison correctly matches nothing. The
|
|
192
|
+
// uuid-to-uuid compare also uses `conversions_contact_idx`; the old
|
|
193
|
+
// `c.user_key = js.user_id` compare had no index at all. What it FIXES:
|
|
194
|
+
// the string join missed every conversion recorded under a key the person
|
|
195
|
+
// moved off after enrolling (anon → external), silently under-counting
|
|
196
|
+
// converters and inflating apparent lift.
|
|
197
|
+
const convBySubject = sql`c.contact_id = js.contact_id`;
|
|
198
|
+
|
|
179
199
|
// (a) Journey base rollup — one grouped query, any-definition outcome,
|
|
180
200
|
// latest-by-created_at version label. No deleted_at filter (matches the
|
|
181
201
|
// /lift cohort SQL precedent).
|
|
@@ -185,20 +205,20 @@ export const impactOverviewRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
185
205
|
(array_agg(js.journey_version_label order by js.created_at desc)
|
|
186
206
|
filter (where js.journey_version_label is not null))[1]
|
|
187
207
|
as version_label,
|
|
188
|
-
count(distinct
|
|
208
|
+
count(distinct ${jsSubject})
|
|
189
209
|
filter (where js.status != 'held_out')::int as enrollments,
|
|
190
|
-
(count(distinct
|
|
210
|
+
(count(distinct ${jsSubject}) filter (where js.status != 'held_out'
|
|
191
211
|
and exists (
|
|
192
212
|
select 1 from conversions c
|
|
193
|
-
where
|
|
213
|
+
where ${convBySubject}
|
|
194
214
|
and c.occurred_at >= js.created_at
|
|
195
215
|
)))::int as converters,
|
|
196
|
-
count(distinct
|
|
216
|
+
count(distinct ${jsSubject})
|
|
197
217
|
filter (where js.status = 'held_out')::int as control_contacts,
|
|
198
|
-
(count(distinct
|
|
218
|
+
(count(distinct ${jsSubject}) filter (where js.status = 'held_out'
|
|
199
219
|
and exists (
|
|
200
220
|
select 1 from conversions c
|
|
201
|
-
where
|
|
221
|
+
where ${convBySubject}
|
|
202
222
|
and c.occurred_at >= js.created_at
|
|
203
223
|
)))::int as control_converters
|
|
204
224
|
from journey_states js
|
|
@@ -265,17 +285,17 @@ export const impactOverviewRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
265
285
|
control_converters: number;
|
|
266
286
|
}>(sql`
|
|
267
287
|
select
|
|
268
|
-
(count(distinct
|
|
288
|
+
(count(distinct ${jsSubject})
|
|
269
289
|
filter (where js.status != 'held_out' and exists (
|
|
270
290
|
select 1 from conversions c
|
|
271
|
-
where
|
|
291
|
+
where ${convBySubject}
|
|
272
292
|
and c.occurred_at >= js.created_at
|
|
273
293
|
and c.definition_id = ${goal}
|
|
274
294
|
)))::int as converters,
|
|
275
|
-
(count(distinct
|
|
295
|
+
(count(distinct ${jsSubject})
|
|
276
296
|
filter (where js.status = 'held_out' and exists (
|
|
277
297
|
select 1 from conversions c
|
|
278
|
-
where
|
|
298
|
+
where ${convBySubject}
|
|
279
299
|
and c.occurred_at >= js.created_at
|
|
280
300
|
and c.definition_id = ${goal}
|
|
281
301
|
)))::int as control_converters
|
|
@@ -31,6 +31,7 @@ import { journeyLogsRouter } from "./journey-logs.js";
|
|
|
31
31
|
import { journeysRouter } from "./journeys.js";
|
|
32
32
|
import { linksRouter } from "./links.js";
|
|
33
33
|
import { adminListsRouter } from "./lists.js";
|
|
34
|
+
import { maintenanceRouter } from "./maintenance.js";
|
|
34
35
|
import { metricsRouter } from "./metrics.js";
|
|
35
36
|
import { openEditorRouter } from "./open-editor.js";
|
|
36
37
|
import { preferencesRouter } from "./preferences.js";
|
|
@@ -87,4 +88,5 @@ adminRouter.route("/alerts", alertsRouter);
|
|
|
87
88
|
adminRouter.route("/dlq", dlqRouter);
|
|
88
89
|
adminRouter.route("/domain", domainRouter);
|
|
89
90
|
adminRouter.route("/readiness", readinessRouter);
|
|
91
|
+
adminRouter.route("/maintenance", maintenanceRouter);
|
|
90
92
|
adminRouter.route("/agent", agentRouter);
|
|
@@ -243,6 +243,15 @@ export const journeyImpactRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
243
243
|
? sql` and c.definition_id = ${definitionId}`
|
|
244
244
|
: sql``;
|
|
245
245
|
const sinceTs = sql`${since.toISOString()}::timestamptz`;
|
|
246
|
+
// Same subject/cohort pair as /impact/overview — see the long rationale on
|
|
247
|
+
// `jsSubject`/`convBySubject` in `routes/admin/impact.ts`. In short: the
|
|
248
|
+
// coalesce is load-bearing (a bare `count(distinct js.contact_id)` drops
|
|
249
|
+
// every contactless enrollment, since `count(distinct …)` skips NULLs),
|
|
250
|
+
// and the cohort side needs no fallback because `conversions.contact_id`
|
|
251
|
+
// is NOT NULL. Both readouts MUST use the identical pair or /impact and
|
|
252
|
+
// /journeys/:id/impact report different enrollments for one cohort.
|
|
253
|
+
const jsSubject = sql`coalesce(js.contact_id::text, js.user_id)`;
|
|
254
|
+
const convBySubject = sql`c.contact_id = js.contact_id`;
|
|
246
255
|
const versionRows = await db.execute<{
|
|
247
256
|
hash: string | null;
|
|
248
257
|
label: string | null;
|
|
@@ -261,20 +270,20 @@ export const journeyImpactRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
261
270
|
as first_enrolled_at,
|
|
262
271
|
max(js.created_at) filter (where js.status != 'held_out')
|
|
263
272
|
as last_enrolled_at,
|
|
264
|
-
count(distinct
|
|
273
|
+
count(distinct ${jsSubject})
|
|
265
274
|
filter (where js.status != 'held_out')::int as enrollments,
|
|
266
|
-
(count(distinct
|
|
275
|
+
(count(distinct ${jsSubject}) filter (where js.status != 'held_out'
|
|
267
276
|
and exists (
|
|
268
277
|
select 1 from conversions c
|
|
269
|
-
where
|
|
278
|
+
where ${convBySubject}
|
|
270
279
|
and c.occurred_at >= js.created_at${definitionSql}
|
|
271
280
|
)))::int as converters,
|
|
272
|
-
count(distinct
|
|
281
|
+
count(distinct ${jsSubject})
|
|
273
282
|
filter (where js.status = 'held_out')::int as control_contacts,
|
|
274
|
-
(count(distinct
|
|
283
|
+
(count(distinct ${jsSubject}) filter (where js.status = 'held_out'
|
|
275
284
|
and exists (
|
|
276
285
|
select 1 from conversions c
|
|
277
|
-
where
|
|
286
|
+
where ${convBySubject}
|
|
278
287
|
and c.occurred_at >= js.created_at${definitionSql}
|
|
279
288
|
)))::int as control_converters
|
|
280
289
|
from journey_states js
|
|
@@ -339,10 +348,10 @@ export const journeyImpactRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
339
348
|
select
|
|
340
349
|
v.key as variant_key,
|
|
341
350
|
v.arm as arm,
|
|
342
|
-
count(distinct
|
|
343
|
-
(count(distinct
|
|
351
|
+
count(distinct ${jsSubject})::int as enrollments,
|
|
352
|
+
(count(distinct ${jsSubject}) filter (where exists (
|
|
344
353
|
select 1 from conversions c
|
|
345
|
-
where
|
|
354
|
+
where ${convBySubject}
|
|
346
355
|
and c.occurred_at >= js.created_at${definitionSql}
|
|
347
356
|
)))::int as converters
|
|
348
357
|
from journey_states js
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import {
|
|
2
|
+
bySubject,
|
|
2
3
|
type JourneyGraph,
|
|
3
4
|
journeyGraphSchema,
|
|
4
5
|
resolveTemplateKeyFromConst,
|
|
@@ -24,6 +25,7 @@ import {
|
|
|
24
25
|
} from "drizzle-orm";
|
|
25
26
|
import type { AppEnv } from "../../app.js";
|
|
26
27
|
import { buildJourneyGraph } from "../../journeys/graph/build-graph.js";
|
|
28
|
+
import { lookupContactIdByKey } from "../../lib/contacts.js";
|
|
27
29
|
import { ingestEvent } from "../../lib/ingestion.js";
|
|
28
30
|
import {
|
|
29
31
|
computeJourneyLift,
|
|
@@ -744,8 +746,13 @@ export const journeysRouter = new OpenAPIHono<AppEnv>()
|
|
|
744
746
|
if (status) {
|
|
745
747
|
conditions.push(eq(journeyStates.status, status));
|
|
746
748
|
}
|
|
749
|
+
// Resolve the operator-supplied text key to its owning contact so the
|
|
750
|
+
// filter finds enrollments stamped under keys that person has since moved
|
|
751
|
+
// off. Falls back to the literal string match when no contact owns the
|
|
752
|
+
// key (pre-flip behaviour).
|
|
747
753
|
if (userId) {
|
|
748
|
-
|
|
754
|
+
const contactId = await lookupContactIdByKey(db, userId);
|
|
755
|
+
conditions.push(bySubject(journeyStates, { contactId, userKey: userId }));
|
|
749
756
|
}
|
|
750
757
|
|
|
751
758
|
const where = and(...conditions);
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
import { importJobs } from "@hogsend/db";
|
|
2
|
+
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
+
import { and, desc, eq } from "drizzle-orm";
|
|
4
|
+
import type { AppEnv } from "../../app.js";
|
|
5
|
+
import {
|
|
6
|
+
CONTACT_ID_BACKFILL_FORMAT,
|
|
7
|
+
contactIdBackfillTask,
|
|
8
|
+
verifyContactIdBackfill,
|
|
9
|
+
} from "../../workflows/backfill-contact-id.js";
|
|
10
|
+
|
|
11
|
+
// PRD 04 T5 — the operator force-run for the `contact_id` reconcile sweep. The
|
|
12
|
+
// worker already enqueues it at boot and re-enqueues once the newest completed
|
|
13
|
+
// sweep goes stale (D6); this route forces one NOW, e.g. after a failed run or
|
|
14
|
+
// straight after a dual-write fix. Fire-and-forget 202 + a job id, mirroring
|
|
15
|
+
// `identity.ts`'s alias-backfill trigger. Behind `requireAdmin` with the rest
|
|
16
|
+
// of `/v1/admin`.
|
|
17
|
+
|
|
18
|
+
const backfillContactIdRoute = createRoute({
|
|
19
|
+
method: "post",
|
|
20
|
+
path: "/backfill-contact-id",
|
|
21
|
+
tags: ["Admin — Maintenance"],
|
|
22
|
+
summary: "Force a contact_id backfill sweep",
|
|
23
|
+
description:
|
|
24
|
+
"Fills `contact_id` on user_events / journey_states / " +
|
|
25
|
+
"bucket_memberships / email_sends / email_preferences for rows whose " +
|
|
26
|
+
"user_id is the canonical key of a live contact (or a stale key that " +
|
|
27
|
+
"contact_aliases resolves). Chunked, paced and idempotent — every UPDATE " +
|
|
28
|
+
"is guarded by `contact_id IS NULL`, so re-running is safe and cheap. " +
|
|
29
|
+
"Rows whose key owns no live contact stay NULL by design (PRD 04 D5); " +
|
|
30
|
+
"this job is NOT a repair tool for a wrong non-NULL value.",
|
|
31
|
+
request: {
|
|
32
|
+
body: {
|
|
33
|
+
content: {
|
|
34
|
+
"application/json": {
|
|
35
|
+
schema: z.object({
|
|
36
|
+
/** Overrides for the pacing rule — measure before deploying. */
|
|
37
|
+
contactsPerChunk: z.number().int().min(1).optional(),
|
|
38
|
+
rowsPerStatement: z.number().int().min(1).optional(),
|
|
39
|
+
pauseMs: z.number().int().min(0).optional(),
|
|
40
|
+
}),
|
|
41
|
+
},
|
|
42
|
+
},
|
|
43
|
+
required: false,
|
|
44
|
+
},
|
|
45
|
+
},
|
|
46
|
+
responses: {
|
|
47
|
+
202: {
|
|
48
|
+
content: {
|
|
49
|
+
"application/json": {
|
|
50
|
+
schema: z.object({ jobId: z.string(), status: z.string() }),
|
|
51
|
+
},
|
|
52
|
+
},
|
|
53
|
+
description: "Backfill sweep queued",
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
// PRD 04 T6 — the invariant probe surfaced. Modelled on `identity.ts`'s
|
|
59
|
+
// alias-parity route: read-only, no body, counts only.
|
|
60
|
+
|
|
61
|
+
const verifyCountsSchema = z.object({
|
|
62
|
+
missing: z.number(),
|
|
63
|
+
duplicates: z.number(),
|
|
64
|
+
mismatched: z.number(),
|
|
65
|
+
orphaned: z.number(),
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
const verifyContactIdRoute = createRoute({
|
|
69
|
+
method: "get",
|
|
70
|
+
path: "/contact-id-verify",
|
|
71
|
+
tags: ["Admin — Maintenance"],
|
|
72
|
+
summary: "contact_id invariant probe — the read-flip readiness gate",
|
|
73
|
+
description:
|
|
74
|
+
"Read-only. Per table: `missing` = a live contact owns the row's user_id " +
|
|
75
|
+
"(canonically OR via an external/anonymous contact_aliases row) but " +
|
|
76
|
+
"contact_id is NULL; `mismatched` = contact_id points at a contact that " +
|
|
77
|
+
"owns that user_id NEITHER way (including a pointer at no contact at " +
|
|
78
|
+
"all) — the corruption detector an FK could never give, since an FK " +
|
|
79
|
+
"proves the uuid exists while this proves it is the RIGHT uuid; " +
|
|
80
|
+
"`orphaned` = contact_id is NULL and no live contact owns the key, which " +
|
|
81
|
+
"is EXPECTED and permitted forever (PRD 04 D5) and is never a failure; " +
|
|
82
|
+
"`duplicates` = a live contact owns the key but stamping the row would " +
|
|
83
|
+
"violate a contact-scoped unique index (PRD 05 T3) because that contact " +
|
|
84
|
+
"already holds a row for the same live journey / live bucket / address — " +
|
|
85
|
+
"the backfill skips those on purpose rather than cancelling a live " +
|
|
86
|
+
"enrollment, so they are a triage list, not a failure. " +
|
|
87
|
+
"`flipReady` is true iff every table reports missing = mismatched = 0 — " +
|
|
88
|
+
"that is the entry gate for the release that flips reads onto the " +
|
|
89
|
+
"column. Judge it only once `lastSweepAt` is non-null: before a sweep " +
|
|
90
|
+
"completes, `missing` is just pending backfill. " +
|
|
91
|
+
"**Alert posture (D6):** when `flipReady` is false AND a sweep has " +
|
|
92
|
+
"already completed, this route also emits a structured `error` log line " +
|
|
93
|
+
"— the response stays 200, the log line is what alerting hooks. A " +
|
|
94
|
+
"growing `missing` after the flip is live data loss, and a number in a " +
|
|
95
|
+
"report nobody reads is not a control. " +
|
|
96
|
+
"This is a full-table probe by nature; call it on demand, not on a timer.",
|
|
97
|
+
responses: {
|
|
98
|
+
200: {
|
|
99
|
+
content: {
|
|
100
|
+
"application/json": {
|
|
101
|
+
schema: z.object({
|
|
102
|
+
flipReady: z.boolean(),
|
|
103
|
+
totals: verifyCountsSchema,
|
|
104
|
+
tables: z.object({
|
|
105
|
+
user_events: verifyCountsSchema,
|
|
106
|
+
journey_states: verifyCountsSchema,
|
|
107
|
+
bucket_memberships: verifyCountsSchema,
|
|
108
|
+
email_sends: verifyCountsSchema,
|
|
109
|
+
email_preferences: verifyCountsSchema,
|
|
110
|
+
}),
|
|
111
|
+
/** Newest completed sweep, or null if none has ever finished. */
|
|
112
|
+
lastSweepAt: z.string().nullable(),
|
|
113
|
+
}),
|
|
114
|
+
},
|
|
115
|
+
},
|
|
116
|
+
description: "Per-table invariant counts + flip readiness",
|
|
117
|
+
},
|
|
118
|
+
},
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
export const maintenanceRouter = new OpenAPIHono<AppEnv>()
|
|
122
|
+
.openapi(verifyContactIdRoute, async (c) => {
|
|
123
|
+
const { db, logger } = c.get("container");
|
|
124
|
+
|
|
125
|
+
// Read the sweep marker BEFORE the scans: it decides whether a hole is a
|
|
126
|
+
// live signal or a pending-backfill artifact, and it should describe the
|
|
127
|
+
// world as of the moment the probe started, not several full-table scans
|
|
128
|
+
// later.
|
|
129
|
+
const [sweep] = await db
|
|
130
|
+
.select({ updatedAt: importJobs.updatedAt })
|
|
131
|
+
.from(importJobs)
|
|
132
|
+
.where(
|
|
133
|
+
and(
|
|
134
|
+
eq(importJobs.format, CONTACT_ID_BACKFILL_FORMAT),
|
|
135
|
+
eq(importJobs.status, "completed"),
|
|
136
|
+
),
|
|
137
|
+
)
|
|
138
|
+
.orderBy(desc(importJobs.updatedAt))
|
|
139
|
+
.limit(1);
|
|
140
|
+
const lastSweepAt = sweep?.updatedAt ?? null;
|
|
141
|
+
|
|
142
|
+
const result = await verifyContactIdBackfill({ db });
|
|
143
|
+
|
|
144
|
+
// D6 (revised) — REPORT is not enough. A completed sweep plus a broken
|
|
145
|
+
// invariant means the dual-write or the backfill has a hole that will not
|
|
146
|
+
// heal itself, so say so at `error` level with the counts inline. The
|
|
147
|
+
// request still succeeds: this endpoint's job is to answer, and the log
|
|
148
|
+
// line is the control.
|
|
149
|
+
if (!result.flipReady && lastSweepAt) {
|
|
150
|
+
logger.error("contact_id invariant broken after a completed sweep", {
|
|
151
|
+
route: "admin/maintenance/contact-id-verify",
|
|
152
|
+
flipReady: false,
|
|
153
|
+
totals: result.totals,
|
|
154
|
+
tables: result.tables,
|
|
155
|
+
lastSweepAt: lastSweepAt.toISOString(),
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return c.json(
|
|
160
|
+
{ ...result, lastSweepAt: lastSweepAt?.toISOString() ?? null },
|
|
161
|
+
200,
|
|
162
|
+
);
|
|
163
|
+
})
|
|
164
|
+
.openapi(backfillContactIdRoute, async (c) => {
|
|
165
|
+
const { db, logger } = c.get("container");
|
|
166
|
+
const body = c.req.valid("json") ?? {};
|
|
167
|
+
|
|
168
|
+
const [job] = await db
|
|
169
|
+
.insert(importJobs)
|
|
170
|
+
.values({
|
|
171
|
+
fileName: CONTACT_ID_BACKFILL_FORMAT,
|
|
172
|
+
format: CONTACT_ID_BACKFILL_FORMAT,
|
|
173
|
+
status: "pending",
|
|
174
|
+
})
|
|
175
|
+
.returning({ id: importJobs.id });
|
|
176
|
+
if (!job) throw new Error("Failed to create backfill job");
|
|
177
|
+
|
|
178
|
+
// Fire-and-forget 202 (the bulk.ts contract this mirrors). A failed enqueue
|
|
179
|
+
// marks the durable row `failed` so it does not sit "pending" forever and
|
|
180
|
+
// block the boot sweep's in-flight guard.
|
|
181
|
+
void contactIdBackfillTask
|
|
182
|
+
.runNoWait({
|
|
183
|
+
jobId: job.id,
|
|
184
|
+
contactsPerChunk: body.contactsPerChunk,
|
|
185
|
+
rowsPerStatement: body.rowsPerStatement,
|
|
186
|
+
pauseMs: body.pauseMs,
|
|
187
|
+
})
|
|
188
|
+
.catch(async (error: unknown) => {
|
|
189
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
190
|
+
logger.warn("maintenance/backfill-contact-id: enqueue failed", {
|
|
191
|
+
jobId: job.id,
|
|
192
|
+
error: message,
|
|
193
|
+
});
|
|
194
|
+
try {
|
|
195
|
+
await db
|
|
196
|
+
.update(importJobs)
|
|
197
|
+
.set({
|
|
198
|
+
status: "failed",
|
|
199
|
+
errors: [{ row: 0, error: `Task enqueue failed: ${message}` }],
|
|
200
|
+
updatedAt: new Date(),
|
|
201
|
+
})
|
|
202
|
+
.where(eq(importJobs.id, job.id));
|
|
203
|
+
} catch (dbError: unknown) {
|
|
204
|
+
logger.warn(
|
|
205
|
+
"maintenance/backfill-contact-id: could not mark job failed",
|
|
206
|
+
{
|
|
207
|
+
jobId: job.id,
|
|
208
|
+
error:
|
|
209
|
+
dbError instanceof Error ? dbError.message : String(dbError),
|
|
210
|
+
},
|
|
211
|
+
);
|
|
212
|
+
}
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
return c.json({ jobId: job.id, status: "pending" }, 202);
|
|
216
|
+
});
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { emailPreferences } from "@hogsend/db";
|
|
2
2
|
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
-
import { eq } from "drizzle-orm";
|
|
3
|
+
import { and, eq, sql } from "drizzle-orm";
|
|
4
4
|
import type { AppEnv } from "../../app.js";
|
|
5
5
|
import { resolveContact, serializePrefs } from "../../lib/contacts.js";
|
|
6
|
+
import {
|
|
7
|
+
isUniqueViolationOn,
|
|
8
|
+
UQ_CONTACT_EMAIL_PREFERENCES,
|
|
9
|
+
} from "../../lib/unique-violation.js";
|
|
6
10
|
|
|
7
11
|
const preferencesResponseSchema = z.object({
|
|
8
12
|
id: z.string(),
|
|
@@ -101,10 +105,12 @@ export const preferencesRouter = new OpenAPIHono<AppEnv>()
|
|
|
101
105
|
return c.json({ error: "Contact not found" }, 404);
|
|
102
106
|
}
|
|
103
107
|
|
|
108
|
+
// PRD 05 T6 — the contact is in hand: read by ownership stamp, never by
|
|
109
|
+
// the mutable string key (which goes stale on adoption).
|
|
104
110
|
const rows = await db
|
|
105
111
|
.select()
|
|
106
112
|
.from(emailPreferences)
|
|
107
|
-
.where(eq(emailPreferences.
|
|
113
|
+
.where(eq(emailPreferences.contactId, contact.id))
|
|
108
114
|
.limit(1);
|
|
109
115
|
|
|
110
116
|
if (rows.length === 0) {
|
|
@@ -127,43 +133,84 @@ export const preferencesRouter = new OpenAPIHono<AppEnv>()
|
|
|
127
133
|
if (!contact.email) {
|
|
128
134
|
return c.json({ error: "Contact has no email address" }, 400);
|
|
129
135
|
}
|
|
136
|
+
// Bound once: the narrowing from the guard above does not survive into the
|
|
137
|
+
// `upsertOnStringKey` closure below.
|
|
138
|
+
const email = contact.email;
|
|
130
139
|
|
|
131
|
-
const
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
140
|
+
const setClause = {
|
|
141
|
+
// Fill-if-known, never null-out — the same conflict arm
|
|
142
|
+
// `upsertEmailPreference` uses, so the two writers to this table
|
|
143
|
+
// cannot disagree about whether a stamp may be erased.
|
|
144
|
+
contactId: sql`coalesce(excluded.contact_id, ${emailPreferences.contactId})`,
|
|
145
|
+
...(body.unsubscribedAll !== undefined
|
|
146
|
+
? { unsubscribedAll: body.unsubscribedAll }
|
|
147
|
+
: {}),
|
|
148
|
+
...(body.suppressed !== undefined
|
|
149
|
+
? {
|
|
150
|
+
suppressed: body.suppressed,
|
|
151
|
+
suppressedAt: body.suppressed ? new Date() : null,
|
|
152
|
+
// Un-suppressing clears the bounce slate. `bounceCount` only
|
|
153
|
+
// drives the auto-suppress threshold (the send-gate keys off
|
|
154
|
+
// `suppressed`/`unsubscribedAll`), so a leftover count would
|
|
155
|
+
// otherwise keep a bounced recipient pinned to the suppression
|
|
156
|
+
// list with no way to remove them.
|
|
157
|
+
...(body.suppressed ? {} : { bounceCount: 0, lastBounceAt: null }),
|
|
158
|
+
}
|
|
159
|
+
: {}),
|
|
160
|
+
...(body.categories !== undefined ? { categories: body.categories } : {}),
|
|
161
|
+
updatedAt: new Date(),
|
|
162
|
+
};
|
|
163
|
+
|
|
164
|
+
const upsertOnStringKey = () =>
|
|
165
|
+
db
|
|
166
|
+
.insert(emailPreferences)
|
|
167
|
+
.values({
|
|
168
|
+
userId: contact.externalId ?? contact.id,
|
|
169
|
+
email,
|
|
170
|
+
// PRD 04 dual-write: this handler resolved the contact one statement
|
|
171
|
+
// ago, so the owning id is in hand — zero queries, no D6 wrapper
|
|
172
|
+
// needed (there is no new call that could throw).
|
|
173
|
+
contactId: contact.id,
|
|
174
|
+
unsubscribedAll: body.unsubscribedAll ?? false,
|
|
175
|
+
suppressed: body.suppressed ?? false,
|
|
176
|
+
categories: body.categories ?? {},
|
|
177
|
+
})
|
|
178
|
+
.onConflictDoUpdate({
|
|
179
|
+
target: [emailPreferences.userId, emailPreferences.email],
|
|
180
|
+
set: setClause,
|
|
181
|
+
})
|
|
182
|
+
.returning();
|
|
183
|
+
|
|
184
|
+
let upserted: typeof emailPreferences.$inferSelect | undefined;
|
|
185
|
+
try {
|
|
186
|
+
[upserted] = await upsertOnStringKey();
|
|
187
|
+
} catch (err) {
|
|
188
|
+
// PRD 05 T3 — the twin of the conversion in `upsertEmailPreference`; see
|
|
189
|
+
// its comment for why the arbiter stays on the string key. This route
|
|
190
|
+
// reaches the same row from the OTHER side (it keys the insert off
|
|
191
|
+
// `external_id ?? id` while an adopted row may still carry the old anon
|
|
192
|
+
// string), so it needs the same fallback or an admin preference edit
|
|
193
|
+
// would 500 on exactly the contacts adoption has touched.
|
|
194
|
+
if (!isUniqueViolationOn(err, UQ_CONTACT_EMAIL_PREFERENCES)) throw err;
|
|
195
|
+
const { contactId: _excludedCoalesce, ...contactScopedSet } = setClause;
|
|
196
|
+
[upserted] = await db
|
|
197
|
+
.update(emailPreferences)
|
|
198
|
+
.set(contactScopedSet)
|
|
199
|
+
.where(
|
|
200
|
+
and(
|
|
201
|
+
eq(emailPreferences.contactId, contact.id),
|
|
202
|
+
eq(emailPreferences.email, email),
|
|
203
|
+
),
|
|
204
|
+
)
|
|
205
|
+
.returning();
|
|
206
|
+
// The conflicting row vanished between the INSERT and the UPDATE (a
|
|
207
|
+
// concurrent merge folded it away): zero rows back, `upserted` undefined,
|
|
208
|
+
// and the throw below would turn an ordinary race into a 500 the old
|
|
209
|
+
// single-statement path could not produce. The row is gone, so the
|
|
210
|
+
// arbiter INSERT now lands — retry it ONCE. See the twin in
|
|
211
|
+
// `lib/preferences.ts`.
|
|
212
|
+
if (!upserted) [upserted] = await upsertOnStringKey();
|
|
213
|
+
}
|
|
167
214
|
|
|
168
215
|
if (!upserted) {
|
|
169
216
|
throw new Error("Failed to upsert preferences");
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { bySubject } from "@hogsend/core";
|
|
1
2
|
import { emailSends, userEvents } from "@hogsend/db";
|
|
2
3
|
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
4
|
import {
|
|
@@ -12,7 +13,7 @@ import {
|
|
|
12
13
|
sql,
|
|
13
14
|
} from "drizzle-orm";
|
|
14
15
|
import type { AppEnv } from "../../app.js";
|
|
15
|
-
import {
|
|
16
|
+
import { lookupContactIdByKey, resolveContact } from "../../lib/contacts.js";
|
|
16
17
|
import { rate, TRUNC_SQL } from "../../lib/metrics-sql.js";
|
|
17
18
|
import { errorSchema } from "../../lib/schemas.js";
|
|
18
19
|
|
|
@@ -283,9 +284,11 @@ export const reportingRouter = new OpenAPIHono<AppEnv>()
|
|
|
283
284
|
return c.json({ error: "Contact not found" }, 404);
|
|
284
285
|
}
|
|
285
286
|
|
|
286
|
-
//
|
|
287
|
-
//
|
|
288
|
-
|
|
287
|
+
// PRD 05 T6 — sends are matched by ownership stamp (the send-time string
|
|
288
|
+
// key is frozen and goes stale on adoption). The email leg stays: a
|
|
289
|
+
// journeyless/raw send carries no userId, so its stamp is NULL by design
|
|
290
|
+
// (D7) and only the address can surface it.
|
|
291
|
+
const idConds = [eq(emailSends.contactId, contact.id)];
|
|
289
292
|
if (contact.email) idConds.push(eq(emailSends.userEmail, contact.email));
|
|
290
293
|
const where = or(...idConds);
|
|
291
294
|
|
|
@@ -417,7 +420,24 @@ reportingRouter.get("/sends/export", async (c) => {
|
|
|
417
420
|
eq(emailSends.status, q.status as typeof emailSends.$inferSelect.status),
|
|
418
421
|
);
|
|
419
422
|
if (q.category) conditions.push(eq(emailSends.category, q.category));
|
|
420
|
-
if (q.userId)
|
|
423
|
+
if (q.userId) {
|
|
424
|
+
// PRD 05 T6 — resolve the caller's key to its contact so the export
|
|
425
|
+
// follows the subject across adoption; an unresolvable key degrades to
|
|
426
|
+
// the frozen string leg (bySubject's else-arm). D6-wrapped: a bookkeeping
|
|
427
|
+
// probe may never fail the export.
|
|
428
|
+
let exportContactId: string | null = null;
|
|
429
|
+
try {
|
|
430
|
+
exportContactId = await lookupContactIdByKey(db, q.userId);
|
|
431
|
+
} catch {
|
|
432
|
+
exportContactId = null;
|
|
433
|
+
}
|
|
434
|
+
conditions.push(
|
|
435
|
+
bySubject(emailSends, {
|
|
436
|
+
contactId: exportContactId,
|
|
437
|
+
userKey: q.userId,
|
|
438
|
+
}),
|
|
439
|
+
);
|
|
440
|
+
}
|
|
421
441
|
if (q.engagement && q.engagement in engagementColumn)
|
|
422
442
|
conditions.push(
|
|
423
443
|
isNotNull(
|