@hogsend/engine 0.57.0 → 0.59.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -11
- package/src/buckets/check-membership.ts +6 -0
- package/src/index.ts +16 -0
- package/src/journeys/execute-journey-run.ts +83 -1
- package/src/lib/contacts.ts +114 -0
- package/src/lib/ingestion.ts +9 -0
- package/src/lib/preferences.ts +39 -2
- package/src/lib/tracked.ts +31 -0
- package/src/routes/admin/bulk.ts +4 -0
- package/src/routes/admin/index.ts +2 -0
- package/src/routes/admin/maintenance.ts +210 -0
- package/src/routes/admin/preferences.ts +9 -1
- package/src/routes/lists/index.ts +5 -0
- package/src/worker.ts +20 -0
- package/src/workflows/backfill-contact-id.ts +771 -0
- package/src/workflows/bucket-backfill.ts +9 -1
- package/src/workflows/bucket-reconcile.ts +4 -0
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import { importJobs } from "@hogsend/db";
|
|
2
|
+
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
+
import { and, desc, eq } from "drizzle-orm";
|
|
4
|
+
import type { AppEnv } from "../../app.js";
|
|
5
|
+
import {
|
|
6
|
+
CONTACT_ID_BACKFILL_FORMAT,
|
|
7
|
+
contactIdBackfillTask,
|
|
8
|
+
verifyContactIdBackfill,
|
|
9
|
+
} from "../../workflows/backfill-contact-id.js";
|
|
10
|
+
|
|
11
|
+
// PRD 04 T5 — the operator force-run for the `contact_id` reconcile sweep. The
|
|
12
|
+
// worker already enqueues it at boot and re-enqueues once the newest completed
|
|
13
|
+
// sweep goes stale (D6); this route forces one NOW, e.g. after a failed run or
|
|
14
|
+
// straight after a dual-write fix. Fire-and-forget 202 + a job id, mirroring
|
|
15
|
+
// `identity.ts`'s alias-backfill trigger. Behind `requireAdmin` with the rest
|
|
16
|
+
// of `/v1/admin`.
|
|
17
|
+
|
|
18
|
+
const backfillContactIdRoute = createRoute({
|
|
19
|
+
method: "post",
|
|
20
|
+
path: "/backfill-contact-id",
|
|
21
|
+
tags: ["Admin — Maintenance"],
|
|
22
|
+
summary: "Force a contact_id backfill sweep",
|
|
23
|
+
description:
|
|
24
|
+
"Fills `contact_id` on user_events / journey_states / " +
|
|
25
|
+
"bucket_memberships / email_sends / email_preferences for rows whose " +
|
|
26
|
+
"user_id is the canonical key of a live contact (or a stale key that " +
|
|
27
|
+
"contact_aliases resolves). Chunked, paced and idempotent — every UPDATE " +
|
|
28
|
+
"is guarded by `contact_id IS NULL`, so re-running is safe and cheap. " +
|
|
29
|
+
"Rows whose key owns no live contact stay NULL by design (PRD 04 D5); " +
|
|
30
|
+
"this job is NOT a repair tool for a wrong non-NULL value.",
|
|
31
|
+
request: {
|
|
32
|
+
body: {
|
|
33
|
+
content: {
|
|
34
|
+
"application/json": {
|
|
35
|
+
schema: z.object({
|
|
36
|
+
/** Overrides for the pacing rule — measure before deploying. */
|
|
37
|
+
contactsPerChunk: z.number().int().min(1).optional(),
|
|
38
|
+
rowsPerStatement: z.number().int().min(1).optional(),
|
|
39
|
+
pauseMs: z.number().int().min(0).optional(),
|
|
40
|
+
}),
|
|
41
|
+
},
|
|
42
|
+
},
|
|
43
|
+
required: false,
|
|
44
|
+
},
|
|
45
|
+
},
|
|
46
|
+
responses: {
|
|
47
|
+
202: {
|
|
48
|
+
content: {
|
|
49
|
+
"application/json": {
|
|
50
|
+
schema: z.object({ jobId: z.string(), status: z.string() }),
|
|
51
|
+
},
|
|
52
|
+
},
|
|
53
|
+
description: "Backfill sweep queued",
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
// PRD 04 T6 — the invariant probe surfaced. Modelled on `identity.ts`'s
|
|
59
|
+
// alias-parity route: read-only, no body, counts only.
|
|
60
|
+
|
|
61
|
+
const verifyCountsSchema = z.object({
|
|
62
|
+
missing: z.number(),
|
|
63
|
+
mismatched: z.number(),
|
|
64
|
+
orphaned: z.number(),
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
const verifyContactIdRoute = createRoute({
|
|
68
|
+
method: "get",
|
|
69
|
+
path: "/contact-id-verify",
|
|
70
|
+
tags: ["Admin — Maintenance"],
|
|
71
|
+
summary: "contact_id invariant probe — the read-flip readiness gate",
|
|
72
|
+
description:
|
|
73
|
+
"Read-only. Per table: `missing` = a live contact owns the row's user_id " +
|
|
74
|
+
"(canonically OR via an external/anonymous contact_aliases row) but " +
|
|
75
|
+
"contact_id is NULL; `mismatched` = contact_id points at a contact that " +
|
|
76
|
+
"owns that user_id NEITHER way (including a pointer at no contact at " +
|
|
77
|
+
"all) — the corruption detector an FK could never give, since an FK " +
|
|
78
|
+
"proves the uuid exists while this proves it is the RIGHT uuid; " +
|
|
79
|
+
"`orphaned` = contact_id is NULL and no live contact owns the key, which " +
|
|
80
|
+
"is EXPECTED and permitted forever (PRD 04 D5) and is never a failure. " +
|
|
81
|
+
"`flipReady` is true iff every table reports missing = mismatched = 0 — " +
|
|
82
|
+
"that is the entry gate for the release that flips reads onto the " +
|
|
83
|
+
"column. Judge it only once `lastSweepAt` is non-null: before a sweep " +
|
|
84
|
+
"completes, `missing` is just pending backfill. " +
|
|
85
|
+
"**Alert posture (D6):** when `flipReady` is false AND a sweep has " +
|
|
86
|
+
"already completed, this route also emits a structured `error` log line " +
|
|
87
|
+
"— the response stays 200, the log line is what alerting hooks. A " +
|
|
88
|
+
"growing `missing` after the flip is live data loss, and a number in a " +
|
|
89
|
+
"report nobody reads is not a control. " +
|
|
90
|
+
"This is a full-table probe by nature; call it on demand, not on a timer.",
|
|
91
|
+
responses: {
|
|
92
|
+
200: {
|
|
93
|
+
content: {
|
|
94
|
+
"application/json": {
|
|
95
|
+
schema: z.object({
|
|
96
|
+
flipReady: z.boolean(),
|
|
97
|
+
totals: verifyCountsSchema,
|
|
98
|
+
tables: z.object({
|
|
99
|
+
user_events: verifyCountsSchema,
|
|
100
|
+
journey_states: verifyCountsSchema,
|
|
101
|
+
bucket_memberships: verifyCountsSchema,
|
|
102
|
+
email_sends: verifyCountsSchema,
|
|
103
|
+
email_preferences: verifyCountsSchema,
|
|
104
|
+
}),
|
|
105
|
+
/** Newest completed sweep, or null if none has ever finished. */
|
|
106
|
+
lastSweepAt: z.string().nullable(),
|
|
107
|
+
}),
|
|
108
|
+
},
|
|
109
|
+
},
|
|
110
|
+
description: "Per-table invariant counts + flip readiness",
|
|
111
|
+
},
|
|
112
|
+
},
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
export const maintenanceRouter = new OpenAPIHono<AppEnv>()
|
|
116
|
+
.openapi(verifyContactIdRoute, async (c) => {
|
|
117
|
+
const { db, logger } = c.get("container");
|
|
118
|
+
|
|
119
|
+
// Read the sweep marker BEFORE the scans: it decides whether a hole is a
|
|
120
|
+
// live signal or a pending-backfill artifact, and it should describe the
|
|
121
|
+
// world as of the moment the probe started, not several full-table scans
|
|
122
|
+
// later.
|
|
123
|
+
const [sweep] = await db
|
|
124
|
+
.select({ updatedAt: importJobs.updatedAt })
|
|
125
|
+
.from(importJobs)
|
|
126
|
+
.where(
|
|
127
|
+
and(
|
|
128
|
+
eq(importJobs.format, CONTACT_ID_BACKFILL_FORMAT),
|
|
129
|
+
eq(importJobs.status, "completed"),
|
|
130
|
+
),
|
|
131
|
+
)
|
|
132
|
+
.orderBy(desc(importJobs.updatedAt))
|
|
133
|
+
.limit(1);
|
|
134
|
+
const lastSweepAt = sweep?.updatedAt ?? null;
|
|
135
|
+
|
|
136
|
+
const result = await verifyContactIdBackfill({ db });
|
|
137
|
+
|
|
138
|
+
// D6 (revised) — REPORT is not enough. A completed sweep plus a broken
|
|
139
|
+
// invariant means the dual-write or the backfill has a hole that will not
|
|
140
|
+
// heal itself, so say so at `error` level with the counts inline. The
|
|
141
|
+
// request still succeeds: this endpoint's job is to answer, and the log
|
|
142
|
+
// line is the control.
|
|
143
|
+
if (!result.flipReady && lastSweepAt) {
|
|
144
|
+
logger.error("contact_id invariant broken after a completed sweep", {
|
|
145
|
+
route: "admin/maintenance/contact-id-verify",
|
|
146
|
+
flipReady: false,
|
|
147
|
+
totals: result.totals,
|
|
148
|
+
tables: result.tables,
|
|
149
|
+
lastSweepAt: lastSweepAt.toISOString(),
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
return c.json(
|
|
154
|
+
{ ...result, lastSweepAt: lastSweepAt?.toISOString() ?? null },
|
|
155
|
+
200,
|
|
156
|
+
);
|
|
157
|
+
})
|
|
158
|
+
.openapi(backfillContactIdRoute, async (c) => {
|
|
159
|
+
const { db, logger } = c.get("container");
|
|
160
|
+
const body = c.req.valid("json") ?? {};
|
|
161
|
+
|
|
162
|
+
const [job] = await db
|
|
163
|
+
.insert(importJobs)
|
|
164
|
+
.values({
|
|
165
|
+
fileName: CONTACT_ID_BACKFILL_FORMAT,
|
|
166
|
+
format: CONTACT_ID_BACKFILL_FORMAT,
|
|
167
|
+
status: "pending",
|
|
168
|
+
})
|
|
169
|
+
.returning({ id: importJobs.id });
|
|
170
|
+
if (!job) throw new Error("Failed to create backfill job");
|
|
171
|
+
|
|
172
|
+
// Fire-and-forget 202 (the bulk.ts contract this mirrors). A failed enqueue
|
|
173
|
+
// marks the durable row `failed` so it does not sit "pending" forever and
|
|
174
|
+
// block the boot sweep's in-flight guard.
|
|
175
|
+
void contactIdBackfillTask
|
|
176
|
+
.runNoWait({
|
|
177
|
+
jobId: job.id,
|
|
178
|
+
contactsPerChunk: body.contactsPerChunk,
|
|
179
|
+
rowsPerStatement: body.rowsPerStatement,
|
|
180
|
+
pauseMs: body.pauseMs,
|
|
181
|
+
})
|
|
182
|
+
.catch(async (error: unknown) => {
|
|
183
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
184
|
+
logger.warn("maintenance/backfill-contact-id: enqueue failed", {
|
|
185
|
+
jobId: job.id,
|
|
186
|
+
error: message,
|
|
187
|
+
});
|
|
188
|
+
try {
|
|
189
|
+
await db
|
|
190
|
+
.update(importJobs)
|
|
191
|
+
.set({
|
|
192
|
+
status: "failed",
|
|
193
|
+
errors: [{ row: 0, error: `Task enqueue failed: ${message}` }],
|
|
194
|
+
updatedAt: new Date(),
|
|
195
|
+
})
|
|
196
|
+
.where(eq(importJobs.id, job.id));
|
|
197
|
+
} catch (dbError: unknown) {
|
|
198
|
+
logger.warn(
|
|
199
|
+
"maintenance/backfill-contact-id: could not mark job failed",
|
|
200
|
+
{
|
|
201
|
+
jobId: job.id,
|
|
202
|
+
error:
|
|
203
|
+
dbError instanceof Error ? dbError.message : String(dbError),
|
|
204
|
+
},
|
|
205
|
+
);
|
|
206
|
+
}
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
return c.json({ jobId: job.id, status: "pending" }, 202);
|
|
210
|
+
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { emailPreferences } from "@hogsend/db";
|
|
2
2
|
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
-
import { eq } from "drizzle-orm";
|
|
3
|
+
import { eq, sql } from "drizzle-orm";
|
|
4
4
|
import type { AppEnv } from "../../app.js";
|
|
5
5
|
import { resolveContact, serializePrefs } from "../../lib/contacts.js";
|
|
6
6
|
|
|
@@ -133,6 +133,10 @@ export const preferencesRouter = new OpenAPIHono<AppEnv>()
|
|
|
133
133
|
.values({
|
|
134
134
|
userId: contact.externalId ?? contact.id,
|
|
135
135
|
email: contact.email,
|
|
136
|
+
// PRD 04 dual-write: this handler resolved the contact one statement
|
|
137
|
+
// ago, so the owning id is in hand — zero queries, no D6 wrapper needed
|
|
138
|
+
// (there is no new call that could throw).
|
|
139
|
+
contactId: contact.id,
|
|
136
140
|
unsubscribedAll: body.unsubscribedAll ?? false,
|
|
137
141
|
suppressed: body.suppressed ?? false,
|
|
138
142
|
categories: body.categories ?? {},
|
|
@@ -140,6 +144,10 @@ export const preferencesRouter = new OpenAPIHono<AppEnv>()
|
|
|
140
144
|
.onConflictDoUpdate({
|
|
141
145
|
target: [emailPreferences.userId, emailPreferences.email],
|
|
142
146
|
set: {
|
|
147
|
+
// Fill-if-known, never null-out — the same conflict arm
|
|
148
|
+
// `upsertEmailPreference` uses, so the two writers to this table
|
|
149
|
+
// cannot disagree about whether a stamp may be erased.
|
|
150
|
+
contactId: sql`coalesce(excluded.contact_id, ${emailPreferences.contactId})`,
|
|
143
151
|
...(body.unsubscribedAll !== undefined
|
|
144
152
|
? { unsubscribedAll: body.unsubscribedAll }
|
|
145
153
|
: {}),
|
|
@@ -541,6 +541,11 @@ export const listsRouter = new OpenAPIHono<AppEnv>()
|
|
|
541
541
|
externalId,
|
|
542
542
|
email: recipient.email,
|
|
543
543
|
update: { unsubscribedAll },
|
|
544
|
+
// PRD 04 dual-write: the id is ALREADY in hand from the
|
|
545
|
+
// `resolveOrCreateContact` above (this handler resolves/creates the row
|
|
546
|
+
// before writing preferences), so pass it rather than making
|
|
547
|
+
// `upsertEmailPreference` re-probe the same contact.
|
|
548
|
+
contactId,
|
|
544
549
|
});
|
|
545
550
|
|
|
546
551
|
return c.json({ unsubscribedAll }, 200);
|
package/src/worker.ts
CHANGED
|
@@ -18,6 +18,10 @@ import { reportWorkerReady } from "./lib/boot.js";
|
|
|
18
18
|
import { hatchet } from "./lib/hatchet.js";
|
|
19
19
|
import { getRedisIfConnected } from "./lib/redis.js";
|
|
20
20
|
import { startWorkerHeartbeat } from "./lib/worker-heartbeat.js";
|
|
21
|
+
import {
|
|
22
|
+
contactIdBackfillTask,
|
|
23
|
+
enqueueContactIdBackfill,
|
|
24
|
+
} from "./workflows/backfill-contact-id.js";
|
|
21
25
|
import {
|
|
22
26
|
bucketBackfillTask,
|
|
23
27
|
enqueueBucketBackfills,
|
|
@@ -137,6 +141,7 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
|
|
|
137
141
|
bucketReconcileTask,
|
|
138
142
|
bucketBackfillTask,
|
|
139
143
|
identityAliasBackfillTask,
|
|
144
|
+
contactIdBackfillTask,
|
|
140
145
|
crmReconcileTask,
|
|
141
146
|
dispatchConversionTask,
|
|
142
147
|
...journeyTasks,
|
|
@@ -246,6 +251,21 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
|
|
|
246
251
|
});
|
|
247
252
|
});
|
|
248
253
|
|
|
254
|
+
// PRD 04 T5: fill the five history tables' `contact_id` for rows the
|
|
255
|
+
// dual-write never saw. Unlike the alias backfill this is a PERIODIC
|
|
256
|
+
// reconcile sweep (D6) — the enqueue re-fires once the newest completed
|
|
257
|
+
// sweep is older than CONTACT_ID_BACKFILL_RESWEEP_HOURS (default 24h),
|
|
258
|
+
// because the dual-write is best-effort and a miss is otherwise permanent.
|
|
259
|
+
// Cheap by construction: every UPDATE is guarded by `contact_id IS NULL`.
|
|
260
|
+
enqueueContactIdBackfill({
|
|
261
|
+
db: container.db,
|
|
262
|
+
logger: container.logger,
|
|
263
|
+
}).catch((err) => {
|
|
264
|
+
container.logger.warn("Contact-id backfill enqueue (boot) failed", {
|
|
265
|
+
error: err instanceof Error ? err.message : String(err),
|
|
266
|
+
});
|
|
267
|
+
});
|
|
268
|
+
|
|
249
269
|
// Reconcile code-defined campaigns into scheduled rows (create / sync /
|
|
250
270
|
// expire — see `reconcileDefinedCampaigns`). Same fire-and-forget stance
|
|
251
271
|
// as the bucket backfills above: best-effort, never blocks the listener.
|