@hogsend/engine 0.55.0 → 0.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -11
- package/src/buckets/bucket-access.ts +5 -10
- package/src/buckets/check-membership.ts +91 -6
- package/src/index.ts +31 -1
- package/src/journeys/execute-journey-run.ts +56 -1
- package/src/journeys/journey-context.ts +59 -0
- package/src/lib/bucket-emit.test.ts +83 -2
- package/src/lib/bucket-emit.ts +16 -0
- package/src/lib/contacts.ts +1031 -141
- package/src/lib/crm-ingest.ts +9 -0
- package/src/lib/enrichment-ledger.ts +1 -1
- package/src/lib/feed.ts +64 -3
- package/src/lib/identity-service.ts +15 -2
- package/src/lib/ingestion.ts +191 -14
- package/src/lib/outbound.ts +3 -3
- package/src/lib/refine-chain.test.ts +83 -25
- package/src/lib/refine-chain.ts +124 -23
- package/src/lib/refine-traits.test.ts +74 -41
- package/src/lib/refine-traits.ts +107 -52
- package/src/lib/refine.ts +28 -27
- package/src/lib/tracking-events.ts +33 -6
- package/src/routes/admin/agent.ts +26 -0
- package/src/routes/admin/contacts.ts +65 -5
- package/src/routes/admin/identity.ts +197 -0
- package/src/routes/admin/index.ts +2 -0
- package/src/routes/contacts/index.ts +50 -7
- package/src/routes/events/index.ts +94 -6
- package/src/routes/feed/index.ts +36 -0
- package/src/routes/feed/recipient.ts +44 -2
- package/src/routes/lists/index.ts +21 -0
- package/src/routes/tracking/arrive.ts +42 -3
- package/src/worker.ts +18 -0
- package/src/workflows/bucket-backfill.ts +1 -1
- package/src/workflows/bucket-reconcile.ts +129 -4
- package/src/workflows/identity-alias-backfill.ts +448 -0
- package/src/workflows/import-contacts.ts +10 -0
- package/src/workflows/send-campaign.ts +31 -13
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
import { importJobs } from "@hogsend/db";
|
|
2
|
+
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
+
import { eq } from "drizzle-orm";
|
|
4
|
+
import type { AppEnv } from "../../app.js";
|
|
5
|
+
import { errorSchema } from "../../lib/schemas.js";
|
|
6
|
+
import {
|
|
7
|
+
IDENTITY_ALIAS_BACKFILL_FORMAT,
|
|
8
|
+
identityAliasBackfillTask,
|
|
9
|
+
identityAliasParity,
|
|
10
|
+
} from "../../workflows/identity-alias-backfill.js";
|
|
11
|
+
|
|
12
|
+
// PRD 02 T3/T4 — the identity-table admin surface: trigger/poll the alias
|
|
13
|
+
// backfill, and the read-only parity verifier that gates trusting alias-first
|
|
14
|
+
// resolution. Mirrors the `/contacts/import` job-trigger + status-poll pattern
|
|
15
|
+
// in `bulk.ts`. Behind `requireAdmin` like the rest of `/v1/admin`.
|
|
16
|
+
|
|
17
|
+
const triggerRoute = createRoute({
|
|
18
|
+
method: "post",
|
|
19
|
+
path: "/identity/alias-backfill",
|
|
20
|
+
tags: ["Admin — Identity"],
|
|
21
|
+
summary: "Run the contact_aliases backfill",
|
|
22
|
+
description:
|
|
23
|
+
"Fills contact_aliases with a row per identity key of every live " +
|
|
24
|
+
"contact (chunked, idempotent, resumable — safe to re-run). The worker " +
|
|
25
|
+
"also enqueues this once at boot; this route forces a re-run, e.g. after " +
|
|
26
|
+
"a failed job or to re-verify. `dryRun: true` classifies and counts " +
|
|
27
|
+
"without writing.",
|
|
28
|
+
request: {
|
|
29
|
+
body: {
|
|
30
|
+
content: {
|
|
31
|
+
"application/json": {
|
|
32
|
+
schema: z.object({ dryRun: z.boolean().default(false) }),
|
|
33
|
+
},
|
|
34
|
+
},
|
|
35
|
+
required: false,
|
|
36
|
+
},
|
|
37
|
+
},
|
|
38
|
+
responses: {
|
|
39
|
+
202: {
|
|
40
|
+
content: {
|
|
41
|
+
"application/json": {
|
|
42
|
+
schema: z.object({ jobId: z.string(), status: z.string() }),
|
|
43
|
+
},
|
|
44
|
+
},
|
|
45
|
+
description: "Backfill job queued",
|
|
46
|
+
},
|
|
47
|
+
},
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
const statusRoute = createRoute({
|
|
51
|
+
method: "get",
|
|
52
|
+
path: "/identity/alias-backfill/{jobId}",
|
|
53
|
+
tags: ["Admin — Identity"],
|
|
54
|
+
summary: "Get alias-backfill job status",
|
|
55
|
+
description:
|
|
56
|
+
"processedRows = live contacts scanned; failedRows = pairs whose " +
|
|
57
|
+
"(kind, value) is claimed by a DIFFERENT contact (the divergence metric, " +
|
|
58
|
+
"not an error count). The full breakdown is in the worker log line.",
|
|
59
|
+
request: {
|
|
60
|
+
params: z.object({ jobId: z.string().uuid() }),
|
|
61
|
+
},
|
|
62
|
+
responses: {
|
|
63
|
+
200: {
|
|
64
|
+
content: {
|
|
65
|
+
"application/json": {
|
|
66
|
+
schema: z.object({
|
|
67
|
+
id: z.string(),
|
|
68
|
+
status: z.string(),
|
|
69
|
+
totalRows: z.number().nullable(),
|
|
70
|
+
processedRows: z.number(),
|
|
71
|
+
failedRows: z.number(),
|
|
72
|
+
errors: z
|
|
73
|
+
.array(z.object({ row: z.number(), error: z.string() }))
|
|
74
|
+
.nullable(),
|
|
75
|
+
createdAt: z.string(),
|
|
76
|
+
updatedAt: z.string(),
|
|
77
|
+
}),
|
|
78
|
+
},
|
|
79
|
+
},
|
|
80
|
+
description: "Backfill job details",
|
|
81
|
+
},
|
|
82
|
+
404: {
|
|
83
|
+
content: { "application/json": { schema: errorSchema } },
|
|
84
|
+
description: "Job not found",
|
|
85
|
+
},
|
|
86
|
+
},
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
const parityRoute = createRoute({
|
|
90
|
+
method: "get",
|
|
91
|
+
path: "/identity/alias-parity",
|
|
92
|
+
tags: ["Admin — Identity"],
|
|
93
|
+
summary: "Alias vs column resolution parity, per kind",
|
|
94
|
+
description:
|
|
95
|
+
"Read-only. `diverged` MUST be 0 before trusting alias-first " +
|
|
96
|
+
"resolution — a non-zero value is a data bug to fix on its own. " +
|
|
97
|
+
"`aliasDead` is expected (handled by the live-target rule); `aliasOnly` " +
|
|
98
|
+
"is the merge/promote population and, after PRD 03, the point of the " +
|
|
99
|
+
"table.",
|
|
100
|
+
responses: {
|
|
101
|
+
200: {
|
|
102
|
+
content: {
|
|
103
|
+
"application/json": {
|
|
104
|
+
schema: z.object({
|
|
105
|
+
kinds: z.array(
|
|
106
|
+
z.object({
|
|
107
|
+
kind: z.string(),
|
|
108
|
+
diverged: z.number(),
|
|
109
|
+
aliasDead: z.number(),
|
|
110
|
+
aliasOnly: z.number(),
|
|
111
|
+
}),
|
|
112
|
+
),
|
|
113
|
+
}),
|
|
114
|
+
},
|
|
115
|
+
},
|
|
116
|
+
description: "Parity counts per alias kind",
|
|
117
|
+
},
|
|
118
|
+
},
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
export const identityRouter = new OpenAPIHono<AppEnv>()
|
|
122
|
+
.openapi(triggerRoute, async (c) => {
|
|
123
|
+
const { db, logger } = c.get("container");
|
|
124
|
+
const body = c.req.valid("json") ?? { dryRun: false };
|
|
125
|
+
|
|
126
|
+
const [job] = await db
|
|
127
|
+
.insert(importJobs)
|
|
128
|
+
.values({
|
|
129
|
+
fileName: IDENTITY_ALIAS_BACKFILL_FORMAT,
|
|
130
|
+
format: IDENTITY_ALIAS_BACKFILL_FORMAT,
|
|
131
|
+
status: "pending",
|
|
132
|
+
})
|
|
133
|
+
.returning({ id: importJobs.id });
|
|
134
|
+
if (!job) throw new Error("Failed to create backfill job");
|
|
135
|
+
|
|
136
|
+
// Fire-and-forget 202 + status poll (the bulk.ts contract). A failed
|
|
137
|
+
// enqueue marks the durable row `failed` so pollers get a terminal state.
|
|
138
|
+
void identityAliasBackfillTask
|
|
139
|
+
.runNoWait({ jobId: job.id, dryRun: body.dryRun })
|
|
140
|
+
.catch(async (error: unknown) => {
|
|
141
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
142
|
+
logger.warn("identity/alias-backfill: task enqueue failed", {
|
|
143
|
+
jobId: job.id,
|
|
144
|
+
error: message,
|
|
145
|
+
});
|
|
146
|
+
try {
|
|
147
|
+
await db
|
|
148
|
+
.update(importJobs)
|
|
149
|
+
.set({
|
|
150
|
+
status: "failed",
|
|
151
|
+
errors: [{ row: 0, error: `Task enqueue failed: ${message}` }],
|
|
152
|
+
updatedAt: new Date(),
|
|
153
|
+
})
|
|
154
|
+
.where(eq(importJobs.id, job.id));
|
|
155
|
+
} catch (dbError: unknown) {
|
|
156
|
+
logger.warn("identity/alias-backfill: could not mark job failed", {
|
|
157
|
+
jobId: job.id,
|
|
158
|
+
error: dbError instanceof Error ? dbError.message : String(dbError),
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
return c.json({ jobId: job.id, status: "pending" }, 202);
|
|
164
|
+
})
|
|
165
|
+
.openapi(statusRoute, async (c) => {
|
|
166
|
+
const { db } = c.get("container");
|
|
167
|
+
const { jobId } = c.req.valid("param");
|
|
168
|
+
|
|
169
|
+
const rows = await db
|
|
170
|
+
.select()
|
|
171
|
+
.from(importJobs)
|
|
172
|
+
.where(eq(importJobs.id, jobId))
|
|
173
|
+
.limit(1);
|
|
174
|
+
const job = rows[0];
|
|
175
|
+
if (!job || job.format !== IDENTITY_ALIAS_BACKFILL_FORMAT) {
|
|
176
|
+
return c.json({ error: "Backfill job not found" }, 404);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
return c.json(
|
|
180
|
+
{
|
|
181
|
+
id: job.id,
|
|
182
|
+
status: job.status,
|
|
183
|
+
totalRows: job.totalRows,
|
|
184
|
+
processedRows: job.processedRows,
|
|
185
|
+
failedRows: job.failedRows,
|
|
186
|
+
errors: job.errors as Array<{ row: number; error: string }> | null,
|
|
187
|
+
createdAt: job.createdAt.toISOString(),
|
|
188
|
+
updatedAt: job.updatedAt.toISOString(),
|
|
189
|
+
},
|
|
190
|
+
200,
|
|
191
|
+
);
|
|
192
|
+
})
|
|
193
|
+
.openapi(parityRoute, async (c) => {
|
|
194
|
+
const { db } = c.get("container");
|
|
195
|
+
const kinds = await identityAliasParity(db);
|
|
196
|
+
return c.json({ kinds }, 200);
|
|
197
|
+
});
|
|
@@ -24,6 +24,7 @@ import { eventsRouter } from "./events.js";
|
|
|
24
24
|
import { adminFlagsRouter } from "./flags.js";
|
|
25
25
|
import { adminFunnelsRouter } from "./funnels.js";
|
|
26
26
|
import { groupsRouter } from "./groups.js";
|
|
27
|
+
import { identityRouter } from "./identity.js";
|
|
27
28
|
import { impactOverviewRouter } from "./impact.js";
|
|
28
29
|
import { journeyImpactRouter } from "./journey-impact.js";
|
|
29
30
|
import { journeyLogsRouter } from "./journey-logs.js";
|
|
@@ -48,6 +49,7 @@ adminRouter.use("*", requireAdmin);
|
|
|
48
49
|
adminRouter.use("*", rateLimit);
|
|
49
50
|
adminRouter.use("*", auditMiddleware);
|
|
50
51
|
adminRouter.route("/", bulkRouter);
|
|
52
|
+
adminRouter.route("/", identityRouter);
|
|
51
53
|
adminRouter.route("/contacts", contactsRouter);
|
|
52
54
|
adminRouter.route("/contacts", preferencesRouter);
|
|
53
55
|
adminRouter.route("/contacts", timelineRouter);
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
2
2
|
import type { AppEnv } from "../../app.js";
|
|
3
3
|
import {
|
|
4
|
+
ALL_IDENTITY_KINDS,
|
|
4
5
|
findContacts,
|
|
5
6
|
PublishableAnonymousMergeError,
|
|
6
7
|
resolveContact,
|
|
@@ -45,8 +46,25 @@ const upsertRoute = createRoute({
|
|
|
45
46
|
email: z.string().email().optional(),
|
|
46
47
|
userId: z.string().min(1).optional(),
|
|
47
48
|
// §4: caller's analytics anon id — the resolver's 2nd-precedence
|
|
48
|
-
// key
|
|
49
|
-
//
|
|
49
|
+
// key, normally an EXTRA alongside email/userId rather than an
|
|
50
|
+
// identity arm of its own.
|
|
51
|
+
//
|
|
52
|
+
// NOT a guarantee, on the publishable path. The handler gates with
|
|
53
|
+
// `gatePublishableIdentity`, NOT `requireIdentity`, and that gate
|
|
54
|
+
// returns early for a pk_ caller who claims nothing
|
|
55
|
+
// (`routes/_shared.ts` — "No claimed identity → anon-only,
|
|
56
|
+
// allowed"). The create arm of `resolveContactShared` carries no
|
|
57
|
+
// `restrictToAnonymous` guard either — that flag bites only
|
|
58
|
+
// fill-in-link and collide-merge. So a hand-rolled `pk_` fetch
|
|
59
|
+
// sending ONLY `anonymousId` still MINTS an anonymous-only contact.
|
|
60
|
+
//
|
|
61
|
+
// Known and deliberately deferred (ghost-contacts BACKLOG): refusing
|
|
62
|
+
// here would force `id: z.string().nullable()` on the published
|
|
63
|
+
// response schema plus a `@hogsend/client` type widening. No
|
|
64
|
+
// first-party SDK emits that shape — `@hogsend/js` `identify()`
|
|
65
|
+
// always sends a `userId` — so the residual is a hand-rolled
|
|
66
|
+
// request, not an SDK path. Do not "fix" this comment back into a
|
|
67
|
+
// guarantee the code does not provide.
|
|
50
68
|
anonymousId: z.string().min(1).max(200).optional(),
|
|
51
69
|
properties: z.record(z.string(), z.unknown()).optional(),
|
|
52
70
|
lists: z.record(z.string(), z.boolean()).optional(),
|
|
@@ -164,11 +182,36 @@ export const contactsRouter = new OpenAPIHono<AppEnv>()
|
|
|
164
182
|
// §4: 2nd-precedence resolver key (zero-merge stitch).
|
|
165
183
|
anonymousId: body.anonymousId,
|
|
166
184
|
contactProperties: body.properties,
|
|
167
|
-
//
|
|
168
|
-
//
|
|
169
|
-
// an
|
|
170
|
-
//
|
|
171
|
-
|
|
185
|
+
// PRD 06 T3 (L5 rows 4-5): trust is DECLARED by this caller, not
|
|
186
|
+
// re-inferred inside the resolver.
|
|
187
|
+
// - `create` — an upsert is an identity assertion, never observation:
|
|
188
|
+
// this route NEVER refuses to create, on either arm.
|
|
189
|
+
// - `allowMerge` — §Phase 1 GAP-1: a publishable (pk_) browser upsert
|
|
190
|
+
// is anon-clamped — its browser-readable `anonymousId` may NOT
|
|
191
|
+
// attach to / merge / poison an already-identified victim contact.
|
|
192
|
+
// Kept "anonymous-only" even when a verified token proves a
|
|
193
|
+
// `userId`: the clamp is inert there by DERIVATION (a non-anon key
|
|
194
|
+
// is present), never by hard-coding "any". Secret-key upserts are
|
|
195
|
+
// never clamped.
|
|
196
|
+
// - `trustedKinds` — from the gate's own evidence: a publishable
|
|
197
|
+
// `userId` reaching this call is token-proven
|
|
198
|
+
// (`gatePublishableIdentity` 403'd every other identity-claiming
|
|
199
|
+
// shape), so pk_ = `anonymous` (+`external` with that proof);
|
|
200
|
+
// secret = all four. Declared now, enforced by T5.
|
|
201
|
+
policy:
|
|
202
|
+
c.get("publishable") === true
|
|
203
|
+
? {
|
|
204
|
+
create: "on-miss",
|
|
205
|
+
allowMerge: "anonymous-only",
|
|
206
|
+
trustedKinds: body.userId
|
|
207
|
+
? ["anonymous", "external"]
|
|
208
|
+
: ["anonymous"],
|
|
209
|
+
}
|
|
210
|
+
: {
|
|
211
|
+
create: "on-miss",
|
|
212
|
+
allowMerge: "any",
|
|
213
|
+
trustedKinds: ALL_IDENTITY_KINDS,
|
|
214
|
+
},
|
|
172
215
|
});
|
|
173
216
|
} catch (err) {
|
|
174
217
|
if (err instanceof PublishableAnonymousMergeError) {
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
2
2
|
import type { AppEnv } from "../../app.js";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
ALL_IDENTITY_KINDS,
|
|
5
|
+
PublishableAnonymousMergeError,
|
|
6
|
+
} from "../../lib/contacts.js";
|
|
4
7
|
import { ingestEvent } from "../../lib/ingestion.js";
|
|
5
8
|
import { applyListMembership } from "../../lib/preferences.js";
|
|
6
9
|
import { errorSchema } from "../../lib/schemas.js";
|
|
@@ -115,6 +118,61 @@ export const eventsRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
115
118
|
const headerKey = c.req.header("idempotency-key");
|
|
116
119
|
const idempotencyKey = headerKey ?? body.idempotencyKey;
|
|
117
120
|
|
|
121
|
+
// D1 — a contact is minted by IDENTITY, not by observation. A publishable
|
|
122
|
+
// (pk_) browser write that asserts nothing is pure traffic: it still stores
|
|
123
|
+
// in `user_events`, still routes to journeys, still checks exits (D2), but
|
|
124
|
+
// it no longer mints a `contacts` row for every unidentified visitor.
|
|
125
|
+
//
|
|
126
|
+
// Every conjunct is load-bearing:
|
|
127
|
+
// - `publishable` — the caller CLASS is the whole guard (D1 is
|
|
128
|
+
// caller-keyed, not endpoint-keyed). A secret-key server caller asserts
|
|
129
|
+
// intent and is never refused.
|
|
130
|
+
// - `!body.userId` — a userId on this path is already token-proven by
|
|
131
|
+
// `gatePublishableIdentity` above, i.e. a server-authorized identity
|
|
132
|
+
// assertion. It keeps creating.
|
|
133
|
+
// - `!body.email` — an asserted email is an identity assertion too. It is
|
|
134
|
+
// also a D8 PRECONDITION: `resolveContactNoCreate` THROWS unless the
|
|
135
|
+
// highest-precedence key is `userId`/`anonymousId`, and `email`
|
|
136
|
+
// outranks `anonymousId` in the resolver. (The gate 403s a publishable
|
|
137
|
+
// email today, so this conjunct is defense in depth against a future
|
|
138
|
+
// email arm silently becoming a 500.)
|
|
139
|
+
// - `body.value === undefined` — D9's escape hatch. `conversions`,
|
|
140
|
+
// `funnel_progress` and `deals` all hold `.notNull()` FKs to
|
|
141
|
+
// `contacts.id` and `packages/db` is out of boundary, so a refused
|
|
142
|
+
// ingest can perform no conversion, funnel or attribution work at all. A
|
|
143
|
+
// money-bearing browser event is therefore treated as an identity
|
|
144
|
+
// assertion so e-commerce revenue conversions and attribution credits
|
|
145
|
+
// keep firing.
|
|
146
|
+
// - `body.groups === undefined` — D10, the same hatch for the same reason.
|
|
147
|
+
// `group_memberships.contact_id` is a fourth `.notNull()` FK to
|
|
148
|
+
// `contacts.id`, so a refused ingest writes NEITHER the `groups` row
|
|
149
|
+
// (minted by `associateGroups` → `resolveGroupId`) NOR the membership.
|
|
150
|
+
// But browser group ASSOCIATION is a documented first-class capability —
|
|
151
|
+
// root `CLAUDE.md`: "Publishable/browser keys may ONLY associate —
|
|
152
|
+
// attach a `groups` map to an event via `hogsend.group()` → /v1/events"
|
|
153
|
+
// — and `@hogsend/js` attaches the map to every capture regardless of
|
|
154
|
+
// identification, so a pre-login `hogsend.group()` sends exactly this
|
|
155
|
+
// shape. A visitor calling `hogsend.group()` is asserting an association
|
|
156
|
+
// INTENT, which is more than pure observation, so it keeps creating.
|
|
157
|
+
// Association-only still: the property writes a group row can receive
|
|
158
|
+
// remain secret-key-only, unchanged by this hatch.
|
|
159
|
+
// - `body.anonymousId` — the other half of D8: refusal is legal only where
|
|
160
|
+
// the key is STABLE. With no key at all the resolve throws either way
|
|
161
|
+
// (today: "requires at least one of…"), so restricting the guard to the
|
|
162
|
+
// shape that has a real refusal key changes no outcome and keeps the
|
|
163
|
+
// thrown message pointing at the actual defect.
|
|
164
|
+
//
|
|
165
|
+
// NOT collapsed into the policy's `allowMerge` clamp even though the
|
|
166
|
+
// conditions overlap here: that leg guards MERGE/poison, this one guards
|
|
167
|
+
// CREATION.
|
|
168
|
+
const observationOnly =
|
|
169
|
+
c.get("publishable") === true &&
|
|
170
|
+
!body.userId &&
|
|
171
|
+
!body.email &&
|
|
172
|
+
body.value === undefined &&
|
|
173
|
+
body.groups === undefined &&
|
|
174
|
+
!!body.anonymousId;
|
|
175
|
+
|
|
118
176
|
let result: Awaited<ReturnType<typeof ingestEvent>>;
|
|
119
177
|
try {
|
|
120
178
|
result = await ingestEvent({
|
|
@@ -125,11 +183,41 @@ export const eventsRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
125
183
|
// §5.3: thread the active analytics provider so a collide-MERGE /
|
|
126
184
|
// key-flip fires the provider-neutral `mergeIdentities` stitch.
|
|
127
185
|
analytics,
|
|
128
|
-
//
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
|
|
186
|
+
// PRD 06 T3 (L5 rows 1-3): trust is DECLARED by this caller, not
|
|
187
|
+
// re-inferred inside the resolver.
|
|
188
|
+
// - `create` — the D1 creation guard (see `observationOnly` above): a
|
|
189
|
+
// token-less pk_ observation refuses to mint; everything else keeps
|
|
190
|
+
// creating. Deliberately a SEPARATE leg from `allowMerge`:
|
|
191
|
+
// merge-clamping and creation-refusal are different questions that
|
|
192
|
+
// only happen to agree on part of this route's input.
|
|
193
|
+
// - `allowMerge` — §Phase 1 GAP-1: a publishable (pk_) browser write
|
|
194
|
+
// is anon-clamped — its browser-readable `anonymousId` may NOT
|
|
195
|
+
// attach to / merge / poison an already-identified victim contact.
|
|
196
|
+
// Declared "anonymous-only" even for a token-proven `userId` (row
|
|
197
|
+
// 2): the clamp is inert there because the DERIVATION sees a
|
|
198
|
+
// non-anon key — hard-coding the conclusion ("any") would silently
|
|
199
|
+
// break on the next key-precedence change. Secret-key ingest is
|
|
200
|
+
// never clamped.
|
|
201
|
+
// - `trustedKinds` — the evidence `gatePublishableIdentity` already
|
|
202
|
+
// computed: a publishable `userId` reaching this call IS
|
|
203
|
+
// token-proven (the gate 403'd every other identity-claiming shape,
|
|
204
|
+
// incl. any pk_ email), so a pk_ caller may assert `anonymous`
|
|
205
|
+
// (+`external` with that proof); a secret caller all four. Declared
|
|
206
|
+
// now, enforced by T5.
|
|
207
|
+
policy:
|
|
208
|
+
c.get("publishable") === true
|
|
209
|
+
? {
|
|
210
|
+
create: observationOnly ? "refuse-on-miss" : "on-miss",
|
|
211
|
+
allowMerge: "anonymous-only",
|
|
212
|
+
trustedKinds: body.userId
|
|
213
|
+
? ["anonymous", "external"]
|
|
214
|
+
: ["anonymous"],
|
|
215
|
+
}
|
|
216
|
+
: {
|
|
217
|
+
create: "on-miss",
|
|
218
|
+
allowMerge: "any",
|
|
219
|
+
trustedKinds: ALL_IDENTITY_KINDS,
|
|
220
|
+
},
|
|
133
221
|
event: {
|
|
134
222
|
event: body.name,
|
|
135
223
|
userId: body.userId,
|
package/src/routes/feed/index.ts
CHANGED
|
@@ -3,6 +3,7 @@ import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
|
3
3
|
import { and, count, desc, eq, gt, inArray, lt, sql } from "drizzle-orm";
|
|
4
4
|
import { streamSSE } from "hono/streaming";
|
|
5
5
|
import type { AppEnv } from "../../app.js";
|
|
6
|
+
import { ALL_IDENTITY_KINDS } from "../../lib/contacts.js";
|
|
6
7
|
import { ingestEvent } from "../../lib/ingestion.js";
|
|
7
8
|
import { getRedis } from "../../lib/redis.js";
|
|
8
9
|
import { errorSchema } from "../../lib/schemas.js";
|
|
@@ -50,12 +51,34 @@ function emitMarkEvents(
|
|
|
50
51
|
feedId: string;
|
|
51
52
|
recipientKey: string;
|
|
52
53
|
contactId?: string;
|
|
54
|
+
allowCreate?: boolean;
|
|
53
55
|
},
|
|
54
56
|
): Promise<unknown> {
|
|
55
57
|
return Promise.allSettled(
|
|
56
58
|
args.ids.map((r) =>
|
|
57
59
|
ingestEvent({
|
|
58
60
|
...deps,
|
|
61
|
+
// PRD 06 T3 (L5 rows 8-9) / L4: this is an ENGINE-INTERNAL re-emit.
|
|
62
|
+
// `trustedKinds` describes the keys as supplied by the caller OF THE
|
|
63
|
+
// RESOLVER — here the engine itself, whose subject (`recipientKey` +
|
|
64
|
+
// the `contactId` pin below) was derived server-side by
|
|
65
|
+
// `resolveFeedRecipient` — NOT by whatever authenticated the
|
|
66
|
+
// originating HTTP request. So the policy declares full server trust
|
|
67
|
+
// (`allowMerge: "any"`, all four kinds; deriving them from the pk_
|
|
68
|
+
// request would break every anon bell mark/clear once T5 arms) and
|
|
69
|
+
// inherits exactly ONE thing from that request: the D1 `create`
|
|
70
|
+
// refusal for an UNSEEN anonymous visitor. `resolveFeedRecipient` sets
|
|
71
|
+
// `allowCreate: false` only on the anon arm where BOTH the column
|
|
72
|
+
// probe and the alias probe missed — and there `recipientKey` IS the
|
|
73
|
+
// raw anon id, so D8 holds and the refused re-ingest keys
|
|
74
|
+
// `user_events` exactly as a creating one would. The `contactId` pin
|
|
75
|
+
// is a SUBJECT declaration, orthogonal to this policy — never folded
|
|
76
|
+
// into it.
|
|
77
|
+
policy: {
|
|
78
|
+
create: args.allowCreate === false ? "refuse-on-miss" : "on-miss",
|
|
79
|
+
allowMerge: "any",
|
|
80
|
+
trustedKinds: ALL_IDENTITY_KINDS,
|
|
81
|
+
},
|
|
59
82
|
event: {
|
|
60
83
|
event: args.eventType,
|
|
61
84
|
// recipientKey IS the canonical contact key — pass it as userId so
|
|
@@ -419,6 +442,7 @@ export const feedRouter = new OpenAPIHono<AppEnv>()
|
|
|
419
442
|
feedId,
|
|
420
443
|
recipientKey: rec.recipientKey,
|
|
421
444
|
contactId: rec.contactId,
|
|
445
|
+
allowCreate: rec.allowCreate,
|
|
422
446
|
},
|
|
423
447
|
);
|
|
424
448
|
|
|
@@ -452,6 +476,17 @@ export const feedRouter = new OpenAPIHono<AppEnv>()
|
|
|
452
476
|
hatchet,
|
|
453
477
|
logger,
|
|
454
478
|
analytics,
|
|
479
|
+
// PRD 06 T3 / L4 (see emitMarkEvents): engine-internal re-emit — full
|
|
480
|
+
// server trust, inheriting ONLY the D1 `create` refusal from the
|
|
481
|
+
// recipient verdict. This emit fires UNCONDITIONALLY — even when
|
|
482
|
+
// `updated` is empty — so a first-time visitor clearing a bell that
|
|
483
|
+
// never held an item drives one re-ingest, and that alone used to
|
|
484
|
+
// mint the ghost that locks them out.
|
|
485
|
+
policy: {
|
|
486
|
+
create: rec.allowCreate === false ? "refuse-on-miss" : "on-miss",
|
|
487
|
+
allowMerge: "any",
|
|
488
|
+
trustedKinds: ALL_IDENTITY_KINDS,
|
|
489
|
+
},
|
|
455
490
|
event: {
|
|
456
491
|
event: "inapp.feed_cleared",
|
|
457
492
|
userId: rec.recipientKey,
|
|
@@ -472,6 +507,7 @@ export const feedRouter = new OpenAPIHono<AppEnv>()
|
|
|
472
507
|
feedId,
|
|
473
508
|
recipientKey: rec.recipientKey,
|
|
474
509
|
contactId: rec.contactId,
|
|
510
|
+
allowCreate: rec.allowCreate,
|
|
475
511
|
},
|
|
476
512
|
);
|
|
477
513
|
}
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
contactKey,
|
|
8
8
|
normalizeEmail,
|
|
9
9
|
resolveContact,
|
|
10
|
+
resolveViaAlias,
|
|
10
11
|
} from "../../lib/contacts.js";
|
|
11
12
|
import {
|
|
12
13
|
InvalidUserTokenError,
|
|
@@ -14,7 +15,31 @@ import {
|
|
|
14
15
|
} from "../../lib/user-token.js";
|
|
15
16
|
|
|
16
17
|
export type FeedRecipient =
|
|
17
|
-
| {
|
|
18
|
+
| {
|
|
19
|
+
ok: true;
|
|
20
|
+
recipientKey: string;
|
|
21
|
+
contactId?: string;
|
|
22
|
+
/**
|
|
23
|
+
* D1 REFUSAL verdict for the feed's OWN re-ingests (`emitMarkEvents`,
|
|
24
|
+
* `inapp.feed_cleared`), threaded into their ingest policy's `create`
|
|
25
|
+
* leg (`false` ⇒ `create: "refuse-on-miss"` — PRD 06 L5 row 8). It is
|
|
26
|
+
* the ONLY thing those engine-internal re-emits inherit from the
|
|
27
|
+
* originating request: their `allowMerge`/`trustedKinds` stay full
|
|
28
|
+
* server trust, because the resolver's caller is the engine with a
|
|
29
|
+
* server-derived subject, not the pk_ HTTP caller (L4).
|
|
30
|
+
* `false` ⇒ this recipient is an UNSEEN anonymous visitor: no live contact
|
|
31
|
+
* owns their anon id, so a re-ingest keyed on it is pure OBSERVATION and
|
|
32
|
+
* must not mint. Omitted (⇒ create as before) on every identity-asserting
|
|
33
|
+
* arm — a token/secret `userId`/`email`, or an anon id that already owns a
|
|
34
|
+
* row (`contactId` is then set and the resolve pins to it).
|
|
35
|
+
*
|
|
36
|
+
* Set ONLY on the anon arm, where D8 holds: `recipientKey` IS the raw
|
|
37
|
+
* `anonymousId`, which is exactly the key the create arm would have made
|
|
38
|
+
* canonical, so a refused re-ingest and a creating one agree byte-for-byte
|
|
39
|
+
* and no history is stranded.
|
|
40
|
+
*/
|
|
41
|
+
allowCreate?: boolean;
|
|
42
|
+
}
|
|
18
43
|
| { ok: false; status: 400 | 403; error: string };
|
|
19
44
|
|
|
20
45
|
export interface FeedRecipientParams {
|
|
@@ -116,10 +141,27 @@ export async function resolveFeedRecipient(
|
|
|
116
141
|
),
|
|
117
142
|
)
|
|
118
143
|
.limit(1);
|
|
144
|
+
// PRD 03 T5: a SECOND device's anon id lives ONLY as an identity row —
|
|
145
|
+
// the column holds the first device's. Without this fallback the resolver
|
|
146
|
+
// read the person as having no contact, forced `allowCreate: false`, and
|
|
147
|
+
// the device's mark/clear re-ingests were refused: the events stranded
|
|
148
|
+
// under the raw device id instead of folding into the contact. Trust-wise
|
|
149
|
+
// this is the same class as the column hit: the id is the caller's OWN
|
|
150
|
+
// browser id held by the contact, never an identified canonical key (those
|
|
151
|
+
// were 403'd by `collidesWithIdentified` above).
|
|
152
|
+
const contactId =
|
|
153
|
+
anonRow[0]?.id ??
|
|
154
|
+
(await resolveViaAlias(db, "anonymous", params.anonymousId))?.id;
|
|
119
155
|
return {
|
|
120
156
|
ok: true,
|
|
121
157
|
recipientKey: params.anonymousId,
|
|
122
|
-
contactId
|
|
158
|
+
contactId,
|
|
159
|
+
// No row to pin to ⇒ the provenance pin cannot save us: an unpinned
|
|
160
|
+
// re-ingest keyed on this anon id would MINT `external_id = <anonId>`,
|
|
161
|
+
// the very ghost `sendFeedItem` just refused (D1/PRD 02 site 3), and the
|
|
162
|
+
// next poll would 403 on it. Refuse instead — legal here and only here,
|
|
163
|
+
// because `recipientKey` is the raw `anonymousId` (D8).
|
|
164
|
+
...(contactId ? {} : { allowCreate: false }),
|
|
123
165
|
};
|
|
124
166
|
}
|
|
125
167
|
|
|
@@ -317,6 +317,17 @@ async function applyListSubscription(opts: {
|
|
|
317
317
|
db,
|
|
318
318
|
userId,
|
|
319
319
|
email,
|
|
320
|
+
// PRD 06 T3 (L5 row 12): a subscribe/unsubscribe reaches here only
|
|
321
|
+
// server-trusted or token-proven (`gatePublishableIdentity` at both call
|
|
322
|
+
// sites) and the body carries no `anonymousId`, so the clamp would be
|
|
323
|
+
// inert — `allowMerge: "any"` is behaviour-identical, and the narrow
|
|
324
|
+
// `trustedKinds` is the honest statement of what this caller may
|
|
325
|
+
// assert. Declared now, enforced by T5.
|
|
326
|
+
policy: {
|
|
327
|
+
create: "on-miss",
|
|
328
|
+
allowMerge: "any",
|
|
329
|
+
trustedKinds: ["external", "email"],
|
|
330
|
+
},
|
|
320
331
|
});
|
|
321
332
|
|
|
322
333
|
emitContactCreatedIfNew({ db, hatchet, logger, contactId, created });
|
|
@@ -499,6 +510,16 @@ export const listsRouter = new OpenAPIHono<AppEnv>()
|
|
|
499
510
|
db,
|
|
500
511
|
userId,
|
|
501
512
|
email,
|
|
513
|
+
// PRD 06 T3 (L5 row 13): same trust statement as the subscribe path —
|
|
514
|
+
// server or token-proven caller, no `anonymousId` in the body (the
|
|
515
|
+
// handler 400s without an email/userId), so `allowMerge: "any"` is
|
|
516
|
+
// behaviour-identical and the narrow `trustedKinds` is honest. Enforced
|
|
517
|
+
// by T5.
|
|
518
|
+
policy: {
|
|
519
|
+
create: "on-miss",
|
|
520
|
+
allowMerge: "any",
|
|
521
|
+
trustedKinds: ["external", "email"],
|
|
522
|
+
},
|
|
502
523
|
});
|
|
503
524
|
|
|
504
525
|
emitContactCreatedIfNew({ db, hatchet, logger, contactId, created });
|
|
@@ -28,8 +28,9 @@ import {
|
|
|
28
28
|
* a concrete userId. `visitor_kind = 'token'`.
|
|
29
29
|
* - a raw `anonymousId` is PROVENANCE-ONLY: rejected if it collides with an
|
|
30
30
|
* identified contact's canonical key (`collidesWithIdentified`), and the
|
|
31
|
-
* bus ingest
|
|
32
|
-
* merge into an identified
|
|
31
|
+
* bus ingest declares the anon-clamped policy (`allowMerge:
|
|
32
|
+
* "anonymous-only"`) — it can never attach to / merge into an identified
|
|
33
|
+
* contact. `visitor_kind = 'anon'`.
|
|
33
34
|
* - a bare asserted email/userId is not even in the schema.
|
|
34
35
|
* - INVARIANT: nothing the ref resolves to (above all `links.distinct_id`)
|
|
35
36
|
* ever enters the contact resolver as a subject — identity inputs come
|
|
@@ -198,7 +199,45 @@ export const arriveRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
198
199
|
hatchet,
|
|
199
200
|
logger,
|
|
200
201
|
analytics,
|
|
201
|
-
|
|
202
|
+
// PRD 06 T3 (L5 rows 6-7): trust is DECLARED by this caller, not
|
|
203
|
+
// re-inferred inside the resolver.
|
|
204
|
+
//
|
|
205
|
+
// `create` — the D1 creation guard. An anon arrival is pure
|
|
206
|
+
// OBSERVATION — a visitor landing from a tracked hit with nothing but
|
|
207
|
+
// a browser id is not grounds for a CRM row. The stamp, the
|
|
208
|
+
// `link.arrived` event and the outbound emit all still happen (D2);
|
|
209
|
+
// only the `contacts` INSERT is refused. A token leg is a
|
|
210
|
+
// server-minted identity assertion and creates exactly as before. D8
|
|
211
|
+
// holds on both legs: the ingest key below is `userId` on the token
|
|
212
|
+
// leg and `anonymousId` on the anon one, and `stamp.visitorDistinctId`
|
|
213
|
+
// is guaranteed non-empty by the guard above — both are canonical-key
|
|
214
|
+
// shapes, so the refusal keys history on the same string the create
|
|
215
|
+
// arm would have.
|
|
216
|
+
//
|
|
217
|
+
// POLARITY — the one place in this stack where the two legs are
|
|
218
|
+
// OPPOSITES rather than coinciding: the anon leg clamps merges
|
|
219
|
+
// (`allowMerge: "anonymous-only"`, no token) while REFUSING creation;
|
|
220
|
+
// the token leg permits creation while never clamping. They read as
|
|
221
|
+
// complements here only because the token is simultaneously what makes
|
|
222
|
+
// a merge safe and what makes creation warranted; collapsing the two
|
|
223
|
+
// legs into one is forbidden (they answer different questions and
|
|
224
|
+
// diverge at every other call site).
|
|
225
|
+
//
|
|
226
|
+
// `trustedKinds` — derived from the first-write-wins STAMP, never the
|
|
227
|
+
// current request: exactly one key per leg, matching the declared
|
|
228
|
+
// kinds (`userId`/external on the token leg, `anonymousId` on the anon
|
|
229
|
+
// one). Declared now, enforced by T5.
|
|
230
|
+
policy: isToken
|
|
231
|
+
? {
|
|
232
|
+
create: "on-miss",
|
|
233
|
+
allowMerge: "any",
|
|
234
|
+
trustedKinds: ["anonymous", "external"],
|
|
235
|
+
}
|
|
236
|
+
: {
|
|
237
|
+
create: "refuse-on-miss",
|
|
238
|
+
allowMerge: "anonymous-only",
|
|
239
|
+
trustedKinds: ["anonymous"],
|
|
240
|
+
},
|
|
202
241
|
event: {
|
|
203
242
|
event: LINK_ARRIVED,
|
|
204
243
|
...(isToken
|