@hogsend/engine 0.52.1 → 0.54.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +12 -11
- package/src/connectors/define-action.ts +15 -2
- package/src/container.ts +85 -0
- package/src/env.ts +21 -0
- package/src/index.ts +20 -0
- package/src/journeys/durable-law-harness.test.ts +107 -0
- package/src/journeys/durable-law-harness.ts +139 -0
- package/src/journeys/journey-boundary.test.ts +70 -0
- package/src/journeys/journey-boundary.ts +10 -1
- package/src/journeys/with-durable-gate.test.ts +258 -0
- package/src/journeys/with-durable-gate.ts +141 -0
- package/src/lib/connector-actions.test.ts +292 -1
- package/src/lib/connector-actions.ts +65 -6
- package/src/lib/enrichment-ledger.ts +207 -0
- package/src/lib/enrichment-provider-registry.test.ts +64 -0
- package/src/lib/enrichment-provider-registry.ts +35 -0
- package/src/lib/enrichment-providers-from-env.apollo.test.ts +30 -0
- package/src/lib/enrichment-providers-from-env.test.ts +67 -0
- package/src/lib/enrichment-providers-from-env.ts +85 -0
- package/src/lib/enrichment-registry-singleton.ts +45 -0
- package/src/lib/links.ts +118 -7
- package/src/lib/outbound.ts +27 -0
- package/src/lib/refine-chain.test.ts +764 -0
- package/src/lib/refine-chain.ts +583 -0
- package/src/lib/refine-traits.test.ts +141 -0
- package/src/lib/refine-traits.ts +128 -0
- package/src/lib/refine.ts +287 -0
- package/src/lib/webhook-catalog-sync.test.ts +112 -0
- package/src/lib/webhook-signing.ts +5 -1
- package/src/routes/admin/contacts.ts +113 -13
- package/src/routes/admin/links.ts +57 -25
|
@@ -1,10 +1,32 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
2
|
import test from "node:test";
|
|
3
|
+
import type { Database } from "@hogsend/db";
|
|
4
|
+
import {
|
|
5
|
+
ConnectorActionRegistry,
|
|
6
|
+
resetConnectorActionRegistry,
|
|
7
|
+
setConnectorActionRegistry,
|
|
8
|
+
} from "../connectors/action-registry-singleton.js";
|
|
9
|
+
import type {
|
|
10
|
+
DefinedConnectorAction,
|
|
11
|
+
ResolvedActionContact,
|
|
12
|
+
} from "../connectors/define-action.js";
|
|
3
13
|
import {
|
|
4
14
|
type JourneyBoundary,
|
|
5
15
|
runWithJourneyBoundary,
|
|
6
16
|
} from "../journeys/journey-boundary.js";
|
|
7
|
-
import {
|
|
17
|
+
import {
|
|
18
|
+
type ColdPosture,
|
|
19
|
+
defineContactSource,
|
|
20
|
+
} from "../sources/define-contact-source.js";
|
|
21
|
+
import {
|
|
22
|
+
buildContactSourceRegistry,
|
|
23
|
+
ContactSourceRegistry,
|
|
24
|
+
setContactSourceRegistry,
|
|
25
|
+
} from "../sources/registry.js";
|
|
26
|
+
import {
|
|
27
|
+
checkActionAudience,
|
|
28
|
+
sendConnectorAction,
|
|
29
|
+
} from "./connector-actions.js";
|
|
8
30
|
|
|
9
31
|
test("a scoped connector override requires registration validation before capture", async () => {
|
|
10
32
|
let captures = 0;
|
|
@@ -34,3 +56,272 @@ test("a scoped connector override requires registration validation before captur
|
|
|
34
56
|
);
|
|
35
57
|
assert.equal(captures, 0);
|
|
36
58
|
});
|
|
59
|
+
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
// Cold-channel gate (PRD 05). The engine suite is pure/unit (no DB), so these
|
|
62
|
+
// drive `checkActionAudience` directly with a fake preference DB + a fake
|
|
63
|
+
// contact resolver; `sendConnectorAction` end-to-end coverage is limited to the
|
|
64
|
+
// paths that touch no DB (the no-audience ops action). The gate's placement
|
|
65
|
+
// inside the durable memo closure is structural (it runs from `gate()` inside
|
|
66
|
+
// `boundary.memoize`) and is not re-derived here.
|
|
67
|
+
// ---------------------------------------------------------------------------
|
|
68
|
+
|
|
69
|
+
/** Fake `Database` serving `readRecipientPreferences` an empty result set. */
|
|
70
|
+
function fakePrefDb(rows: unknown[] = []): Database {
|
|
71
|
+
return {
|
|
72
|
+
select: () => ({
|
|
73
|
+
from: () => ({
|
|
74
|
+
where: () => Promise.resolve(rows),
|
|
75
|
+
}),
|
|
76
|
+
}),
|
|
77
|
+
} as unknown as Database;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
function prospect(source: string | null): ResolvedActionContact {
|
|
81
|
+
return {
|
|
82
|
+
id: "11111111-1111-1111-1111-111111111111",
|
|
83
|
+
email: "prospect@example.com",
|
|
84
|
+
discordId: null,
|
|
85
|
+
externalId: null,
|
|
86
|
+
source,
|
|
87
|
+
properties: {},
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/** A member-directed DM action whose `run` counts invocations. */
|
|
92
|
+
function memberAction(counter: { runs: number }): DefinedConnectorAction {
|
|
93
|
+
return {
|
|
94
|
+
connectorId: "discord",
|
|
95
|
+
name: "dmMember",
|
|
96
|
+
audience: {
|
|
97
|
+
kind: "member",
|
|
98
|
+
ref: (args) => (args as { ref: string }).ref,
|
|
99
|
+
},
|
|
100
|
+
run: async () => {
|
|
101
|
+
counter.runs += 1;
|
|
102
|
+
return { delivered: true };
|
|
103
|
+
},
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function claySource(coldPosture?: ColdPosture) {
|
|
108
|
+
return defineContactSource({
|
|
109
|
+
meta: { id: "clay", name: "Clay" },
|
|
110
|
+
auth: { type: "match", header: "x-clay-secret", envKey: "CLAY_SECRET" },
|
|
111
|
+
transform: async () => null,
|
|
112
|
+
...(coldPosture ? { coldPosture } : {}),
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
/** Install a registry for the test, restoring an empty one afterwards. */
|
|
117
|
+
function withSources<T>(
|
|
118
|
+
sources: ReturnType<typeof claySource>[],
|
|
119
|
+
fn: () => Promise<T>,
|
|
120
|
+
): Promise<T> {
|
|
121
|
+
buildContactSourceRegistry(sources);
|
|
122
|
+
return fn().finally(() => {
|
|
123
|
+
setContactSourceRegistry(new ContactSourceRegistry());
|
|
124
|
+
});
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/** Capture stdout (winston Console transport) around `fn`. */
|
|
128
|
+
async function captureStdout<T>(
|
|
129
|
+
fn: () => Promise<T>,
|
|
130
|
+
): Promise<{ result: T; output: string }> {
|
|
131
|
+
const chunks: string[] = [];
|
|
132
|
+
const original = process.stdout.write.bind(process.stdout);
|
|
133
|
+
// biome-ignore lint/suspicious/noExplicitAny: stdout.write overload capture
|
|
134
|
+
(process.stdout as any).write = (chunk: any, ...rest: any[]) => {
|
|
135
|
+
chunks.push(String(chunk));
|
|
136
|
+
return original(chunk, ...rest);
|
|
137
|
+
};
|
|
138
|
+
try {
|
|
139
|
+
const result = await fn();
|
|
140
|
+
// Let winston's stream pipeline flush before restoring.
|
|
141
|
+
await new Promise((resolve) => setImmediate(resolve));
|
|
142
|
+
return { result, output: chunks.join("") };
|
|
143
|
+
} finally {
|
|
144
|
+
// biome-ignore lint/suspicious/noExplicitAny: restore the original writer
|
|
145
|
+
(process.stdout as any).write = original;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
test("AC1: a prospect from a registered source is blocked on a channel the posture blocks", async () => {
|
|
150
|
+
await withSources([claySource()], async () => {
|
|
151
|
+
const counter = { runs: 0 };
|
|
152
|
+
const action = memberAction(counter);
|
|
153
|
+
const verdict = await checkActionAudience(
|
|
154
|
+
fakePrefDb(),
|
|
155
|
+
async () => prospect("clay"),
|
|
156
|
+
action,
|
|
157
|
+
"discord",
|
|
158
|
+
"dmMember",
|
|
159
|
+
{ ref: "prospect@example.com" },
|
|
160
|
+
);
|
|
161
|
+
assert.deepEqual(verdict, {
|
|
162
|
+
skipped: true,
|
|
163
|
+
reason: "cold_channel_blocked",
|
|
164
|
+
connectorId: "discord",
|
|
165
|
+
action: "dmMember",
|
|
166
|
+
});
|
|
167
|
+
// The connector's run was never invoked by the gate.
|
|
168
|
+
assert.equal(counter.runs, 0);
|
|
169
|
+
});
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
test("AC2: the same prospect is allowed on a channel the posture explicitly allows", async () => {
|
|
173
|
+
await withSources([claySource({ discord: "allow" })], async () => {
|
|
174
|
+
const verdict = await checkActionAudience(
|
|
175
|
+
fakePrefDb(),
|
|
176
|
+
async () => prospect("clay"),
|
|
177
|
+
memberAction({ runs: 0 }),
|
|
178
|
+
"discord",
|
|
179
|
+
"dmMember",
|
|
180
|
+
{ ref: "prospect@example.com" },
|
|
181
|
+
);
|
|
182
|
+
assert.equal(verdict, null);
|
|
183
|
+
});
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
test("AC3: a contact with a null source is not a prospect and proceeds", async () => {
|
|
187
|
+
await withSources([claySource()], async () => {
|
|
188
|
+
const verdict = await checkActionAudience(
|
|
189
|
+
fakePrefDb(),
|
|
190
|
+
async () => prospect(null),
|
|
191
|
+
memberAction({ runs: 0 }),
|
|
192
|
+
"discord",
|
|
193
|
+
"dmMember",
|
|
194
|
+
{ ref: "prospect@example.com" },
|
|
195
|
+
);
|
|
196
|
+
assert.equal(verdict, null);
|
|
197
|
+
});
|
|
198
|
+
});
|
|
199
|
+
|
|
200
|
+
test("AC4: a source not in the registry is not a prospect and proceeds", async () => {
|
|
201
|
+
await withSources([claySource()], async () => {
|
|
202
|
+
const verdict = await checkActionAudience(
|
|
203
|
+
fakePrefDb(),
|
|
204
|
+
async () => prospect("api"),
|
|
205
|
+
memberAction({ runs: 0 }),
|
|
206
|
+
"discord",
|
|
207
|
+
"dmMember",
|
|
208
|
+
{ ref: "prospect@example.com" },
|
|
209
|
+
);
|
|
210
|
+
assert.equal(verdict, null);
|
|
211
|
+
});
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
test("AC5: an unresolvable contact fails open (allowed, unchanged)", async () => {
|
|
215
|
+
await withSources([claySource()], async () => {
|
|
216
|
+
const verdict = await checkActionAudience(
|
|
217
|
+
fakePrefDb(),
|
|
218
|
+
async () => null,
|
|
219
|
+
memberAction({ runs: 0 }),
|
|
220
|
+
"discord",
|
|
221
|
+
"dmMember",
|
|
222
|
+
{ ref: "nobody@example.com" },
|
|
223
|
+
);
|
|
224
|
+
assert.equal(verdict, null);
|
|
225
|
+
});
|
|
226
|
+
});
|
|
227
|
+
|
|
228
|
+
test("AC6: an ops action (no audience) is never gated — even with prospect sources registered", async () => {
|
|
229
|
+
await withSources([claySource()], async () => {
|
|
230
|
+
// Direct: the gate returns null WITHOUT consulting the resolver.
|
|
231
|
+
let resolves = 0;
|
|
232
|
+
const opsAction: DefinedConnectorAction = {
|
|
233
|
+
connectorId: "discord",
|
|
234
|
+
name: "sendChannelMessage",
|
|
235
|
+
run: async () => ({ messageId: "m1" }),
|
|
236
|
+
};
|
|
237
|
+
const verdict = await checkActionAudience(
|
|
238
|
+
fakePrefDb(),
|
|
239
|
+
async () => {
|
|
240
|
+
resolves += 1;
|
|
241
|
+
return prospect("clay");
|
|
242
|
+
},
|
|
243
|
+
opsAction,
|
|
244
|
+
"discord",
|
|
245
|
+
"sendChannelMessage",
|
|
246
|
+
{ channelId: "c1" },
|
|
247
|
+
);
|
|
248
|
+
assert.equal(verdict, null);
|
|
249
|
+
assert.equal(resolves, 0);
|
|
250
|
+
|
|
251
|
+
// End-to-end: sendConnectorAction runs the ops action to completion. This
|
|
252
|
+
// path touches no DB (the lazy postgres client never issues a query).
|
|
253
|
+
process.env.DATABASE_URL ??=
|
|
254
|
+
"postgresql://never:connects@localhost:9/never";
|
|
255
|
+
let runs = 0;
|
|
256
|
+
setConnectorActionRegistry(
|
|
257
|
+
new ConnectorActionRegistry([
|
|
258
|
+
{
|
|
259
|
+
connectorId: "discord",
|
|
260
|
+
name: "opsAlert",
|
|
261
|
+
run: async () => {
|
|
262
|
+
runs += 1;
|
|
263
|
+
return { delivered: true };
|
|
264
|
+
},
|
|
265
|
+
},
|
|
266
|
+
]),
|
|
267
|
+
);
|
|
268
|
+
try {
|
|
269
|
+
const result = await sendConnectorAction({
|
|
270
|
+
connectorId: "discord",
|
|
271
|
+
action: "opsAlert",
|
|
272
|
+
args: { channelId: "c1" },
|
|
273
|
+
});
|
|
274
|
+
assert.deepEqual(result, { delivered: true });
|
|
275
|
+
assert.equal(runs, 1);
|
|
276
|
+
} finally {
|
|
277
|
+
resetConnectorActionRegistry();
|
|
278
|
+
}
|
|
279
|
+
});
|
|
280
|
+
});
|
|
281
|
+
|
|
282
|
+
test("AC7: a cold skip logs once at info with the source id and blocked channel", async () => {
|
|
283
|
+
await withSources([claySource()], async () => {
|
|
284
|
+
const { result, output } = await captureStdout(() =>
|
|
285
|
+
checkActionAudience(
|
|
286
|
+
fakePrefDb(),
|
|
287
|
+
async () => prospect("clay"),
|
|
288
|
+
memberAction({ runs: 0 }),
|
|
289
|
+
"discord",
|
|
290
|
+
"dmMember",
|
|
291
|
+
{ ref: "prospect@example.com" },
|
|
292
|
+
),
|
|
293
|
+
);
|
|
294
|
+
assert.equal(
|
|
295
|
+
(result as { reason?: string } | null)?.reason,
|
|
296
|
+
"cold_channel_blocked",
|
|
297
|
+
);
|
|
298
|
+
const lines = output
|
|
299
|
+
.split("\n")
|
|
300
|
+
.filter((line) => line.includes("cold channel blocked"));
|
|
301
|
+
assert.equal(lines.length, 1, `expected exactly one log line:\n${output}`);
|
|
302
|
+
assert.match(lines[0] as string, /"clay"/);
|
|
303
|
+
assert.match(lines[0] as string, /"discord"/);
|
|
304
|
+
});
|
|
305
|
+
});
|
|
306
|
+
|
|
307
|
+
test("a throwing cold-posture lookup fails open (allowed)", async () => {
|
|
308
|
+
const throwing = new (class extends ContactSourceRegistry {
|
|
309
|
+
override isProspectSource(): boolean {
|
|
310
|
+
throw new Error("registry exploded");
|
|
311
|
+
}
|
|
312
|
+
})();
|
|
313
|
+
setContactSourceRegistry(throwing);
|
|
314
|
+
try {
|
|
315
|
+
const verdict = await checkActionAudience(
|
|
316
|
+
fakePrefDb(),
|
|
317
|
+
async () => prospect("clay"),
|
|
318
|
+
memberAction({ runs: 0 }),
|
|
319
|
+
"discord",
|
|
320
|
+
"dmMember",
|
|
321
|
+
{ ref: "prospect@example.com" },
|
|
322
|
+
);
|
|
323
|
+
assert.equal(verdict, null);
|
|
324
|
+
} finally {
|
|
325
|
+
setContactSourceRegistry(new ContactSourceRegistry());
|
|
326
|
+
}
|
|
327
|
+
});
|
|
@@ -12,6 +12,11 @@ import {
|
|
|
12
12
|
registerKey,
|
|
13
13
|
} from "../journeys/journey-boundary.js";
|
|
14
14
|
import { getListRegistry } from "../lists/registry-singleton.js";
|
|
15
|
+
import {
|
|
16
|
+
defaultColdPosture,
|
|
17
|
+
isColdChannelAllowed,
|
|
18
|
+
} from "../sources/define-contact-source.js";
|
|
19
|
+
import { getContactSourceRegistry } from "../sources/registry.js";
|
|
15
20
|
import { getDb } from "./db.js";
|
|
16
21
|
import { createLogger } from "./logger.js";
|
|
17
22
|
import { readRecipientPreferences } from "./recipient-preferences.js";
|
|
@@ -83,6 +88,7 @@ async function resolveContact(
|
|
|
83
88
|
email: contacts.email,
|
|
84
89
|
discordId: contacts.discordId,
|
|
85
90
|
externalId: contacts.externalId,
|
|
91
|
+
source: contacts.source,
|
|
86
92
|
properties: contacts.properties,
|
|
87
93
|
})
|
|
88
94
|
.from(contacts)
|
|
@@ -107,6 +113,7 @@ async function resolveContact(
|
|
|
107
113
|
email: row.email ?? null,
|
|
108
114
|
discordId: row.discordId ?? null,
|
|
109
115
|
externalId: row.externalId ?? null,
|
|
116
|
+
source: row.source ?? null,
|
|
110
117
|
properties: (row.properties ?? {}) as Record<string, unknown>,
|
|
111
118
|
};
|
|
112
119
|
}
|
|
@@ -127,6 +134,12 @@ async function resolveContact(
|
|
|
127
134
|
* wins. A candidate that resolves NO contact (a raw snowflake, a group-chat
|
|
128
135
|
* id, an unlinked ref) means there is NO preference surface to consult — the
|
|
129
136
|
* send is ALLOWED (null). Only once a contact is resolved do preferences gate.
|
|
137
|
+
* - On a resolved contact the COLD-CHANNEL GATE runs first: when the
|
|
138
|
+
* contact's `source` names a registered Contact Source (⇒ a cold prospect),
|
|
139
|
+
* the source's resolved {@link ColdPosture} must allow this connector's
|
|
140
|
+
* channel or the action skips with `cold_channel_blocked` (logged once at
|
|
141
|
+
* info with the source id + blocked channel). Null/unregistered `source` ⇒
|
|
142
|
+
* not a prospect ⇒ unaffected; a throwing posture lookup fails OPEN.
|
|
130
143
|
* - On a resolved contact we read the aggregated {@link readRecipientPreferences}
|
|
131
144
|
* keyed by its email + `externalId ?? id` (the SAME key form preference writes
|
|
132
145
|
* use), then: `unsubscribedAll` → skip `unsubscribed_all`; else channel
|
|
@@ -135,8 +148,13 @@ async function resolveContact(
|
|
|
135
148
|
*
|
|
136
149
|
* Deliberately does NOT consume `prefs.suppressed` — that is an email-transport
|
|
137
150
|
* hard-bounce/complaint signal, irrelevant to a chat channel.
|
|
151
|
+
*
|
|
152
|
+
* Exported ONLY for unit tests (the engine suite is pure/unit and cannot drive
|
|
153
|
+
* the DB-backed `sendConnectorAction` path) — production callers go through
|
|
154
|
+
* {@link sendConnectorAction}, which runs this gate INSIDE the durable memo
|
|
155
|
+
* closure so the verdict replays verbatim.
|
|
138
156
|
*/
|
|
139
|
-
async function checkActionAudience(
|
|
157
|
+
export async function checkActionAudience(
|
|
140
158
|
db: Database,
|
|
141
159
|
resolveCachedContact: (ref: string) => Promise<ResolvedActionContact | null>,
|
|
142
160
|
action: DefinedConnectorAction,
|
|
@@ -170,6 +188,42 @@ async function checkActionAudience(
|
|
|
170
188
|
// fully-unresolved recipient (raw id / group chat) falls through to allow.
|
|
171
189
|
if (!contact) continue;
|
|
172
190
|
|
|
191
|
+
// COLD-CHANNEL GATE (PRD 05): a contact whose `source` names a registered
|
|
192
|
+
// Contact Source is a cold prospect; its source's resolved posture decides
|
|
193
|
+
// which channels may reach it before consent (default: email only). The
|
|
194
|
+
// connector's id IS the channel id. Checked BEFORE the preference read so a
|
|
195
|
+
// blocked prospect costs no extra round-trip. A null/unregistered `source`
|
|
196
|
+
// is not a prospect and is unaffected; a THROWING posture lookup fails
|
|
197
|
+
// OPEN (warn + fall through), matching this gate's failure posture.
|
|
198
|
+
try {
|
|
199
|
+
const sources = getContactSourceRegistry();
|
|
200
|
+
if (sources.isProspectSource(contact.source)) {
|
|
201
|
+
const posture =
|
|
202
|
+
sources.get(contact.source as string)?.coldPosture ??
|
|
203
|
+
defaultColdPosture();
|
|
204
|
+
if (!isColdChannelAllowed(posture, connectorId)) {
|
|
205
|
+
logger.info("connector action skipped: cold channel blocked", {
|
|
206
|
+
connectorId,
|
|
207
|
+
action: actionName,
|
|
208
|
+
source: contact.source,
|
|
209
|
+
channel: connectorId,
|
|
210
|
+
});
|
|
211
|
+
return {
|
|
212
|
+
skipped: true,
|
|
213
|
+
reason: "cold_channel_blocked",
|
|
214
|
+
connectorId,
|
|
215
|
+
action: actionName,
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
} catch (error) {
|
|
220
|
+
logger.warn("cold-channel gate threw; sending anyway", {
|
|
221
|
+
connectorId,
|
|
222
|
+
action: actionName,
|
|
223
|
+
error: error instanceof Error ? error.message : String(error),
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
|
|
173
227
|
const prefs = await readRecipientPreferences(db, {
|
|
174
228
|
email: contact.email,
|
|
175
229
|
userId: contact.externalId ?? contact.id,
|
|
@@ -327,11 +381,16 @@ export async function sendConnectorAction(
|
|
|
327
381
|
input.args,
|
|
328
382
|
);
|
|
329
383
|
if (skip) {
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
384
|
+
// A cold-channel skip already logged once (with the source id + blocked
|
|
385
|
+
// channel) inside checkActionAudience — don't log it a second time under
|
|
386
|
+
// the misleading "recipient preferences" banner.
|
|
387
|
+
if (skip.reason !== "cold_channel_blocked") {
|
|
388
|
+
logger.info("connector action skipped: recipient preferences", {
|
|
389
|
+
connectorId: input.connectorId,
|
|
390
|
+
action: input.action,
|
|
391
|
+
reason: skip.reason,
|
|
392
|
+
});
|
|
393
|
+
}
|
|
335
394
|
return skip;
|
|
336
395
|
}
|
|
337
396
|
return run();
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
import { type Database, enrichmentLookups } from "@hogsend/db";
|
|
2
|
+
import { and, eq, sql, sum } from "drizzle-orm";
|
|
3
|
+
|
|
4
|
+
/** The lookup subject's kind — mirrors the `enrichment_lookup_kind` pgEnum. */
|
|
5
|
+
export type EnrichmentLookupKind = "email" | "domain";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The lookup's outcome — mirrors the `enrichment_lookup_status` pgEnum.
|
|
9
|
+
* `found`/`not_found` are both PAID answers and suppress re-spend until the row
|
|
10
|
+
* expires; `error` is not a paid answer and must never suppress a retry.
|
|
11
|
+
*/
|
|
12
|
+
export type EnrichmentLookupStatus = "found" | "not_found" | "error";
|
|
13
|
+
|
|
14
|
+
/** A ledger row as the refinement gate chain reads it. */
|
|
15
|
+
export interface EnrichmentLookupRow {
|
|
16
|
+
id: string;
|
|
17
|
+
provider: string;
|
|
18
|
+
lookupKind: EnrichmentLookupKind;
|
|
19
|
+
lookupKey: string;
|
|
20
|
+
status: EnrichmentLookupStatus;
|
|
21
|
+
contactId: string | null;
|
|
22
|
+
refinedAt: Date;
|
|
23
|
+
expiresAt: Date;
|
|
24
|
+
raw: Record<string, unknown> | null;
|
|
25
|
+
/** The normalized `refined_*` patch the paid answer produced. */
|
|
26
|
+
traits: Record<string, unknown> | null;
|
|
27
|
+
spendWindow: Date | null;
|
|
28
|
+
spendCount: number;
|
|
29
|
+
lastErrorAt: Date | null;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The ledger row for a `(provider, lookupKind, lookupKey)` triple, or null when
|
|
34
|
+
* this subject has never been looked up by this provider.
|
|
35
|
+
*
|
|
36
|
+
* Deliberately returns the row REGARDLESS of status and expiry: the gate chain
|
|
37
|
+
* owns the "unexpired AND paid" verdict so the `error`-never-short-circuits rule
|
|
38
|
+
* lives next to the rest of the gate ordering rather than being buried in a
|
|
39
|
+
* WHERE clause.
|
|
40
|
+
*/
|
|
41
|
+
export async function findEnrichmentLookup(opts: {
|
|
42
|
+
db: Database;
|
|
43
|
+
provider: string;
|
|
44
|
+
lookupKind: EnrichmentLookupKind;
|
|
45
|
+
lookupKey: string;
|
|
46
|
+
}): Promise<EnrichmentLookupRow | null> {
|
|
47
|
+
const rows = await opts.db
|
|
48
|
+
.select()
|
|
49
|
+
.from(enrichmentLookups)
|
|
50
|
+
.where(
|
|
51
|
+
and(
|
|
52
|
+
eq(enrichmentLookups.provider, opts.provider),
|
|
53
|
+
eq(enrichmentLookups.lookupKind, opts.lookupKind),
|
|
54
|
+
eq(enrichmentLookups.lookupKey, opts.lookupKey),
|
|
55
|
+
),
|
|
56
|
+
)
|
|
57
|
+
.limit(1);
|
|
58
|
+
return (rows[0] as EnrichmentLookupRow | undefined) ?? null;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Record a lookup against the `(provider, lookup_kind, lookup_key)` unique index
|
|
63
|
+
* — the Layer-2, version-independent exactly-once backstop for refinement — AND
|
|
64
|
+
* the spend it cost.
|
|
65
|
+
*
|
|
66
|
+
* Conflict behaviour is status-dependent, and deliberately asymmetric:
|
|
67
|
+
*
|
|
68
|
+
* - a PAID answer (`found` / `not_found`) UPDATES the existing row, so a
|
|
69
|
+
* `force: true` re-lookup refreshes the TTL + the stored traits in place
|
|
70
|
+
* instead of inserting a duplicate (the unique index would reject one
|
|
71
|
+
* anyway);
|
|
72
|
+
* - an `error` never touches the CACHE columns (`status`/`expiresAt`/`raw`/
|
|
73
|
+
* `traits`/`refinedAt`), so a transient vendor 5xx during a forced re-lookup
|
|
74
|
+
* can never clobber a live, good cached row.
|
|
75
|
+
*
|
|
76
|
+
* What BOTH branches always do is bump the spend counter and stamp
|
|
77
|
+
* `spend_window`. That is the whole point: the row is one-per-key, so counting
|
|
78
|
+
* rows counts SUBJECTS, not vendor calls — a `force` loop, or an outage
|
|
79
|
+
* retrying an already-refined base, would otherwise spend real money without
|
|
80
|
+
* ever moving the budget cap. `spend_count` resets to 1 whenever the call lands
|
|
81
|
+
* in a window later than the one on the row, so a previous month's attempts can
|
|
82
|
+
* never be pulled into this month's ceiling.
|
|
83
|
+
*/
|
|
84
|
+
export async function upsertEnrichmentLookup(opts: {
|
|
85
|
+
db: Database;
|
|
86
|
+
provider: string;
|
|
87
|
+
lookupKind: EnrichmentLookupKind;
|
|
88
|
+
lookupKey: string;
|
|
89
|
+
status: EnrichmentLookupStatus;
|
|
90
|
+
contactId?: string | null;
|
|
91
|
+
refinedAt: Date;
|
|
92
|
+
expiresAt: Date;
|
|
93
|
+
raw?: unknown;
|
|
94
|
+
traits?: Record<string, unknown> | null;
|
|
95
|
+
/** First instant of the budget window this call is charged to. */
|
|
96
|
+
spendWindow: Date;
|
|
97
|
+
}): Promise<void> {
|
|
98
|
+
const isError = opts.status === "error";
|
|
99
|
+
const values = {
|
|
100
|
+
provider: opts.provider,
|
|
101
|
+
lookupKind: opts.lookupKind,
|
|
102
|
+
lookupKey: opts.lookupKey,
|
|
103
|
+
status: opts.status,
|
|
104
|
+
contactId: opts.contactId ?? null,
|
|
105
|
+
refinedAt: opts.refinedAt,
|
|
106
|
+
expiresAt: opts.expiresAt,
|
|
107
|
+
raw: toJsonbRecord(opts.raw),
|
|
108
|
+
traits: opts.traits ?? null,
|
|
109
|
+
spendWindow: opts.spendWindow,
|
|
110
|
+
spendCount: 1,
|
|
111
|
+
lastErrorAt: isError ? opts.refinedAt : null,
|
|
112
|
+
};
|
|
113
|
+
|
|
114
|
+
// A call that lands in the SAME window as the one already on the row extends
|
|
115
|
+
// that window's tally; a call in a LATER window starts a fresh tally. Written
|
|
116
|
+
// as SQL so the whole thing is one atomic statement — a read-modify-write
|
|
117
|
+
// here would lose concurrent lookups, which is exactly the spend the cap
|
|
118
|
+
// exists to bound.
|
|
119
|
+
const bumpedSpend = sql`case
|
|
120
|
+
when ${enrichmentLookups.spendWindow} = excluded.spend_window
|
|
121
|
+
then ${enrichmentLookups.spendCount} + 1
|
|
122
|
+
else 1
|
|
123
|
+
end`;
|
|
124
|
+
|
|
125
|
+
const spend = {
|
|
126
|
+
spendWindow: opts.spendWindow,
|
|
127
|
+
spendCount: bumpedSpend,
|
|
128
|
+
updatedAt: new Date(),
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
const insert = opts.db.insert(enrichmentLookups).values(values);
|
|
132
|
+
const target = [
|
|
133
|
+
enrichmentLookups.provider,
|
|
134
|
+
enrichmentLookups.lookupKind,
|
|
135
|
+
enrichmentLookups.lookupKey,
|
|
136
|
+
];
|
|
137
|
+
|
|
138
|
+
if (isError) {
|
|
139
|
+
// Spend + forensics ONLY. Every cache column is left exactly as it was.
|
|
140
|
+
await insert.onConflictDoUpdate({
|
|
141
|
+
target,
|
|
142
|
+
set: { ...spend, lastErrorAt: opts.refinedAt },
|
|
143
|
+
});
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
await insert.onConflictDoUpdate({
|
|
148
|
+
target,
|
|
149
|
+
set: {
|
|
150
|
+
...spend,
|
|
151
|
+
status: values.status,
|
|
152
|
+
refinedAt: values.refinedAt,
|
|
153
|
+
expiresAt: values.expiresAt,
|
|
154
|
+
raw: values.raw,
|
|
155
|
+
traits: values.traits,
|
|
156
|
+
// Only ever ATTACH a contact, never detach one: a later lookup that
|
|
157
|
+
// resolved no contact must not erase the contact the first one attributed
|
|
158
|
+
// the spend to.
|
|
159
|
+
...(values.contactId ? { contactId: values.contactId } : {}),
|
|
160
|
+
},
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* How many provider LOOKUPS the ledger recorded in the budget window starting
|
|
166
|
+
* at `since` — the number behind `ENRICHMENT_MONTHLY_LOOKUPS`.
|
|
167
|
+
*
|
|
168
|
+
* A SUM of `spend_count` over the rows stamped with this window, not a COUNT of
|
|
169
|
+
* rows: the table holds one row per subject, so a row count would answer "how
|
|
170
|
+
* many distinct subjects were touched", which equals spend only when nothing is
|
|
171
|
+
* ever re-looked-up. Errors are included — an `error` row still represents a
|
|
172
|
+
* request that left the building, and the operator's cap is a ceiling on
|
|
173
|
+
* requests, not on successful matches. Served by
|
|
174
|
+
* `enrichment_lookups_spend_window_idx`.
|
|
175
|
+
*/
|
|
176
|
+
export async function countEnrichmentLookups(opts: {
|
|
177
|
+
db: Database;
|
|
178
|
+
since: Date;
|
|
179
|
+
}): Promise<number> {
|
|
180
|
+
const rows = await opts.db
|
|
181
|
+
.select({ total: sum(enrichmentLookups.spendCount) })
|
|
182
|
+
.from(enrichmentLookups)
|
|
183
|
+
.where(eq(enrichmentLookups.spendWindow, opts.since));
|
|
184
|
+
return Number(rows[0]?.total ?? 0);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* The first instant of `at`'s calendar month in UTC — the lower bound of the
|
|
189
|
+
* monthly budget window. UTC (not local) so the cap resets at the same instant
|
|
190
|
+
* for every process regardless of the host's timezone.
|
|
191
|
+
*/
|
|
192
|
+
export function startOfUtcMonth(at: Date): Date {
|
|
193
|
+
return new Date(Date.UTC(at.getUTCFullYear(), at.getUTCMonth(), 1));
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Narrow a provider's verbatim payload to something the `raw` jsonb column
|
|
198
|
+
* accepts. A plain object is stored as-is; anything else (array, scalar) is
|
|
199
|
+
* wrapped so it stays inspectable without widening the column's type.
|
|
200
|
+
*/
|
|
201
|
+
function toJsonbRecord(raw: unknown): Record<string, unknown> | null {
|
|
202
|
+
if (raw === null || raw === undefined) return null;
|
|
203
|
+
if (typeof raw === "object" && !Array.isArray(raw)) {
|
|
204
|
+
return raw as Record<string, unknown>;
|
|
205
|
+
}
|
|
206
|
+
return { value: raw as never };
|
|
207
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import test from "node:test";
|
|
3
|
+
import type { EnrichmentProvider } from "@hogsend/core";
|
|
4
|
+
import { EnrichmentProviderRegistry } from "./enrichment-provider-registry.js";
|
|
5
|
+
import {
|
|
6
|
+
getEnrichmentProvider,
|
|
7
|
+
getEnrichmentProviderRegistry,
|
|
8
|
+
resetEnrichmentProviders,
|
|
9
|
+
setEnrichmentProviders,
|
|
10
|
+
} from "./enrichment-registry-singleton.js";
|
|
11
|
+
|
|
12
|
+
function fakeProvider(id: string, name = id): EnrichmentProvider {
|
|
13
|
+
return {
|
|
14
|
+
meta: { id, name },
|
|
15
|
+
capabilities: { personLookup: true, companyLookup: false },
|
|
16
|
+
async enrichPerson() {
|
|
17
|
+
return { found: false, raw: null };
|
|
18
|
+
},
|
|
19
|
+
};
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
test("registry keys providers by meta.id", () => {
|
|
23
|
+
const apollo = fakeProvider("apollo");
|
|
24
|
+
const clay = fakeProvider("clay");
|
|
25
|
+
const registry = new EnrichmentProviderRegistry([apollo, clay]);
|
|
26
|
+
|
|
27
|
+
assert.equal(registry.get("apollo"), apollo);
|
|
28
|
+
assert.equal(registry.get("clay"), clay);
|
|
29
|
+
assert.equal(registry.get("clearbit"), undefined);
|
|
30
|
+
assert.equal(registry.count(), 2);
|
|
31
|
+
assert.deepEqual(registry.getAll(), [apollo, clay]);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test("duplicate meta.id is last-writer-wins (constructor order + register)", () => {
|
|
35
|
+
const first = fakeProvider("apollo", "first");
|
|
36
|
+
const second = fakeProvider("apollo", "second");
|
|
37
|
+
const registry = new EnrichmentProviderRegistry([first, second]);
|
|
38
|
+
|
|
39
|
+
assert.equal(registry.count(), 1);
|
|
40
|
+
assert.equal(registry.get("apollo"), second);
|
|
41
|
+
|
|
42
|
+
const third = fakeProvider("apollo", "third");
|
|
43
|
+
registry.register(third);
|
|
44
|
+
assert.equal(registry.count(), 1);
|
|
45
|
+
assert.equal(registry.get("apollo"), third);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("singleton: set → get (registry + active), reset clears", () => {
|
|
49
|
+
const apollo = fakeProvider("apollo");
|
|
50
|
+
const registry = new EnrichmentProviderRegistry([apollo]);
|
|
51
|
+
|
|
52
|
+
setEnrichmentProviders(registry, apollo);
|
|
53
|
+
assert.equal(getEnrichmentProviderRegistry(), registry);
|
|
54
|
+
assert.equal(getEnrichmentProvider(), apollo);
|
|
55
|
+
|
|
56
|
+
// Active is legitimately undefined (registered-but-inert deploy).
|
|
57
|
+
setEnrichmentProviders(registry, undefined);
|
|
58
|
+
assert.equal(getEnrichmentProviderRegistry(), registry);
|
|
59
|
+
assert.equal(getEnrichmentProvider(), undefined);
|
|
60
|
+
|
|
61
|
+
resetEnrichmentProviders();
|
|
62
|
+
assert.equal(getEnrichmentProviderRegistry(), undefined);
|
|
63
|
+
assert.equal(getEnrichmentProvider(), undefined);
|
|
64
|
+
});
|