@hogsend/engine 0.31.1 → 0.32.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +7 -7
- package/src/cold-connect/index.ts +1 -0
- package/src/connectors/define-connector.ts +12 -2
- package/src/connectors/runtime.ts +10 -8
- package/src/index.ts +8 -0
- package/src/journeys/define-journey.ts +111 -41
- package/src/journeys/journey-boundary.ts +172 -0
- package/src/journeys/journey-context.ts +391 -23
- package/src/lib/boot.ts +8 -4
- package/src/lib/bot-prefetch.ts +48 -0
- package/src/lib/bucket-emit.ts +2 -0
- package/src/lib/connector-actions.ts +197 -5
- package/src/lib/domain-status.ts +27 -12
- package/src/lib/email.ts +63 -5
- package/src/lib/ingestion.ts +71 -1
- package/src/lib/links.ts +1 -1
- package/src/lib/tracked.ts +111 -37
- package/src/lib/tracking-event-names.ts +9 -0
- package/src/lib/tracking-events.ts +73 -0
- package/src/routes/admin/bulk.ts +2 -0
- package/src/routes/admin/events.ts +51 -11
- package/src/routes/admin/index.ts +2 -0
- package/src/routes/admin/journeys.ts +95 -1
- package/src/routes/admin/links.ts +51 -8
- package/src/routes/admin/readiness.ts +181 -0
- package/src/routes/connectors/index.ts +22 -5
- package/src/routes/events/index.ts +2 -0
- package/src/routes/tracking/click.ts +51 -1
- package/src/routes/webhooks/sources.ts +31 -4
|
@@ -1,8 +1,13 @@
|
|
|
1
|
-
import { contacts, type Database } from "@hogsend/db";
|
|
1
|
+
import { connectorDeliveries, contacts, type Database } from "@hogsend/db";
|
|
2
2
|
import { and, eq, isNull, or } from "drizzle-orm";
|
|
3
3
|
import { getConnectorActionRegistry } from "../connectors/action-registry-singleton.js";
|
|
4
4
|
import type { ResolvedActionContact } from "../connectors/define-action.js";
|
|
5
5
|
import { env } from "../env.js";
|
|
6
|
+
import {
|
|
7
|
+
deriveJourneyKey,
|
|
8
|
+
getJourneyBoundary,
|
|
9
|
+
registerKey,
|
|
10
|
+
} from "../journeys/journey-boundary.js";
|
|
6
11
|
import { getDb } from "./db.js";
|
|
7
12
|
import { createLogger } from "./logger.js";
|
|
8
13
|
|
|
@@ -55,6 +60,18 @@ export interface SendConnectorActionArgs {
|
|
|
55
60
|
action: string;
|
|
56
61
|
/** The action's own args object (shape defined by the action). */
|
|
57
62
|
args?: unknown;
|
|
63
|
+
/**
|
|
64
|
+
* Disambiguates the exactly-once dedupe key when the SAME action is sent more
|
|
65
|
+
* than once in one journey run. The action's `args` are NOT part of the
|
|
66
|
+
* derived key, so two genuinely-distinct sends of the same action within one
|
|
67
|
+
* run — without an intervening `ctx.sleep`/`waitForEvent`/`checkpoint` to
|
|
68
|
+
* advance the nearest label — would otherwise derive an IDENTICAL key and the
|
|
69
|
+
* second would fail-fast via `registerKey`. Pass a distinct label per call
|
|
70
|
+
* (e.g. broadcasting the same message to two channels back-to-back). Mirrors
|
|
71
|
+
* `idempotencyLabel` on `sendEmail()` and `ctx.trigger()`. Additive and
|
|
72
|
+
* optional.
|
|
73
|
+
*/
|
|
74
|
+
idempotencyLabel?: string;
|
|
58
75
|
}
|
|
59
76
|
|
|
60
77
|
/**
|
|
@@ -82,9 +99,184 @@ export async function sendConnectorAction(
|
|
|
82
99
|
);
|
|
83
100
|
}
|
|
84
101
|
const db = getDb();
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
102
|
+
const doRun = () =>
|
|
103
|
+
action.run(input.args, {
|
|
104
|
+
db,
|
|
105
|
+
logger: createLogger(env.LOG_LEVEL),
|
|
106
|
+
resolveContact: (ref: string) => resolveContact(db, ref),
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
// Outside a journey run (an admin/manual send) there is no replay to defend
|
|
110
|
+
// against and no boundary to key from — run the action directly.
|
|
111
|
+
const boundary = getJourneyBoundary();
|
|
112
|
+
if (!boundary) return doRun();
|
|
113
|
+
|
|
114
|
+
// The replay-stable, branch-derived key shared by BOTH defense layers. `site`
|
|
115
|
+
// = the explicit `idempotencyLabel` ?? the nearest authored wait/checkpoint
|
|
116
|
+
// label (so two of the SAME action on divergent branches derive distinct keys
|
|
117
|
+
// for free) ?? the connector:action. Mirrors sendEmail()/ctx.trigger().
|
|
118
|
+
const site =
|
|
119
|
+
input.idempotencyLabel ??
|
|
120
|
+
boundary.currentLabel ??
|
|
121
|
+
`${input.connectorId}:${input.action}`;
|
|
122
|
+
const discriminant = `${input.connectorId}:${input.action}`;
|
|
123
|
+
const key = deriveJourneyKey({
|
|
124
|
+
kind: "connector",
|
|
125
|
+
anchor: boundary.runAnchor,
|
|
126
|
+
site,
|
|
127
|
+
discriminant,
|
|
89
128
|
});
|
|
129
|
+
registerKey(boundary, key);
|
|
130
|
+
|
|
131
|
+
// Layer 1 (eviction-gated, FREE) fast path wrapping the Layer-2-backed run.
|
|
132
|
+
// When the engine supports eviction the durable `memo()` skips the action —
|
|
133
|
+
// and the DB round-trip below — entirely on a replay. When it does NOT (a
|
|
134
|
+
// pre-eviction / ':latest'-drifted engine), `memoize` falls through to the
|
|
135
|
+
// closure and Layer 2 (the `connector_deliveries` short-circuit) is the
|
|
136
|
+
// version-INDEPENDENT exactly-once guarantee. Belt-and-suspenders, mirroring
|
|
137
|
+
// the email send path (tracked.ts).
|
|
138
|
+
return boundary.memoize([key], () =>
|
|
139
|
+
runWithConnectorDelivery({
|
|
140
|
+
db,
|
|
141
|
+
connectorId: input.connectorId,
|
|
142
|
+
action: input.action,
|
|
143
|
+
dedupeKey: key,
|
|
144
|
+
run: doRun,
|
|
145
|
+
}),
|
|
146
|
+
);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
interface ConnectorDeliveryArgs {
|
|
150
|
+
db: Database;
|
|
151
|
+
connectorId: string;
|
|
152
|
+
action: string;
|
|
153
|
+
dedupeKey: string;
|
|
154
|
+
run: () => Promise<unknown>;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Layer-2 (version-independent) DB backstop for a connector action, mirroring
|
|
159
|
+
* the `email_sends.idempotencyKey` short-circuit in tracked.ts:
|
|
160
|
+
*
|
|
161
|
+
* 1. SELECT by `(connectorId, dedupeKey)` — a TERMINAL-success ("sent") prior
|
|
162
|
+
* row is a satisfied duplicate: return its stored `result` WITHOUT
|
|
163
|
+
* re-running the action. A "queued" prior row is NOT satisfied (a prior
|
|
164
|
+
* attempt claimed the key but may have died before the action returned) —
|
|
165
|
+
* re-drive it (safer missed>doubled), matching the MF-2 fix in tracked.ts.
|
|
166
|
+
* 2. INSERT `onConflictDoNothing` to claim the key; on a concurrent loser,
|
|
167
|
+
* re-read the winner and apply the same satisfied/queued rule.
|
|
168
|
+
* 3. Run the action, then UPDATE the claimed row with the JSON-round-tripped
|
|
169
|
+
* result + "sent". On a thrown action, stamp "failed" AND release the key
|
|
170
|
+
* (set null) so a retry genuinely re-attempts rather than deduping to it.
|
|
171
|
+
*/
|
|
172
|
+
async function runWithConnectorDelivery(
|
|
173
|
+
args: ConnectorDeliveryArgs,
|
|
174
|
+
): Promise<unknown> {
|
|
175
|
+
const { db, connectorId, action, dedupeKey, run } = args;
|
|
176
|
+
|
|
177
|
+
// 1. Up-front short-circuit: a prior terminal-success row replays its result.
|
|
178
|
+
const existing = await db
|
|
179
|
+
.select({
|
|
180
|
+
id: connectorDeliveries.id,
|
|
181
|
+
status: connectorDeliveries.status,
|
|
182
|
+
result: connectorDeliveries.result,
|
|
183
|
+
})
|
|
184
|
+
.from(connectorDeliveries)
|
|
185
|
+
.where(
|
|
186
|
+
and(
|
|
187
|
+
eq(connectorDeliveries.connectorId, connectorId),
|
|
188
|
+
eq(connectorDeliveries.dedupeKey, dedupeKey),
|
|
189
|
+
),
|
|
190
|
+
)
|
|
191
|
+
.limit(1);
|
|
192
|
+
|
|
193
|
+
const prior = existing[0];
|
|
194
|
+
let rowId: string;
|
|
195
|
+
if (prior) {
|
|
196
|
+
if (prior.status === "sent") return prior.result;
|
|
197
|
+
// Only a "queued" row can reach here: this SELECT is keyed on the (non-null)
|
|
198
|
+
// dedupeKey, and the failure path nulls the key (see below), so a "failed"
|
|
199
|
+
// row is never returned by the keyed lookup. A post-failure retry therefore
|
|
200
|
+
// takes the INSERT-new-row branch below — it never re-drives a failed row.
|
|
201
|
+
// Re-drive the queued row against the same id (the unique index is honored).
|
|
202
|
+
rowId = prior.id;
|
|
203
|
+
} else {
|
|
204
|
+
// 2. Claim the key. Swallow a concurrent-insert collision on the unique
|
|
205
|
+
// index (the select above is not atomic) and resolve the winner.
|
|
206
|
+
const inserted = await db
|
|
207
|
+
.insert(connectorDeliveries)
|
|
208
|
+
.values({ connectorId, action, dedupeKey, status: "queued" })
|
|
209
|
+
.onConflictDoNothing({
|
|
210
|
+
target: [
|
|
211
|
+
connectorDeliveries.connectorId,
|
|
212
|
+
connectorDeliveries.dedupeKey,
|
|
213
|
+
],
|
|
214
|
+
})
|
|
215
|
+
.returning({ id: connectorDeliveries.id });
|
|
216
|
+
|
|
217
|
+
const insertedRow = inserted[0];
|
|
218
|
+
if (insertedRow) {
|
|
219
|
+
rowId = insertedRow.id;
|
|
220
|
+
} else {
|
|
221
|
+
// A concurrent send claimed the key first — resolve its row, applying the
|
|
222
|
+
// same satisfied-vs-requeue rule.
|
|
223
|
+
const winnerRows = await db
|
|
224
|
+
.select({
|
|
225
|
+
id: connectorDeliveries.id,
|
|
226
|
+
status: connectorDeliveries.status,
|
|
227
|
+
result: connectorDeliveries.result,
|
|
228
|
+
})
|
|
229
|
+
.from(connectorDeliveries)
|
|
230
|
+
.where(
|
|
231
|
+
and(
|
|
232
|
+
eq(connectorDeliveries.connectorId, connectorId),
|
|
233
|
+
eq(connectorDeliveries.dedupeKey, dedupeKey),
|
|
234
|
+
),
|
|
235
|
+
)
|
|
236
|
+
.limit(1);
|
|
237
|
+
const winner = winnerRows[0];
|
|
238
|
+
if (!winner) throw new Error("Failed to claim connector_deliveries row");
|
|
239
|
+
if (winner.status === "sent") return winner.result;
|
|
240
|
+
// As above, the winner can only be "sent" or "queued": a "failed" row has
|
|
241
|
+
// a null dedupeKey and is never returned by this keyed lookup.
|
|
242
|
+
rowId = winner.id;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// 3. Run the action and persist the (JSON-round-tripped) result. `result`
|
|
247
|
+
// survives the jsonb column, so a later replay returns the SAME value.
|
|
248
|
+
// Normalize an `undefined` action result to `null` on write: an action may
|
|
249
|
+
// legitimately return nothing (a fire-and-forget send with no message id),
|
|
250
|
+
// and Drizzle's `mapUpdateSet` drops `undefined` values from the UPDATE — so
|
|
251
|
+
// an unnormalized `undefined` would leave the column at its NULL default and a
|
|
252
|
+
// later short-circuit would replay `null`, NOT the `undefined` the first run
|
|
253
|
+
// observed. Coercing here makes both the first run and every replay observe
|
|
254
|
+
// the SAME value (`null`), keeping the round-trip claim literally true.
|
|
255
|
+
try {
|
|
256
|
+
const result = await run();
|
|
257
|
+
const now = new Date();
|
|
258
|
+
await db
|
|
259
|
+
.update(connectorDeliveries)
|
|
260
|
+
.set({
|
|
261
|
+
result: (result === undefined ? null : result) as unknown,
|
|
262
|
+
status: "sent",
|
|
263
|
+
sentAt: now,
|
|
264
|
+
updatedAt: now,
|
|
265
|
+
})
|
|
266
|
+
.where(eq(connectorDeliveries.id, rowId));
|
|
267
|
+
return result === undefined ? null : result;
|
|
268
|
+
} catch (error) {
|
|
269
|
+
// Release the key (set null) so a retry genuinely re-attempts rather than
|
|
270
|
+
// deduping to this failed row — exactly like the tracked-mailer failed path.
|
|
271
|
+
await db
|
|
272
|
+
.update(connectorDeliveries)
|
|
273
|
+
.set({
|
|
274
|
+
status: "failed",
|
|
275
|
+
dedupeKey: null,
|
|
276
|
+
lastError: error instanceof Error ? error.message : String(error),
|
|
277
|
+
updatedAt: new Date(),
|
|
278
|
+
})
|
|
279
|
+
.where(eq(connectorDeliveries.id, rowId));
|
|
280
|
+
throw error;
|
|
281
|
+
}
|
|
90
282
|
}
|
package/src/lib/domain-status.ts
CHANGED
|
@@ -303,7 +303,23 @@ export function createDomainStatusService(deps: {
|
|
|
303
303
|
|
|
304
304
|
// biome-ignore lint/style/noNonNullAssertion: `supported` guarantees it.
|
|
305
305
|
const capability = provider.domains!;
|
|
306
|
-
|
|
306
|
+
let providerStatus: DomainStatus | null;
|
|
307
|
+
try {
|
|
308
|
+
providerStatus = await capability.get(domain);
|
|
309
|
+
} catch (err) {
|
|
310
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
311
|
+
// A permission-denied (401/403) from the provider's domains API — a
|
|
312
|
+
// missing or send-only restricted key — is a CONFIGURATION state, not a
|
|
313
|
+
// server error. Engage the same warn-once + back-off the per-send refresh
|
|
314
|
+
// uses and return a degraded (`status: null`) snapshot, so an explicit
|
|
315
|
+
// getStatus({ refresh }) (admin route, Studio Setup, readiness) resolves
|
|
316
|
+
// to a graceful 200 "not configured" instead of throwing a scary 502.
|
|
317
|
+
// Transient failures (network/5xx) still throw → the caller surfaces 502.
|
|
318
|
+
if (isPermissionDeniedMessage(message)) {
|
|
319
|
+
return markPermissionBlocked(message);
|
|
320
|
+
}
|
|
321
|
+
throw err;
|
|
322
|
+
}
|
|
307
323
|
// The key CAN read domains after all — clear any permission back-off so a
|
|
308
324
|
// key swap recovers immediately and a future restriction warns once again.
|
|
309
325
|
permissionBlocked = false;
|
|
@@ -342,7 +358,7 @@ export function createDomainStatusService(deps: {
|
|
|
342
358
|
* cold cache commits the assumed-verified `null` snapshot (fail-open
|
|
343
359
|
* verified, the existing contract — production mail is never redirected).
|
|
344
360
|
*/
|
|
345
|
-
const markPermissionBlocked = (message: string):
|
|
361
|
+
const markPermissionBlocked = (message: string): EngineDomainStatus => {
|
|
346
362
|
permissionBlocked = true;
|
|
347
363
|
if (!permissionWarned) {
|
|
348
364
|
permissionWarned = true;
|
|
@@ -361,9 +377,12 @@ export function createDomainStatusService(deps: {
|
|
|
361
377
|
// Preserve the last real snapshot as the truth; just push its
|
|
362
378
|
// freshness window out to the back-off TTL.
|
|
363
379
|
cache.fetchedAt = Date.now();
|
|
364
|
-
return;
|
|
380
|
+
return cache.snapshot;
|
|
365
381
|
}
|
|
366
|
-
|
|
382
|
+
// Cold cache: commit the assumed-verified `null` snapshot (fail-open) and
|
|
383
|
+
// hand it back so an explicit getStatus() resolves to a degraded 200
|
|
384
|
+
// instead of throwing.
|
|
385
|
+
return commitSnapshot(null, { assumedVerified: true });
|
|
367
386
|
};
|
|
368
387
|
|
|
369
388
|
return {
|
|
@@ -397,14 +416,10 @@ export function createDomainStatusService(deps: {
|
|
|
397
416
|
if (isFresh()) return;
|
|
398
417
|
void fetchDeduped().catch((error: unknown) => {
|
|
399
418
|
const message = error instanceof Error ? error.message : String(error);
|
|
400
|
-
// Permission-
|
|
401
|
-
//
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
return;
|
|
405
|
-
}
|
|
406
|
-
// Transient failures (network, 5xx) keep the existing behavior: warn
|
|
407
|
-
// every stale refresh, short TTL, fail-open verified via the cache.
|
|
419
|
+
// Permission-denied (401/403) is handled INSIDE fetchSnapshot now
|
|
420
|
+
// (warn-once + long back-off, resolves rather than rejects), so only
|
|
421
|
+
// transient failures (network, 5xx) reach here: warn every stale
|
|
422
|
+
// refresh, short TTL, fail-open verified via the cache.
|
|
408
423
|
logger.warn("domain-status refresh failed", {
|
|
409
424
|
domain,
|
|
410
425
|
providerId,
|
package/src/lib/email.ts
CHANGED
|
@@ -1,4 +1,9 @@
|
|
|
1
1
|
import { generateUnsubscribeUrl } from "@hogsend/email";
|
|
2
|
+
import {
|
|
3
|
+
deriveJourneyKey,
|
|
4
|
+
getJourneyBoundary,
|
|
5
|
+
registerKey,
|
|
6
|
+
} from "../journeys/journey-boundary.js";
|
|
2
7
|
import type {
|
|
3
8
|
EmailService,
|
|
4
9
|
EmailServiceSendOptions,
|
|
@@ -29,6 +34,21 @@ export interface SendEmailOptions {
|
|
|
29
34
|
journeyName?: string;
|
|
30
35
|
journeyStateId?: string;
|
|
31
36
|
props?: Record<string, unknown>;
|
|
37
|
+
/**
|
|
38
|
+
* Explicit idempotency key. A public caller (e.g. POST /v1/emails) sets this
|
|
39
|
+
* directly; it always wins over the engine's auto-derivation. Journey sends
|
|
40
|
+
* leave it unset — the engine derives a deterministic key from the active
|
|
41
|
+
* journey boundary (see {@link idempotencyLabel}).
|
|
42
|
+
*/
|
|
43
|
+
idempotencyKey?: string;
|
|
44
|
+
/**
|
|
45
|
+
* Disambiguates a send's exactly-once idempotency key when the SAME template
|
|
46
|
+
* is sent more than once in one journey enrollment on divergent branches.
|
|
47
|
+
* Normally the engine auto-derives the key from the nearest authored wait
|
|
48
|
+
* label, so this is rarely needed; pass a distinct label per call if the
|
|
49
|
+
* engine throws an intra-run key-collision error. Additive and optional.
|
|
50
|
+
*/
|
|
51
|
+
idempotencyLabel?: string;
|
|
32
52
|
}
|
|
33
53
|
|
|
34
54
|
export interface SendEmailResult {
|
|
@@ -41,6 +61,31 @@ export async function sendEmail(
|
|
|
41
61
|
): Promise<SendEmailResult> {
|
|
42
62
|
const service = getEmailService();
|
|
43
63
|
|
|
64
|
+
// Exactly-once across a durable replay. When inside a journey run, derive a
|
|
65
|
+
// deterministic, branch-stable key (anchored on the replay-stable
|
|
66
|
+
// `boundary.runAnchor` = Hatchet run id, NOT the freshly-minted stateId) so a
|
|
67
|
+
// replay re-firing the same logical send re-derives the SAME key and the unique
|
|
68
|
+
// `email_sends.idempotencyKey` index absorbs the duplicate provider call
|
|
69
|
+
// (Layer 2 — version-independent). The boundary's nearest authored wait label
|
|
70
|
+
// is the "site" discriminant, so two sends of the SAME template on different
|
|
71
|
+
// branches derive distinct keys for free. An explicit key (public callers)
|
|
72
|
+
// always wins; outside a journey (admin bulk / POST /v1/emails) the key is
|
|
73
|
+
// whatever the caller passed (NULL for journey-less sends — Postgres treats
|
|
74
|
+
// NULLs as distinct, so unchanged).
|
|
75
|
+
const boundary = getJourneyBoundary();
|
|
76
|
+
let resolvedIdempotencyKey: string | undefined = opts.idempotencyKey;
|
|
77
|
+
if (!resolvedIdempotencyKey && boundary) {
|
|
78
|
+
const site =
|
|
79
|
+
opts.idempotencyLabel ?? boundary.currentLabel ?? opts.template;
|
|
80
|
+
resolvedIdempotencyKey = deriveJourneyKey({
|
|
81
|
+
kind: "send",
|
|
82
|
+
anchor: boundary.runAnchor,
|
|
83
|
+
site,
|
|
84
|
+
discriminant: opts.template,
|
|
85
|
+
});
|
|
86
|
+
registerKey(boundary, resolvedIdempotencyKey);
|
|
87
|
+
}
|
|
88
|
+
|
|
44
89
|
let unsubscribeUrl: string | undefined;
|
|
45
90
|
if (process.env.API_PUBLIC_URL && process.env.BETTER_AUTH_SECRET) {
|
|
46
91
|
unsubscribeUrl = generateUnsubscribeUrl({
|
|
@@ -78,6 +123,7 @@ export async function sendEmail(
|
|
|
78
123
|
to: opts.to,
|
|
79
124
|
subject: opts.subject,
|
|
80
125
|
journeyStateId: opts.journeyStateId,
|
|
126
|
+
idempotencyKey: resolvedIdempotencyKey,
|
|
81
127
|
userId: opts.userId,
|
|
82
128
|
userEmail: opts.to,
|
|
83
129
|
category: "journey",
|
|
@@ -89,10 +135,22 @@ export async function sendEmail(
|
|
|
89
135
|
headers,
|
|
90
136
|
} as unknown as EmailServiceSendOptions;
|
|
91
137
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
138
|
+
// Layer 1 (fast path): when inside a journey on an eviction-capable engine,
|
|
139
|
+
// run the provider send through Hatchet's durable `memo` so a replay returns
|
|
140
|
+
// the recorded `{ emailSendId, sentAt }` WITHOUT re-hitting the provider or
|
|
141
|
+
// the DB. JSON-safe (both fields are strings). Degrades to a bare send when
|
|
142
|
+
// eviction is unavailable — Layer 2 (the DB key above) still guarantees
|
|
143
|
+
// exactly-once. Outside a journey, send directly (no boundary).
|
|
144
|
+
const doSend = async (): Promise<SendEmailResult> => {
|
|
145
|
+
const result = await service.send(sendOptions);
|
|
146
|
+
return {
|
|
147
|
+
emailSendId: result.emailSendId,
|
|
148
|
+
sentAt: new Date().toISOString(),
|
|
149
|
+
};
|
|
97
150
|
};
|
|
151
|
+
|
|
152
|
+
if (boundary && resolvedIdempotencyKey) {
|
|
153
|
+
return boundary.memoize([resolvedIdempotencyKey], doSend);
|
|
154
|
+
}
|
|
155
|
+
return doSend();
|
|
98
156
|
}
|
package/src/lib/ingestion.ts
CHANGED
|
@@ -35,6 +35,13 @@ export interface IngestEvent {
|
|
|
35
35
|
* the ingest instant. Accepts a `Date` or an ISO-8601 string.
|
|
36
36
|
*/
|
|
37
37
|
occurredAt?: Date | string;
|
|
38
|
+
/**
|
|
39
|
+
* Where the event entered the pipeline — a webhook source id ("posthog",
|
|
40
|
+
* "stripe", …), "api" (public data plane), "studio" (Debug panel), a connector
|
|
41
|
+
* id, "journey" (cross-journey trigger), etc. Stored on `user_events.source`
|
|
42
|
+
* so the Events feed can show + filter by origin. Optional (null when unset).
|
|
43
|
+
*/
|
|
44
|
+
source?: string;
|
|
38
45
|
}
|
|
39
46
|
|
|
40
47
|
export interface ExitResult {
|
|
@@ -57,6 +64,64 @@ export interface IngestResult {
|
|
|
57
64
|
contactKey: string;
|
|
58
65
|
}
|
|
59
66
|
|
|
67
|
+
/**
|
|
68
|
+
* Ingest a connector transform RESULT — a single {@link IngestEvent}, an ARRAY
|
|
69
|
+
* of events (a dual-side fan-out, e.g. a Discord reaction yields a reactor-keyed
|
|
70
|
+
* `reaction_added` AND an author-keyed `reaction_received`), or null. Each
|
|
71
|
+
* element ingests INDEPENDENTLY with per-element error isolation, so one bad
|
|
72
|
+
* element never aborts its siblings, plus a defensive zero-key skip mirroring
|
|
73
|
+
* `pushLinkClickEvent` (`resolveOrCreateContact` throws on a zero-key event).
|
|
74
|
+
* `source` stamps every element's `user_events.source`.
|
|
75
|
+
*/
|
|
76
|
+
export async function ingestTransformResult(opts: {
|
|
77
|
+
result: IngestEvent | IngestEvent[] | null;
|
|
78
|
+
db: Database;
|
|
79
|
+
registry: JourneyRegistry;
|
|
80
|
+
hatchet: HatchetClient;
|
|
81
|
+
logger: Logger;
|
|
82
|
+
source: string;
|
|
83
|
+
analytics?: AnalyticsProvider;
|
|
84
|
+
}): Promise<{ ingested: number; exits: number }> {
|
|
85
|
+
const { result, db, registry, hatchet, logger, source, analytics } = opts;
|
|
86
|
+
if (!result) return { ingested: 0, exits: 0 };
|
|
87
|
+
const events = Array.isArray(result) ? result : [result];
|
|
88
|
+
let ingested = 0;
|
|
89
|
+
let exits = 0;
|
|
90
|
+
for (const event of events) {
|
|
91
|
+
if (
|
|
92
|
+
!event.userId &&
|
|
93
|
+
!event.userEmail &&
|
|
94
|
+
!event.anonymousId &&
|
|
95
|
+
!event.discordId
|
|
96
|
+
) {
|
|
97
|
+
logger.warn("ingestTransformResult: skipping zero-key event", {
|
|
98
|
+
event: event.event,
|
|
99
|
+
source,
|
|
100
|
+
});
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
try {
|
|
104
|
+
const r = await ingestEvent({
|
|
105
|
+
db,
|
|
106
|
+
registry,
|
|
107
|
+
hatchet,
|
|
108
|
+
logger,
|
|
109
|
+
event: { ...event, source },
|
|
110
|
+
analytics,
|
|
111
|
+
});
|
|
112
|
+
ingested++;
|
|
113
|
+
exits += r.exits.length;
|
|
114
|
+
} catch (err) {
|
|
115
|
+
logger.warn("ingestTransformResult: element ingest failed", {
|
|
116
|
+
event: event.event,
|
|
117
|
+
source,
|
|
118
|
+
error: err instanceof Error ? err.message : String(err),
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
return { ingested, exits };
|
|
123
|
+
}
|
|
124
|
+
|
|
60
125
|
export async function ingestEvent(opts: {
|
|
61
126
|
db: Database;
|
|
62
127
|
registry: JourneyRegistry;
|
|
@@ -102,7 +167,10 @@ export async function ingestEvent(opts: {
|
|
|
102
167
|
const occurredAt = event.occurredAt ? new Date(event.occurredAt) : undefined;
|
|
103
168
|
|
|
104
169
|
// (2) Idempotency dedup + `user_events` insert keyed on the resolved key, with
|
|
105
|
-
// ONLY eventProperties in the properties bag (D2).
|
|
170
|
+
// ONLY eventProperties in the properties bag (D2). `ctx.trigger` now supplies a
|
|
171
|
+
// deterministic key (`journeyTrigger:<runAnchor>:<site>:<event>`), so a journey
|
|
172
|
+
// replay re-pushing the same trigger hits the onConflictDoNothing early-return
|
|
173
|
+
// below — the push, checkExits, contact upsert, and alias never re-fire.
|
|
106
174
|
let idempotentInsertId: string | undefined;
|
|
107
175
|
if (event.idempotencyKey) {
|
|
108
176
|
const result = await db
|
|
@@ -111,6 +179,7 @@ export async function ingestEvent(opts: {
|
|
|
111
179
|
userId: resolvedKey,
|
|
112
180
|
event: event.event,
|
|
113
181
|
properties: event.eventProperties,
|
|
182
|
+
source: event.source ?? null,
|
|
114
183
|
idempotencyKey: event.idempotencyKey,
|
|
115
184
|
...(occurredAt ? { occurredAt } : {}),
|
|
116
185
|
})
|
|
@@ -128,6 +197,7 @@ export async function ingestEvent(opts: {
|
|
|
128
197
|
userId: resolvedKey,
|
|
129
198
|
event: event.event,
|
|
130
199
|
properties: event.eventProperties,
|
|
200
|
+
source: event.source ?? null,
|
|
131
201
|
...(occurredAt ? { occurredAt } : {}),
|
|
132
202
|
});
|
|
133
203
|
}
|
package/src/lib/links.ts
CHANGED
|
@@ -56,7 +56,7 @@ export interface MintedLink {
|
|
|
56
56
|
* stored URL verbatim, so giving operators a UI to mint these would otherwise
|
|
57
57
|
* widen the latent open-redirect into `javascript:`/`data:` territory.
|
|
58
58
|
*/
|
|
59
|
-
function assertHttpUrl(url: string): void {
|
|
59
|
+
export function assertHttpUrl(url: string): void {
|
|
60
60
|
let parsed: URL;
|
|
61
61
|
try {
|
|
62
62
|
parsed = new URL(url);
|