@hogsend/engine 0.50.0 → 0.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +10 -10
- package/src/campaigns/cohort-sql.ts +8 -9
- package/src/container.ts +1 -1
- package/src/destinations/define-destination.ts +2 -2
- package/src/flags/reconcile.ts +7 -2
- package/src/index.ts +9 -1
- package/src/journeys/execute-journey-run.ts +15 -0
- package/src/journeys/group-scope.ts +153 -0
- package/src/journeys/journey-context.ts +152 -19
- package/src/journeys/record-once.ts +27 -2
- package/src/lib/campaign-send-key.ts +9 -18
- package/src/lib/conversions.ts +1 -1
- package/src/lib/crm-deals.ts +1 -1
- package/src/lib/crm-ingest.ts +1 -1
- package/src/lib/event-names.ts +7 -1
- package/src/lib/frequency-cap.ts +4 -2
- package/src/lib/funnel-progress.ts +1 -1
- package/src/lib/holdout.ts +1 -1
- package/src/lib/ingestion.ts +6 -0
- package/src/lib/journey-suppress.ts +4 -2
- package/src/lib/lift-stats.ts +2 -2
- package/src/lib/tracked.ts +9 -2
- package/src/routes/admin/attribution.ts +1 -2
- package/src/routes/admin/campaigns.ts +84 -45
- package/src/routes/admin/conversions.ts +1 -1
- package/src/routes/admin/deals.ts +43 -34
- package/src/routes/admin/emails.ts +4 -6
- package/src/routes/admin/events.ts +128 -1
- package/src/routes/admin/groups.ts +34 -0
- package/src/routes/admin/impact.ts +19 -19
- package/src/routes/admin/targeting.ts +4 -1
- package/src/workflows/crm-reconcile.ts +1 -1
- package/src/middleware/auth.ts +0 -24
|
@@ -17,12 +17,10 @@ import {
|
|
|
17
17
|
inArray,
|
|
18
18
|
isNotNull,
|
|
19
19
|
isNull,
|
|
20
|
-
like,
|
|
21
20
|
lte,
|
|
22
21
|
or,
|
|
23
22
|
} from "drizzle-orm";
|
|
24
23
|
import type { AppEnv } from "../../app.js";
|
|
25
|
-
import { campaignSendKeyPattern } from "../../lib/campaign-send-key.js";
|
|
26
24
|
|
|
27
25
|
const emailSchema = z.object({
|
|
28
26
|
id: z.string(),
|
|
@@ -142,7 +140,7 @@ const listRoute = createRoute({
|
|
|
142
140
|
* Only the sends of one campaign — matched on the deterministic
|
|
143
141
|
* `campaign:<id>:<email>` idempotency key the blast wrote per recipient.
|
|
144
142
|
*/
|
|
145
|
-
campaignId: z.string().optional(),
|
|
143
|
+
campaignId: z.string().uuid().optional(),
|
|
146
144
|
userId: z.string().optional(),
|
|
147
145
|
category: z.string().optional(),
|
|
148
146
|
engagement: z
|
|
@@ -278,10 +276,10 @@ export const emailsRouter = new OpenAPIHono<AppEnv>()
|
|
|
278
276
|
if (status) conditions.push(eq(emailSends.status, status));
|
|
279
277
|
if (category) conditions.push(eq(emailSends.category, category));
|
|
280
278
|
if (journeyId) conditions.push(eq(journeyStates.journeyId, journeyId));
|
|
279
|
+
// The indexed campaign_id column is the attribution — a key scan would
|
|
280
|
+
// miss suppressed sends, which write no idempotency key at all.
|
|
281
281
|
if (campaignId) {
|
|
282
|
-
conditions.push(
|
|
283
|
-
like(emailSends.idempotencyKey, campaignSendKeyPattern(campaignId)),
|
|
284
|
-
);
|
|
282
|
+
conditions.push(eq(emailSends.campaignId, campaignId));
|
|
285
283
|
}
|
|
286
284
|
// Match the denormalized identity OR the journey-state join, so journeyless
|
|
287
285
|
// sends (which only carry the denormalized userId) are still filterable.
|
|
@@ -1,6 +1,19 @@
|
|
|
1
1
|
import { contacts, userEvents } from "@hogsend/db";
|
|
2
2
|
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
3
|
-
import {
|
|
3
|
+
import {
|
|
4
|
+
and,
|
|
5
|
+
count,
|
|
6
|
+
desc,
|
|
7
|
+
eq,
|
|
8
|
+
gte,
|
|
9
|
+
isNotNull,
|
|
10
|
+
isNull,
|
|
11
|
+
lte,
|
|
12
|
+
max,
|
|
13
|
+
min,
|
|
14
|
+
or,
|
|
15
|
+
sql,
|
|
16
|
+
} from "drizzle-orm";
|
|
4
17
|
import type { AppEnv } from "../../app.js";
|
|
5
18
|
import type { HogsendClient } from "../../container.js";
|
|
6
19
|
import {
|
|
@@ -169,6 +182,59 @@ const namesRoute = createRoute({
|
|
|
169
182
|
},
|
|
170
183
|
});
|
|
171
184
|
|
|
185
|
+
/**
|
|
186
|
+
* One-line explanations for the engine-stamped ingest origins, so the Studio
|
|
187
|
+
* source picker can say what a source IS rather than just echo its slug.
|
|
188
|
+
* Anything not listed here and not a registered connector is "observed" —
|
|
189
|
+
* sources are an open vocabulary exactly like event names.
|
|
190
|
+
*/
|
|
191
|
+
const BUILTIN_SOURCES: Record<string, string> = {
|
|
192
|
+
api: "Server-side ingest — the secret-key /v1/events endpoint and SDKs.",
|
|
193
|
+
inapp: "Browser SDK ingest (publishable key).",
|
|
194
|
+
studio: "Fired from the Studio (debug drawer, admin tools).",
|
|
195
|
+
journey: "Emitted by a journey via ctx.trigger().",
|
|
196
|
+
connector: "Inbound connector engagement (Discord, Telegram, …).",
|
|
197
|
+
tracking: "First-party email tracking — opens, clicks, semantic actions.",
|
|
198
|
+
bucket: "Bucket entry/exit transitions.",
|
|
199
|
+
import: "Contact import jobs.",
|
|
200
|
+
sms: "SMS pipeline — delivery status, link clicks, inbound messages.",
|
|
201
|
+
};
|
|
202
|
+
|
|
203
|
+
const sourceEntrySchema = z.object({
|
|
204
|
+
name: z.string(),
|
|
205
|
+
occurrences: z.number(),
|
|
206
|
+
firstSeenAt: z.string().nullable(),
|
|
207
|
+
lastSeenAt: z.string().nullable(),
|
|
208
|
+
// "connector" = a registered inbound connector (label = its display name),
|
|
209
|
+
// "builtin" = an engine-stamped origin (label = what it means),
|
|
210
|
+
// "observed" = seen in the event store but not otherwise known.
|
|
211
|
+
kind: z.enum(["connector", "builtin", "observed"]),
|
|
212
|
+
label: z.string().nullable(),
|
|
213
|
+
});
|
|
214
|
+
|
|
215
|
+
const sourcesRoute = createRoute({
|
|
216
|
+
method: "get",
|
|
217
|
+
path: "/sources",
|
|
218
|
+
tags: ["Admin — Events"],
|
|
219
|
+
summary: "List event sources (observed + registered)",
|
|
220
|
+
description:
|
|
221
|
+
"Best-effort source vocabulary for the events filter. Merges sources " +
|
|
222
|
+
"actually observed on stored events (with occurrence counts and " +
|
|
223
|
+
"first/last seen) with registered inbound connectors that haven't " +
|
|
224
|
+
"fired yet (zero-backfilled). Sources are an open vocabulary — any " +
|
|
225
|
+
"other value is also valid.",
|
|
226
|
+
responses: {
|
|
227
|
+
200: {
|
|
228
|
+
content: {
|
|
229
|
+
"application/json": {
|
|
230
|
+
schema: z.object({ sources: z.array(sourceEntrySchema) }),
|
|
231
|
+
},
|
|
232
|
+
},
|
|
233
|
+
description: "Merged observed + registered source vocabulary",
|
|
234
|
+
},
|
|
235
|
+
},
|
|
236
|
+
});
|
|
237
|
+
|
|
172
238
|
const exitSchema = z.object({
|
|
173
239
|
journeyId: z.string(),
|
|
174
240
|
stateId: z.string(),
|
|
@@ -292,6 +358,67 @@ export const eventsRouter = new OpenAPIHono<AppEnv>()
|
|
|
292
358
|
});
|
|
293
359
|
return c.json({ note, events }, 200);
|
|
294
360
|
})
|
|
361
|
+
// Registered before getRoute so the literal "sources" is never captured as
|
|
362
|
+
// an {id}. One grouped scan over the indexed source column (sources number
|
|
363
|
+
// in the dozens at most), then registered-connector + builtin labeling.
|
|
364
|
+
.openapi(sourcesRoute, async (c) => {
|
|
365
|
+
const container = c.get("container");
|
|
366
|
+
const { db } = container;
|
|
367
|
+
|
|
368
|
+
const observed = await db
|
|
369
|
+
.select({
|
|
370
|
+
name: userEvents.source,
|
|
371
|
+
occurrences: count(),
|
|
372
|
+
firstSeenAt: min(userEvents.occurredAt),
|
|
373
|
+
lastSeenAt: max(userEvents.occurredAt),
|
|
374
|
+
})
|
|
375
|
+
.from(userEvents)
|
|
376
|
+
.where(isNotNull(userEvents.source))
|
|
377
|
+
.groupBy(userEvents.source)
|
|
378
|
+
.orderBy(desc(count()));
|
|
379
|
+
|
|
380
|
+
const connectors = new Map(
|
|
381
|
+
container.connectorRegistry
|
|
382
|
+
.getAll()
|
|
383
|
+
.map((conn) => [conn.meta.id, conn.meta.name ?? conn.meta.id]),
|
|
384
|
+
);
|
|
385
|
+
|
|
386
|
+
const entries = observed
|
|
387
|
+
.filter((row) => row.name !== null)
|
|
388
|
+
.map((row) => {
|
|
389
|
+
const name = row.name as string;
|
|
390
|
+
const connectorLabel = connectors.get(name);
|
|
391
|
+
return {
|
|
392
|
+
name,
|
|
393
|
+
occurrences: row.occurrences,
|
|
394
|
+
firstSeenAt: row.firstSeenAt?.toISOString() ?? null,
|
|
395
|
+
lastSeenAt: row.lastSeenAt?.toISOString() ?? null,
|
|
396
|
+
kind: connectorLabel
|
|
397
|
+
? ("connector" as const)
|
|
398
|
+
: BUILTIN_SOURCES[name]
|
|
399
|
+
? ("builtin" as const)
|
|
400
|
+
: ("observed" as const),
|
|
401
|
+
label: connectorLabel ?? BUILTIN_SOURCES[name] ?? null,
|
|
402
|
+
};
|
|
403
|
+
});
|
|
404
|
+
|
|
405
|
+
// Zero-backfill registered connectors that haven't fired yet, so a
|
|
406
|
+
// freshly-wired source is pickable before its first event arrives.
|
|
407
|
+
const seen = new Set(entries.map((e) => e.name));
|
|
408
|
+
for (const [id, label] of connectors) {
|
|
409
|
+
if (seen.has(id)) continue;
|
|
410
|
+
entries.push({
|
|
411
|
+
name: id,
|
|
412
|
+
occurrences: 0,
|
|
413
|
+
firstSeenAt: null,
|
|
414
|
+
lastSeenAt: null,
|
|
415
|
+
kind: "connector" as const,
|
|
416
|
+
label,
|
|
417
|
+
});
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
return c.json({ sources: entries }, 200);
|
|
421
|
+
})
|
|
295
422
|
.openapi(getRoute, async (c) => {
|
|
296
423
|
const { db } = c.get("container");
|
|
297
424
|
const { id } = c.req.valid("param");
|
|
@@ -575,6 +575,28 @@ const listRoute = createRoute({
|
|
|
575
575
|
},
|
|
576
576
|
});
|
|
577
577
|
|
|
578
|
+
const listTypesRoute = createRoute({
|
|
579
|
+
method: "get",
|
|
580
|
+
path: "/types",
|
|
581
|
+
tags: ["Admin — Groups"],
|
|
582
|
+
summary: "List distinct group types",
|
|
583
|
+
responses: {
|
|
584
|
+
200: {
|
|
585
|
+
content: {
|
|
586
|
+
"application/json": {
|
|
587
|
+
schema: z.object({
|
|
588
|
+
types: z.array(
|
|
589
|
+
z.object({ groupType: z.string(), count: z.number() }),
|
|
590
|
+
),
|
|
591
|
+
}),
|
|
592
|
+
},
|
|
593
|
+
},
|
|
594
|
+
description:
|
|
595
|
+
"Distinct group types across live groups with per-type counts, largest first",
|
|
596
|
+
},
|
|
597
|
+
},
|
|
598
|
+
});
|
|
599
|
+
|
|
578
600
|
const getRoute = createRoute({
|
|
579
601
|
method: "get",
|
|
580
602
|
path: "/{groupType}/{groupKey}",
|
|
@@ -697,6 +719,18 @@ export const groupsRouter = new OpenAPIHono<AppEnv>()
|
|
|
697
719
|
200,
|
|
698
720
|
);
|
|
699
721
|
})
|
|
722
|
+
// Registered before the `/{groupType}/{groupKey}` param route so the
|
|
723
|
+
// literal "types" isn't captured as a groupType.
|
|
724
|
+
.openapi(listTypesRoute, async (c) => {
|
|
725
|
+
const { db } = c.get("container");
|
|
726
|
+
const rows = await db
|
|
727
|
+
.select({ groupType: groups.groupType, count: count() })
|
|
728
|
+
.from(groups)
|
|
729
|
+
.where(isNull(groups.deletedAt))
|
|
730
|
+
.groupBy(groups.groupType)
|
|
731
|
+
.orderBy(desc(count()), asc(groups.groupType));
|
|
732
|
+
return c.json({ types: rows }, 200);
|
|
733
|
+
})
|
|
700
734
|
.openapi(listMembersRoute, async (c) => {
|
|
701
735
|
const { db } = c.get("container");
|
|
702
736
|
const { groupType, groupKey } = c.req.valid("param");
|
|
@@ -353,12 +353,15 @@ export const impactOverviewRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
353
353
|
);
|
|
354
354
|
|
|
355
355
|
// (d) Campaigns — correlational only, ACTIVITY-windowed enumeration:
|
|
356
|
-
// ids come from the in-window email_sends
|
|
356
|
+
// ids come from the in-window email_sends campaign_id rollup ∪ in-window
|
|
357
357
|
// attribution_credits.campaign_id (NOT "newest 50 created in window" —
|
|
358
358
|
// that drops older-but-active multi-step/scheduled campaigns whose
|
|
359
|
-
// sends fall inside the window). Cap 50 by send volume desc.
|
|
360
|
-
|
|
361
|
-
|
|
359
|
+
// sends fall inside the window). Cap 50 by send volume desc. The
|
|
360
|
+
// indexed campaign_id column is the attribution (stamped at send time,
|
|
361
|
+
// backfilled from legacy idempotency keys by migration 0051) — never
|
|
362
|
+
// key parsing. Policy-suppressed rows (status 'suppressed') carry the FK
|
|
363
|
+
// but were never dispatched, so they stay out of the funnel; provider
|
|
364
|
+
// failures (status 'failed', key released) stay IN it as attempts.
|
|
362
365
|
const [campaignSendRows, campaignCreditRows] = await Promise.all([
|
|
363
366
|
db.execute<{
|
|
364
367
|
campaign_id: string;
|
|
@@ -367,13 +370,14 @@ export const impactOverviewRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
367
370
|
opened: number;
|
|
368
371
|
clicked: number;
|
|
369
372
|
}>(sql`
|
|
370
|
-
select
|
|
373
|
+
select campaign_id::text as campaign_id,
|
|
371
374
|
count(*)::int as sends,
|
|
372
375
|
count(delivered_at)::int as delivered,
|
|
373
376
|
count(opened_at)::int as opened,
|
|
374
377
|
count(clicked_at)::int as clicked
|
|
375
378
|
from email_sends
|
|
376
|
-
where
|
|
379
|
+
where campaign_id is not null
|
|
380
|
+
and status <> 'suppressed'
|
|
377
381
|
and created_at >= ${sinceTs}
|
|
378
382
|
group by 1
|
|
379
383
|
`),
|
|
@@ -395,19 +399,15 @@ export const impactOverviewRouter = new OpenAPIHono<AppEnv>().openapi(
|
|
|
395
399
|
]);
|
|
396
400
|
|
|
397
401
|
const sendsByCampaign = new Map(
|
|
398
|
-
[...campaignSendRows]
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
r.
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
opened: Number(r.opened),
|
|
408
|
-
clicked: Number(r.clicked),
|
|
409
|
-
},
|
|
410
|
-
]),
|
|
402
|
+
[...campaignSendRows].map((r) => [
|
|
403
|
+
r.campaign_id,
|
|
404
|
+
{
|
|
405
|
+
sends: Number(r.sends),
|
|
406
|
+
delivered: Number(r.delivered),
|
|
407
|
+
opened: Number(r.opened),
|
|
408
|
+
clicked: Number(r.clicked),
|
|
409
|
+
},
|
|
410
|
+
]),
|
|
411
411
|
);
|
|
412
412
|
const creditsByCampaign = new Map<
|
|
413
413
|
string,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { CANONICAL_STAGES, flagTargetingSchema } from "@hogsend/core";
|
|
2
2
|
import { campaigns, contacts } from "@hogsend/db";
|
|
3
|
+
import { getTemplateNames } from "@hogsend/email";
|
|
3
4
|
import { createRoute, OpenAPIHono, z } from "@hono/zod-openapi";
|
|
4
5
|
import { desc, isNull, sql } from "drizzle-orm";
|
|
5
6
|
import type { AppEnv } from "../../app.js";
|
|
@@ -93,6 +94,7 @@ const catalogSchema = z.object({
|
|
|
93
94
|
dealStages: z.array(z.string()),
|
|
94
95
|
events: z.array(eventNameEntrySchema),
|
|
95
96
|
campaigns: z.array(idNameSchema),
|
|
97
|
+
templates: z.array(z.string()),
|
|
96
98
|
});
|
|
97
99
|
|
|
98
100
|
const catalogRoute = createRoute({
|
|
@@ -146,7 +148,7 @@ const countRoute = createRoute({
|
|
|
146
148
|
|
|
147
149
|
export const targetingRouter = new OpenAPIHono<AppEnv>()
|
|
148
150
|
.openapi(catalogRoute, async (c) => {
|
|
149
|
-
const { db, registry, bucketRegistry } = c.get("container");
|
|
151
|
+
const { db, registry, bucketRegistry, templates } = c.get("container");
|
|
150
152
|
|
|
151
153
|
// Distinct top-level property keys across a bounded SAMPLE of the most
|
|
152
154
|
// recently-updated live contacts, sorted + capped. Sampling first (inner
|
|
@@ -191,6 +193,7 @@ export const targetingRouter = new OpenAPIHono<AppEnv>()
|
|
|
191
193
|
dealStages: DEAL_STAGES,
|
|
192
194
|
events: events.events,
|
|
193
195
|
campaigns: campaignRows,
|
|
196
|
+
templates: getTemplateNames(templates),
|
|
194
197
|
},
|
|
195
198
|
200,
|
|
196
199
|
);
|
|
@@ -9,7 +9,7 @@ import { hatchet } from "../lib/hatchet.js";
|
|
|
9
9
|
import { createLogger, type Logger } from "../lib/logger.js";
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
|
-
* CRM reconciliation poll
|
|
12
|
+
* CRM reconciliation poll — the pull
|
|
13
13
|
* half of the webhook+poll hybrid. Every provider implementing `poll` is
|
|
14
14
|
* walked on a cron: read the persisted cursor, pull changed deals, feed them
|
|
15
15
|
* through the SAME `ingestCrmStageEvents` sink the webhook route uses (the
|
package/src/middleware/auth.ts
DELETED
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
import type { Context } from "hono";
|
|
2
|
-
import { createMiddleware } from "hono/factory";
|
|
3
|
-
import type { AppEnv } from "../app.js";
|
|
4
|
-
|
|
5
|
-
async function resolveSession(c: Context<AppEnv>) {
|
|
6
|
-
const { auth } = c.get("container");
|
|
7
|
-
const session = await auth.api.getSession({ headers: c.req.raw.headers });
|
|
8
|
-
c.set("user", session?.user ?? null);
|
|
9
|
-
c.set("session", session?.session ?? null);
|
|
10
|
-
return session;
|
|
11
|
-
}
|
|
12
|
-
|
|
13
|
-
export const sessionMiddleware = createMiddleware<AppEnv>(async (c, next) => {
|
|
14
|
-
await resolveSession(c);
|
|
15
|
-
return next();
|
|
16
|
-
});
|
|
17
|
-
|
|
18
|
-
export const requireAuth = createMiddleware<AppEnv>(async (c, next) => {
|
|
19
|
-
const session = await resolveSession(c);
|
|
20
|
-
if (!session) {
|
|
21
|
-
return c.json({ error: "Unauthorized" }, 401);
|
|
22
|
-
}
|
|
23
|
-
return next();
|
|
24
|
-
});
|