@hogsend/engine 0.42.0 → 0.44.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +10 -7
- package/src/container.ts +419 -4
- package/src/destinations/define-destination.ts +7 -3
- package/src/destinations/presets/posthog.ts +31 -0
- package/src/env.ts +28 -0
- package/src/index.ts +138 -0
- package/src/journeys/execute-journey-run.ts +93 -0
- package/src/journeys/journey-boundary.ts +5 -1
- package/src/journeys/journey-context.ts +30 -0
- package/src/lib/attribution-backfill.ts +285 -0
- package/src/lib/attribution.ts +239 -0
- package/src/lib/contacts.ts +71 -0
- package/src/lib/conversion-dispatch.ts +326 -0
- package/src/lib/conversions.ts +197 -0
- package/src/lib/crm-deals.ts +320 -0
- package/src/lib/crm-ingest.ts +341 -0
- package/src/lib/crm-provider-registry.ts +41 -0
- package/src/lib/crm-registry-singleton.ts +22 -0
- package/src/lib/deal-money-events.ts +85 -0
- package/src/lib/email-service-types.ts +16 -1
- package/src/lib/event-names.ts +1 -1
- package/src/lib/funnel-progress.ts +76 -0
- package/src/lib/funnel-registry.ts +108 -0
- package/src/lib/funnel-transitions.ts +280 -0
- package/src/lib/holdout.ts +62 -0
- package/src/lib/ingestion.ts +218 -9
- package/src/lib/journey-suppress.ts +33 -2
- package/src/lib/lift-stats.ts +125 -0
- package/src/lib/mailer.ts +1 -0
- package/src/lib/operator.ts +31 -0
- package/src/lib/outbound.ts +109 -0
- package/src/lib/phone.ts +19 -0
- package/src/lib/preferences.ts +32 -0
- package/src/lib/revenue.ts +116 -0
- package/src/lib/sms-frequency-cap.ts +47 -0
- package/src/lib/sms-inbound.ts +200 -0
- package/src/lib/sms-link-tracking.ts +160 -0
- package/src/lib/sms-mailer.ts +338 -0
- package/src/lib/sms-provider-registry.ts +35 -0
- package/src/lib/sms-providers-from-env.ts +100 -0
- package/src/lib/sms-service-types.ts +175 -0
- package/src/lib/sms-tracked.ts +625 -0
- package/src/lib/sms.ts +117 -0
- package/src/lib/tracked.ts +40 -0
- package/src/lib/tracking-event-names.ts +6 -0
- package/src/lib/tracking-events.ts +133 -1
- package/src/lib/tracking.ts +2 -1
- package/src/lib/webhook-signing.ts +24 -1
- package/src/lists/channels.ts +21 -0
- package/src/routes/admin/attribution.ts +414 -0
- package/src/routes/admin/contacts.ts +71 -11
- package/src/routes/admin/conversions.ts +247 -0
- package/src/routes/admin/deals.ts +472 -0
- package/src/routes/admin/funnels.ts +228 -0
- package/src/routes/admin/index.ts +8 -0
- package/src/routes/admin/journeys.ts +158 -1
- package/src/routes/admin/readiness.ts +97 -2
- package/src/routes/admin/templates.ts +28 -1
- package/src/routes/email/unsubscribe.ts +1 -0
- package/src/routes/events/index.ts +12 -0
- package/src/routes/index.ts +6 -0
- package/src/routes/lists/index.ts +54 -7
- package/src/routes/tracking/click-pipeline.ts +105 -16
- package/src/routes/tracking/short.ts +61 -0
- package/src/routes/webhooks/crm-provider.ts +117 -0
- package/src/routes/webhooks/index.ts +9 -1
- package/src/routes/webhooks/sms-provider.ts +110 -0
- package/src/sources/define-contact-source.ts +159 -0
- package/src/sources/registry.ts +71 -0
- package/src/sources/webhook-source.ts +109 -0
- package/src/worker.ts +4 -0
- package/src/workflows/crm-reconcile.ts +129 -0
- package/src/workflows/dispatch-conversion.ts +31 -0
- package/src/workflows/import-contacts.ts +61 -4
- package/src/workflows/send-campaign.ts +5 -0
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { DefinedContactSource } from "./define-contact-source.js";
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* In-process index of the registered contact sources (Clay, Attio, generic
|
|
5
|
+
* webhook, …), keyed by `meta.id`. Mirrors {@link ListRegistry} /
|
|
6
|
+
* `ConnectorRegistry`: a plain id-keyed map plus the one predicate the rest of
|
|
7
|
+
* the engine needs — {@link ContactSourceRegistry.isProspectSource}, which
|
|
8
|
+
* decides whether a contact's stamped `contacts.source` names a Contact Source
|
|
9
|
+
* (⇒ a cold **prospect**) versus an ordinary inbound origin ("api"/"posthog").
|
|
10
|
+
* The cold posture + write-back adapter travel with each entry so the cold gate
|
|
11
|
+
* and the write-back step can resolve them by source id.
|
|
12
|
+
*/
|
|
13
|
+
export class ContactSourceRegistry {
|
|
14
|
+
private sources: Map<string, DefinedContactSource> = new Map();
|
|
15
|
+
|
|
16
|
+
register(source: DefinedContactSource): void {
|
|
17
|
+
this.sources.set(source.meta.id, source);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
get(id: string): DefinedContactSource | undefined {
|
|
21
|
+
return this.sources.get(id);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
getAll(): DefinedContactSource[] {
|
|
25
|
+
return Array.from(this.sources.values());
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
has(id: string): boolean {
|
|
29
|
+
return this.sources.has(id);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
count(): number {
|
|
33
|
+
return this.sources.size;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Does `source` (a `contacts.source` value) name a registered Contact Source?
|
|
38
|
+
* A contact whose provenance is a Contact Source id is a cold **prospect**;
|
|
39
|
+
* one stamped with a plain pipeline origin ("api"/"posthog"/…) or null is not.
|
|
40
|
+
* Null/undefined ⇒ false (an un-sourced contact is never a prospect).
|
|
41
|
+
*/
|
|
42
|
+
isProspectSource(source: string | null | undefined): boolean {
|
|
43
|
+
return source != null && this.sources.has(source);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
// The process singleton — installed by `createHogsendClient` at container build
|
|
48
|
+
// so the cold gate + write-back can resolve a source without threading the
|
|
49
|
+
// container. Defaults to an empty registry (no sources ⇒ nothing is a prospect
|
|
50
|
+
// source), mirroring the lists registry-singleton.
|
|
51
|
+
let currentRegistry: ContactSourceRegistry | undefined;
|
|
52
|
+
|
|
53
|
+
export function setContactSourceRegistry(
|
|
54
|
+
registry: ContactSourceRegistry,
|
|
55
|
+
): void {
|
|
56
|
+
currentRegistry = registry;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function getContactSourceRegistry(): ContactSourceRegistry {
|
|
60
|
+
return currentRegistry ?? new ContactSourceRegistry();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Build a {@link ContactSourceRegistry} from sources and install the singleton. */
|
|
64
|
+
export function buildContactSourceRegistry(
|
|
65
|
+
sources: DefinedContactSource[] = [],
|
|
66
|
+
): ContactSourceRegistry {
|
|
67
|
+
const registry = new ContactSourceRegistry();
|
|
68
|
+
for (const source of sources) registry.register(source);
|
|
69
|
+
setContactSourceRegistry(registry);
|
|
70
|
+
return registry;
|
|
71
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { z } from "zod";
|
|
2
|
+
import type { IngestEvent } from "../lib/ingestion.js";
|
|
3
|
+
import type { WebhookSourceAuth } from "../webhook-sources/define-webhook-source.js";
|
|
4
|
+
import {
|
|
5
|
+
type ColdPosture,
|
|
6
|
+
type ContactWriteBack,
|
|
7
|
+
type DefinedContactSource,
|
|
8
|
+
defineContactSource,
|
|
9
|
+
} from "./define-contact-source.js";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Normalized inbound payload for the built-in GENERIC webhook contact source.
|
|
13
|
+
* Anything that can POST JSON — a HubSpot/Salesforce workflow, Zapier/Make/n8n,
|
|
14
|
+
* a bespoke CRM — maps its record to this shape and gets a sourced prospect with
|
|
15
|
+
* no bespoke adapter. At least one of `email` / `external_id` is required (an
|
|
16
|
+
* event with no resolvable key can't create a contact).
|
|
17
|
+
*/
|
|
18
|
+
export const webhookContactPayloadSchema = z
|
|
19
|
+
.object({
|
|
20
|
+
/** The sourcing event name (e.g. "prospect.sourced"). */
|
|
21
|
+
event: z.string().min(1),
|
|
22
|
+
/** The anchor identity key for a sourced lead. */
|
|
23
|
+
email: z.string().email().optional(),
|
|
24
|
+
/** Alternative/again identity key (a CRM record id, etc.). */
|
|
25
|
+
external_id: z.string().min(1).optional(),
|
|
26
|
+
anonymous_id: z.string().min(1).optional(),
|
|
27
|
+
/** Enrichment → `contacts.properties` (merged additively). */
|
|
28
|
+
properties: z.record(z.string(), z.unknown()).optional(),
|
|
29
|
+
/** Event payload → `user_events` only. */
|
|
30
|
+
event_properties: z.record(z.string(), z.unknown()).optional(),
|
|
31
|
+
/** Dedup key so source retries / re-sends don't re-ingest (Layer 2). */
|
|
32
|
+
idempotency_key: z.string().min(1).optional(),
|
|
33
|
+
/** Caller-supplied event time (ISO-8601) for backfill/replay. */
|
|
34
|
+
occurred_at: z.string().datetime().optional(),
|
|
35
|
+
})
|
|
36
|
+
.refine((p) => Boolean(p.email || p.external_id), {
|
|
37
|
+
message: "at least one of email / external_id is required",
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
export type WebhookContactPayload = z.infer<typeof webhookContactPayloadSchema>;
|
|
41
|
+
|
|
42
|
+
export interface WebhookContactSourceOptions {
|
|
43
|
+
/** The source id — keys the registry AND the route `/v1/webhooks/:id`. */
|
|
44
|
+
id: string;
|
|
45
|
+
name?: string;
|
|
46
|
+
description?: string;
|
|
47
|
+
/** Header carrying the shared secret. Default `x-hogsend-secret`. */
|
|
48
|
+
header?: string;
|
|
49
|
+
/**
|
|
50
|
+
* Env var holding the shared secret the header is matched against. NOTE: a
|
|
51
|
+
* `match` auth is OPEN when the env value is unset — set the secret in
|
|
52
|
+
* production (this source mints identified contacts).
|
|
53
|
+
*/
|
|
54
|
+
envKey: string;
|
|
55
|
+
coldPosture?: ColdPosture;
|
|
56
|
+
writeBack?: ContactWriteBack;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Map the normalized payload to an {@link IngestEvent}. Pure + unit-tested; the
|
|
61
|
+
* `source` (provenance) is stamped by the webhook route from the source id, so
|
|
62
|
+
* it is intentionally NOT set here.
|
|
63
|
+
*/
|
|
64
|
+
export function normalizeWebhookContactEvent(
|
|
65
|
+
payload: WebhookContactPayload,
|
|
66
|
+
): IngestEvent {
|
|
67
|
+
return {
|
|
68
|
+
event: payload.event,
|
|
69
|
+
userEmail: payload.email,
|
|
70
|
+
userId: payload.external_id,
|
|
71
|
+
anonymousId: payload.anonymous_id,
|
|
72
|
+
eventProperties: payload.event_properties ?? {},
|
|
73
|
+
contactProperties: payload.properties,
|
|
74
|
+
idempotencyKey: payload.idempotency_key,
|
|
75
|
+
occurredAt: payload.occurred_at,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Build the built-in GENERIC webhook contact source — the day-one path for any
|
|
81
|
+
* CRM/tool that isn't Clay/Attio. Point its outbound webhook at
|
|
82
|
+
* `POST /v1/webhooks/:id` with the shared-secret header and the normalized
|
|
83
|
+
* {@link webhookContactPayloadSchema}. Auth is a shared-secret header MATCH
|
|
84
|
+
* ({@link WebhookSourceAuth} `type: "match"`); the cold posture defaults to
|
|
85
|
+
* email-only (see {@link defineContactSource}).
|
|
86
|
+
*/
|
|
87
|
+
export function webhookContactSource(
|
|
88
|
+
opts: WebhookContactSourceOptions,
|
|
89
|
+
): DefinedContactSource<WebhookContactPayload> {
|
|
90
|
+
const auth: WebhookSourceAuth = {
|
|
91
|
+
type: "match",
|
|
92
|
+
header: opts.header ?? "x-hogsend-secret",
|
|
93
|
+
envKey: opts.envKey,
|
|
94
|
+
};
|
|
95
|
+
return defineContactSource<WebhookContactPayload>({
|
|
96
|
+
meta: {
|
|
97
|
+
id: opts.id,
|
|
98
|
+
name: opts.name ?? "Webhook",
|
|
99
|
+
description:
|
|
100
|
+
opts.description ??
|
|
101
|
+
"Generic webhook contact source (normalized CRM/tool payload).",
|
|
102
|
+
},
|
|
103
|
+
auth,
|
|
104
|
+
schema: webhookContactPayloadSchema,
|
|
105
|
+
coldPosture: opts.coldPosture,
|
|
106
|
+
writeBack: opts.writeBack,
|
|
107
|
+
transform: async (payload) => normalizeWebhookContactEvent(payload),
|
|
108
|
+
});
|
|
109
|
+
}
|
package/src/worker.ts
CHANGED
|
@@ -24,10 +24,12 @@ import {
|
|
|
24
24
|
import { bucketReconcileTask } from "./workflows/bucket-reconcile.js";
|
|
25
25
|
import { checkAlertsTask } from "./workflows/check-alerts.js";
|
|
26
26
|
import { confirmSemanticClickTask } from "./workflows/confirm-semantic-click.js";
|
|
27
|
+
import { crmReconcileTask } from "./workflows/crm-reconcile.js";
|
|
27
28
|
import {
|
|
28
29
|
deliverWebhookTask,
|
|
29
30
|
reapDueWebhookDeliveriesTask,
|
|
30
31
|
} from "./workflows/deliver-webhook.js";
|
|
32
|
+
import { dispatchConversionTask } from "./workflows/dispatch-conversion.js";
|
|
31
33
|
import { importContactsTask } from "./workflows/import-contacts.js";
|
|
32
34
|
import { importSuppressionsTask } from "./workflows/import-suppressions.js";
|
|
33
35
|
import { journeyBlueprintInterpreter } from "./workflows/journey-blueprint-interpreter.js";
|
|
@@ -127,6 +129,8 @@ export function createWorker(opts: CreateWorkerOptions): Worker {
|
|
|
127
129
|
checkAlertsTask,
|
|
128
130
|
bucketReconcileTask,
|
|
129
131
|
bucketBackfillTask,
|
|
132
|
+
crmReconcileTask,
|
|
133
|
+
dispatchConversionTask,
|
|
130
134
|
...journeyTasks,
|
|
131
135
|
...bucketTasks,
|
|
132
136
|
...bucketReactionTasks,
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
import { ConcurrencyLimitStrategy } from "@hatchet-dev/typescript-sdk/v1/index.js";
|
|
2
|
+
import { createDatabase, crmSyncCursors, type Database } from "@hogsend/db";
|
|
3
|
+
import { eq } from "drizzle-orm";
|
|
4
|
+
import { getJourneyRegistrySingleton } from "../journeys/registry-singleton.js";
|
|
5
|
+
import { getAnalytics } from "../lib/analytics-singleton.js";
|
|
6
|
+
import { ingestCrmStageEvents } from "../lib/crm-ingest.js";
|
|
7
|
+
import { getCrmSyncConfig } from "../lib/crm-registry-singleton.js";
|
|
8
|
+
import { hatchet } from "../lib/hatchet.js";
|
|
9
|
+
import { createLogger, type Logger } from "../lib/logger.js";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* CRM reconciliation poll (docs/revenue-attribution-plan.md §4.3) — the pull
|
|
13
|
+
* half of the webhook+poll hybrid. Every provider implementing `poll` is
|
|
14
|
+
* walked on a cron: read the persisted cursor, pull changed deals, feed them
|
|
15
|
+
* through the SAME `ingestCrmStageEvents` sink the webhook route uses (the
|
|
16
|
+
* spine's idempotency dedups the overlap), persist the next cursor. A failing
|
|
17
|
+
* provider records `lastError` on its cursor row and never blocks the others
|
|
18
|
+
* — webhooks give latency, this poll guarantees eventual consistency.
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
/** The sweep body — exported for direct testing; the cron task wraps it. */
|
|
22
|
+
export async function runCrmReconcile(deps: {
|
|
23
|
+
db: Database;
|
|
24
|
+
logger: Logger;
|
|
25
|
+
/** Test seam — defaults to the process Hatchet client. */
|
|
26
|
+
hatchet?: typeof hatchet;
|
|
27
|
+
}): Promise<{ polled: number; ingested: number }> {
|
|
28
|
+
const { db, logger } = deps;
|
|
29
|
+
const hatchetClient = deps.hatchet ?? hatchet;
|
|
30
|
+
const config = getCrmSyncConfig();
|
|
31
|
+
if (!config || config.registry.count() === 0) {
|
|
32
|
+
return { polled: 0, ingested: 0 };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
const registry = getJourneyRegistrySingleton();
|
|
36
|
+
const analytics = getAnalytics();
|
|
37
|
+
|
|
38
|
+
let polled = 0;
|
|
39
|
+
let ingested = 0;
|
|
40
|
+
|
|
41
|
+
for (const provider of config.registry.getAll()) {
|
|
42
|
+
if (!provider.poll) continue;
|
|
43
|
+
const providerId = provider.meta.id;
|
|
44
|
+
|
|
45
|
+
const cursorRows = await db
|
|
46
|
+
.select()
|
|
47
|
+
.from(crmSyncCursors)
|
|
48
|
+
.where(eq(crmSyncCursors.provider, providerId))
|
|
49
|
+
.limit(1);
|
|
50
|
+
const cursor = cursorRows[0]?.cursor ?? null;
|
|
51
|
+
|
|
52
|
+
try {
|
|
53
|
+
const result = await provider.poll(cursor);
|
|
54
|
+
const sink = await ingestCrmStageEvents({
|
|
55
|
+
db,
|
|
56
|
+
registry,
|
|
57
|
+
hatchet: hatchetClient,
|
|
58
|
+
logger,
|
|
59
|
+
analytics,
|
|
60
|
+
providerId,
|
|
61
|
+
events: result.events,
|
|
62
|
+
funnels: config.funnels,
|
|
63
|
+
});
|
|
64
|
+
ingested += sink.ingested;
|
|
65
|
+
polled++;
|
|
66
|
+
|
|
67
|
+
await db
|
|
68
|
+
.insert(crmSyncCursors)
|
|
69
|
+
.values({
|
|
70
|
+
provider: providerId,
|
|
71
|
+
cursor: result.nextCursor,
|
|
72
|
+
lastPolledAt: new Date(),
|
|
73
|
+
lastError: null,
|
|
74
|
+
updatedAt: new Date(),
|
|
75
|
+
})
|
|
76
|
+
.onConflictDoUpdate({
|
|
77
|
+
target: crmSyncCursors.provider,
|
|
78
|
+
set: {
|
|
79
|
+
cursor: result.nextCursor,
|
|
80
|
+
lastPolledAt: new Date(),
|
|
81
|
+
lastError: null,
|
|
82
|
+
updatedAt: new Date(),
|
|
83
|
+
},
|
|
84
|
+
});
|
|
85
|
+
} catch (err) {
|
|
86
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
87
|
+
logger.warn("crm-reconcile provider poll failed", {
|
|
88
|
+
provider: providerId,
|
|
89
|
+
error: message,
|
|
90
|
+
});
|
|
91
|
+
await db
|
|
92
|
+
.insert(crmSyncCursors)
|
|
93
|
+
.values({
|
|
94
|
+
provider: providerId,
|
|
95
|
+
cursor,
|
|
96
|
+
lastError: { message, at: new Date().toISOString() },
|
|
97
|
+
updatedAt: new Date(),
|
|
98
|
+
})
|
|
99
|
+
.onConflictDoUpdate({
|
|
100
|
+
target: crmSyncCursors.provider,
|
|
101
|
+
set: {
|
|
102
|
+
lastError: { message, at: new Date().toISOString() },
|
|
103
|
+
updatedAt: new Date(),
|
|
104
|
+
},
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
return { polled, ingested };
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export const crmReconcileTask = hatchet.task({
|
|
113
|
+
name: "crm-reconcile",
|
|
114
|
+
onCrons: [process.env.CRM_RECONCILE_CRON ?? "*/10 * * * *"],
|
|
115
|
+
retries: 1,
|
|
116
|
+
executionTimeout: "300s",
|
|
117
|
+
concurrency: {
|
|
118
|
+
// Single global key → at most one sweep runs; the next QUEUES rather than
|
|
119
|
+
// cancelling the in-flight run.
|
|
120
|
+
expression: "'crm-reconcile'",
|
|
121
|
+
maxRuns: 1,
|
|
122
|
+
limitStrategy: ConcurrencyLimitStrategy.GROUP_ROUND_ROBIN,
|
|
123
|
+
},
|
|
124
|
+
fn: async () => {
|
|
125
|
+
const { db } = createDatabase({ url: process.env.DATABASE_URL ?? "" });
|
|
126
|
+
const logger = createLogger(process.env.LOG_LEVEL ?? "info");
|
|
127
|
+
return runCrmReconcile({ db, logger });
|
|
128
|
+
},
|
|
129
|
+
});
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { createDatabase } from "@hogsend/db";
|
|
2
|
+
import { deliverConversionDispatch } from "../lib/conversion-dispatch.js";
|
|
3
|
+
import { hatchet } from "../lib/hatchet.js";
|
|
4
|
+
import { createLogger } from "../lib/logger.js";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Durable delivery of ONE conversion dispatch row (plan §5.2). Retries ride
|
|
8
|
+
* Hatchet (the deterministic event_id makes platform-side dedup safe);
|
|
9
|
+
* `deliverConversionDispatch` marks the row failed once attempts exhaust and
|
|
10
|
+
* stops throwing, so the task run ends cleanly.
|
|
11
|
+
*/
|
|
12
|
+
export interface DispatchConversionInput {
|
|
13
|
+
dispatchId: string;
|
|
14
|
+
[key: string]: string;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export const dispatchConversionTask = hatchet.task({
|
|
18
|
+
name: "dispatch-conversion",
|
|
19
|
+
retries: 4,
|
|
20
|
+
executionTimeout: "60s",
|
|
21
|
+
fn: async (input: DispatchConversionInput) => {
|
|
22
|
+
const { db } = createDatabase({ url: process.env.DATABASE_URL ?? "" });
|
|
23
|
+
const logger = createLogger(process.env.LOG_LEVEL ?? "info");
|
|
24
|
+
return deliverConversionDispatch({
|
|
25
|
+
db,
|
|
26
|
+
logger,
|
|
27
|
+
dispatchId: input.dispatchId,
|
|
28
|
+
maxAttempts: 5,
|
|
29
|
+
});
|
|
30
|
+
},
|
|
31
|
+
});
|
|
@@ -1,17 +1,70 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import type { Database } from "@hogsend/db";
|
|
2
|
+
import { contacts, createDatabase, importJobs } from "@hogsend/db";
|
|
3
|
+
import { and, eq, isNull } from "drizzle-orm";
|
|
3
4
|
import Papa from "papaparse";
|
|
4
5
|
import { resolveOrCreateContact } from "../lib/contacts.js";
|
|
5
6
|
import { hatchet } from "../lib/hatchet.js";
|
|
7
|
+
import { normalizePhone } from "../lib/phone.js";
|
|
6
8
|
|
|
7
9
|
const BATCH_SIZE = 500;
|
|
8
10
|
|
|
9
11
|
interface ImportRow {
|
|
10
12
|
externalId?: string;
|
|
11
13
|
email?: string;
|
|
14
|
+
/** Optional E.164 phone; attached to the resolved contact (SMS channel). */
|
|
15
|
+
phone?: string;
|
|
12
16
|
properties?: Record<string, unknown>;
|
|
13
17
|
}
|
|
14
18
|
|
|
19
|
+
/**
|
|
20
|
+
* True when `err` is the `contacts_phone_unique_idx` partial-unique violation
|
|
21
|
+
* (another live contact already owns the number). Walks the drizzle cause
|
|
22
|
+
* chain for PG 23505, mirroring `isSlugUniqueViolation` in lib/links.ts.
|
|
23
|
+
*/
|
|
24
|
+
function isPhoneUniqueViolation(err: unknown): boolean {
|
|
25
|
+
let candidate: unknown = err;
|
|
26
|
+
while (candidate && typeof candidate === "object") {
|
|
27
|
+
const c = candidate as {
|
|
28
|
+
code?: string;
|
|
29
|
+
constraint_name?: string;
|
|
30
|
+
constraint?: string;
|
|
31
|
+
cause?: unknown;
|
|
32
|
+
};
|
|
33
|
+
if (c.code === "23505") {
|
|
34
|
+
const constraint = c.constraint_name ?? c.constraint ?? "";
|
|
35
|
+
return constraint.includes("contacts_phone_unique");
|
|
36
|
+
}
|
|
37
|
+
candidate = c.cause;
|
|
38
|
+
}
|
|
39
|
+
return false;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Best-effort attach a normalized phone to a resolved contact's `phone` column
|
|
44
|
+
* without minting an identity conflict. Only fills a NULL column, and swallows
|
|
45
|
+
* ONLY the partial-unique conflict (another live contact already owns that
|
|
46
|
+
* number) so one duplicate phone never fails the whole import row — any other
|
|
47
|
+
* DB error rethrows into the row's error accounting instead of silently
|
|
48
|
+
* dropping the phone (the task runs with retries: 0).
|
|
49
|
+
*/
|
|
50
|
+
async function attachPhone(
|
|
51
|
+
db: Database,
|
|
52
|
+
contactId: string,
|
|
53
|
+
rawPhone: string | undefined,
|
|
54
|
+
): Promise<void> {
|
|
55
|
+
const phone = normalizePhone(rawPhone);
|
|
56
|
+
if (!phone) return;
|
|
57
|
+
try {
|
|
58
|
+
await db
|
|
59
|
+
.update(contacts)
|
|
60
|
+
.set({ phone, updatedAt: new Date() })
|
|
61
|
+
.where(and(eq(contacts.id, contactId), isNull(contacts.phone)));
|
|
62
|
+
} catch (err) {
|
|
63
|
+
if (!isPhoneUniqueViolation(err)) throw err;
|
|
64
|
+
// Unique-index conflict (another contact owns this phone) — leave unset.
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
15
68
|
export const importContactsTask = hatchet.task({
|
|
16
69
|
name: "import-contacts",
|
|
17
70
|
retries: 0,
|
|
@@ -77,7 +130,10 @@ export const importContactsTask = hatchet.task({
|
|
|
77
130
|
userId: row.externalId,
|
|
78
131
|
email: row.email,
|
|
79
132
|
contactProperties: row.properties,
|
|
80
|
-
}).then(() =>
|
|
133
|
+
}).then(async (result) => {
|
|
134
|
+
await attachPhone(db, result.id, row.phone);
|
|
135
|
+
return { index: i + idx, ok: true };
|
|
136
|
+
});
|
|
81
137
|
}),
|
|
82
138
|
);
|
|
83
139
|
|
|
@@ -131,11 +187,12 @@ function parseCsv(data: string): ImportRow[] {
|
|
|
131
187
|
}
|
|
132
188
|
|
|
133
189
|
return result.data.map((row) => {
|
|
134
|
-
const { externalId, email, ...rest } = row;
|
|
190
|
+
const { externalId, email, phone, ...rest } = row;
|
|
135
191
|
const properties = Object.keys(rest).length > 0 ? rest : undefined;
|
|
136
192
|
return {
|
|
137
193
|
externalId: externalId || undefined,
|
|
138
194
|
email: email || undefined,
|
|
195
|
+
phone: phone || undefined,
|
|
139
196
|
properties: properties as Record<string, unknown> | undefined,
|
|
140
197
|
};
|
|
141
198
|
});
|
|
@@ -598,6 +598,11 @@ export const sendCampaignTask = hatchet.task({
|
|
|
598
598
|
// real, honored List-Unsubscribe target. The rule applies to
|
|
599
599
|
// EVERY wave, not just the first.
|
|
600
600
|
category: audienceKind === "bucket" ? undefined : audienceId,
|
|
601
|
+
// Queryable campaign identity on the email_sends row — the
|
|
602
|
+
// idempotency key below still encodes it for dedup, but
|
|
603
|
+
// suppressed sends write no key, so the column is the carrier
|
|
604
|
+
// stats + attribution can trust.
|
|
605
|
+
campaignId: input.campaignId,
|
|
601
606
|
// The idempotency key dedupes a retried send to its prior row.
|
|
602
607
|
// Legacy format for single-step campaigns; step-scoped for every
|
|
603
608
|
// step of a multi-step campaign (campaignSendKey contract).
|