fullstackgtm 0.34.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +91 -0
- package/README.md +25 -0
- package/dist/acquireMeter.d.ts +67 -0
- package/dist/acquireMeter.js +145 -0
- package/dist/acquireSeen.d.ts +5 -0
- package/dist/acquireSeen.js +54 -0
- package/dist/cli.js +732 -21
- package/dist/connectors/hubspot.js +135 -0
- package/dist/connectors/prospectSources.d.ts +91 -0
- package/dist/connectors/prospectSources.js +227 -0
- package/dist/enrich.d.ts +87 -0
- package/dist/enrich.js +188 -4
- package/dist/format.d.ts +3 -1
- package/dist/format.js +14 -2
- package/dist/health.d.ts +71 -0
- package/dist/health.js +172 -0
- package/dist/icp.d.ts +96 -0
- package/dist/icp.js +256 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mappings.js +3 -0
- package/dist/types.d.ts +17 -1
- package/docs/api.md +29 -2
- package/docs/architecture.md +8 -1
- package/docs/dx-punch-list.md +87 -0
- package/llms.txt +16 -0
- package/package.json +1 -1
- package/skills/fullstackgtm/SKILL.md +2 -1
- package/src/acquireMeter.ts +186 -0
- package/src/acquireSeen.ts +57 -0
- package/src/cli.ts +870 -20
- package/src/connectors/hubspot.ts +140 -0
- package/src/connectors/prospectSources.ts +324 -0
- package/src/enrich.ts +275 -3
- package/src/format.ts +20 -2
- package/src/health.ts +238 -0
- package/src/icp.ts +313 -0
- package/src/index.ts +8 -0
- package/src/mappings.ts +3 -0
- package/src/types.ts +24 -0
|
@@ -26,6 +26,10 @@ export function createHubspotConnector(options) {
|
|
|
26
26
|
// search API is eventually consistent, so a just-created company is
|
|
27
27
|
// invisible to search — this map is the authoritative same-run record.
|
|
28
28
|
const createdCompaniesByName = new Map();
|
|
29
|
+
// Same-run dedup for `create_record` contact creates, keyed by lowercased
|
|
30
|
+
// match value (email): HubSpot search is eventually consistent, so a contact
|
|
31
|
+
// created earlier in this apply run is invisible to a later search.
|
|
32
|
+
const createdContactsByMatch = new Map();
|
|
29
33
|
async function request(path, init = {}) {
|
|
30
34
|
const token = await options.getAccessToken();
|
|
31
35
|
let response;
|
|
@@ -161,6 +165,7 @@ export function createHubspotConnector(options) {
|
|
|
161
165
|
email: stringOrUndefined(readMapped(props, "contacts", "email", "email")),
|
|
162
166
|
phone: stringOrUndefined(readMapped(props, "contacts", "phone", "phone")),
|
|
163
167
|
title: stringOrUndefined(readMapped(props, "contacts", "title", "jobtitle")),
|
|
168
|
+
linkedin: stringOrUndefined(readMapped(props, "contacts", "linkedin", "hs_linkedin_url")),
|
|
164
169
|
ownerId: stringOrUndefined(readMapped(props, "contacts", "ownerId", "hubspot_owner_id")),
|
|
165
170
|
provenance: provenanceFrom(props),
|
|
166
171
|
lastSyncAt: stringOrUndefined(contact.updatedAt),
|
|
@@ -413,6 +418,134 @@ export function createHubspotConnector(options) {
|
|
|
413
418
|
});
|
|
414
419
|
return (data?.results ?? []).map((row) => String(row.id));
|
|
415
420
|
}
|
|
421
|
+
/** Exact-value contact lookup (by any searchable property) for resolve-first creates. */
|
|
422
|
+
async function searchContactsBy(property, value) {
|
|
423
|
+
const data = await request(`/crm/v3/objects/contacts/search`, {
|
|
424
|
+
method: "POST",
|
|
425
|
+
body: JSON.stringify({
|
|
426
|
+
filterGroups: [{ filters: [{ propertyName: property, operator: "EQ", value }] }],
|
|
427
|
+
properties: [property],
|
|
428
|
+
limit: 3,
|
|
429
|
+
}),
|
|
430
|
+
});
|
|
431
|
+
return (data?.results ?? []).map((row) => String(row.id));
|
|
432
|
+
}
|
|
433
|
+
/**
|
|
434
|
+
* Resolve a company by exact name, creating it on a confirmed miss. Returns
|
|
435
|
+
* the company id, or null on ambiguity (≥2 existing) so the caller skips the
|
|
436
|
+
* association rather than guessing. Same-run safe via createdCompaniesByName.
|
|
437
|
+
*/
|
|
438
|
+
async function resolveOrCreateCompanyByName(name, opId) {
|
|
439
|
+
const nameKey = name.toLowerCase();
|
|
440
|
+
const already = createdCompaniesByName.get(nameKey);
|
|
441
|
+
if (already)
|
|
442
|
+
return already;
|
|
443
|
+
const matches = await searchCompaniesByName(name);
|
|
444
|
+
if (matches.length > 1)
|
|
445
|
+
return null;
|
|
446
|
+
if (matches.length === 1) {
|
|
447
|
+
createdCompaniesByName.set(nameKey, matches[0]);
|
|
448
|
+
return matches[0];
|
|
449
|
+
}
|
|
450
|
+
let created;
|
|
451
|
+
try {
|
|
452
|
+
created = await request(`/crm/v3/objects/companies`, {
|
|
453
|
+
method: "POST",
|
|
454
|
+
body: JSON.stringify({
|
|
455
|
+
properties: { name, hs_object_source_detail_2: `fullstackgtm acquire (${opId})` },
|
|
456
|
+
}),
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
catch {
|
|
460
|
+
created = await request(`/crm/v3/objects/companies`, {
|
|
461
|
+
method: "POST",
|
|
462
|
+
body: JSON.stringify({ properties: { name } }),
|
|
463
|
+
});
|
|
464
|
+
}
|
|
465
|
+
const id = String(created.id);
|
|
466
|
+
createdCompaniesByName.set(nameKey, id);
|
|
467
|
+
return id;
|
|
468
|
+
}
|
|
469
|
+
/**
|
|
470
|
+
* Create a NET-NEW record (a sourced lead). Resolve-first: re-checks the
|
|
471
|
+
* dedupe key against the live CRM (the plan-time snapshot can be stale) and
|
|
472
|
+
* creates ONLY on a confirmed miss — a record a concurrent writer already
|
|
473
|
+
* added is returned as `skipped`, never duplicated. Contacts can be linked
|
|
474
|
+
* to a resolved-or-created company in the same step.
|
|
475
|
+
*/
|
|
476
|
+
async function createRecord(operation) {
|
|
477
|
+
const payload = operation.afterValue;
|
|
478
|
+
if (!payload || typeof payload !== "object" || !payload.properties) {
|
|
479
|
+
return { operationId: operation.id, status: "skipped", detail: "create_record needs a CreateRecordPayload afterValue." };
|
|
480
|
+
}
|
|
481
|
+
const objectPath = OBJECT_PATHS[operation.objectType];
|
|
482
|
+
if (operation.objectType !== "contact" && operation.objectType !== "account") {
|
|
483
|
+
return { operationId: operation.id, status: "skipped", detail: "create_record supports contacts and accounts." };
|
|
484
|
+
}
|
|
485
|
+
const matchValue = String(payload.matchValue ?? "").trim();
|
|
486
|
+
if (!matchValue) {
|
|
487
|
+
return { operationId: operation.id, status: "skipped", detail: "create_record needs a non-empty matchValue to resolve-first." };
|
|
488
|
+
}
|
|
489
|
+
if (operation.objectType === "account") {
|
|
490
|
+
const id = await resolveOrCreateCompanyByName(matchValue, operation.id);
|
|
491
|
+
if (id === null) {
|
|
492
|
+
return { operationId: operation.id, status: "skipped", detail: `create_record: ambiguous — multiple companies named "${matchValue}". Not creating.` };
|
|
493
|
+
}
|
|
494
|
+
return { operationId: operation.id, status: "applied", detail: `Resolved/created company "${matchValue}" (${id}).`, providerData: { id } };
|
|
495
|
+
}
|
|
496
|
+
// contact — resolve-first on the dedupe key (email by default)
|
|
497
|
+
const matchKey = payload.matchKey || "email";
|
|
498
|
+
const matchKeyLower = `${matchKey}:${matchValue.toLowerCase()}`;
|
|
499
|
+
if (createdContactsByMatch.has(matchKeyLower)) {
|
|
500
|
+
return { operationId: operation.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already created earlier in this run; not duplicating.`, providerData: { id: createdContactsByMatch.get(matchKeyLower), existing: true } };
|
|
501
|
+
}
|
|
502
|
+
const existing = await searchContactsBy(matchKey, matchValue);
|
|
503
|
+
if (existing.length > 0) {
|
|
504
|
+
createdContactsByMatch.set(matchKeyLower, existing[0]);
|
|
505
|
+
return { operationId: operation.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already exists (${existing.join(", ")}); resolve-first declined to create.`, providerData: { id: existing[0], existing: true } };
|
|
506
|
+
}
|
|
507
|
+
const properties = { ...payload.properties };
|
|
508
|
+
let created;
|
|
509
|
+
try {
|
|
510
|
+
created = await request(`/crm/v3/objects/${objectPath}`, {
|
|
511
|
+
method: "POST",
|
|
512
|
+
body: JSON.stringify({ properties: { ...properties, hs_object_source_detail_2: `fullstackgtm acquire (${operation.id})` } }),
|
|
513
|
+
});
|
|
514
|
+
}
|
|
515
|
+
catch {
|
|
516
|
+
// Some portals reject writes to source-detail properties — the provenance
|
|
517
|
+
// stamp is best-effort, the create is not.
|
|
518
|
+
created = await request(`/crm/v3/objects/${objectPath}`, {
|
|
519
|
+
method: "POST",
|
|
520
|
+
body: JSON.stringify({ properties }),
|
|
521
|
+
});
|
|
522
|
+
}
|
|
523
|
+
const contactId = String(created.id);
|
|
524
|
+
createdContactsByMatch.set(matchKeyLower, contactId);
|
|
525
|
+
let companyNote = "";
|
|
526
|
+
if (payload.associateCompanyName) {
|
|
527
|
+
try {
|
|
528
|
+
const companyId = await resolveOrCreateCompanyByName(payload.associateCompanyName, operation.id);
|
|
529
|
+
if (companyId) {
|
|
530
|
+
await request(`/crm/v4/objects/contacts/${encodeURIComponent(contactId)}/associations/default/companies/${encodeURIComponent(companyId)}`, { method: "PUT" });
|
|
531
|
+
companyNote = ` Linked to company "${payload.associateCompanyName}" (${companyId}).`;
|
|
532
|
+
}
|
|
533
|
+
else {
|
|
534
|
+
companyNote = ` Company "${payload.associateCompanyName}" was ambiguous; left unlinked.`;
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
catch (error) {
|
|
538
|
+
// Association is best-effort — the contact create already succeeded.
|
|
539
|
+
companyNote = ` (company link failed: ${error instanceof Error ? error.message : String(error)})`;
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
return {
|
|
543
|
+
operationId: operation.id,
|
|
544
|
+
status: "applied",
|
|
545
|
+
detail: `Created contact ${matchKey}=${matchValue} (${contactId}).${companyNote}`,
|
|
546
|
+
providerData: { id: contactId, created: true },
|
|
547
|
+
};
|
|
548
|
+
}
|
|
416
549
|
async function createTask(operation) {
|
|
417
550
|
const associationTypeId = TASK_ASSOCIATION_TYPE_IDS[operation.objectType];
|
|
418
551
|
if (associationTypeId === undefined) {
|
|
@@ -602,6 +735,8 @@ export function createHubspotConnector(options) {
|
|
|
602
735
|
return await linkRecord(operation);
|
|
603
736
|
case "create_task":
|
|
604
737
|
return await createTask(operation);
|
|
738
|
+
case "create_record":
|
|
739
|
+
return await createRecord(operation);
|
|
605
740
|
case "merge_records":
|
|
606
741
|
return await mergeRecords(operation);
|
|
607
742
|
case "archive_record":
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* API prospect sources for `enrich acquire` — net-new lead discovery + email
|
|
3
|
+
* resolution, behind one normalized shape the acquire builder consumes.
|
|
4
|
+
*
|
|
5
|
+
* Two confirmed providers (others slot in the same way):
|
|
6
|
+
* - Explorium — net-new discovery (POST /v1/prospects). Returns names, titles,
|
|
7
|
+
* company + website, LinkedIn; the email it returns is HASHED, so it is a
|
|
8
|
+
* discovery source, not an email source.
|
|
9
|
+
* - pipe0 — work-email resolution (POST /v1/pipes/run/sync with the
|
|
10
|
+
* `person:workemail:waterfall@1` block). Turns {name, company_domain} into a
|
|
11
|
+
* real work email. This is the email leg for Explorium-discovered people.
|
|
12
|
+
*
|
|
13
|
+
* Zero runtime deps: global fetch only, injectable for tests. Keys arrive via
|
|
14
|
+
* env/credential store, never argv.
|
|
15
|
+
*/
|
|
16
|
+
import type { CanonicalGtmSnapshot } from "../types.ts";
|
|
17
|
+
export type Prospect = {
|
|
18
|
+
firstName?: string;
|
|
19
|
+
lastName?: string;
|
|
20
|
+
fullName?: string;
|
|
21
|
+
jobTitle?: string;
|
|
22
|
+
/** normalized seniority, e.g. "cxo","vp","director" (Explorium job_level_main) */
|
|
23
|
+
jobLevel?: string;
|
|
24
|
+
/** normalized department, e.g. "sales" (Explorium job_department_main) */
|
|
25
|
+
jobDepartment?: string;
|
|
26
|
+
companyName?: string;
|
|
27
|
+
/** bare domain, e.g. "microsoft.com" — feeds pipe0's company_domain */
|
|
28
|
+
companyDomain?: string;
|
|
29
|
+
linkedin?: string;
|
|
30
|
+
/** real work email once resolved (pipe0); never the hashed value */
|
|
31
|
+
email?: string;
|
|
32
|
+
/** ICP fit score 0..1, set by the acquire scorer */
|
|
33
|
+
fitScore?: number;
|
|
34
|
+
/** provider-native id for traceability */
|
|
35
|
+
sourceId?: string;
|
|
36
|
+
};
|
|
37
|
+
type FetchImpl = typeof fetch;
|
|
38
|
+
export type ExploriumFilters = Record<string, {
|
|
39
|
+
values?: string[];
|
|
40
|
+
value?: boolean;
|
|
41
|
+
}>;
|
|
42
|
+
export declare function fetchExploriumProspects(opts: {
|
|
43
|
+
apiKey: string;
|
|
44
|
+
filters: ExploriumFilters;
|
|
45
|
+
size?: number;
|
|
46
|
+
apiBaseUrl?: string;
|
|
47
|
+
fetchImpl?: FetchImpl;
|
|
48
|
+
}): Promise<Prospect[]>;
|
|
49
|
+
export declare function fetchPipe0CrustdataProspects(opts: {
|
|
50
|
+
apiKey: string;
|
|
51
|
+
filters: Record<string, unknown>;
|
|
52
|
+
limit?: number;
|
|
53
|
+
apiBaseUrl?: string;
|
|
54
|
+
fetchImpl?: FetchImpl;
|
|
55
|
+
}): Promise<Prospect[]>;
|
|
56
|
+
/**
|
|
57
|
+
* Resolve real work emails for people via pipe0's waterfall. Input rows need a
|
|
58
|
+
* full name + a company domain (or name). Returns the prospects with `email`
|
|
59
|
+
* filled where the waterfall found one; rows with no hit are returned unchanged.
|
|
60
|
+
*/
|
|
61
|
+
export declare function pipe0ResolveWorkEmails(opts: {
|
|
62
|
+
apiKey: string;
|
|
63
|
+
prospects: Prospect[];
|
|
64
|
+
apiBaseUrl?: string;
|
|
65
|
+
fetchImpl?: FetchImpl;
|
|
66
|
+
/** Rows per pipe0 call. Small chunks are resilient to the waterfall's
|
|
67
|
+
* batch rate-limit (a throttled chunk returns work_email status "failed"
|
|
68
|
+
* for every row, so one big batch is all-or-nothing). Default 3. */
|
|
69
|
+
chunkSize?: number;
|
|
70
|
+
}): Promise<Prospect[]>;
|
|
71
|
+
/**
|
|
72
|
+
* Stable identity keys for a prospect (prefixed by kind). A match on ANY key
|
|
73
|
+
* means "same person". LinkedIn is strongest; name+domain is the only key
|
|
74
|
+
* shareable with the canonical CRM snapshot (which has no LinkedIn); email is
|
|
75
|
+
* available only after resolution (so it can't pre-filter, but it strengthens
|
|
76
|
+
* the seen cache).
|
|
77
|
+
*/
|
|
78
|
+
export declare function prospectIdentityKeys(p: Prospect): string[];
|
|
79
|
+
/** Identity keys for every contact already in the CRM snapshot (email + name|domain). */
|
|
80
|
+
export declare function crmContactKeys(snapshot: CanonicalGtmSnapshot): Set<string>;
|
|
81
|
+
/**
|
|
82
|
+
* Split prospects into those worth paying to enrich vs. drop. A prospect is
|
|
83
|
+
* dropped if any identity key is already in the CRM (`crmKeys`) or in the
|
|
84
|
+
* cross-run `seen` cache. Pure + deterministic.
|
|
85
|
+
*/
|
|
86
|
+
export declare function partitionFreshProspects(prospects: Prospect[], crmKeys: Set<string>, seen: Set<string>): {
|
|
87
|
+
fresh: Prospect[];
|
|
88
|
+
skippedCrm: number;
|
|
89
|
+
skippedSeen: number;
|
|
90
|
+
};
|
|
91
|
+
export {};
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
function splitName(full) {
|
|
2
|
+
if (!full)
|
|
3
|
+
return {};
|
|
4
|
+
const parts = full.trim().split(/\s+/);
|
|
5
|
+
if (parts.length === 1)
|
|
6
|
+
return { firstName: parts[0] };
|
|
7
|
+
return { firstName: parts[0], lastName: parts.slice(1).join(" ") };
|
|
8
|
+
}
|
|
9
|
+
function bareDomain(value) {
|
|
10
|
+
if (!value)
|
|
11
|
+
return undefined;
|
|
12
|
+
return value
|
|
13
|
+
.trim()
|
|
14
|
+
.replace(/^https?:\/\//i, "")
|
|
15
|
+
.replace(/^www\./i, "")
|
|
16
|
+
.replace(/\/.*$/, "")
|
|
17
|
+
.toLowerCase() || undefined;
|
|
18
|
+
}
|
|
19
|
+
export async function fetchExploriumProspects(opts) {
|
|
20
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
21
|
+
const base = (opts.apiBaseUrl ?? "https://api.explorium.ai").replace(/\/$/, "");
|
|
22
|
+
const size = Math.min(opts.size ?? 25, 100);
|
|
23
|
+
const response = await fetchImpl(`${base}/v1/prospects`, {
|
|
24
|
+
method: "POST",
|
|
25
|
+
headers: { "api_key": opts.apiKey, "Content-Type": "application/json" },
|
|
26
|
+
body: JSON.stringify({ mode: "full", size, page_size: size, page: 1, filters: opts.filters }),
|
|
27
|
+
});
|
|
28
|
+
if (!response.ok) {
|
|
29
|
+
throw new Error(`Explorium /v1/prospects failed: HTTP ${response.status} ${await safeText(response)}`);
|
|
30
|
+
}
|
|
31
|
+
const data = (await response.json());
|
|
32
|
+
return (data.data ?? []).map((row) => ({
|
|
33
|
+
firstName: row.first_name,
|
|
34
|
+
lastName: row.last_name,
|
|
35
|
+
fullName: row.full_name ?? ([row.first_name, row.last_name].filter(Boolean).join(" ") || undefined),
|
|
36
|
+
jobTitle: row.job_title,
|
|
37
|
+
jobLevel: row.job_level_main,
|
|
38
|
+
jobDepartment: row.job_department_main,
|
|
39
|
+
companyName: row.company_name,
|
|
40
|
+
companyDomain: bareDomain(row.company_website),
|
|
41
|
+
linkedin: normalizeLinkedin(row.linkedin),
|
|
42
|
+
sourceId: row.prospect_id,
|
|
43
|
+
}));
|
|
44
|
+
}
|
|
45
|
+
// ---------------------------------------------------------------------------
|
|
46
|
+
// pipe0 / Crustdata — net-new discovery (search). Crustdata is built into pipe0
|
|
47
|
+
// (no separate connection needed). Returns profiles; pair with
|
|
48
|
+
// pipe0ResolveWorkEmails to get real emails.
|
|
49
|
+
export async function fetchPipe0CrustdataProspects(opts) {
|
|
50
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
51
|
+
const base = (opts.apiBaseUrl ?? "https://api.pipe0.com").replace(/\/$/, "");
|
|
52
|
+
const limit = Math.min(opts.limit ?? 25, 100);
|
|
53
|
+
const response = await fetchImpl(`${base}/v1/searches/run/sync`, {
|
|
54
|
+
method: "POST",
|
|
55
|
+
headers: { Authorization: `Bearer ${opts.apiKey}`, "Content-Type": "application/json" },
|
|
56
|
+
body: JSON.stringify({
|
|
57
|
+
searches: [{ search_id: "people:profiles:crustdata@1", config: { limit, filters: opts.filters } }],
|
|
58
|
+
}),
|
|
59
|
+
});
|
|
60
|
+
if (!response.ok) {
|
|
61
|
+
throw new Error(`pipe0 /v1/searches/run/sync failed: HTTP ${response.status} ${await safeText(response)}`);
|
|
62
|
+
}
|
|
63
|
+
const body = (await response.json());
|
|
64
|
+
// Surface upstream provider errors (e.g. CreditBalanceInsufficient) instead of
|
|
65
|
+
// silently returning [] — a throttle/credit failure must not look like "no ICP matches".
|
|
66
|
+
const upstreamErrors = (body.search_statuses ?? []).flatMap((s) => s.errors ?? []);
|
|
67
|
+
if (upstreamErrors.length > 0 && !(body.results ?? []).length) {
|
|
68
|
+
throw new Error(`pipe0/Crustdata search error: ${upstreamErrors.map((e) => `${e.code ?? "error"}: ${e.message ?? ""}`).join("; ")}`);
|
|
69
|
+
}
|
|
70
|
+
return (body.results ?? []).map((row) => {
|
|
71
|
+
const fullName = strField(row.name);
|
|
72
|
+
const { firstName, lastName } = splitName(fullName);
|
|
73
|
+
return {
|
|
74
|
+
firstName,
|
|
75
|
+
lastName,
|
|
76
|
+
fullName,
|
|
77
|
+
jobTitle: strField(row.job_title),
|
|
78
|
+
companyDomain: bareDomain(strField(row.company_website_url)),
|
|
79
|
+
linkedin: normalizeLinkedin(strField(row.profile_url)),
|
|
80
|
+
sourceId: strField(row.profile_url) ?? fullName,
|
|
81
|
+
};
|
|
82
|
+
});
|
|
83
|
+
}
|
|
84
|
+
function strField(field) {
|
|
85
|
+
const v = field?.value;
|
|
86
|
+
return typeof v === "string" && v.trim() ? v : undefined;
|
|
87
|
+
}
|
|
88
|
+
function normalizeLinkedin(value) {
|
|
89
|
+
if (!value)
|
|
90
|
+
return undefined;
|
|
91
|
+
const v = value.trim().replace(/^https?:\/\//i, "").replace(/\/$/, "");
|
|
92
|
+
return v ? `https://${v}` : undefined;
|
|
93
|
+
}
|
|
94
|
+
// ---------------------------------------------------------------------------
|
|
95
|
+
// pipe0 — work-email resolution (waterfall)
|
|
96
|
+
/**
|
|
97
|
+
* Resolve real work emails for people via pipe0's waterfall. Input rows need a
|
|
98
|
+
* full name + a company domain (or name). Returns the prospects with `email`
|
|
99
|
+
* filled where the waterfall found one; rows with no hit are returned unchanged.
|
|
100
|
+
*/
|
|
101
|
+
export async function pipe0ResolveWorkEmails(opts) {
|
|
102
|
+
const fetchImpl = opts.fetchImpl ?? fetch;
|
|
103
|
+
const base = (opts.apiBaseUrl ?? "https://api.pipe0.com").replace(/\/$/, "");
|
|
104
|
+
const chunkSize = Math.max(1, opts.chunkSize ?? 3);
|
|
105
|
+
// Only rows with the waterfall's required inputs (name + company_domain|name).
|
|
106
|
+
const resolvable = opts.prospects.filter((p) => p.fullName && (p.companyDomain || p.companyName));
|
|
107
|
+
if (resolvable.length === 0)
|
|
108
|
+
return opts.prospects;
|
|
109
|
+
const emailByKey = new Map();
|
|
110
|
+
for (let i = 0; i < resolvable.length; i += chunkSize) {
|
|
111
|
+
const chunk = resolvable.slice(i, i + chunkSize);
|
|
112
|
+
const input = chunk.map((p) => ({
|
|
113
|
+
name: p.fullName,
|
|
114
|
+
...(p.companyDomain ? { company_domain: p.companyDomain } : { company_name: p.companyName }),
|
|
115
|
+
}));
|
|
116
|
+
let body;
|
|
117
|
+
try {
|
|
118
|
+
const response = await fetchImpl(`${base}/v1/pipes/run/sync`, {
|
|
119
|
+
method: "POST",
|
|
120
|
+
headers: { Authorization: `Bearer ${opts.apiKey}`, "Content-Type": "application/json" },
|
|
121
|
+
body: JSON.stringify({ pipes: [{ pipe_id: "person:workemail:waterfall@1" }], input }),
|
|
122
|
+
});
|
|
123
|
+
if (!response.ok)
|
|
124
|
+
continue; // throttled/5xx chunk — skip, keep the rest
|
|
125
|
+
body = (await response.json());
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
continue; // network hiccup on one chunk must not sink the whole run
|
|
129
|
+
}
|
|
130
|
+
for (const recordId of body.order ?? []) {
|
|
131
|
+
const fields = body.records?.[recordId]?.fields ?? {};
|
|
132
|
+
const email = fields.work_email?.status === "completed" ? fieldValue(fields.work_email) : undefined;
|
|
133
|
+
if (email) {
|
|
134
|
+
const domain = fieldValue(fields.company_domain) ?? fieldValue(fields.company_name);
|
|
135
|
+
emailByKey.set(personKey(fieldValue(fields.name), domain), email);
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
return opts.prospects.map((p) => {
|
|
140
|
+
const email = emailByKey.get(personKey(p.fullName, p.companyDomain ?? p.companyName));
|
|
141
|
+
return email ? { ...p, email } : p;
|
|
142
|
+
});
|
|
143
|
+
}
|
|
144
|
+
function personKey(name, company) {
|
|
145
|
+
return `${(name ?? "").trim().toLowerCase()}|${(company ?? "").trim().toLowerCase()}`;
|
|
146
|
+
}
|
|
147
|
+
function fieldValue(field) {
|
|
148
|
+
const v = field?.value;
|
|
149
|
+
return typeof v === "string" && v.trim() ? v : undefined;
|
|
150
|
+
}
|
|
151
|
+
async function safeText(response) {
|
|
152
|
+
try {
|
|
153
|
+
return (await response.text()).slice(0, 300);
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
return "";
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
// ---------------------------------------------------------------------------
|
|
160
|
+
// Pre-email dedup: identity keys shared between prospects and CRM contacts, so
|
|
161
|
+
// we can drop already-in-CRM / already-seen people BEFORE paying for emails.
|
|
162
|
+
function normName(value) {
|
|
163
|
+
return (value ?? "").trim().toLowerCase().replace(/\s+/g, " ");
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Stable identity keys for a prospect (prefixed by kind). A match on ANY key
|
|
167
|
+
* means "same person". LinkedIn is strongest; name+domain is the only key
|
|
168
|
+
* shareable with the canonical CRM snapshot (which has no LinkedIn); email is
|
|
169
|
+
* available only after resolution (so it can't pre-filter, but it strengthens
|
|
170
|
+
* the seen cache).
|
|
171
|
+
*/
|
|
172
|
+
export function prospectIdentityKeys(p) {
|
|
173
|
+
const keys = [];
|
|
174
|
+
if (p.linkedin)
|
|
175
|
+
keys.push(`li:${p.linkedin.trim().toLowerCase().replace(/\/+$/, "")}`);
|
|
176
|
+
const name = normName(p.fullName ?? [p.firstName, p.lastName].filter(Boolean).join(" "));
|
|
177
|
+
const domain = bareDomain(p.companyDomain);
|
|
178
|
+
if (name && domain)
|
|
179
|
+
keys.push(`nd:${name}|${domain}`);
|
|
180
|
+
if (p.email)
|
|
181
|
+
keys.push(`em:${p.email.trim().toLowerCase()}`);
|
|
182
|
+
return keys;
|
|
183
|
+
}
|
|
184
|
+
/** Identity keys for every contact already in the CRM snapshot (email + name|domain). */
|
|
185
|
+
export function crmContactKeys(snapshot) {
|
|
186
|
+
const domainByAccount = new Map();
|
|
187
|
+
for (const account of snapshot.accounts ?? []) {
|
|
188
|
+
const d = bareDomain(account.domain);
|
|
189
|
+
if (d)
|
|
190
|
+
domainByAccount.set(account.id, d);
|
|
191
|
+
}
|
|
192
|
+
const set = new Set();
|
|
193
|
+
for (const contact of snapshot.contacts ?? []) {
|
|
194
|
+
if (contact.linkedin)
|
|
195
|
+
set.add(`li:${contact.linkedin.trim().toLowerCase().replace(/\/+$/, "")}`);
|
|
196
|
+
if (contact.email)
|
|
197
|
+
set.add(`em:${contact.email.trim().toLowerCase()}`);
|
|
198
|
+
const name = normName([contact.firstName, contact.lastName].filter(Boolean).join(" "));
|
|
199
|
+
const domain = contact.accountId ? domainByAccount.get(contact.accountId) : undefined;
|
|
200
|
+
if (name && domain)
|
|
201
|
+
set.add(`nd:${name}|${domain}`);
|
|
202
|
+
}
|
|
203
|
+
return set;
|
|
204
|
+
}
|
|
205
|
+
/**
|
|
206
|
+
* Split prospects into those worth paying to enrich vs. drop. A prospect is
|
|
207
|
+
* dropped if any identity key is already in the CRM (`crmKeys`) or in the
|
|
208
|
+
* cross-run `seen` cache. Pure + deterministic.
|
|
209
|
+
*/
|
|
210
|
+
export function partitionFreshProspects(prospects, crmKeys, seen) {
|
|
211
|
+
const fresh = [];
|
|
212
|
+
let skippedCrm = 0;
|
|
213
|
+
let skippedSeen = 0;
|
|
214
|
+
for (const p of prospects) {
|
|
215
|
+
const keys = prospectIdentityKeys(p);
|
|
216
|
+
if (keys.some((k) => crmKeys.has(k))) {
|
|
217
|
+
skippedCrm += 1;
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
if (keys.some((k) => seen.has(k))) {
|
|
221
|
+
skippedSeen += 1;
|
|
222
|
+
continue;
|
|
223
|
+
}
|
|
224
|
+
fresh.push(p);
|
|
225
|
+
}
|
|
226
|
+
return { fresh, skippedCrm, skippedSeen };
|
|
227
|
+
}
|
package/dist/enrich.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { AcquireBudget } from "./acquireMeter.ts";
|
|
1
2
|
import type { CanonicalGtmSnapshot, PatchPlan } from "./types.ts";
|
|
2
3
|
/**
|
|
3
4
|
* The enrich layer: governed append/refresh of third-party data into the CRM.
|
|
@@ -55,6 +56,39 @@ export type EnrichConfig = {
|
|
|
55
56
|
match: Partial<Record<EnrichObjectType, EnrichMatchConfig>>;
|
|
56
57
|
fields: Partial<Record<EnrichObjectType, EnrichFieldConfig[]>>;
|
|
57
58
|
policy: EnrichPolicyConfig;
|
|
59
|
+
/** Net-new lead creation (`enrich acquire`). Optional; absent = acquire off. */
|
|
60
|
+
acquire?: AcquireConfig;
|
|
61
|
+
};
|
|
62
|
+
/** Maps a sourced record to the CRM properties a net-new record is created with. */
|
|
63
|
+
export type AcquireCreateMap = {
|
|
64
|
+
/** CRM property name → source payload path (read via sourceValueAt). */
|
|
65
|
+
properties: Record<string, string>;
|
|
66
|
+
/** Source path whose value becomes the dedupe key (e.g. "email"). */
|
|
67
|
+
matchKey: string;
|
|
68
|
+
/** Optional source path to a company name; the connector resolves-or-creates it. */
|
|
69
|
+
associateCompanyFrom?: string;
|
|
70
|
+
};
|
|
71
|
+
/**
|
|
72
|
+
* Net-new discovery for an API acquire source. `provider` selects the adapter
|
|
73
|
+
* (explorium = prospect search; pipe0 = work-email waterfall). `filters` is the
|
|
74
|
+
* provider-native query; `resolveEmailsWith` chains a second provider to fill
|
|
75
|
+
* real emails (e.g. explorium discovers, pipe0 resolves the work email).
|
|
76
|
+
*/
|
|
77
|
+
export type AcquireDiscoveryConfig = {
|
|
78
|
+
provider: "explorium" | "pipe0";
|
|
79
|
+
filters?: Record<string, unknown>;
|
|
80
|
+
size?: number;
|
|
81
|
+
resolveEmailsWith?: "pipe0";
|
|
82
|
+
};
|
|
83
|
+
export type AcquireConfig = {
|
|
84
|
+
/** Windowed budget enforced by the acquire meter; absent = unmetered. */
|
|
85
|
+
budget?: AcquireBudget;
|
|
86
|
+
/** Estimated provider cost (USD) per created record, by source id. */
|
|
87
|
+
costPerRecord?: Record<string, number>;
|
|
88
|
+
/** How to build a net-new record from a sourced row, per object type. */
|
|
89
|
+
create: Partial<Record<EnrichObjectType, AcquireCreateMap>>;
|
|
90
|
+
/** Net-new discovery params for API sources, by source id. */
|
|
91
|
+
discovery?: Record<string, AcquireDiscoveryConfig>;
|
|
58
92
|
};
|
|
59
93
|
export declare const ENRICH_CONFIG_FILE_NAME = "enrich.config.json";
|
|
60
94
|
export declare const DEFAULT_STALE_DAYS = 90;
|
|
@@ -68,6 +102,25 @@ export declare function resolveCrmField(objectType: EnrichObjectType, name: stri
|
|
|
68
102
|
* the accepted values.
|
|
69
103
|
*/
|
|
70
104
|
export declare function parseEnrichConfig(raw: string): EnrichConfig;
|
|
105
|
+
/**
|
|
106
|
+
* Built-in zero-config presets for well-known sources, so the common Mode-A
|
|
107
|
+
* loop — "feed a Clay export, get the dedup/routing verdict" — needs no
|
|
108
|
+
* hand-authored config. These are **match-only** (empty `fields`): the value is
|
|
109
|
+
* the matched / unmatched / ambiguous routing and collision detection, not field
|
|
110
|
+
* writes. Drop an `enrich.config.json` next to your run to map fields for actual
|
|
111
|
+
* field enrichment; an explicit config always wins over a preset. Constructed
|
|
112
|
+
* directly (not via `parseEnrichConfig`) so the match-only shape is allowed.
|
|
113
|
+
*/
|
|
114
|
+
export declare const BUILTIN_ENRICH_PRESETS: Record<string, EnrichConfig>;
|
|
115
|
+
export declare function builtinEnrichPreset(source: string | undefined): EnrichConfig | undefined;
|
|
116
|
+
/**
|
|
117
|
+
* Zero-config `enrich acquire` preset so every client gets targeted, deduped,
|
|
118
|
+
* metered lead-fill out of the box — no hand-authored enrich.config.json. The
|
|
119
|
+
* ICP (icp.json) supplies the targeting; this supplies sensible plumbing. The
|
|
120
|
+
* create mapping writes the prospect's LinkedIn URL into hs_linkedin_url, so
|
|
121
|
+
* each created contact strengthens future dedup. An explicit config always wins.
|
|
122
|
+
*/
|
|
123
|
+
export declare function builtinAcquirePreset(source: string | undefined): EnrichConfig | undefined;
|
|
71
124
|
export declare function loadEnrichConfig(path: string): EnrichConfig;
|
|
72
125
|
export declare function parseCsv(text: string): Array<Record<string, string>>;
|
|
73
126
|
export type EnrichSourceRecord = {
|
|
@@ -154,6 +207,40 @@ export type EnrichPlanResult = {
|
|
|
154
207
|
* current CRM value → apply-time compare-and-set rejects drifted records).
|
|
155
208
|
*/
|
|
156
209
|
export declare function buildEnrichPlan(options: BuildEnrichPlanOptions): EnrichPlanResult;
|
|
210
|
+
export type AcquireCounts = {
|
|
211
|
+
fetched: number;
|
|
212
|
+
matched: number;
|
|
213
|
+
ambiguous: number;
|
|
214
|
+
unmatched: number;
|
|
215
|
+
/** create_record ops emitted. */
|
|
216
|
+
created: number;
|
|
217
|
+
/** unmatched rows that would have been created but for the meter ceiling. */
|
|
218
|
+
withheldByMeter: number;
|
|
219
|
+
};
|
|
220
|
+
export type AcquirePlanResult = {
|
|
221
|
+
plan: PatchPlan;
|
|
222
|
+
counts: AcquireCounts;
|
|
223
|
+
estCostUsd: number;
|
|
224
|
+
};
|
|
225
|
+
export type BuildAcquirePlanOptions = {
|
|
226
|
+
config: EnrichConfig;
|
|
227
|
+
source: string;
|
|
228
|
+
snapshot: CanonicalGtmSnapshot;
|
|
229
|
+
records: EnrichSourceRecord[];
|
|
230
|
+
runLabel: string;
|
|
231
|
+
/** Meter ceiling: max create ops to emit. null/undefined = unlimited. */
|
|
232
|
+
maxRecords?: number | null;
|
|
233
|
+
now?: () => Date;
|
|
234
|
+
};
|
|
235
|
+
/**
|
|
236
|
+
* Match each sourced record against the snapshot and route it: matched =
|
|
237
|
+
* already in the CRM (skip), ambiguous = a possible duplicate exists (skip —
|
|
238
|
+
* resolve-first never creates over ambiguity), unmatched = a net-new lead.
|
|
239
|
+
* Unmatched rows with a dedupe key and at least one mapped property become
|
|
240
|
+
* `create_record` operations, capped at `maxRecords` (the meter's headroom).
|
|
241
|
+
* Always a dry-run plan; nothing is written until apply.
|
|
242
|
+
*/
|
|
243
|
+
export declare function buildAcquirePlan(options: BuildAcquirePlanOptions): AcquirePlanResult;
|
|
157
244
|
/** Latest stamp per (objectType, objectId, field) across a source's runs. */
|
|
158
245
|
export declare function latestStamps(runs: EnrichRun[], source: string): Map<string, EnrichStamp>;
|
|
159
246
|
export declare function staleDaysFor(config: EnrichConfig, objectType: EnrichObjectType, field: string): number;
|