fullstackgtm 0.34.0 → 0.37.0

package/src/connectors/hubspot.ts

@@ -13,6 +13,7 @@ import type {
   RecordProvenance,
   CanonicalGtmSnapshot,
   CanonicalUser,
+  CreateRecordPayload,
   GtmConnector,
   GtmObjectType,
   PatchOperation,
@@ -59,6 +60,10 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
   // search API is eventually consistent, so a just-created company is
   // invisible to search — this map is the authoritative same-run record.
   const createdCompaniesByName = new Map<string, string>();
+  // Same-run dedup for `create_record` contact creates, keyed by lowercased
+  // match value (email): HubSpot search is eventually consistent, so a contact
+  // created earlier in this apply run is invisible to a later search.
+  const createdContactsByMatch = new Map<string, string>();
 
   async function request(path: string, init: RequestInit = {}): Promise<any> {
     const token = await options.getAccessToken();
@@ -220,6 +225,7 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
           email: stringOrUndefined(readMapped(props, "contacts", "email", "email")),
           phone: stringOrUndefined(readMapped(props, "contacts", "phone", "phone")),
           title: stringOrUndefined(readMapped(props, "contacts", "title", "jobtitle")),
+          linkedin: stringOrUndefined(readMapped(props, "contacts", "linkedin", "hs_linkedin_url")),
           ownerId: stringOrUndefined(
             readMapped(props, "contacts", "ownerId", "hubspot_owner_id"),
           ),
@@ -513,6 +519,138 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
     return ((data?.results ?? []) as Array<{ id: string }>).map((row) => String(row.id));
   }
 
+  /** Exact-value contact lookup (by any searchable property) for resolve-first creates. */
+  async function searchContactsBy(property: string, value: string): Promise<string[]> {
+    const data = await request(`/crm/v3/objects/contacts/search`, {
+      method: "POST",
+      body: JSON.stringify({
+        filterGroups: [{ filters: [{ propertyName: property, operator: "EQ", value }] }],
+        properties: [property],
+        limit: 3,
+      }),
+    });
+    return ((data?.results ?? []) as Array<{ id: string }>).map((row) => String(row.id));
+  }
+
+  /**
+   * Resolve a company by exact name, creating it on a confirmed miss. Returns
+   * the company id, or null on ambiguity (≥2 existing) so the caller skips the
+   * association rather than guessing. Same-run safe via createdCompaniesByName.
+   */
+  async function resolveOrCreateCompanyByName(name: string, opId: string): Promise<string | null> {
+    const nameKey = name.toLowerCase();
+    const already = createdCompaniesByName.get(nameKey);
+    if (already) return already;
+    const matches = await searchCompaniesByName(name);
+    if (matches.length > 1) return null;
+    if (matches.length === 1) {
+      createdCompaniesByName.set(nameKey, matches[0]);
+      return matches[0];
+    }
+    let created;
+    try {
+      created = await request(`/crm/v3/objects/companies`, {
+        method: "POST",
+        body: JSON.stringify({
+          properties: { name, hs_object_source_detail_2: `fullstackgtm acquire (${opId})` },
+        }),
+      });
+    } catch {
+      created = await request(`/crm/v3/objects/companies`, {
+        method: "POST",
+        body: JSON.stringify({ properties: { name } }),
+      });
+    }
+    const id = String(created.id);
+    createdCompaniesByName.set(nameKey, id);
+    return id;
+  }
+
+  /**
+   * Create a NET-NEW record (a sourced lead). Resolve-first: re-checks the
+   * dedupe key against the live CRM (the plan-time snapshot can be stale) and
+   * creates ONLY on a confirmed miss — a record a concurrent writer already
+   * added is returned as `skipped`, never duplicated. Contacts can be linked
+   * to a resolved-or-created company in the same step.
+   */
+  async function createRecord(operation: PatchOperation): Promise<PatchOperationResult> {
+    const payload = operation.afterValue as CreateRecordPayload | undefined;
+    if (!payload || typeof payload !== "object" || !payload.properties) {
+      return { operationId: operation.id, status: "skipped", detail: "create_record needs a CreateRecordPayload afterValue." };
+    }
+    const objectPath = OBJECT_PATHS[operation.objectType];
+    if (operation.objectType !== "contact" && operation.objectType !== "account") {
+      return { operationId: operation.id, status: "skipped", detail: "create_record supports contacts and accounts." };
+    }
+    const matchValue = String(payload.matchValue ?? "").trim();
+    if (!matchValue) {
+      return { operationId: operation.id, status: "skipped", detail: "create_record needs a non-empty matchValue to resolve-first." };
+    }
+
+    if (operation.objectType === "account") {
+      const id = await resolveOrCreateCompanyByName(matchValue, operation.id);
+      if (id === null) {
+        return { operationId: operation.id, status: "skipped", detail: `create_record: ambiguous — multiple companies named "${matchValue}". Not creating.` };
+      }
+      return { operationId: operation.id, status: "applied", detail: `Resolved/created company "${matchValue}" (${id}).`, providerData: { id } };
+    }
+
+    // contact — resolve-first on the dedupe key (email by default)
+    const matchKey = payload.matchKey || "email";
+    const matchKeyLower = `${matchKey}:${matchValue.toLowerCase()}`;
+    if (createdContactsByMatch.has(matchKeyLower)) {
+      return { operationId: operation.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already created earlier in this run; not duplicating.`, providerData: { id: createdContactsByMatch.get(matchKeyLower), existing: true } };
+    }
+    const existing = await searchContactsBy(matchKey, matchValue);
+    if (existing.length > 0) {
+      createdContactsByMatch.set(matchKeyLower, existing[0]);
+      return { operationId: operation.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already exists (${existing.join(", ")}); resolve-first declined to create.`, providerData: { id: existing[0], existing: true } };
+    }
+
+    const properties: Record<string, string> = { ...payload.properties };
+    let created;
+    try {
+      created = await request(`/crm/v3/objects/${objectPath}`, {
+        method: "POST",
+        body: JSON.stringify({ properties: { ...properties, hs_object_source_detail_2: `fullstackgtm acquire (${operation.id})` } }),
+      });
+    } catch {
+      // Some portals reject writes to source-detail properties — the provenance
+      // stamp is best-effort, the create is not.
+      created = await request(`/crm/v3/objects/${objectPath}`, {
+        method: "POST",
+        body: JSON.stringify({ properties }),
+      });
+    }
+    const contactId = String(created.id);
+    createdContactsByMatch.set(matchKeyLower, contactId);
+
+    let companyNote = "";
+    if (payload.associateCompanyName) {
+      try {
+        const companyId = await resolveOrCreateCompanyByName(payload.associateCompanyName, operation.id);
+        if (companyId) {
+          await request(
+            `/crm/v4/objects/contacts/${encodeURIComponent(contactId)}/associations/default/companies/${encodeURIComponent(companyId)}`,
+            { method: "PUT" },
+          );
+          companyNote = ` Linked to company "${payload.associateCompanyName}" (${companyId}).`;
+        } else {
+          companyNote = ` Company "${payload.associateCompanyName}" was ambiguous; left unlinked.`;
+        }
+      } catch (error) {
+        // Association is best-effort — the contact create already succeeded.
+        companyNote = ` (company link failed: ${error instanceof Error ? error.message : String(error)})`;
+      }
+    }
+    return {
+      operationId: operation.id,
+      status: "applied",
+      detail: `Created contact ${matchKey}=${matchValue} (${contactId}).${companyNote}`,
+      providerData: { id: contactId, created: true },
+    };
+  }
+
   async function createTask(operation: PatchOperation): Promise<PatchOperationResult> {
     const associationTypeId = TASK_ASSOCIATION_TYPE_IDS[operation.objectType];
     if (associationTypeId === undefined) {
@@ -707,6 +845,8 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
           return await linkRecord(operation);
         case "create_task":
           return await createTask(operation);
+        case "create_record":
+          return await createRecord(operation);
         case "merge_records":
           return await mergeRecords(operation);
         case "archive_record":

package/src/connectors/prospectSources.ts

@@ -0,0 +1,324 @@
+/**
+ * API prospect sources for `enrich acquire` — net-new lead discovery + email
+ * resolution, behind one normalized shape the acquire builder consumes.
+ *
+ * Two confirmed providers (others slot in the same way):
+ *  - Explorium  — net-new discovery (POST /v1/prospects). Returns names, titles,
+ *    company + website, LinkedIn; the email it returns is HASHED, so it is a
+ *    discovery source, not an email source.
+ *  - pipe0      — work-email resolution (POST /v1/pipes/run/sync with the
+ *    `person:workemail:waterfall@1` block). Turns {name, company_domain} into a
+ *    real work email. This is the email leg for Explorium-discovered people.
+ *
+ * Zero runtime deps: global fetch only, injectable for tests. Keys arrive via
+ * env/credential store, never argv.
+ */
+import type { CanonicalGtmSnapshot } from "../types.ts";
+
+export type Prospect = {
+  firstName?: string;
+  lastName?: string;
+  fullName?: string;
+  jobTitle?: string;
+  /** normalized seniority, e.g. "cxo","vp","director" (Explorium job_level_main) */
+  jobLevel?: string;
+  /** normalized department, e.g. "sales" (Explorium job_department_main) */
+  jobDepartment?: string;
+  companyName?: string;
+  /** bare domain, e.g. "microsoft.com" — feeds pipe0's company_domain */
+  companyDomain?: string;
+  linkedin?: string;
+  /** real work email once resolved (pipe0); never the hashed value */
+  email?: string;
+  /** ICP fit score 0..1, set by the acquire scorer */
+  fitScore?: number;
+  /** provider-native id for traceability */
+  sourceId?: string;
+};
+
+function splitName(full: string | undefined): { firstName?: string; lastName?: string } {
+  if (!full) return {};
+  const parts = full.trim().split(/\s+/);
+  if (parts.length === 1) return { firstName: parts[0] };
+  return { firstName: parts[0], lastName: parts.slice(1).join(" ") };
+}
+
+type FetchImpl = typeof fetch;
+
+function bareDomain(value: string | undefined): string | undefined {
+  if (!value) return undefined;
+  return value
+    .trim()
+    .replace(/^https?:\/\//i, "")
+    .replace(/^www\./i, "")
+    .replace(/\/.*$/, "")
+    .toLowerCase() || undefined;
+}
+
+// ---------------------------------------------------------------------------
+// Explorium — net-new discovery
+
+export type ExploriumFilters = Record<string, { values?: string[]; value?: boolean }>;
+
+export async function fetchExploriumProspects(opts: {
+  apiKey: string;
+  filters: ExploriumFilters;
+  size?: number;
+  apiBaseUrl?: string;
+  fetchImpl?: FetchImpl;
+}): Promise<Prospect[]> {
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const base = (opts.apiBaseUrl ?? "https://api.explorium.ai").replace(/\/$/, "");
+  const size = Math.min(opts.size ?? 25, 100);
+  const response = await fetchImpl(`${base}/v1/prospects`, {
+    method: "POST",
+    headers: { "api_key": opts.apiKey, "Content-Type": "application/json" },
+    body: JSON.stringify({ mode: "full", size, page_size: size, page: 1, filters: opts.filters }),
+  });
+  if (!response.ok) {
+    throw new Error(`Explorium /v1/prospects failed: HTTP ${response.status} ${await safeText(response)}`);
+  }
+  const data = (await response.json()) as { data?: ExploriumRow[] };
+  return (data.data ?? []).map((row) => ({
+    firstName: row.first_name,
+    lastName: row.last_name,
+    fullName: row.full_name ?? ([row.first_name, row.last_name].filter(Boolean).join(" ") || undefined),
+    jobTitle: row.job_title,
+    jobLevel: row.job_level_main,
+    jobDepartment: row.job_department_main,
+    companyName: row.company_name,
+    companyDomain: bareDomain(row.company_website),
+    linkedin: normalizeLinkedin(row.linkedin),
+    sourceId: row.prospect_id,
+  }));
+}
+
+type ExploriumRow = {
+  prospect_id?: string;
+  first_name?: string;
+  last_name?: string;
+  full_name?: string;
+  job_title?: string;
+  job_level_main?: string;
+  job_department_main?: string;
+  company_name?: string;
+  company_website?: string;
+  linkedin?: string;
+};
+
+// ---------------------------------------------------------------------------
+// pipe0 / Crustdata — net-new discovery (search). Crustdata is built into pipe0
+// (no separate connection needed). Returns profiles; pair with
+// pipe0ResolveWorkEmails to get real emails.
+
+export async function fetchPipe0CrustdataProspects(opts: {
+  apiKey: string;
+  filters: Record<string, unknown>;
+  limit?: number;
+  apiBaseUrl?: string;
+  fetchImpl?: FetchImpl;
+}): Promise<Prospect[]> {
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const base = (opts.apiBaseUrl ?? "https://api.pipe0.com").replace(/\/$/, "");
+  const limit = Math.min(opts.limit ?? 25, 100);
+  const response = await fetchImpl(`${base}/v1/searches/run/sync`, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${opts.apiKey}`, "Content-Type": "application/json" },
+    body: JSON.stringify({
+      searches: [{ search_id: "people:profiles:crustdata@1", config: { limit, filters: opts.filters } }],
+    }),
+  });
+  if (!response.ok) {
+    throw new Error(`pipe0 /v1/searches/run/sync failed: HTTP ${response.status} ${await safeText(response)}`);
+  }
+  const body = (await response.json()) as {
+    results?: Array<Record<string, { value?: unknown }>>;
+    search_statuses?: Array<{ errors?: Array<{ code?: string; message?: string }> }>;
+  };
+  // Surface upstream provider errors (e.g. CreditBalanceInsufficient) instead of
+  // silently returning [] — a throttle/credit failure must not look like "no ICP matches".
+  const upstreamErrors = (body.search_statuses ?? []).flatMap((s) => s.errors ?? []);
+  if (upstreamErrors.length > 0 && !(body.results ?? []).length) {
+    throw new Error(
+      `pipe0/Crustdata search error: ${upstreamErrors.map((e) => `${e.code ?? "error"}: ${e.message ?? ""}`).join("; ")}`,
+    );
+  }
+  return (body.results ?? []).map((row) => {
+    const fullName = strField(row.name);
+    const { firstName, lastName } = splitName(fullName);
+    return {
+      firstName,
+      lastName,
+      fullName,
+      jobTitle: strField(row.job_title),
+      companyDomain: bareDomain(strField(row.company_website_url)),
+      linkedin: normalizeLinkedin(strField(row.profile_url)),
+      sourceId: strField(row.profile_url) ?? fullName,
+    };
+  });
+}
+
+function strField(field: { value?: unknown } | undefined): string | undefined {
+  const v = field?.value;
+  return typeof v === "string" && v.trim() ? v : undefined;
+}
+
+function normalizeLinkedin(value: string | undefined): string | undefined {
+  if (!value) return undefined;
+  const v = value.trim().replace(/^https?:\/\//i, "").replace(/\/$/, "");
+  return v ? `https://${v}` : undefined;
+}
+
+// ---------------------------------------------------------------------------
+// pipe0 — work-email resolution (waterfall)
+
+/**
+ * Resolve real work emails for people via pipe0's waterfall. Input rows need a
+ * full name + a company domain (or name). Returns the prospects with `email`
+ * filled where the waterfall found one; rows with no hit are returned unchanged.
+ */
+export async function pipe0ResolveWorkEmails(opts: {
+  apiKey: string;
+  prospects: Prospect[];
+  apiBaseUrl?: string;
+  fetchImpl?: FetchImpl;
+  /** Rows per pipe0 call. Small chunks are resilient to the waterfall's
+   *  batch rate-limit (a throttled chunk returns work_email status "failed"
+   *  for every row, so one big batch is all-or-nothing). Default 3. */
+  chunkSize?: number;
+}): Promise<Prospect[]> {
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const base = (opts.apiBaseUrl ?? "https://api.pipe0.com").replace(/\/$/, "");
+  const chunkSize = Math.max(1, opts.chunkSize ?? 3);
+
+  // Only rows with the waterfall's required inputs (name + company_domain|name).
+  const resolvable = opts.prospects.filter((p) => p.fullName && (p.companyDomain || p.companyName));
+  if (resolvable.length === 0) return opts.prospects;
+
+  const emailByKey = new Map<string, string>();
+  for (let i = 0; i < resolvable.length; i += chunkSize) {
+    const chunk = resolvable.slice(i, i + chunkSize);
+    const input = chunk.map((p) => ({
+      name: p.fullName,
+      ...(p.companyDomain ? { company_domain: p.companyDomain } : { company_name: p.companyName }),
+    }));
+    let body: Pipe0RunResponse;
+    try {
+      const response = await fetchImpl(`${base}/v1/pipes/run/sync`, {
+        method: "POST",
+        headers: { Authorization: `Bearer ${opts.apiKey}`, "Content-Type": "application/json" },
+        body: JSON.stringify({ pipes: [{ pipe_id: "person:workemail:waterfall@1" }], input }),
+      });
+      if (!response.ok) continue; // throttled/5xx chunk — skip, keep the rest
+      body = (await response.json()) as Pipe0RunResponse;
+    } catch {
+      continue; // network hiccup on one chunk must not sink the whole run
+    }
+    for (const recordId of body.order ?? []) {
+      const fields = body.records?.[recordId]?.fields ?? {};
+      const email = fields.work_email?.status === "completed" ? fieldValue(fields.work_email) : undefined;
+      if (email) {
+        const domain = fieldValue(fields.company_domain) ?? fieldValue(fields.company_name);
+        emailByKey.set(personKey(fieldValue(fields.name), domain), email);
+      }
+    }
+  }
+  return opts.prospects.map((p) => {
+    const email = emailByKey.get(personKey(p.fullName, p.companyDomain ?? p.companyName));
+    return email ? { ...p, email } : p;
+  });
+}
+
+function personKey(name: string | undefined, company: string | undefined): string {
+  return `${(name ?? "").trim().toLowerCase()}|${(company ?? "").trim().toLowerCase()}`;
+}
+
+type Pipe0Field = { value?: unknown; status?: string };
+type Pipe0RunResponse = {
+  order?: string[];
+  records?: Record<string, { fields?: Record<string, Pipe0Field> }>;
+};
+
+function fieldValue(field: Pipe0Field | undefined): string | undefined {
+  const v = field?.value;
+  return typeof v === "string" && v.trim() ? v : undefined;
+}
+
+async function safeText(response: Response): Promise<string> {
+  try {
+    return (await response.text()).slice(0, 300);
+  } catch {
+    return "";
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Pre-email dedup: identity keys shared between prospects and CRM contacts, so
+// we can drop already-in-CRM / already-seen people BEFORE paying for emails.
+
+function normName(value: string | undefined): string {
+  return (value ?? "").trim().toLowerCase().replace(/\s+/g, " ");
+}
+
+/**
+ * Stable identity keys for a prospect (prefixed by kind). A match on ANY key
+ * means "same person". LinkedIn is strongest; name+domain is the only key
+ * shareable with the canonical CRM snapshot (which has no LinkedIn); email is
+ * available only after resolution (so it can't pre-filter, but it strengthens
+ * the seen cache).
+ */
+export function prospectIdentityKeys(p: Prospect): string[] {
+  const keys: string[] = [];
+  if (p.linkedin) keys.push(`li:${p.linkedin.trim().toLowerCase().replace(/\/+$/, "")}`);
+  const name = normName(p.fullName ?? [p.firstName, p.lastName].filter(Boolean).join(" "));
+  const domain = bareDomain(p.companyDomain);
+  if (name && domain) keys.push(`nd:${name}|${domain}`);
+  if (p.email) keys.push(`em:${p.email.trim().toLowerCase()}`);
+  return keys;
+}
+
+/** Identity keys for every contact already in the CRM snapshot (email + name|domain). */
+export function crmContactKeys(snapshot: CanonicalGtmSnapshot): Set<string> {
+  const domainByAccount = new Map<string, string>();
+  for (const account of snapshot.accounts ?? []) {
+    const d = bareDomain(account.domain);
+    if (d) domainByAccount.set(account.id, d);
+  }
+  const set = new Set<string>();
+  for (const contact of snapshot.contacts ?? []) {
+    if (contact.linkedin) set.add(`li:${contact.linkedin.trim().toLowerCase().replace(/\/+$/, "")}`);
+    if (contact.email) set.add(`em:${contact.email.trim().toLowerCase()}`);
+    const name = normName([contact.firstName, contact.lastName].filter(Boolean).join(" "));
+    const domain = contact.accountId ? domainByAccount.get(contact.accountId) : undefined;
+    if (name && domain) set.add(`nd:${name}|${domain}`);
+  }
+  return set;
+}
+
+/**
+ * Split prospects into those worth paying to enrich vs. drop. A prospect is
+ * dropped if any identity key is already in the CRM (`crmKeys`) or in the
+ * cross-run `seen` cache. Pure + deterministic.
+ */
+export function partitionFreshProspects(
+  prospects: Prospect[],
+  crmKeys: Set<string>,
+  seen: Set<string>,
+): { fresh: Prospect[]; skippedCrm: number; skippedSeen: number } {
+  const fresh: Prospect[] = [];
+  let skippedCrm = 0;
+  let skippedSeen = 0;
+  for (const p of prospects) {
+    const keys = prospectIdentityKeys(p);
+    if (keys.some((k) => crmKeys.has(k))) {
+      skippedCrm += 1;
+      continue;
+    }
+    if (keys.some((k) => seen.has(k))) {
+      skippedSeen += 1;
+      continue;
+    }
+    fresh.push(p);
+  }
+  return { fresh, skippedCrm, skippedSeen };
+}