fullstackgtm 0.50.0 → 0.51.0

package/src/connectors/hubspot.ts

@@ -42,6 +42,10 @@ export type HubspotConnectorOptions = {
    * alongside the legacy `onProgress` callback; both are presentation-only.
    */
   progress?: ProgressEmitter;
+  /** Maximum retries for HubSpot 429/5xx responses (default 5). */
+  maxRetries?: number;
+  /** Injectable delay for deterministic retry tests. */
+  sleep?: (milliseconds: number) => Promise<void>;
 };
 
 const OBJECT_PATHS: Partial<Record<GtmObjectType, string>> = {
@@ -76,6 +80,8 @@ const PULL_STAGE_BY_TYPE: Record<SnapshotProgress["objectType"], (typeof SNAPSHO
 export function createHubspotConnector(options: HubspotConnectorOptions): HubspotWritableConnector {
   const baseUrl = (options.apiBaseUrl ?? DEFAULT_API_BASE_URL).replace(/\/$/, "");
   const fetchImpl = options.fetchImpl ?? fetch;
+  const maxRetries = options.maxRetries ?? 5;
+  const sleep = options.sleep ?? ((milliseconds: number) => new Promise<void>((resolve) => setTimeout(resolve, milliseconds)));
   const mappings = options.fieldMappings;
   // create:<Name> dedup within one connector lifetime (one apply run): the
   // search API is eventually consistent, so a just-created company is
@@ -113,27 +119,45 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Hubspo
   };
 
   async function request(path: string, init: RequestInit = {}): Promise<any> {
-    const token = await options.getAccessToken();
-    let response: Response;
-    try {
-      response = await fetchImpl(`${baseUrl}${path}`, {
-        ...init,
-        headers: {
-          Authorization: `Bearer ${token}`,
-          "Content-Type": "application/json",
-          ...(init.headers ?? {}),
-        },
-      });
-    } catch (error) {
-      const cause = error instanceof Error && error.cause instanceof Error ? `: ${error.cause.message}` : "";
-      throw new Error(`Cannot reach HubSpot at ${baseUrl}${cause}. Check network access.`);
+    let response: Response | undefined;
+    for (let attempt = 0; attempt <= maxRetries; attempt += 1) {
+      const token = await options.getAccessToken();
+      try {
+        response = await fetchImpl(`${baseUrl}${path}`, {
+          ...init,
+          headers: {
+            Authorization: `Bearer ${token}`,
+            "Content-Type": "application/json",
+            ...(init.headers ?? {}),
+          },
+        });
+      } catch (error) {
+        const cause = error instanceof Error && error.cause instanceof Error ? `: ${error.cause.message}` : "";
+        throw new Error(`Cannot reach HubSpot at ${baseUrl}${cause}. Check network access.`);
+      }
+      const retryable = response.status === 429 || response.status >= 500;
+      if (!retryable || attempt === maxRetries) break;
+      await response.text().catch(() => undefined);
+      const retryAfter = response.headers.get("Retry-After");
+      const seconds = retryAfter === null ? NaN : Number(retryAfter);
+      const dateDelay = retryAfter && !Number.isFinite(seconds) ? Date.parse(retryAfter) - Date.now() : NaN;
+      const delayMs = Math.min(
+        30_000,
+        Math.max(0, Number.isFinite(seconds) ? seconds * 1_000 : Number.isFinite(dateDelay) ? dateDelay : 1_000 * 2 ** attempt),
+      );
+      const reason = response.status === 429 ? "rate limited" : `temporarily unavailable (${response.status})`;
+      options.progress?.note(`HubSpot ${reason}; retrying in ${Math.ceil(delayMs / 1_000)}s (${attempt + 1}/${maxRetries})`);
+      await sleep(delayMs);
     }
-    if (!response.ok) {
+    if (!response || !response.ok) {
       // Status line only — HubSpot 4xx bodies echo submitted property values
       // (contact emails, company domains) and the request payload, and these
       // errors are persisted into scheduled-run records. Never interpolate it.
-      await response.text().catch(() => undefined);
-      throw new Error(`HubSpot API error ${response.status}. Check the token scopes and request.`);
+      await response?.text().catch(() => undefined);
+      if (response?.status === 429) {
+        throw new Error(`HubSpot rate limit (429) persisted after ${maxRetries} retries. Wait for the portal limit to reset, then retry; no CRM writes were made.`);
+      }
+      throw new Error(`HubSpot API error ${response?.status ?? "unknown"}. Check the token scopes and request.`);
     }
     // DELETE and some association writes return 204 with an empty body.
     const text = await response.text();

package/src/connectors/prospectSources.ts

@@ -34,6 +34,12 @@ export type Prospect = {
   linkedin?: string;
   /** real work email once resolved (pipe0); never the hashed value */
   email?: string;
+  /** Validated mobile number when a contact provider explicitly returns one. */
+  mobile?: string;
+  /** Validated business direct dial when distinct from mobile. */
+  directDial?: string;
+  /** Person geography returned by identity-search providers. */
+  location?: { city?: string; state?: string; region?: string; country?: string; countryCode?: string };
   /** ICP fit score 0..1, set by the acquire scorer */
   fitScore?: number;
   /** provider-native id for traceability */
@@ -471,7 +477,7 @@ function fieldValue(field: Pipe0Field | undefined): string | undefined {
 }
 
 // ---------------------------------------------------------------------------
-// Pre-email dedup: identity keys shared between prospects and CRM contacts, so
+// Pre-enrichment dedup: identity keys shared between prospects and CRM contacts, so
 // we can drop already-in-CRM / already-seen people BEFORE paying for emails.
 
 function normName(value: string | undefined): string {

package/src/contactProviders.ts

@@ -0,0 +1,141 @@
+import type { Prospect } from "./connectors/prospectSources.ts";
+
+export type ContactField = "work_email" | "mobile" | "direct_dial";
+export type ContactInputShape = "linkedin" | "name_domain" | "email" | "phone" | "company_domain";
+export type ContactProviderExecution = "sync" | "poll" | "webhook" | "batch";
+export type ContactProviderBilling = "per_attempt" | "per_match" | "per_field" | "subscription" | "unknown";
+
+export type ContactProviderCapability = {
+  provider: string;
+  operation: string;
+  inputShapes: ContactInputShape[];
+  outputFields: ContactField[];
+  execution: ContactProviderExecution;
+  billing: ContactProviderBilling;
+  chargesOn: Array<"request" | "match" | "valid" | "accept_all" | "unknown">;
+  supportsBalance: boolean;
+  supportsIdempotency: boolean;
+  maxBatchSize?: number;
+};
+
+/** Only implemented adapters belong here; planned providers stay in the strategy document. */
+export const CONTACT_PROVIDER_CAPABILITIES: Readonly<Record<string, readonly ContactProviderCapability[]>> = {
+  pipe0: [{
+    provider: "pipe0",
+    operation: "person:workemail:waterfall@1",
+    inputShapes: ["name_domain"],
+    outputFields: ["work_email"],
+    execution: "batch",
+    billing: "per_attempt",
+    chargesOn: ["request"],
+    supportsBalance: false,
+    supportsIdempotency: false,
+    maxBatchSize: 100,
+  }],
+};
+
+export type ContactWaterfallStep = { provider: string; fields: ContactField[] };
+
+export type ContactProviderAdapter = {
+  provider: string;
+  resolve(prospects: Prospect[], fields: ContactField[]): Promise<Prospect[]>;
+};
+
+export type ContactWaterfallAttempt = {
+  provider: string;
+  fields: ContactField[];
+  attempted: number;
+  added: Partial<Record<ContactField, number>>;
+};
+
+export type ContactWaterfallResult = { prospects: Prospect[]; attempts: ContactWaterfallAttempt[] };
+
+export function validateContactWaterfall(steps: ContactWaterfallStep[]): ContactWaterfallStep[] {
+  if (!Array.isArray(steps) || steps.length === 0) {
+    throw new Error("contact waterfall must contain at least one provider step");
+  }
+  return steps.map((step, index) => {
+    if (!step || typeof step.provider !== "string" || !step.provider.trim()) {
+      throw new Error(`contact waterfall step ${index + 1}: provider must be a non-empty string`);
+    }
+    const capabilities = CONTACT_PROVIDER_CAPABILITIES[step.provider];
+    if (!capabilities) {
+      throw new Error(
+        `contact waterfall step ${index + 1}: unsupported provider "${step.provider}" ` +
+          `(implemented: ${Object.keys(CONTACT_PROVIDER_CAPABILITIES).join(", ")})`,
+      );
+    }
+    if (!Array.isArray(step.fields) || step.fields.length === 0) {
+      throw new Error(`contact waterfall step ${index + 1}: fields must be a non-empty array`);
+    }
+    const fields = [...new Set(step.fields)];
+    for (const field of fields) {
+      if (field !== "work_email" && field !== "mobile" && field !== "direct_dial") {
+        throw new Error(`contact waterfall step ${index + 1}: unsupported field "${String(field)}"`);
+      }
+      if (!capabilities.some((capability) => capability.outputFields.includes(field))) {
+        const available = [...new Set(capabilities.flatMap((capability) => capability.outputFields))];
+        throw new Error(
+          `contact waterfall step ${index + 1}: ${step.provider} does not implement "${field}" ` +
+            `(available: ${available.join(", ")})`,
+        );
+      }
+    }
+    return { provider: step.provider, fields };
+  });
+}
+
+/** Run providers in order. Later steps receive only records still missing a requested field. */
+export async function runContactWaterfall(opts: {
+  prospects: Prospect[];
+  steps: ContactWaterfallStep[];
+  adapters: Readonly<Record<string, ContactProviderAdapter>>;
+}): Promise<ContactWaterfallResult> {
+  const steps = validateContactWaterfall(opts.steps);
+  const prospects = opts.prospects.map((prospect) => ({ ...prospect }));
+  const attempts: ContactWaterfallAttempt[] = [];
+  for (const step of steps) {
+    const adapter = opts.adapters[step.provider];
+    if (!adapter) throw new Error(`contact waterfall: adapter "${step.provider}" is not configured`);
+    const indexes = prospects
+      .map((prospect, index) => step.fields.some((field) => !fieldValue(prospect, field)) ? index : -1)
+      .filter((index) => index >= 0);
+    if (indexes.length === 0) {
+      attempts.push({ provider: step.provider, fields: step.fields, attempted: 0, added: {} });
+      continue;
+    }
+    const inputs = indexes.map((index) => prospects[index]);
+    const outputs = await adapter.resolve(inputs, step.fields);
+    if (outputs.length !== inputs.length) {
+      throw new Error(`contact waterfall: ${step.provider} returned ${outputs.length} result(s) for ${inputs.length} input(s)`);
+    }
+    const added: Partial<Record<ContactField, number>> = {};
+    outputs.forEach((output, outputIndex) => {
+      const prospectIndex = indexes[outputIndex];
+      const current = prospects[prospectIndex];
+      let merged = current;
+      for (const field of step.fields) {
+        if (fieldValue(current, field)) continue;
+        const value = fieldValue(output, field);
+        if (!value) continue;
+        merged = setFieldValue(merged, field, value);
+        added[field] = (added[field] ?? 0) + 1;
+      }
+      prospects[prospectIndex] = merged;
+    });
+    attempts.push({ provider: step.provider, fields: step.fields, attempted: inputs.length, added });
+  }
+  return { prospects, attempts };
+}
+
+function fieldValue(prospect: Prospect, field: ContactField): string | undefined {
+  if (field === "work_email") return prospect.email;
+  if (field === "mobile") return prospect.mobile;
+  return prospect.directDial;
+}
+
+function setFieldValue(prospect: Prospect, field: ContactField, value: string): Prospect {
+  if (field === "work_email") return { ...prospect, email: value };
+  if (field === "mobile") return { ...prospect, mobile: value };
+  return { ...prospect, directDial: value };
+}

package/src/enrich.ts

@@ -6,6 +6,7 @@ import type { AcquireBudget } from "./acquireMeter.ts";
 import type { ProgressEmitter } from "./progress.ts";
 import { parseAssignmentPolicy, resolveAssignment } from "./assign.ts";
 import type { AssignmentContext, AssignmentPolicy } from "./assign.ts";
+import { validateContactWaterfall, type ContactWaterfallStep } from "./contactProviders.ts";
 import type {
   CanonicalGtmSnapshot,
   CreateRecordPayload,
@@ -108,11 +109,16 @@ export type AcquireCreateMap = {
  * real emails (e.g. explorium discovers, pipe0 resolves the work email).
  */
 export type AcquireDiscoveryConfig = {
-  provider: "explorium" | "pipe0" | "linkedin" | "heyreach";
+  provider: "explorium" | "pipe0" | "clay" | "linkedin" | "heyreach";
+  /** Clay search collection. Phase 1 supports people; companies follows separately. */
+  sourceType?: "people" | "companies";
   filters?: Record<string, unknown>;
   size?: number;
   /** Maximum raw provider candidates scanned per run while seeking fresh leads. */
   scanLimit?: number;
+  /** Ordered fill-only contact providers. Later steps receive only unresolved fields. */
+  contactWaterfall?: ContactWaterfallStep[];
+  /** @deprecated Use contactWaterfall. Retained for existing configurations. */
   resolveEmailsWith?: "pipe0";
   /** LinkedIn sources only: the provider-native list id to read (HeyReach lead-list id). */
   listId?: string;
@@ -148,7 +154,7 @@ const MATCH_KEYS: Record<EnrichObjectType, string[]> = {
 };
 
 /** API source ids the MVP can pull from. */
-export const SUPPORTED_API_SOURCES = ["apollo", "explorium", "pipe0", "linkedin"];
+export const SUPPORTED_API_SOURCES = ["apollo", "explorium", "pipe0", "clay", "linkedin"];
 
 /**
  * Canonical fields enrich may target, plus the HubSpot property spellings the
@@ -339,6 +345,21 @@ export function parseEnrichConfig(raw: string): EnrichConfig {
       fail(error instanceof Error ? error.message : String(error));
     }
   }
+  for (const [sourceId, discovery] of Object.entries(config.acquire?.discovery ?? {})) {
+    if (discovery.provider === "clay" && (discovery.sourceType ?? "people") !== "people") {
+      fail(`acquire.discovery.${sourceId}.sourceType currently supports only "people" for Clay`);
+    }
+    if (discovery.contactWaterfall !== undefined) {
+      try {
+        discovery.contactWaterfall = validateContactWaterfall(discovery.contactWaterfall);
+      } catch (error) {
+        fail(`acquire.discovery.${sourceId}.${error instanceof Error ? error.message : String(error)}`);
+      }
+    }
+    if (discovery.resolveEmailsWith !== undefined && discovery.resolveEmailsWith !== "pipe0") {
+      fail(`acquire.discovery.${sourceId}.resolveEmailsWith must be "pipe0"`);
+    }
+  }
 
   return config as EnrichConfig;
 }
@@ -408,6 +429,34 @@ export function builtinAcquirePreset(source: string | undefined): EnrichConfig |
       },
     };
   }
+  if (provider === "clay") {
+    return {
+      sources: { clay: { kind: "api" } },
+      match: { contact: { keys: ["linkedin", "email"], onAmbiguous: "skip" } },
+      fields: {},
+      policy: { overwrite: "never" },
+      acquire: {
+        budget: { records: { perDay: 50, perMonth: 500 } },
+        costPerRecord: { clay: 0 },
+        discovery: { clay: { provider: "clay", sourceType: "people", size: 25 } },
+        create: {
+          contact: {
+            matchKey: "linkedin",
+            properties: {
+              hs_linkedin_url: "linkedin",
+              firstname: "firstName",
+              lastname: "lastName",
+              jobtitle: "jobTitle",
+              company: "companyName",
+              email: "email",
+            },
+            associateCompanyFrom: "companyName",
+            associateCompanyDomainFrom: "companyDomain",
+          },
+        },
+      },
+    };
+  }
   if (provider !== "pipe0" && provider !== "explorium") return undefined;
   return {
     sources: { [provider]: { kind: "api" } },

package/src/hostedPatchPlan.ts

@@ -207,7 +207,12 @@ export async function reportHostedPlanLifecycle(stored: StoredPlan, options: Opt
   if (!paired) return { status: "unpaired" };
   const status = stored.status === "applied" ? "applied" : stored.status === "rejected" ? "rejected" : "approved";
   const runRecord = status === "applied" ? stored.runs.at(-1) : undefined;
-  const operationReceipts = runRecord?.results.map((result) => ({
+  // The hosted terminal transition is authority-bound to the exact approved
+  // subset. Local runs also record excluded operations as `skipped` for a
+  // complete human audit trail; those are not execution receipts and must not
+  // be echoed as though they were authorized provider work.
+  const approved = new Set(stored.approvedOperationIds);
+  const operationReceipts = runRecord?.results.filter((result) => approved.has(result.operationId)).map((result) => ({
     packageOpId: result.operationId,
     status: result.status,
     ...(result.detail ? { error: result.detail.slice(0, 1000) } : {}),

package/src/icp.ts

@@ -240,6 +240,57 @@ export function icpToCrustdataFilters(icp: Icp): Record<string, unknown> {
   return f;
 }
 
+const CLAY_INDUSTRY: Record<string, string[]> = {
+  software: ["Software Development"],
+  saas: ["Software Development"],
+  internet: ["Technology, Information and Internet"],
+  fintech: ["Financial Services"],
+  "financial services": ["Financial Services"],
+  "information technology & services": ["IT Services and IT Consulting"],
+  "information technology and services": ["IT Services and IT Consulting"],
+};
+
+const CLAY_EMPLOYEE_BAND: Record<string, string> = {
+  "1-10": "2-10",
+  "11-50": "11-50",
+  "51-200": "51-200",
+  "201-500": "201-500",
+  "501-1000": "501-1,000",
+  "1001-5000": "1,001-5,000",
+  "5001-10000": "5,001-10,000",
+  "10001+": "10,001+",
+};
+
+const CLAY_SENIORITY: Record<string, string> = {
+  cxo: "c-suite",
+  "c-suite": "c-suite",
+  vp: "vp",
+  director: "director",
+  head: "head",
+  manager: "manager",
+  owner: "owner",
+  founder: "founder",
+  senior: "senior",
+};
+
+/** Clay people-search filters using only fields confirmed in the live catalog. */
+export function icpToClayPeopleFilters(icp: Icp): Record<string, unknown> {
+  const filters: Record<string, unknown> = {};
+  if (icp.persona.titleKeywords?.length) filters.job_title_keywords = icp.persona.titleKeywords;
+  const seniorities = [...new Set((icp.persona.jobLevels ?? []).map((level) => CLAY_SENIORITY[level.toLowerCase()]).filter(Boolean))];
+  if (seniorities.length) filters.job_title_seniority_levels_v2 = seniorities;
+  const sizes = [...new Set((icp.firmographics.employeeBands ?? []).map((band) => CLAY_EMPLOYEE_BAND[band.replace(/,/g, "")]).filter(Boolean))];
+  if (sizes.length) filters.company_sizes = sizes;
+  if (icp.firmographics.geos?.length) {
+    filters.location_countries_include = icp.firmographics.geos.map((geo) => COUNTRY_NAMES[geo.toLowerCase()] ?? geo);
+  }
+  const industries = [...new Set((icp.firmographics.industries ?? []).flatMap((industry) =>
+    CLAY_INDUSTRY[industry.toLowerCase()] ?? [titleCase(industry)]
+  ))];
+  if (industries.length) filters.company_industries_include = industries;
+  return filters;
+}
+
 const ACRONYMS = new Set(["cro", "coo", "ceo", "cfo", "vp", "crm", "gtm", "saas"]);
 function titleCase(value: string): string {
   return value

package/src/index.ts

@@ -55,6 +55,20 @@ export {
   type RulePackageTrust,
 } from "./config.ts";
 export { applyPatchPlan, type ApplyPatchPlanOptions } from "./connector.ts";
+export {
+  CONTACT_PROVIDER_CAPABILITIES,
+  runContactWaterfall,
+  validateContactWaterfall,
+  type ContactField,
+  type ContactInputShape,
+  type ContactProviderAdapter,
+  type ContactProviderBilling,
+  type ContactProviderCapability,
+  type ContactProviderExecution,
+  type ContactWaterfallAttempt,
+  type ContactWaterfallResult,
+  type ContactWaterfallStep,
+} from "./contactProviders.ts";
 export {
   APPLY_STAGES,
   BACKFILL_STRIPE_STAGES,
@@ -70,6 +84,17 @@ export {
   type ProgressSnapshot,
 } from "./progress.ts";
 export { createHubspotConnector, type HubspotConnectorOptions } from "./connectors/hubspot.ts";
+export {
+  CLAY_PUBLIC_API_BASE,
+  clayApiKey,
+  createClaySearch,
+  normalizeClayPerson,
+  runClayPeopleSearchPage,
+  validateClayApiKey,
+  type ClayPeopleSearchPage,
+  type ClaySearchSourceType,
+  type ClayKeyValidation,
+} from "./connectors/clay.ts";
 export {
   DEFAULT_LOOPBACK_PORT,
   DEFAULT_OAUTH_SCOPES,

package/src/init.ts

@@ -19,7 +19,7 @@ import { builtinAcquirePreset, ENRICH_CONFIG_FILE_NAME, type EnrichConfig } from
 import { DEFAULT_FIT_THRESHOLD, type Icp } from "./icp.ts";
 
 export type InitProvider = "hubspot" | "salesforce";
-export type InitSource = "pipe0" | "explorium" | "linkedin";
+export type InitSource = "pipe0" | "explorium" | "clay" | "linkedin";
 
 export type ScaffoldOptions = {
   /** discovery source the acquire preset + playbook are wired for. Default "pipe0". */
@@ -63,7 +63,7 @@ export function starterIcp(): Icp {
 export function starterEnrichConfig(source: InitSource): EnrichConfig {
   const preset = builtinAcquirePreset(source);
   if (!preset?.acquire) {
-    // builtinAcquirePreset covers pipe0/explorium/linkedin, so this is unreachable
+    // builtinAcquirePreset covers pipe0/explorium/clay/linkedin, so this is unreachable
     // for the typed InitSource set — guard anyway rather than emit a broken file.
     throw new Error(`init: no acquire preset for source "${source}"`);
   }