fullstackgtm 0.34.0 → 0.38.0

package/src/enrich.ts

@@ -2,8 +2,12 @@ import { mkdirSync, readFileSync, readdirSync } from "node:fs";
 import { join } from "node:path";
 import { credentialsDir, ensureSecureHomeDir, writeSecureFile } from "./credentials.ts";
 import { HUBSPOT_DEFAULT_FIELD_MAPPINGS } from "./mappings.ts";
+import type { AcquireBudget } from "./acquireMeter.ts";
+import { parseAssignmentPolicy, resolveAssignment } from "./assign.ts";
+import type { AssignmentContext, AssignmentPolicy } from "./assign.ts";
 import type {
   CanonicalGtmSnapshot,
+  CreateRecordPayload,
   GtmEvidence,
   PatchOperation,
   PatchPlan,
@@ -76,6 +80,50 @@ export type EnrichConfig = {
   match: Partial<Record<EnrichObjectType, EnrichMatchConfig>>;
   fields: Partial<Record<EnrichObjectType, EnrichFieldConfig[]>>;
   policy: EnrichPolicyConfig;
+  /** Net-new lead creation (`enrich acquire`). Optional; absent = acquire off. */
+  acquire?: AcquireConfig;
+};
+
+/** Maps a sourced record to the CRM properties a net-new record is created with. */
+export type AcquireCreateMap = {
+  /** CRM property name → source payload path (read via sourceValueAt). */
+  properties: Record<string, string>;
+  /** Source path whose value becomes the dedupe key (e.g. "email"). */
+  matchKey: string;
+  /** Optional source path to a company name; the connector resolves-or-creates it. */
+  associateCompanyFrom?: string;
+};
+
+/**
+ * Net-new discovery for an API acquire source. `provider` selects the adapter
+ * (explorium = prospect search; pipe0 = work-email waterfall). `filters` is the
+ * provider-native query; `resolveEmailsWith` chains a second provider to fill
+ * real emails (e.g. explorium discovers, pipe0 resolves the work email).
+ */
+export type AcquireDiscoveryConfig = {
+  provider: "explorium" | "pipe0" | "linkedin" | "heyreach";
+  filters?: Record<string, unknown>;
+  size?: number;
+  resolveEmailsWith?: "pipe0";
+  /** LinkedIn sources only: the provider-native list id to read (HeyReach lead-list id). */
+  listId?: string;
+};
+
+export type AcquireConfig = {
+  /** Windowed budget enforced by the acquire meter; absent = unmetered. */
+  budget?: AcquireBudget;
+  /** Estimated provider cost (USD) per created record, by source id. */
+  costPerRecord?: Record<string, number>;
+  /** How to build a net-new record from a sourced row, per object type. */
+  create: Partial<Record<EnrichObjectType, AcquireCreateMap>>;
+  /** Net-new discovery params for API sources, by source id. */
+  discovery?: Record<string, AcquireDiscoveryConfig>;
+  /**
+   * Ownership rule stamped onto every created lead so it is never born
+   * ownerless. Absent = no auto-assignment (the CLI may still default to the
+   * portal's sole active owner). Shared with `reassign --assign-unowned`.
+   */
+  assign?: AssignmentPolicy;
 };
 
 export const ENRICH_CONFIG_FILE_NAME = "enrich.config.json";
@@ -87,11 +135,11 @@ const OBJECT_TYPES: EnrichObjectType[] = ["company", "contact"];
 /** Match keys the matcher knows how to read off canonical snapshot records. */
 const MATCH_KEYS: Record<EnrichObjectType, string[]> = {
   company: ["domain", "name"],
-  contact: ["email", "name"],
+  contact: ["email", "name", "linkedin"],
 };
 
 /** API source ids the MVP can pull from. */
-export const SUPPORTED_API_SOURCES = ["apollo"];
+export const SUPPORTED_API_SOURCES = ["apollo", "explorium", "pipe0"];
 
 /**
  * Canonical fields enrich may target, plus the HubSpot property spellings the
@@ -263,11 +311,119 @@ export function parseEnrichConfig(raw: string): EnrichConfig {
       }
     }
   }
-  if (!anyField) fail('"fields" maps nothing — add at least one field entry');
+  // `fields` drives append/refresh. An acquire-only config (net-new lead
+  // creation, no blank-filling) legitimately maps no fields — accept it when
+  // an acquire.create mapping is present.
+  const hasAcquireCreate = Boolean(
+    config.acquire?.create && Object.keys(config.acquire.create).length > 0,
+  );
+  if (!anyField && !hasAcquireCreate) {
+    fail('"fields" maps nothing — add at least one field entry (or an "acquire.create" mapping)');
+  }
+
+  // Normalize + validate the assignment policy (fails loud on shape errors so a
+  // misconfigured rule never silently mis-routes a created lead).
+  if (config.acquire?.assign !== undefined) {
+    try {
+      config.acquire.assign = parseAssignmentPolicy(config.acquire.assign);
+    } catch (error) {
+      fail(error instanceof Error ? error.message : String(error));
+    }
+  }
 
   return config as EnrichConfig;
 }
 
+/**
+ * Built-in zero-config presets for well-known sources, so the common Mode-A
+ * loop — "feed a Clay export, get the dedup/routing verdict" — needs no
+ * hand-authored config. These are **match-only** (empty `fields`): the value is
+ * the matched / unmatched / ambiguous routing and collision detection, not field
+ * writes. Drop an `enrich.config.json` next to your run to map fields for actual
+ * field enrichment; an explicit config always wins over a preset. Constructed
+ * directly (not via `parseEnrichConfig`) so the match-only shape is allowed.
+ */
+export const BUILTIN_ENRICH_PRESETS: Record<string, EnrichConfig> = {
+  clay: {
+    sources: { clay: { kind: "ingest", format: "csv" } },
+    policy: { overwrite: "never", defaultStaleDays: 90 },
+    match: {
+      company: { keys: ["domain"], onAmbiguous: "skip" },
+      contact: { keys: ["email"], onAmbiguous: "skip" },
+    },
+    fields: {},
+  },
+};
+
+export function builtinEnrichPreset(source: string | undefined): EnrichConfig | undefined {
+  return source ? BUILTIN_ENRICH_PRESETS[source] : undefined;
+}
+
+/**
+ * Zero-config `enrich acquire` preset so every client gets targeted, deduped,
+ * metered lead-fill out of the box — no hand-authored enrich.config.json. The
+ * ICP (icp.json) supplies the targeting; this supplies sensible plumbing. The
+ * create mapping writes the prospect's LinkedIn URL into hs_linkedin_url, so
+ * each created contact strengthens future dedup. An explicit config always wins.
+ */
+export function builtinAcquirePreset(source: string | undefined): EnrichConfig | undefined {
+  const provider = source ?? "pipe0";
+  if (provider === "linkedin" || provider === "heyreach") {
+    // LinkedIn discovery reads our OWN HeyReach lead list, so it costs $0/record
+    // (no spend budget) and keys on the LinkedIn URL — leads are sparse on email,
+    // so email-keying would drop most of them. listId is supplied via --list.
+    return {
+      sources: { linkedin: { kind: "api" } },
+      match: { contact: { keys: ["linkedin", "email"], onAmbiguous: "skip" } },
+      fields: {},
+      policy: { overwrite: "never" },
+      acquire: {
+        budget: { records: { perDay: 50, perMonth: 500 } },
+        costPerRecord: { linkedin: 0 },
+        discovery: { linkedin: { provider: "linkedin", size: 25 } },
+        create: {
+          contact: {
+            matchKey: "linkedin",
+            properties: {
+              hs_linkedin_url: "linkedin",
+              firstname: "firstName",
+              lastname: "lastName",
+              jobtitle: "jobTitle",
+              company: "companyName",
+              email: "email",
+            },
+          },
+        },
+      },
+    };
+  }
+  if (provider !== "pipe0" && provider !== "explorium") return undefined;
+  return {
+    sources: { [provider]: { kind: "api" } },
+    match: { contact: { keys: ["email"], onAmbiguous: "skip" } },
+    fields: {},
+    policy: { overwrite: "never" },
+    acquire: {
+      budget: { records: { perDay: 50, perMonth: 500 }, spend: { perDay: 25, perMonth: 250 } },
+      costPerRecord: { pipe0: 0.1, explorium: 0.4 },
+      discovery: { [provider]: { provider, size: 25 } },
+      create: {
+        contact: {
+          matchKey: "email",
+          properties: {
+            email: "email",
+            firstname: "firstName",
+            lastname: "lastName",
+            jobtitle: "jobTitle",
+            company: "companyName",
+            hs_linkedin_url: "linkedin",
+          },
+        },
+      },
+    },
+  };
+}
+
 export function loadEnrichConfig(path: string): EnrichConfig {
   let raw: string;
   try {
@@ -435,12 +591,16 @@ function normalizeKeyValue(key: string, value: unknown): string {
       .replace(/^www\./, "")
       .replace(/\/.*$/, "");
   }
+  if (key === "linkedin") {
+    // Match crmContactKeys' `li:` normalization (lowercased, trailing slash stripped).
+    return text.trim().replace(/\/+$/, "");
+  }
   return text.replace(/\s+/g, " ");
 }
 
 function crmKeyValue(
   objectType: EnrichObjectType,
-  record: { name?: string; domain?: string; email?: string; firstName?: string; lastName?: string },
+  record: { name?: string; domain?: string; email?: string; firstName?: string; lastName?: string; linkedin?: string },
   key: string,
 ): string {
   if (objectType === "company") {
@@ -449,6 +609,7 @@ function crmKeyValue(
     return "";
   }
   if (key === "email") return normalizeKeyValue("email", record.email);
+  if (key === "linkedin") return normalizeKeyValue("linkedin", record.linkedin);
   if (key === "name") {
     return normalizeKeyValue("name", `${record.firstName ?? ""} ${record.lastName ?? ""}`.trim());
   }
@@ -634,7 +795,12 @@ export function buildEnrichPlan(options: BuildEnrichPlanOptions): EnrichPlanResu
     const fields = (config.fields[record.objectType] ?? []).filter(
       (field) => field.from[source] !== undefined,
     );
-    if (!match || fields.length === 0) {
+    // Run the matcher whenever the object type has match keys; field mappings
+    // only gate whether we EMIT field ops. With no fields (a match-only config,
+    // e.g. the built-in Clay preset) this still reports the matched / ambiguous
+    // / unmatched routing verdict and records collisions — the Mode-A hygiene
+    // gate — it just proposes zero field writes.
+    if (!match) {
       counts.unmatched += 1;
       unmatchedSourceIds.push(record.id);
       continue;
@@ -798,6 +964,246 @@ export function buildEnrichPlan(options: BuildEnrichPlanOptions): EnrichPlanResu
   return { plan, counts, stamps, ambiguities, unmatchedSourceIds };
 }
 
+// ---------------------------------------------------------------------------
+// Acquire: turn sourced-but-unmatched prospects into net-new lead creates.
+
+export type AcquireCounts = {
+  fetched: number;
+  matched: number;
+  ambiguous: number;
+  unmatched: number;
+  /** create_record ops emitted. */
+  created: number;
+  /** unmatched rows that would have been created but for the meter ceiling. */
+  withheldByMeter: number;
+  /** created ops that got an owner stamped (a subset of `created`). */
+  assigned: number;
+  /** created ops left ownerless (no policy, or policy could not place them). */
+  unassigned: number;
+};
+
+export type AcquirePlanResult = {
+  plan: PatchPlan;
+  counts: AcquireCounts;
+  estCostUsd: number;
+};
+
+export type BuildAcquirePlanOptions = {
+  config: EnrichConfig;
+  source: string;
+  snapshot: CanonicalGtmSnapshot;
+  records: EnrichSourceRecord[];
+  runLabel: string;
+  /** Meter ceiling: max create ops to emit. null/undefined = unlimited. */
+  maxRecords?: number | null;
+  now?: () => Date;
+};
+
+/** First non-empty value among several candidate paths in a source payload. */
+function firstSourceValue(payload: Record<string, unknown>, paths: string[]): string | undefined {
+  for (const path of paths) {
+    const value = sourceValueAt(payload, path);
+    if (!isEmptyValue(value)) return String(value).trim();
+  }
+  return undefined;
+}
+
+/**
+ * Lift a prospect/source payload into the routing attributes a territory rule
+ * reads. Tolerant of provider spellings (explorium vs pipe0); absent attributes
+ * stay undefined, so a rule that routes on them simply won't match (falling to
+ * the policy fallback) instead of mis-routing.
+ */
+export function acquireAssignmentContext(
+  payload: Record<string, unknown>,
+  companyName: string | undefined,
+  accountOwnerByName: ReadonlyMap<string, string>,
+): AssignmentContext {
+  const ctx: AssignmentContext = {};
+  const title = firstSourceValue(payload, ["jobTitle", "job_title", "title"]);
+  if (title) ctx.title = title.toLowerCase();
+  const geo = firstSourceValue(payload, ["geo", "country", "company_country_code", "companyCountry"]);
+  if (geo) ctx.geo = geo.toLowerCase();
+  const industry = firstSourceValue(payload, ["industry", "company_industry", "companyIndustry"]);
+  if (industry) ctx.industry = industry.toLowerCase();
+  const employeeBand = firstSourceValue(payload, ["employeeBand", "company_size", "companySize"]);
+  if (employeeBand) ctx.employeeBand = employeeBand;
+  const department = firstSourceValue(payload, ["department", "job_department", "jobDepartment"]);
+  if (department) ctx.department = department.toLowerCase();
+  if (companyName) {
+    const owner = accountOwnerByName.get(companyName.trim().toLowerCase());
+    if (owner) ctx.accountOwnerId = owner;
+  }
+  return ctx;
+}
+
+/**
+ * Match each sourced record against the snapshot and route it: matched =
+ * already in the CRM (skip), ambiguous = a possible duplicate exists (skip —
+ * resolve-first never creates over ambiguity), unmatched = a net-new lead.
+ * Unmatched rows with a dedupe key and at least one mapped property become
+ * `create_record` operations, capped at `maxRecords` (the meter's headroom).
+ * Always a dry-run plan; nothing is written until apply.
+ */
+export function buildAcquirePlan(options: BuildAcquirePlanOptions): AcquirePlanResult {
+  const { config, source, snapshot, records, runLabel } = options;
+  const nowIso = (options.now ?? (() => new Date()))().toISOString();
+  const sourceConfig = config.sources[source];
+  if (!sourceConfig) throw new Error(`enrich: source "${source}" is not declared in the config`);
+  const acquire = config.acquire;
+  if (!acquire) throw new Error('enrich acquire: config has no "acquire" section');
+
+  const cap = options.maxRecords ?? null;
+  const costPerRecord = acquire.costPerRecord?.[source] ?? 0;
+
+  const operations: PatchOperation[] = [];
+  const evidence: GtmEvidence[] = [];
+  const counts: AcquireCounts = {
+    fetched: records.length,
+    matched: 0,
+    ambiguous: 0,
+    unmatched: 0,
+    created: 0,
+    withheldByMeter: 0,
+    assigned: 0,
+    unassigned: 0,
+  };
+  let estCostUsd = 0;
+
+  // Assignment context (built once): the set of owner ids the CRM will accept
+  // (active users), and a company-name → account-owner lookup for the
+  // "account-owner" inherit strategy. Empty when no assign policy is set.
+  const assignPolicy = acquire.assign;
+  const knownOwnerIds = new Set<string>();
+  const accountOwnerByName = new Map<string, string>();
+  if (assignPolicy) {
+    for (const user of snapshot.users ?? []) {
+      if (user.active === false) continue;
+      if (user.crmId) knownOwnerIds.add(user.crmId);
+      if (user.id) knownOwnerIds.add(user.id);
+    }
+    for (const account of snapshot.accounts ?? []) {
+      if (account.name && account.ownerId) {
+        accountOwnerByName.set(account.name.trim().toLowerCase(), account.ownerId);
+      }
+    }
+  }
+
+  for (const record of records) {
+    const createMap = acquire.create[record.objectType];
+    const match = config.match[record.objectType];
+    // No create mapping or no match config for this type — can neither create
+    // nor dedupe it safely; count as unmatched and skip.
+    if (!createMap || !match) {
+      counts.unmatched += 1;
+      continue;
+    }
+    const outcome = matchSourceRecord(snapshot, record.objectType, match.keys, record.keys);
+    if (outcome.status === "matched") {
+      counts.matched += 1;
+      continue;
+    }
+    if (outcome.status === "ambiguous") {
+      // Resolve-first: a possible duplicate exists — never create over it.
+      counts.ambiguous += 1;
+      continue;
+    }
+
+    counts.unmatched += 1;
+    const matchValue = String(record.keys[createMap.matchKey] ?? "").trim();
+    if (!matchValue) continue; // no dedupe key → refuse to create (can't resolve-first)
+
+    if (cap !== null && counts.created >= cap) {
+      counts.withheldByMeter += 1;
+      continue;
+    }
+
+    const properties: Record<string, string> = {};
+    for (const [crmProp, path] of Object.entries(createMap.properties)) {
+      const value = sourceValueAt(record.payload, path);
+      if (!isEmptyValue(value)) properties[crmProp] = String(value);
+    }
+    if (Object.keys(properties).length === 0) continue; // nothing to write
+
+    let associateCompanyName: string | undefined;
+    if (createMap.associateCompanyFrom) {
+      const companyValue = sourceValueAt(record.payload, createMap.associateCompanyFrom);
+      if (!isEmptyValue(companyValue)) associateCompanyName = String(companyValue);
+    }
+
+    const recordEvidence = evidenceFor(
+      source,
+      sourceConfig.kind,
+      sourceConfig.format,
+      record,
+      undefined,
+      nowIso,
+    );
+    evidence.push(recordEvidence);
+
+    // Resolve ownership BEFORE the meter-charged create lands, so the lead is
+    // never born ownerless. `counts.created` is the within-run index, giving
+    // round-robin deterministic, even distribution with no persisted cursor.
+    let ownerId: string | undefined;
+    let assignedBy: string | undefined;
+    if (assignPolicy) {
+      const ctx = acquireAssignmentContext(record.payload, associateCompanyName, accountOwnerByName);
+      const resolved = resolveAssignment(assignPolicy, ctx, counts.created, knownOwnerIds);
+      if (resolved.ownerId) {
+        ownerId = resolved.ownerId;
+        assignedBy = resolved.rule;
+      }
+    }
+
+    const payload: CreateRecordPayload = {
+      properties,
+      matchKey: createMap.matchKey,
+      matchValue,
+      source,
+      estCostUsd: costPerRecord,
+      associateCompanyName,
+      ...(ownerId ? { ownerId, assignedBy } : {}),
+    };
+    operations.push({
+      id: `op_acq_${fnv1a(`${source}:${record.objectType}:${matchValue}`)}`,
+      objectType: canonicalObjectType(record.objectType),
+      objectId: `create:${matchValue}`,
+      operation: "create_record",
+      beforeValue: null,
+      afterValue: payload,
+      reason:
+        `${source} sourced net-new ${record.objectType} "${describeSourceRecord(record)}" ` +
+        `(${createMap.matchKey}=${matchValue}); no CRM match — create as a lead.`,
+      sourceRuleOrPolicy: `acquire:${source}`,
+      riskLevel: "medium",
+      approvalRequired: true,
+      rollback: "Archive the created record (it was net-new).",
+      evidenceIds: [recordEvidence.id],
+    });
+    if (ownerId) counts.assigned += 1;
+    else counts.unassigned += 1;
+    counts.created += 1;
+    estCostUsd += costPerRecord;
+  }
+
+  const plan: PatchPlan = {
+    id: `patch_plan_acq_${fnv1a(`${source}:${runLabel}:${nowIso}`)}`,
+    title: `Acquire leads — ${source}`,
+    createdAt: nowIso,
+    status: operations.length > 0 ? "needs_approval" : "draft",
+    dryRun: true,
+    summary:
+      `${counts.created} net-new lead(s) proposed from ${source} (${counts.fetched} sourced: ` +
+      `${counts.matched} already in CRM, ${counts.ambiguous} ambiguous skipped, ${counts.unmatched} unmatched` +
+      `${counts.withheldByMeter > 0 ? `, ${counts.withheldByMeter} withheld by meter` : ""}). ` +
+      `Est. cost $${estCostUsd.toFixed(2)}.`,
+    findings: [],
+    evidence,
+    operations,
+  };
+  return { plan, counts, estCostUsd };
+}
+
 // ---------------------------------------------------------------------------
 // Staleness: compute the refresh work set from run-store stamps.
 

package/src/format.ts

@@ -1,6 +1,11 @@
 import type { PatchPlan, PatchPlanRun } from "./types.ts";
 
-export function patchPlanToMarkdown(plan: PatchPlan) {
+// `summary: true` renders only the header + the Findings-by-Rule table + an
+// operation count, for read verbs like `audit` where the full per-operation
+// dump (1,400+ lines on a real portal) buries the signal. Defaults to the full
+// view so write-preview callers (bulk-update, fix, dedupe, plans show, MCP) —
+// where you approve specific operations — keep every operation's detail.
+export function patchPlanToMarkdown(plan: PatchPlan, opts: { summary?: boolean } = {}) {
   const lines = [
     `# ${plan.title}`,
     "",
@@ -24,6 +29,19 @@ export function patchPlanToMarkdown(plan: PatchPlan) {
     lines.push("");
   }
 
+  if (opts.summary) {
+    const ops = plan.operations.length;
+    lines.push(
+      ops === 0
+        ? "No patch operations proposed."
+        : `${ops} dry-run patch operation${ops === 1 ? "" : "s"} proposed — nothing written.`,
+      "",
+      "Full plan detail: re-run with `--full`.",
+      "Client-ready report: `fullstackgtm report` (add `--format html` for a printable file).",
+    );
+    return `${lines.join("\n")}\n`;
+  }
+
   lines.push("## Findings", "");
 
   const findings = plan.pipelineFindings ?? [];
@@ -107,7 +125,7 @@ export function patchPlanToMarkdown(plan: PatchPlan) {
 
   lines.push(
     "",
-    "> This prototype is dry-run only. Real CRM writes must require explicit human approval before connector adapters apply any operation.",
+    "> Dry-run plan — read-only. No provider write happens until you approve specific operations and run `apply`; placeholder values are refused without an explicit override. These safety invariants are not beta.",
   );
   return `${lines.join("\n")}\n`;
 }