fullstackgtm 0.45.0 → 0.47.0

package/src/connectors/salesforce.ts

@@ -19,6 +19,7 @@ import type {
   PatchOperationResult,
   SnapshotProgress,
 } from "../types.ts";
+import { SNAPSHOT_PULL_STAGES, type ProgressEmitter } from "../progress.ts";
 
 const DEFAULT_API_VERSION = "v59.0";
 
@@ -38,6 +39,12 @@ export type SalesforceConnectorOptions = {
   fetchImpl?: typeof fetch;
   /** Per-page snapshot-pull progress (presentation only — errors are swallowed). */
   onProgress?: (progress: SnapshotProgress) => void;
+  /**
+   * Shared progress vocabulary (src/progress.ts): the snapshot pull emits a
+   * `stage` per object type and an `items` heartbeat per page. Additive
+   * alongside the legacy `onProgress` callback; both are presentation-only.
+   */
+  progress?: ProgressEmitter;
 };
 
 const SOBJECT_TYPES: Partial<Record<GtmObjectType, string>> = {
@@ -59,6 +66,15 @@ const MAPPING_OBJECT_TYPES: Partial<Record<GtmObjectType, Exclude<CrmObjectType,
   deal: "deals",
 };
 
+// SnapshotProgress object types → the shared SNAPSHOT_PULL_STAGES names, so
+// the CLI checklist and the app StageTimeline read the same stage labels.
+const PULL_STAGE_BY_TYPE: Record<SnapshotProgress["objectType"], (typeof SNAPSHOT_PULL_STAGES)[number]> = {
+  user: "owners",
+  account: "accounts",
+  contact: "contacts",
+  deal: "deals",
+};
+
 /**
  * Reference connector for Salesforce.
  *
@@ -70,7 +86,7 @@ const MAPPING_OBJECT_TYPES: Partial<Record<GtmObjectType, Exclude<CrmObjectType,
  */
 export function createSalesforceConnector(
   options: SalesforceConnectorOptions,
-): Required<GtmConnector> {
+): GtmConnector {
   const apiVersion = options.apiVersion ?? DEFAULT_API_VERSION;
   const fetchImpl = options.fetchImpl ?? fetch;
   const mappings = options.fieldMappings;
@@ -202,8 +218,21 @@ export function createSalesforceConnector(
   }
 
   async function assembleSnapshot(whereClause: string): Promise<CanonicalGtmSnapshot> {
-    const progressFor = (objectType: SnapshotProgress["objectType"]) => (fetched: number) =>
+    // Per-page snapshot-pull progress: the legacy onProgress callback plus the
+    // shared emitter (stage on the first page of each object type, items per
+    // page). Both are presentation-only — `query` already swallows errors.
+    let lastPullStage: string | undefined;
+    const progressFor = (objectType: SnapshotProgress["objectType"]) => (fetched: number) => {
       options.onProgress?.({ objectType, fetched });
+      const emitter = options.progress;
+      if (!emitter) return;
+      const stage = PULL_STAGE_BY_TYPE[objectType];
+      if (stage !== lastPullStage) {
+        lastPullStage = stage;
+        emitter.stage(stage, SNAPSHOT_PULL_STAGES.indexOf(stage), SNAPSHOT_PULL_STAGES.length);
+      }
+      emitter.items(fetched);
+    };
     const sfUsers = await query(
       `SELECT ${selectFields("owners")} FROM User${whereClause}`,
       progressFor("user"),
@@ -322,6 +351,9 @@ export function createSalesforceConnector(
       };
     });
 
+    // Deliver any throttled trailing items heartbeat before the pull returns.
+    options.progress?.flush();
+
     return {
       generatedAt: new Date().toISOString(),
       provider: "salesforce",
@@ -457,6 +489,137 @@ export function createSalesforceConnector(
     };
   }
 
+  function normalizeSalesforceValue(value: unknown): string | null {
+    if (value === undefined || value === null || value === "") return null;
+    return String(value).split("T")[0];
+  }
+
+  function fieldMappingFor(operation: PatchOperation): { sobjectType: string; field: string } | null {
+    const sobjectType = SOBJECT_TYPES[operation.objectType];
+    const mappingType = MAPPING_OBJECT_TYPES[operation.objectType];
+    if (!sobjectType || !mappingType || !operation.field) return null;
+    const defaults = SALESFORCE_DEFAULT_FIELD_MAPPINGS[mappingType] ?? {};
+    return {
+      sobjectType,
+      field: mappedField(mappings, mappingType, operation.field, defaults[operation.field] ?? operation.field),
+    };
+  }
+
+  function compositeErrorDetail(result: any): string {
+    const errors = Array.isArray(result?.errors) ? result.errors : [];
+    const message = errors
+      .map((error: any) => [error?.statusCode, error?.message].filter(Boolean).join(": "))
+      .filter(Boolean)
+      .join("; ");
+    return message ? `Salesforce rejected this record: ${message.slice(0, 300)}.` : "Salesforce rejected this record.";
+  }
+
+  async function applySetFieldBatch(operations: PatchOperation[]): Promise<PatchOperationResult[]> {
+    const out = new Map<string, PatchOperationResult>();
+    const mapped = operations.map((operation) => ({ operation, mapping: fieldMappingFor(operation) }));
+    for (const item of mapped) {
+      if (!item.mapping) {
+        out.set(item.operation.id, {
+          operationId: item.operation.id,
+          status: "skipped",
+          detail: "Field writes are only supported for accounts, contacts, and deals with an explicit field.",
+        });
+      }
+    }
+    const clean = mapped.filter((item): item is { operation: PatchOperation; mapping: { sobjectType: string; field: string } } => Boolean(item.mapping));
+    if (clean.length === 0) return operations.map((operation) => out.get(operation.id)!);
+    const sobjectType = clean[0].mapping.sobjectType;
+    const fields = [...new Set(clean.map((item) => item.mapping.field))];
+    const liveById = new Map<string, Record<string, unknown>>();
+    const ids = [...new Set(clean.map((item) => item.operation.objectId))];
+    for (let i = 0; i < ids.length; i += 200) {
+      const idList = ids.slice(i, i + 200).map((id) => `'${id.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`).join(",");
+      const rows = await query(`SELECT Id, ${fields.join(", ")} FROM ${sobjectType} WHERE Id IN (${idList})`);
+      for (const row of rows) liveById.set(String(row.Id), row);
+    }
+
+    const toWrite: Array<{ operation: PatchOperation; field: string }> = [];
+    for (const { operation, mapping } of clean) {
+      const row = liveById.get(operation.objectId);
+      const current = row?.[mapping.field] ?? null;
+      const expected = normalizeSalesforceValue(operation.beforeValue);
+      const found = normalizeSalesforceValue(current);
+      if (!row || expected !== found) {
+        out.set(operation.id, {
+          operationId: operation.id,
+          status: "conflict",
+          detail: `Value drifted since the plan was proposed: expected ${expected ?? "∅"}, found ${found ?? "∅"}. Re-run the audit.`,
+          providerData: { currentValue: current ?? null },
+        });
+        continue;
+      }
+      toWrite.push({ operation, field: mapping.field });
+    }
+
+    for (let i = 0; i < toWrite.length; i += 200) {
+      const chunk = toWrite.slice(i, i + 200);
+      const records = chunk.map(({ operation, field }) => ({
+        attributes: { type: sobjectType },
+        Id: operation.objectId,
+        [field]: operation.operation === "clear_field" ? null : String(operation.afterValue ?? ""),
+      }));
+      let response: any[];
+      try {
+        response = (await request(`/services/data/${apiVersion}/composite/sobjects`, {
+          method: "PATCH",
+          body: JSON.stringify({ allOrNone: false, records }),
+        })) as any[];
+      } catch (error) {
+        for (const { operation } of chunk) out.set(operation.id, { operationId: operation.id, status: "failed", detail: error instanceof Error ? error.message : String(error) });
+        continue;
+      }
+      for (let j = 0; j < chunk.length; j++) {
+        const { operation, field } = chunk[j];
+        const result = Array.isArray(response) ? response[j] : undefined;
+        out.set(operation.id, result?.success
+          ? { operationId: operation.id, status: "applied", detail: `Set ${field} on ${sobjectType}/${operation.objectId}.`, providerData: { sobjectType, field } }
+          : { operationId: operation.id, status: "failed", detail: compositeErrorDetail(result) });
+      }
+    }
+    return operations.map((operation) => out.get(operation.id) ?? { operationId: operation.id, status: "failed", detail: "Batch update did not process this operation." });
+  }
+
+  async function applyArchiveBatch(operations: PatchOperation[]): Promise<PatchOperationResult[]> {
+    const sobjectType = SOBJECT_TYPES[operations[0]?.objectType];
+    if (!sobjectType) return operations.map((operation) => ({ operationId: operation.id, status: "skipped", detail: "archive_record is supported for accounts, contacts, and deals." }));
+    const out = new Map<string, PatchOperationResult>();
+    for (let i = 0; i < operations.length; i += 200) {
+      const chunk = operations.slice(i, i + 200);
+      const ids = chunk.map((operation) => encodeURIComponent(operation.objectId)).join(",");
+      let response: any[];
+      try {
+        response = (await request(`/services/data/${apiVersion}/composite/sobjects?ids=${ids}&allOrNone=false`, { method: "DELETE" })) as any[];
+      } catch (error) {
+        for (const operation of chunk) out.set(operation.id, { operationId: operation.id, status: "failed", detail: error instanceof Error ? error.message : String(error) });
+        continue;
+      }
+      for (let j = 0; j < chunk.length; j++) {
+        const operation = chunk[j];
+        const result = Array.isArray(response) ? response[j] : undefined;
+        out.set(operation.id, result?.success
+          ? { operationId: operation.id, status: "applied", detail: `Deleted ${sobjectType}/${operation.objectId}.` }
+          : { operationId: operation.id, status: "failed", detail: compositeErrorDetail(result) });
+      }
+    }
+    return operations.map((operation) => out.get(operation.id) ?? { operationId: operation.id, status: "failed", detail: "Batch delete did not process this operation." });
+  }
+
+  async function applyBatch(operations: PatchOperation[]): Promise<PatchOperationResult[]> {
+    if (operations.length === 0) return [];
+    if (operations.every((operation) => (operation.operation === "set_field" || operation.operation === "clear_field") && operation.objectType === operations[0].objectType)) {
+      return applySetFieldBatch(operations);
+    }
+    if (operations.every((operation) => operation.operation === "archive_record" && operation.objectType === operations[0].objectType)) {
+      return applyArchiveBatch(operations);
+    }
+    return operations.map((operation) => ({ operationId: operation.id, status: "skipped", detail: "This mixed operation batch is not supported." }));
+  }
+
   function escapeXml(value: string): string {
     return value
       .replace(/&/g, "&amp;")
@@ -566,7 +729,7 @@ export function createSalesforceConnector(
       return { operationId: operation.id, status: "skipped", detail: "create_record needs a CreateRecordPayload afterValue." };
     }
     if (operation.objectType !== "contact" && operation.objectType !== "account") {
-      return { operationId: operation.id, status: "skipped", detail: "create_record supports contacts and accounts." };
+      return { operationId: operation.id, status: "skipped", detail: "create_record supports contacts and accounts on Salesforce (deal/Opportunity creation is HubSpot-only for now)." };
     }
     const matchValue = String(payload.matchValue ?? "").trim();
     if (!matchValue) {
@@ -837,6 +1000,8 @@ export function createSalesforceConnector(
     fetchChanges,
     applyOperation,
     applyCreateContactsBatch,
+    applyBatch,
+    applyBatchLimit: 200,
     readField,
   };
 }

package/src/connectors/stripe.ts

@@ -7,6 +7,7 @@ import type {
   PatchOperation,
   PatchOperationResult,
 } from "../types.ts";
+import { STRIPE_SNAPSHOT_STAGES, type ProgressEmitter } from "../progress.ts";
 
 const DEFAULT_API_BASE_URL = "https://api.stripe.com";
 
@@ -16,8 +17,59 @@ export type StripeConnectorOptions = {
   apiBaseUrl?: string;
   /** Injectable fetch for testing. */
   fetchImpl?: typeof fetch;
+  /**
+   * Shared progress vocabulary (src/progress.ts): the snapshot pull emits a
+   * `stage` per collection (customers, subscriptions) and an `items`
+   * heartbeat per page. Presentation-only — errors are swallowed.
+   */
+  progress?: ProgressEmitter;
 };
 
+// Module-scope request/list (parameterized by options) so the connector
+// factory and fetchStripePaidInvoices share one implementation.
+async function stripeRequest(options: StripeConnectorOptions, path: string): Promise<any> {
+  const baseUrl = (options.apiBaseUrl ?? DEFAULT_API_BASE_URL).replace(/\/$/, "");
+  const fetchImpl = options.fetchImpl ?? fetch;
+  const apiKey = await options.getApiKey();
+  const response = await fetchImpl(`${baseUrl}${path}`, {
+    headers: { Authorization: `Bearer ${apiKey}` },
+  });
+  if (!response.ok) {
+    // Status line only — the body can echo request details bound to a live
+    // billing key, and these errors land in scheduled-run records.
+    await response.text().catch(() => undefined);
+    throw new Error(`Stripe API error ${response.status}. Check the restricted key and request.`);
+  }
+  return response.json();
+}
+
+/** Stripe list pagination: follow `has_more` with `starting_after=<last id>`. */
+async function stripeList(
+  options: StripeConnectorOptions,
+  path: string,
+  onPage?: (fetched: number) => void,
+): Promise<any[]> {
+  const results: any[] = [];
+  let startingAfter: string | undefined;
+  do {
+    const separator = path.includes("?") ? "&" : "?";
+    const data = await stripeRequest(
+      options,
+      `${path}${startingAfter ? `${separator}starting_after=${encodeURIComponent(startingAfter)}` : ""}`,
+    );
+    const page: any[] = data.data ?? [];
+    results.push(...page);
+    try {
+      onPage?.(results.length);
+    } catch {
+      // progress is presentation-only; never let it fail a pull
+    }
+    startingAfter =
+      data.has_more === true && page.length > 0 ? String(page.at(-1).id) : undefined;
+  } while (startingAfter);
+  return results;
+}
+
 /**
  * Read-only billing connector for Stripe — the first non-CRM connector,
  * proving the GtmConnector contract generalizes to billing systems.
@@ -37,46 +89,24 @@ export type StripeConnectorOptions = {
  *   collections are always empty.
  */
 export function createStripeConnector(options: StripeConnectorOptions): GtmConnector {
-  const baseUrl = (options.apiBaseUrl ?? DEFAULT_API_BASE_URL).replace(/\/$/, "");
-  const fetchImpl = options.fetchImpl ?? fetch;
-
-  async function request(path: string): Promise<any> {
-    const apiKey = await options.getApiKey();
-    const response = await fetchImpl(`${baseUrl}${path}`, {
-      headers: { Authorization: `Bearer ${apiKey}` },
-    });
-    if (!response.ok) {
-      // Status line only — the body can echo request details bound to a live
-      // billing key, and these errors land in scheduled-run records.
-      await response.text().catch(() => undefined);
-      throw new Error(`Stripe API error ${response.status}. Check the restricted key and request.`);
-    }
-    return response.json();
-  }
-
-  /** Stripe list pagination: follow `has_more` with `starting_after=<last id>`. */
-  async function list(path: string): Promise<any[]> {
-    const results: any[] = [];
-    let startingAfter: string | undefined;
-    do {
-      const separator = path.includes("?") ? "&" : "?";
-      const data = await request(
-        `${path}${startingAfter ? `${separator}starting_after=${encodeURIComponent(startingAfter)}` : ""}`,
-      );
-      const page: any[] = data.data ?? [];
-      results.push(...page);
-      startingAfter =
-        data.has_more === true && page.length > 0 ? String(page.at(-1).id) : undefined;
-    } while (startingAfter);
-    return results;
-  }
+  const list = (path: string, onPage?: (fetched: number) => void) =>
+    stripeList(options, path, onPage);
+  // Shared progress vocabulary: stage per collection, items per page.
+  const pullStage = (stage: (typeof STRIPE_SNAPSHOT_STAGES)[number]) =>
+    options.progress?.stage(
+      stage,
+      STRIPE_SNAPSHOT_STAGES.indexOf(stage),
+      STRIPE_SNAPSHOT_STAGES.length,
+    );
+  const pullItems = (fetched: number) => options.progress?.items(fetched);
 
   async function assembleSnapshot(createdGte?: number): Promise<CanonicalGtmSnapshot> {
     // Stripe filters list endpoints on the integer `created` field via
     // created[gte]=<unix seconds>; omitted on a full snapshot.
     const createdFilter =
       createdGte === undefined ? "" : `&created[gte]=${createdGte}`;
-    const customers = await list(`/v1/customers?limit=100${createdFilter}`);
+    pullStage("customers");
+    const customers = await list(`/v1/customers?limit=100${createdFilter}`, pullItems);
     const accounts: CanonicalAccount[] = [];
     const contacts: CanonicalContact[] = [];
     for (const customer of customers) {
@@ -111,8 +141,10 @@ export function createStripeConnector(options: StripeConnectorOptions): GtmConne
       }
     }
 
+    pullStage("subscriptions");
     const subscriptions = await list(
       `/v1/subscriptions?status=all&limit=100${createdFilter}`,
+      pullItems,
     );
     const deals: CanonicalDeal[] = subscriptions
       .filter((subscription) => subscription.id)
@@ -161,6 +193,9 @@ export function createStripeConnector(options: StripeConnectorOptions): GtmConne
         };
       });
 
+    // Deliver any throttled trailing items heartbeat before the pull returns.
+    options.progress?.flush();
+
     return {
       generatedAt: new Date().toISOString(),
       provider: "stripe",
@@ -205,6 +240,91 @@ export function createStripeConnector(options: StripeConnectorOptions): GtmConne
   };
 }
 
+/**
+ * One PAID Stripe invoice, normalized for the backfill plan builder
+ * (`backfill stripe`). Amounts are major units (Stripe returns cents);
+ * `paidAt` is the payment date (YYYY-MM-DD) from status_transitions.paid_at.
+ */
+export type StripePaidInvoice = {
+  id: string;
+  number?: string;
+  customerId?: string;
+  customerName?: string;
+  customerEmail?: string;
+  /** Lowercased domain of the customer's billing email — the CRM match key. */
+  customerDomain?: string;
+  /** Major currency units (amount_paid / 100). */
+  amountPaid: number;
+  currency?: string;
+  /** YYYY-MM-DD from status_transitions.paid_at (unix seconds). */
+  paidAt?: string;
+  description?: string;
+};
+
+/**
+ * Read-only pull of PAID invoices (GET /v1/invoices?status=paid with the
+ * customer expanded), the revenue source of truth the backfill plan builder
+ * turns into closed-won deal proposals. Invoices with a zero amount_paid or
+ * no id are skipped — a $0 invoice is not a won deal.
+ */
+export async function fetchStripePaidInvoices(
+  options: StripeConnectorOptions,
+  opts: { sinceIso?: string; onPage?: (fetched: number) => void } = {},
+): Promise<StripePaidInvoice[]> {
+  let createdFilter = "";
+  if (opts.sinceIso !== undefined) {
+    const sinceMs = Date.parse(opts.sinceIso);
+    if (!Number.isFinite(sinceMs)) throw new Error(`Invalid since timestamp: ${opts.sinceIso}`);
+    createdFilter = `&created[gte]=${Math.floor(sinceMs / 1000)}`;
+  }
+  const invoices = await stripeList(
+    options,
+    `/v1/invoices?status=paid&limit=100&expand[]=data.customer${createdFilter}`,
+    opts.onPage,
+  );
+  const paid: StripePaidInvoice[] = [];
+  for (const invoice of invoices) {
+    if (!invoice?.id) continue;
+    const amountPaidCents = numberOrUndefined(invoice.amount_paid);
+    if (amountPaidCents === undefined || amountPaidCents <= 0) continue;
+    // customer is the expanded object when expand[]=data.customer was honored,
+    // else the plain id string — handle both.
+    const customer = invoice.customer;
+    const customerId =
+      typeof customer === "string" ? customer : stringOrUndefined(customer?.id);
+    // Invoices also carry flat customer_name/customer_email copies — use them
+    // when the customer object wasn't expanded (or lacks the field).
+    const customerName =
+      (typeof customer === "object" && customer !== null
+        ? stringOrUndefined(customer.name)
+        : undefined) ?? stringOrUndefined(invoice.customer_name);
+    const customerEmail =
+      typeof customer === "object" && customer !== null
+        ? stringOrUndefined(customer.email) ?? stringOrUndefined(invoice.customer_email)
+        : stringOrUndefined(invoice.customer_email);
+    const customerDomain = customerEmail?.includes("@")
+      ? customerEmail.split("@").at(-1)!.toLowerCase()
+      : undefined;
+    const paidAtSeconds = numberOrUndefined(invoice.status_transitions?.paid_at);
+    paid.push({
+      id: String(invoice.id),
+      number: stringOrUndefined(invoice.number),
+      customerId,
+      customerName,
+      customerEmail,
+      customerDomain,
+      amountPaid: amountPaidCents / 100,
+      currency: stringOrUndefined(invoice.currency)?.toUpperCase(),
+      paidAt:
+        paidAtSeconds !== undefined
+          ? new Date(paidAtSeconds * 1000).toISOString().split("T")[0]
+          : undefined,
+      description: stringOrUndefined(invoice.description),
+    });
+  }
+  return paid;
+}
+
 function stringOrUndefined(value: unknown): string | undefined {
   if (value === undefined || value === null || value === "") return undefined;
   return String(value);

package/src/enrich.ts

@@ -3,6 +3,7 @@ import { join } from "node:path";
 import { credentialsDir, ensureSecureHomeDir, writeSecureFile } from "./credentials.ts";
 import { HUBSPOT_DEFAULT_FIELD_MAPPINGS } from "./mappings.ts";
 import type { AcquireBudget } from "./acquireMeter.ts";
+import type { ProgressEmitter } from "./progress.ts";
 import { parseAssignmentPolicy, resolveAssignment } from "./assign.ts";
 import type { AssignmentContext, AssignmentPolicy } from "./assign.ts";
 import type {
@@ -1028,6 +1029,12 @@ export type BuildAcquirePlanOptions = {
   /** Meter ceiling: max create ops to emit. null/undefined = unlimited. */
   maxRecords?: number | null;
   now?: () => Date;
+  /**
+   * Shared progress vocabulary (src/progress.ts): `items` over the candidate
+   * rows as they are routed, plus a `meter` reading of creates vs the meter
+   * ceiling. Presentation-only — never changes what the plan proposes.
+   */
+  progress?: ProgressEmitter;
 };
 
 /** First non-empty value among several candidate paths in a source payload. */
@@ -1120,7 +1127,10 @@ export function buildAcquirePlan(options: BuildAcquirePlanOptions): AcquirePlanR
     }
   }
 
+  let routed = 0;
   for (const record of records) {
+    // Progress heartbeat over the candidate rows (throttled by the emitter).
+    options.progress?.items((routed += 1), records.length);
     const createMap = acquire.create[record.objectType];
     const match = config.match[record.objectType];
     // No create mapping or no match config for this type — can neither create
@@ -1235,6 +1245,9 @@ export function buildAcquirePlan(options: BuildAcquirePlanOptions): AcquirePlanR
     counts.created += 1;
     estCostUsd += costPerRecord;
   }
+  options.progress?.flush();
+  // Meter reading: creates proposed this run against the meter's ceiling.
+  if (cap !== null) options.progress?.meter(counts.created, cap, "creates");
 
   const plan: PatchPlan = {
     id: `patch_plan_acq_${fnv1a(`${source}:${runLabel}:${nowIso}`)}`,