fullstackgtm 0.45.0 → 0.46.0

package/src/connectors/stripe.ts

@@ -7,6 +7,7 @@ import type {
   PatchOperation,
   PatchOperationResult,
 } from "../types.ts";
+import { STRIPE_SNAPSHOT_STAGES, type ProgressEmitter } from "../progress.ts";
 
 const DEFAULT_API_BASE_URL = "https://api.stripe.com";
 
@@ -16,8 +17,59 @@ export type StripeConnectorOptions = {
   apiBaseUrl?: string;
   /** Injectable fetch for testing. */
   fetchImpl?: typeof fetch;
+  /**
+   * Shared progress vocabulary (src/progress.ts): the snapshot pull emits a
+   * `stage` per collection (customers, subscriptions) and an `items`
+   * heartbeat per page. Presentation-only — errors are swallowed.
+   */
+  progress?: ProgressEmitter;
 };
 
+// Module-scope request/list (parameterized by options) so the connector
+// factory and fetchStripePaidInvoices share one implementation.
+async function stripeRequest(options: StripeConnectorOptions, path: string): Promise<any> {
+  const baseUrl = (options.apiBaseUrl ?? DEFAULT_API_BASE_URL).replace(/\/$/, "");
+  const fetchImpl = options.fetchImpl ?? fetch;
+  const apiKey = await options.getApiKey();
+  const response = await fetchImpl(`${baseUrl}${path}`, {
+    headers: { Authorization: `Bearer ${apiKey}` },
+  });
+  if (!response.ok) {
+    // Status line only — the body can echo request details bound to a live
+    // billing key, and these errors land in scheduled-run records.
+    await response.text().catch(() => undefined);
+    throw new Error(`Stripe API error ${response.status}. Check the restricted key and request.`);
+  }
+  return response.json();
+}
+
+/** Stripe list pagination: follow `has_more` with `starting_after=<last id>`. */
+async function stripeList(
+  options: StripeConnectorOptions,
+  path: string,
+  onPage?: (fetched: number) => void,
+): Promise<any[]> {
+  const results: any[] = [];
+  let startingAfter: string | undefined;
+  do {
+    const separator = path.includes("?") ? "&" : "?";
+    const data = await stripeRequest(
+      options,
+      `${path}${startingAfter ? `${separator}starting_after=${encodeURIComponent(startingAfter)}` : ""}`,
+    );
+    const page: any[] = data.data ?? [];
+    results.push(...page);
+    try {
+      onPage?.(results.length);
+    } catch {
+      // progress is presentation-only; never let it fail a pull
+    }
+    startingAfter =
+      data.has_more === true && page.length > 0 ? String(page.at(-1).id) : undefined;
+  } while (startingAfter);
+  return results;
+}
+
 /**
  * Read-only billing connector for Stripe — the first non-CRM connector,
  * proving the GtmConnector contract generalizes to billing systems.
@@ -37,46 +89,24 @@ export type StripeConnectorOptions = {
  *   collections are always empty.
  */
 export function createStripeConnector(options: StripeConnectorOptions): GtmConnector {
-  const baseUrl = (options.apiBaseUrl ?? DEFAULT_API_BASE_URL).replace(/\/$/, "");
-  const fetchImpl = options.fetchImpl ?? fetch;
-
-  async function request(path: string): Promise<any> {
-    const apiKey = await options.getApiKey();
-    const response = await fetchImpl(`${baseUrl}${path}`, {
-      headers: { Authorization: `Bearer ${apiKey}` },
-    });
-    if (!response.ok) {
-      // Status line only — the body can echo request details bound to a live
-      // billing key, and these errors land in scheduled-run records.
-      await response.text().catch(() => undefined);
-      throw new Error(`Stripe API error ${response.status}. Check the restricted key and request.`);
-    }
-    return response.json();
-  }
-
-  /** Stripe list pagination: follow `has_more` with `starting_after=<last id>`. */
-  async function list(path: string): Promise<any[]> {
-    const results: any[] = [];
-    let startingAfter: string | undefined;
-    do {
-      const separator = path.includes("?") ? "&" : "?";
-      const data = await request(
-        `${path}${startingAfter ? `${separator}starting_after=${encodeURIComponent(startingAfter)}` : ""}`,
-      );
-      const page: any[] = data.data ?? [];
-      results.push(...page);
-      startingAfter =
-        data.has_more === true && page.length > 0 ? String(page.at(-1).id) : undefined;
-    } while (startingAfter);
-    return results;
-  }
+  const list = (path: string, onPage?: (fetched: number) => void) =>
+    stripeList(options, path, onPage);
+  // Shared progress vocabulary: stage per collection, items per page.
+  const pullStage = (stage: (typeof STRIPE_SNAPSHOT_STAGES)[number]) =>
+    options.progress?.stage(
+      stage,
+      STRIPE_SNAPSHOT_STAGES.indexOf(stage),
+      STRIPE_SNAPSHOT_STAGES.length,
+    );
+  const pullItems = (fetched: number) => options.progress?.items(fetched);
 
   async function assembleSnapshot(createdGte?: number): Promise<CanonicalGtmSnapshot> {
     // Stripe filters list endpoints on the integer `created` field via
     // created[gte]=<unix seconds>; omitted on a full snapshot.
     const createdFilter =
       createdGte === undefined ? "" : `&created[gte]=${createdGte}`;
-    const customers = await list(`/v1/customers?limit=100${createdFilter}`);
+    pullStage("customers");
+    const customers = await list(`/v1/customers?limit=100${createdFilter}`, pullItems);
     const accounts: CanonicalAccount[] = [];
     const contacts: CanonicalContact[] = [];
     for (const customer of customers) {
@@ -111,8 +141,10 @@ export function createStripeConnector(options: StripeConnectorOptions): GtmConne
       }
     }
 
+    pullStage("subscriptions");
     const subscriptions = await list(
       `/v1/subscriptions?status=all&limit=100${createdFilter}`,
+      pullItems,
     );
     const deals: CanonicalDeal[] = subscriptions
       .filter((subscription) => subscription.id)
@@ -161,6 +193,9 @@ export function createStripeConnector(options: StripeConnectorOptions): GtmConne
         };
       });
 
+    // Deliver any throttled trailing items heartbeat before the pull returns.
+    options.progress?.flush();
+
     return {
       generatedAt: new Date().toISOString(),
       provider: "stripe",
@@ -205,6 +240,91 @@ export function createStripeConnector(options: StripeConnectorOptions): GtmConne
   };
 }
 
+/**
+ * One PAID Stripe invoice, normalized for the backfill plan builder
+ * (`backfill stripe`). Amounts are major units (Stripe returns cents);
+ * `paidAt` is the payment date (YYYY-MM-DD) from status_transitions.paid_at.
+ */
+export type StripePaidInvoice = {
+  id: string;
+  number?: string;
+  customerId?: string;
+  customerName?: string;
+  customerEmail?: string;
+  /** Lowercased domain of the customer's billing email — the CRM match key. */
+  customerDomain?: string;
+  /** Major currency units (amount_paid / 100). */
+  amountPaid: number;
+  currency?: string;
+  /** YYYY-MM-DD from status_transitions.paid_at (unix seconds). */
+  paidAt?: string;
+  description?: string;
+};
+
+/**
+ * Read-only pull of PAID invoices (GET /v1/invoices?status=paid with the
+ * customer expanded), the revenue source of truth the backfill plan builder
+ * turns into closed-won deal proposals. Invoices with a zero amount_paid or
+ * no id are skipped — a $0 invoice is not a won deal.
+ */
+export async function fetchStripePaidInvoices(
+  options: StripeConnectorOptions,
+  opts: { sinceIso?: string; onPage?: (fetched: number) => void } = {},
+): Promise<StripePaidInvoice[]> {
+  let createdFilter = "";
+  if (opts.sinceIso !== undefined) {
+    const sinceMs = Date.parse(opts.sinceIso);
+    if (!Number.isFinite(sinceMs)) throw new Error(`Invalid since timestamp: ${opts.sinceIso}`);
+    createdFilter = `&created[gte]=${Math.floor(sinceMs / 1000)}`;
+  }
+  const invoices = await stripeList(
+    options,
+    `/v1/invoices?status=paid&limit=100&expand[]=data.customer${createdFilter}`,
+    opts.onPage,
+  );
+  const paid: StripePaidInvoice[] = [];
+  for (const invoice of invoices) {
+    if (!invoice?.id) continue;
+    const amountPaidCents = numberOrUndefined(invoice.amount_paid);
+    if (amountPaidCents === undefined || amountPaidCents <= 0) continue;
+    // customer is the expanded object when expand[]=data.customer was honored,
+    // else the plain id string — handle both.
+    const customer = invoice.customer;
+    const customerId =
+      typeof customer === "string" ? customer : stringOrUndefined(customer?.id);
+    // Invoices also carry flat customer_name/customer_email copies — use them
+    // when the customer object wasn't expanded (or lacks the field).
+    const customerName =
+      (typeof customer === "object" && customer !== null
+        ? stringOrUndefined(customer.name)
+        : undefined) ?? stringOrUndefined(invoice.customer_name);
+    const customerEmail =
+      typeof customer === "object" && customer !== null
+        ? stringOrUndefined(customer.email) ?? stringOrUndefined(invoice.customer_email)
+        : stringOrUndefined(invoice.customer_email);
+    const customerDomain = customerEmail?.includes("@")
+      ? customerEmail.split("@").at(-1)!.toLowerCase()
+      : undefined;
+    const paidAtSeconds = numberOrUndefined(invoice.status_transitions?.paid_at);
+    paid.push({
+      id: String(invoice.id),
+      number: stringOrUndefined(invoice.number),
+      customerId,
+      customerName,
+      customerEmail,
+      customerDomain,
+      amountPaid: amountPaidCents / 100,
+      currency: stringOrUndefined(invoice.currency)?.toUpperCase(),
+      paidAt:
+        paidAtSeconds !== undefined
+          ? new Date(paidAtSeconds * 1000).toISOString().split("T")[0]
+          : undefined,
+      description: stringOrUndefined(invoice.description),
+    });
+  }
+  return paid;
+}
+
 function stringOrUndefined(value: unknown): string | undefined {
   if (value === undefined || value === null || value === "") return undefined;
   return String(value);

package/src/enrich.ts

@@ -3,6 +3,7 @@ import { join } from "node:path";
 import { credentialsDir, ensureSecureHomeDir, writeSecureFile } from "./credentials.ts";
 import { HUBSPOT_DEFAULT_FIELD_MAPPINGS } from "./mappings.ts";
 import type { AcquireBudget } from "./acquireMeter.ts";
+import type { ProgressEmitter } from "./progress.ts";
 import { parseAssignmentPolicy, resolveAssignment } from "./assign.ts";
 import type { AssignmentContext, AssignmentPolicy } from "./assign.ts";
 import type {
@@ -1028,6 +1029,12 @@ export type BuildAcquirePlanOptions = {
   /** Meter ceiling: max create ops to emit. null/undefined = unlimited. */
   maxRecords?: number | null;
   now?: () => Date;
+  /**
+   * Shared progress vocabulary (src/progress.ts): `items` over the candidate
+   * rows as they are routed, plus a `meter` reading of creates vs the meter
+   * ceiling. Presentation-only — never changes what the plan proposes.
+   */
+  progress?: ProgressEmitter;
 };
 
 /** First non-empty value among several candidate paths in a source payload. */
@@ -1120,7 +1127,10 @@ export function buildAcquirePlan(options: BuildAcquirePlanOptions): AcquirePlanR
     }
   }
 
+  let routed = 0;
   for (const record of records) {
+    // Progress heartbeat over the candidate rows (throttled by the emitter).
+    options.progress?.items((routed += 1), records.length);
     const createMap = acquire.create[record.objectType];
     const match = config.match[record.objectType];
     // No create mapping or no match config for this type — can neither create
@@ -1235,6 +1245,9 @@ export function buildAcquirePlan(options: BuildAcquirePlanOptions): AcquirePlanR
     counts.created += 1;
     estCostUsd += costPerRecord;
   }
+  options.progress?.flush();
+  // Meter reading: creates proposed this run against the meter's ceiling.
+  if (cap !== null) options.progress?.meter(counts.created, cap, "creates");
 
   const plan: PatchPlan = {
     id: `patch_plan_acq_${fnv1a(`${source}:${runLabel}:${nowIso}`)}`,

package/src/health.ts

@@ -2,7 +2,20 @@ import { appendFileSync, mkdirSync, readFileSync } from "node:fs";
 import { join } from "node:path";
 
 import { activeProfile, credentialsDir, ensureSecureHomeDir, writeSecureFile } from "./credentials.ts";
-import type { AuditFindingSeverity, CanonicalGtmSnapshot, PatchPlan } from "./types.ts";
+import type { CanonicalGtmSnapshot } from "./types.ts";
+import type { HealthEntry } from "./healthScore.ts";
+
+// Pure scoring lives in healthScore.ts (node-free, importable from V8 runtimes
+// like the hosted app's Convex queries); this module owns the fs-backed
+// profile timeline. Re-exported so existing imports keep working.
+export {
+  computeHealth,
+  summarizeHealth,
+  healthToMarkdown,
+  type HealthEntry,
+  type HealthRuleDelta,
+  type HealthRollup,
+} from "./healthScore.ts";
 
 /**
  * The engagement workspace: a per-client health timeline that accrues from the
@@ -15,218 +28,6 @@ import type { AuditFindingSeverity, CanonicalGtmSnapshot, PatchPlan } from "./ty
  * so it is stable in CI and comparable run-over-run.
  */
 
-// Severity weights: a critical finding costs ~3× a warning, ~10× an info.
-const SEVERITY_WEIGHT: Record<AuditFindingSeverity, number> = { info: 1, warning: 3, critical: 10 };
-
-export type HealthEntry = {
-  /** ISO timestamp of the audit that produced this entry. */
-  at: string;
-  /** The saved plan this entry summarizes (correlates to plans/ and snapshots/). */
-  planId: string;
-  /** Deterministic 0–100 hygiene score (higher is healthier). */
-  score: number;
-  /** Total finding count across all rules. */
-  findings: number;
-  /** Severity-weighted finding total (drives the score). */
-  weightedFindings: number;
-  /** Record counts at audit time — the denominator that normalizes the score. */
-  records: { accounts: number; contacts: number; deals: number; total: number };
-  /** Finding count per rule id. */
-  byRule: Record<string, number>;
-  /** Finding count per severity. */
-  severityCounts: Record<AuditFindingSeverity, number>;
-  /**
-   * Per-object-type breakdown — so "is my contact data clean but my pipeline
-   * messy?" is answerable without re-auditing. Each type carries its own
-   * record-normalized score (same curve as the overall score, scoped to that
-   * type's records + findings).
-   */
-  byObjectType: Record<"account" | "contact" | "deal", {
-    records: number;
-    findings: number;
-    weightedFindings: number;
-    score: number;
-  }>;
-};
-
-export type HealthRuleDelta = {
-  ruleId: string;
-  current: number;
-  previous: number;
-  /** current − previous: positive = more findings (worse), negative = fewer (better). */
-  delta: number;
-};
-
-export type HealthRollup = {
-  profile: string;
-  auditCount: number;
-  first: string;
-  latest: string;
-  current: HealthEntry;
-  previous: HealthEntry | null;
-  /** current.score − previous.score: positive = improving. null on the first audit. */
-  scoreDelta: number | null;
-  ruleDeltas: HealthRuleDelta[];
-  history: Array<{ at: string; score: number; findings: number }>;
-};
-
-/**
- * Deterministically score a saved audit. score = 100 / (1 + weighted findings
- * per record): 0 findings → 100; one weighted finding per record → 50; the
- * curve is bounded (0, 100], monotonic, and needs no clamping or magic
- * constants. A messy 50-record CRM scores worse than the same finding count
- * across 5,000 records, which is the point.
- */
-export function computeHealth(plan: PatchPlan, snapshot: CanonicalGtmSnapshot, at: string): HealthEntry {
-  const byRule: Record<string, number> = {};
-  const severityCounts: Record<AuditFindingSeverity, number> = { info: 0, warning: 0, critical: 0 };
-  const typeTally: Record<"account" | "contact" | "deal", { findings: number; weightedFindings: number }> = {
-    account: { findings: 0, weightedFindings: 0 },
-    contact: { findings: 0, weightedFindings: 0 },
-    deal: { findings: 0, weightedFindings: 0 },
-  };
-  let weightedFindings = 0;
-  for (const finding of plan.findings) {
-    byRule[finding.ruleId] = (byRule[finding.ruleId] ?? 0) + 1;
-    severityCounts[finding.severity] += 1;
-    weightedFindings += SEVERITY_WEIGHT[finding.severity];
-    const t = finding.objectType;
-    if (t === "account" || t === "contact" || t === "deal") {
-      typeTally[t].findings += 1;
-      typeTally[t].weightedFindings += SEVERITY_WEIGHT[finding.severity];
-    }
-  }
-
-  const records = {
-    accounts: snapshot.accounts.length,
-    contacts: snapshot.contacts.length,
-    deals: snapshot.deals.length,
-    total: snapshot.accounts.length + snapshot.contacts.length + snapshot.deals.length,
-  };
-
-  const scoreFor = (weighted: number, recordCount: number) =>
-    Math.round(100 / (1 + weighted / Math.max(recordCount, 1)));
-  const recordsByType = { account: records.accounts, contact: records.contacts, deal: records.deals };
-  const byObjectType = (["account", "contact", "deal"] as const).reduce(
-    (acc, t) => {
-      acc[t] = {
-        records: recordsByType[t],
-        findings: typeTally[t].findings,
-        weightedFindings: typeTally[t].weightedFindings,
-        score: scoreFor(typeTally[t].weightedFindings, recordsByType[t]),
-      };
-      return acc;
-    },
-    {} as HealthEntry["byObjectType"],
-  );
-
-  const score = scoreFor(weightedFindings, records.total);
-
-  return {
-    at,
-    planId: plan.id,
-    score,
-    findings: plan.findings.length,
-    weightedFindings,
-    records,
-    byRule,
-    severityCounts,
-    byObjectType,
-  };
-}
-
-/** Roll a timeline up into the current state, the change since last time, and per-rule deltas. */
-export function summarizeHealth(entries: HealthEntry[], profile: string): HealthRollup | null {
-  if (entries.length === 0) return null;
-  // Oldest → newest, so `current` is the last entry.
-  const sorted = [...entries].sort((a, b) => a.at.localeCompare(b.at));
-  const current = sorted[sorted.length - 1];
-  const previous = sorted.length > 1 ? sorted[sorted.length - 2] : null;
-
-  const ruleIds = new Set<string>([
-    ...Object.keys(current.byRule),
-    ...(previous ? Object.keys(previous.byRule) : []),
-  ]);
-  const ruleDeltas: HealthRuleDelta[] = [...ruleIds]
-    .map((ruleId) => {
-      const cur = current.byRule[ruleId] ?? 0;
-      const prev = previous?.byRule[ruleId] ?? 0;
-      return { ruleId, current: cur, previous: prev, delta: cur - prev };
-    })
-    .sort((a, b) => b.current - a.current || a.ruleId.localeCompare(b.ruleId));
-
-  return {
-    profile,
-    auditCount: sorted.length,
-    first: sorted[0].at,
-    latest: current.at,
-    current,
-    previous,
-    scoreDelta: previous ? current.score - previous.score : null,
-    ruleDeltas,
-    history: sorted.map((entry) => ({ at: entry.at, score: entry.score, findings: entry.findings })),
-  };
-}
-
-// Sign-based and uniform: ▲ = the number went up, ▼ = down. The reader supplies
-// the meaning (score up = good; findings up = bad) — mixing arrow direction with
-// good/bad reads as contradictory ("▲ -3").
-function arrow(delta: number): string {
-  if (delta === 0) return "·";
-  return delta > 0 ? "▲" : "▼";
-}
-
-/** Human-readable rollup: headline score + trend, then per-rule change since last audit. */
-export function healthToMarkdown(rollup: HealthRollup): string {
-  const { current, scoreDelta } = rollup;
-  const lines = [`# GTM health — profile \`${rollup.profile}\``, ""];
-
-  const deltaText =
-    scoreDelta === null
-      ? "(first audit — no prior reading)"
-      : `${arrow(scoreDelta)} ${scoreDelta >= 0 ? "+" : ""}${scoreDelta} since the previous audit`;
-  lines.push(
-    `Score: **${current.score}/100** ${deltaText}`,
-    `Audits: ${rollup.auditCount} over ${shortDate(rollup.first)} → ${shortDate(rollup.latest)}`,
-    `Findings: ${current.findings} (${current.severityCounts.critical} critical, ${current.severityCounts.warning} warning, ${current.severityCounts.info} info)`,
-    `Records: ${current.records.accounts} accounts, ${current.records.contacts} contacts, ${current.records.deals} deals`,
-    "",
-  );
-
-  lines.push("## By object type", "", "| Type | Score | Findings | Records |", "| --- | --- | --- | --- |");
-  for (const t of ["account", "contact", "deal"] as const) {
-    const b = current.byObjectType[t];
-    lines.push(`| ${t} | ${b.score}/100 | ${b.findings} | ${b.records} |`);
-  }
-  lines.push("");
-
-  if (rollup.history.length > 1) {
-    lines.push("## Trend", "");
-    for (const point of rollup.history) {
-      const marker = point.at === rollup.latest ? "  ← latest" : "";
-      lines.push(`- ${shortDate(point.at)}  score ${point.score}  (${point.findings} findings)${marker}`);
-    }
-    lines.push("");
-  }
-
-  if (rollup.ruleDeltas.length > 0) {
-    lines.push("## By rule (Δ since previous audit)", "", "| Rule | Findings | Δ |", "| --- | --- | --- |");
-    for (const rule of rollup.ruleDeltas) {
-      const sign = rule.delta > 0 ? `+${rule.delta}` : `${rule.delta}`;
-      const cell = rule.delta === 0 ? "·" : `${arrow(rule.delta)} ${sign}`;
-      lines.push(`| ${rule.ruleId} | ${rule.current} | ${cell} |`);
-    }
-    lines.push("");
-  }
-
-  lines.push("> Score is deterministic: 100 / (1 + severity-weighted findings per record). Lower findings or more clean records ⇒ higher score.");
-  return `${lines.join("\n")}\n`;
-}
-
-function shortDate(iso: string): string {
-  return iso.slice(0, 10);
-}
-
 // ── Profile-scoped storage ──────────────────────────────────────────────────
 
 /** `$FSGTM_HOME[/profiles/<name>]/health.jsonl` — append-only, one entry per audit-save. */