fullstackgtm 0.44.0 → 0.46.0

package/src/cli/backfill.ts

@@ -0,0 +1,156 @@
+// `backfill stripe` — paid Stripe invoices → closed-won deal proposals through
+// the governed plan/apply flow. This command NEVER writes to a CRM: it reads
+// paid invoices (Stripe, read-only), reads a CRM snapshot, and builds a
+// dry-run create_record plan. --save persists the plan for the normal
+// plans approve → apply spine, where the HubSpot connector re-resolves each
+// invoice id (resolve-first) and creates only on a confirmed miss.
+
+import { buildStripeBackfillPlan, DEFAULT_BACKFILL_MATCH_PROPERTY } from "../backfill.ts";
+import { fetchStripePaidInvoices } from "../connectors/stripe.ts";
+import { getCredential } from "../credentials.ts";
+import { patchPlanToMarkdown } from "../format.ts";
+import { createFilePlanStore } from "../planStore.ts";
+import { BACKFILL_STRIPE_STAGES, composeListeners, createProgressEmitter } from "../progress.ts";
+import { progressReporter } from "../runReport.ts";
+import { unknownSubcommandError } from "./suggest.ts";
+import { option, readSnapshot, saveRequested } from "./shared.ts";
+import { colorEnabled, createProgressRenderer, paint, stylizePlanMarkdown, table } from "./ui.ts";
+
+/** STRIPE_SECRET_KEY env ∨ stored `login stripe` credential (same ladder as `--provider stripe`). */
+function resolveStripeKey(): string {
+  const key = process.env.STRIPE_SECRET_KEY ?? getCredential("stripe")?.accessToken;
+  if (!key) {
+    throw new Error(
+      "No Stripe credentials. Run `echo \"$STRIPE_KEY\" | fullstackgtm login stripe` or set STRIPE_SECRET_KEY. " +
+        "A restricted key with read access to Invoices and Customers is enough. (`fullstackgtm doctor` shows credential status.)",
+    );
+  }
+  return key;
+}
+
+export async function backfillCommand(args: string[]) {
+  const [subcommand, ...rest] = args;
+  if (subcommand === "runs") {
+    const { backfillRunsCommand } = await import("./backfillRuns.ts");
+    await backfillRunsCommand(rest);
+    return;
+  }
+  if (subcommand !== "stripe") {
+    throw unknownSubcommandError("backfill", subcommand ?? "", ["stripe", "runs"]);
+  }
+
+  const since = option(rest, "--since");
+  const pipeline = option(rest, "--pipeline") ?? undefined;
+  const matchProperty = option(rest, "--match-property") ?? undefined;
+  const skipUnmatched = rest.includes("--skip-unmatched");
+  const save = saveRequested(rest);
+
+  // Shared progress: the BACKFILL_STRIPE_STAGES tick on an interactive stderr
+  // board (inert when piped) and heartbeat to a paired hosted app on long runs.
+  const renderer = createProgressRenderer(BACKFILL_STRIPE_STAGES);
+  const progress = createProgressEmitter(
+    composeListeners(renderer.listener, progressReporter()),
+  );
+  const stage = (name: (typeof BACKFILL_STRIPE_STAGES)[number]) =>
+    progress.stage(name, BACKFILL_STRIPE_STAGES.indexOf(name), BACKFILL_STRIPE_STAGES.length);
+  let result: ReturnType<typeof buildStripeBackfillPlan>;
+  try {
+    // 1. Revenue truth: paid invoices from Stripe (read-only).
+    stage("invoices");
+    const stripeKey = resolveStripeKey();
+    const invoices = await fetchStripePaidInvoices(
+      { getApiKey: () => stripeKey },
+      { ...(since ? { sinceIso: since } : {}), onPage: (fetched) => progress.items(fetched) },
+    );
+
+    // 2. The CRM as it stands (same source options as audit: --provider <crm>,
+    //    --input <snapshot.json>, --demo, --sample). The pull's own stage/items
+    //    events ride the same emitter (the board ignores the nested stages).
+    stage("snapshot");
+    const snapshot = await readSnapshot(rest, progress);
+
+    // 3. Dry-run plan: one closed-won create_record deal per invoice with no
+    //    deal yet; unmatched customers also get a proposed account (unless
+    //    --skip-unmatched restores report-only behavior).
+    stage("matching");
+    result = buildStripeBackfillPlan(invoices, snapshot, {
+      pipeline,
+      matchProperty,
+      source: "stripe:invoices",
+      createMissingAccounts: !skipUnmatched,
+    });
+    stage("plan");
+    progress.flush();
+  } finally {
+    renderer.done();
+  }
+
+  if (rest.includes("--json")) {
+    console.log(
+      JSON.stringify(
+        { plan: result.plan, counts: result.counts, unmatched: result.unmatched, proposedAccounts: result.proposedAccounts },
+        null,
+        2,
+      ),
+    );
+  } else {
+    // TTY-only styling; piped output stays byte-identical plain text.
+    console.log(
+      stylizePlanMarkdown(patchPlanToMarkdown(result.plan), paint(colorEnabled(process.stdout))),
+    );
+    if (result.proposedAccounts.length > 0) {
+      console.log(
+        `\nProposed new accounts (${result.proposedAccounts.length}) — Stripe customers the CRM doesn't know; ` +
+          "created only if this plan is approved and applied:",
+      );
+      const accountRows = [
+        ["account", "domain", "invoices"],
+        ...result.proposedAccounts.map((entry) => [
+          entry.name,
+          entry.domain ?? "—",
+          String(entry.invoiceCount),
+        ]),
+      ];
+      for (const line of table(accountRows)) console.log(`  ${line}`);
+    }
+    if (result.unmatched.length > 0) {
+      console.log(
+        `\nUnmatched invoices (${result.unmatched.length}) — ${
+          skipUnmatched
+            ? "customer matched no CRM account by domain or name; reported only (--skip-unmatched)"
+            : "customer has no usable name or company domain; reported only"
+        }:`,
+      );
+      const rows = [
+        ["invoice", "customer", "domain", "amount", "paid"],
+        ...result.unmatched.map((entry) => [
+          entry.invoiceNumber ?? entry.invoiceId,
+          entry.customerName ?? "—",
+          entry.customerDomain ?? "—",
+          `${entry.amountPaid}${entry.currency ? ` ${entry.currency}` : ""}`,
+          entry.paidAt ?? "—",
+        ]),
+      ];
+      for (const line of table(rows)) console.log(`  ${line}`);
+    }
+  }
+
+  if (!save) {
+    console.error(
+      "\nDry run — nothing written. Re-run with --save to persist the plan, then approve + apply.",
+    );
+    return;
+  }
+  if (result.plan.operations.length === 0) {
+    console.error("No deals to backfill (every paid invoice already matched a deal or its customer was unmatched); nothing saved.");
+    return;
+  }
+  await createFilePlanStore().save(result.plan);
+  console.error(
+    `Saved plan ${result.plan.id} — ${result.counts.planned} closed-won deal(s)` +
+      `${result.counts.accountsProposed > 0 ? ` + ${result.counts.accountsProposed} new account(s)` : ""} from ${result.counts.invoices} paid invoice(s). ` +
+      `Review \`fullstackgtm plans show ${result.plan.id}\`, approve \`fullstackgtm plans approve ${result.plan.id} --operations all\`, ` +
+      `then \`fullstackgtm apply --plan-id ${result.plan.id} --provider hubspot\`. ` +
+      `Apply re-checks ${result.plan.operations.length ? `the ${matchProperty ?? DEFAULT_BACKFILL_MATCH_PROPERTY} dedupe key` : "resolve-first"} before creating.`,
+  );
+}

package/src/cli/backfillRuns.ts

@@ -0,0 +1,198 @@
+// `backfill runs` — replay the LOCAL execution history to the paired hosted
+// app, so an engagement that started CLI-only gets its full trail the moment
+// it pairs (instead of a feed that begins mid-engagement):
+//   - every stored plan's runs[] (apply attempts, per-op result counts)
+//     → POST /api/cli/run, one record per run
+//   - the health.jsonl timeline (per-audit hygiene scores)
+//     → POST /api/cli/health, chunked
+// Idempotent by construction: each run carries a DETERMINISTIC clientRunId
+// (fnv1a of planId + startedAt + index) and health entries dedupe server-side
+// on their timestamp — re-running backfill never duplicates anything.
+// Privacy: replays only what live reporting would have sent — statuses,
+// counts, timestamps, scores. No CRM field values leave the machine.
+
+import { getCredential } from "../credentials.ts";
+import { readHealthTimeline } from "../health.ts";
+import { createFilePlanStore } from "../planStore.ts";
+import { activeProfile } from "../credentials.ts";
+import { composeListeners, createProgressEmitter, RUNS_REPLAY_STAGES } from "../progress.ts";
+import { progressReporter } from "../runReport.ts";
+import { createProgressRenderer } from "./ui.ts";
+import type { PatchPlanRun } from "../types.ts";
+
+function fnv1a(value: string): string {
+  let hash = 0x811c9dc5;
+  for (let i = 0; i < value.length; i += 1) {
+    hash ^= value.charCodeAt(i);
+    hash = Math.imul(hash, 0x01000193);
+  }
+  return (hash >>> 0).toString(16).padStart(8, "0");
+}
+
+const RUN_STATUS: Record<PatchPlanRun["status"], "success" | "partial" | "error"> = {
+  applied: "success",
+  partial: "partial",
+  failed: "error",
+  rejected: "error",
+};
+
+export async function backfillRunsCommand(args: string[]) {
+  const dryRun = args.includes("--dry-run");
+  const json = args.includes("--json");
+
+  const broker = getCredential("broker");
+  if (!broker?.baseUrl || !broker.accessToken) {
+    throw new Error(
+      "Not paired with a hosted app. Run `fullstackgtm login --via <url>` first — backfill replays local history to the paired backend.",
+    );
+  }
+  const baseUrl = broker.baseUrl.replace(/\/+$/, "");
+  const headers = {
+    Authorization: `Bearer ${broker.accessToken}`,
+    "Content-Type": "application/json",
+  };
+  const profile = activeProfile();
+
+  // 1. Gather: every stored plan's runs[] + the health timeline.
+  const plans = await createFilePlanStore().list();
+  const runPayloads = plans.flatMap((stored) =>
+    stored.runs.map((run, index) => {
+      const startedAt = Date.parse(run.startedAt);
+      const finishedAt = Date.parse(run.finishedAt);
+      const tally = { applied: 0, skipped: 0, conflicts: 0, failed: 0 };
+      for (const result of run.results) {
+        if (result.status === "applied") tally.applied += 1;
+        else if (result.status === "skipped") tally.skipped += 1;
+        else if (result.status === "conflict") tally.conflicts += 1;
+        else tally.failed += 1;
+      }
+      return {
+        clientRunId: `bkf_${fnv1a(`${stored.plan.id}:${run.startedAt}:${index}`)}`,
+        command: `apply --plan-id ${stored.plan.id}`,
+        status: RUN_STATUS[run.status] ?? "error",
+        profile,
+        startedAt: Number.isFinite(startedAt) ? startedAt : Date.now(),
+        finishedAt: Number.isFinite(finishedAt) ? finishedAt : startedAt,
+        durationMs:
+          Number.isFinite(startedAt) && Number.isFinite(finishedAt) && finishedAt >= startedAt
+            ? finishedAt - startedAt
+            : 0,
+        counts: { ...tally, provider: run.provider, backfilled: true },
+      };
+    }),
+  );
+  const healthEntries = readHealthTimeline().map((entry) => ({
+    at: Date.parse(entry.at),
+    score: entry.score,
+    findings: entry.findings,
+    weightedFindings: entry.weightedFindings,
+    records: entry.records,
+    severityCounts: entry.severityCounts,
+    byObjectType: entry.byObjectType,
+    byRule: entry.byRule,
+  })).filter((entry) => Number.isFinite(entry.at));
+
+  if (dryRun) {
+    const summary = {
+      ok: true,
+      dryRun: true,
+      plans: plans.length,
+      runs: runPayloads.length,
+      healthEntries: healthEntries.length,
+      target: baseUrl,
+    };
+    if (json) console.log(JSON.stringify(summary, null, 2));
+    else {
+      console.log(
+        `Dry run — would replay ${runPayloads.length} plan run(s) (from ${plans.length} stored plan(s)) ` +
+          `and ${healthEntries.length} health reading(s) to ${baseUrl}. Nothing sent.`,
+      );
+    }
+    return;
+  }
+
+  // Shared progress: replay ticks on an interactive stderr board (inert when
+  // piped) and heartbeats to the paired app — this command is paired by
+  // definition, so a big replay shows live on the dashboard as it lands.
+  const renderer = createProgressRenderer(RUNS_REPLAY_STAGES);
+  const progress = createProgressEmitter(
+    composeListeners(renderer.listener, progressReporter()),
+  );
+
+  // 2. Replay runs (sequential; server dedupes on clientRunId).
+  let reported = 0;
+  let failed = 0;
+  const errors: string[] = [];
+  let healthInserted = 0;
+  let healthDeduped = 0;
+  try {
+    progress.stage(RUNS_REPLAY_STAGES[0], 0, RUNS_REPLAY_STAGES.length);
+    for (const payload of runPayloads) {
+      try {
+        const response = await fetch(`${baseUrl}/api/cli/run`, {
+          method: "POST",
+          headers,
+          body: JSON.stringify(payload),
+          signal: AbortSignal.timeout(10_000),
+        });
+        if (!response.ok) throw new Error(`HTTP ${response.status}`);
+        reported += 1;
+      } catch (error) {
+        failed += 1;
+        if (errors.length < 5) {
+          errors.push(`${payload.command}: ${error instanceof Error ? error.message : String(error)}`);
+        }
+      }
+      progress.items(reported + failed, runPayloads.length);
+    }
+
+    // 3. Replay health history in chunks (server dedupes on timestamp).
+    progress.stage(RUNS_REPLAY_STAGES[1], 1, RUNS_REPLAY_STAGES.length);
+    for (let i = 0; i < healthEntries.length; i += 200) {
+      const chunk = healthEntries.slice(i, i + 200);
+      try {
+        const response = await fetch(`${baseUrl}/api/cli/health`, {
+          method: "POST",
+          headers,
+          body: JSON.stringify({ entries: chunk }),
+          signal: AbortSignal.timeout(15_000),
+        });
+        if (!response.ok) throw new Error(`HTTP ${response.status}`);
+        const result = (await response.json()) as { inserted?: number; deduped?: number };
+        healthInserted += result.inserted ?? 0;
+        healthDeduped += result.deduped ?? 0;
+      } catch (error) {
+        failed += 1;
+        if (errors.length < 5) {
+          errors.push(`health chunk: ${error instanceof Error ? error.message : String(error)}`);
+        }
+      }
+      progress.items(Math.min(i + 200, healthEntries.length), healthEntries.length);
+    }
+    progress.flush();
+  } finally {
+    renderer.done();
+  }
+
+  const summary = {
+    ok: failed === 0,
+    runsReported: reported,
+    runsFailed: failed,
+    healthInserted,
+    healthDeduped,
+    target: baseUrl,
+    ...(errors.length > 0 ? { errors } : {}),
+  };
+  if (json) {
+    console.log(JSON.stringify(summary, null, 2));
+  } else {
+    console.log(
+      `Replayed ${reported}/${runPayloads.length} plan run(s) and ${healthInserted} health reading(s) ` +
+        `to ${baseUrl} (${healthDeduped} readings already there — dedupe is server-side, re-running is safe).`,
+    );
+    for (const line of errors) console.error(`  error: ${line}`);
+  }
+  if (failed > 0) {
+    process.exitCode = 1;
+  }
+}