fullstackgtm 0.45.0 → 0.46.0

package/src/progress.ts

@@ -0,0 +1,197 @@
+/**
+ * The shared progress-event vocabulary — ONE set of semantics for every
+ * long-running process, rendered natively by each surface:
+ *
+ *   CLI  → ui.ts (spinner / checklist / apply-ticker; TTY-only styling)
+ *   app  → Convex heartbeat patches → reactive ProgressChip / StageTimeline
+ *   both → optional broker streaming (a paired CLI POSTs heartbeats to the
+ *          hosted app under its clientRunId, so long local runs tick live in
+ *          the org's activity feed)
+ *
+ * Renderer-agnostic by contract: no event assumes persistence (the CLI drops
+ * them after painting) or ANSI (the app can't use it). Listeners are
+ * best-effort — a throwing renderer must never fail the work — and the
+ * throttle policy lives HERE so both surfaces feel identical.
+ *
+ * Standing convention: if a verb loops, it emits. New long-running verbs wire
+ * an emitter at the package layer; surfaces get feedback for free.
+ */
+
+export type ProgressEvent =
+  | { kind: "stage"; stage: string; stageIndex: number; stageCount: number }
+  | { kind: "items"; done: number; total?: number }
+  | { kind: "note"; note: string }
+  | {
+      kind: "opResult";
+      opId: string;
+      status: "applied" | "skipped" | "failed" | "conflict";
+      detail?: string;
+    }
+  | { kind: "meter"; spent: number; budget: number; unit: string };
+
+/** Accumulated state — what a heartbeat persists / a chip renders. */
+export type ProgressSnapshot = {
+  stage?: string;
+  stageIndex?: number;
+  stageCount?: number;
+  itemsDone?: number;
+  itemsTotal?: number;
+  note?: string;
+  opsApplied: number;
+  opsSkipped: number;
+  opsFailed: number;
+  updatedAt: number;
+};
+
+export type ProgressListener = (event: ProgressEvent, snapshot: ProgressSnapshot) => void;
+
+export type ProgressEmitter = {
+  stage(stage: string, stageIndex: number, stageCount: number): void;
+  items(done: number, total?: number): void;
+  note(note: string): void;
+  opResult(opId: string, status: "applied" | "skipped" | "failed" | "conflict", detail?: string): void;
+  meter(spent: number, budget: number, unit: string): void;
+  /** Current accumulated state (always fresh, ignoring the throttle). */
+  snapshot(): ProgressSnapshot;
+  /** Force-deliver the latest state (call at stage ends / completion). */
+  flush(): void;
+};
+
+const DEFAULT_THROTTLE_MS = 2000;
+
+/**
+ * Throttled emitter: `items` heartbeats coalesce to at most one delivery per
+ * throttle window (terminal `done === total` always delivers); `stage`,
+ * `opResult`, `note`, and `meter` are structural and always deliver.
+ */
+export function createProgressEmitter(
+  listener: ProgressListener,
+  options: { throttleMs?: number; now?: () => number } = {},
+): ProgressEmitter {
+  const throttleMs = options.throttleMs ?? DEFAULT_THROTTLE_MS;
+  const now = options.now ?? Date.now;
+  const state: ProgressSnapshot = {
+    opsApplied: 0,
+    opsSkipped: 0,
+    opsFailed: 0,
+    updatedAt: now(),
+  };
+  let lastItemsDelivery = 0;
+  let pendingItems = false;
+
+  function deliver(event: ProgressEvent) {
+    state.updatedAt = now();
+    try {
+      listener(event, { ...state });
+    } catch {
+      // Renderers are presentation-only; never fail the work.
+    }
+  }
+
+  return {
+    stage(stage, stageIndex, stageCount) {
+      state.stage = stage;
+      state.stageIndex = stageIndex;
+      state.stageCount = stageCount;
+      state.itemsDone = undefined;
+      state.itemsTotal = undefined;
+      state.note = undefined;
+      pendingItems = false;
+      deliver({ kind: "stage", stage, stageIndex, stageCount });
+    },
+    items(done, total) {
+      state.itemsDone = done;
+      state.itemsTotal = total;
+      const terminal = total !== undefined && done >= total;
+      const due = now() - lastItemsDelivery >= throttleMs;
+      if (terminal || due) {
+        lastItemsDelivery = now();
+        pendingItems = false;
+        deliver({ kind: "items", done, total });
+      } else {
+        pendingItems = true;
+      }
+    },
+    note(note) {
+      state.note = note;
+      deliver({ kind: "note", note });
+    },
+    opResult(opId, status, detail) {
+      if (status === "applied") state.opsApplied += 1;
+      else if (status === "skipped") state.opsSkipped += 1;
+      else state.opsFailed += 1;
+      deliver({ kind: "opResult", opId, status, detail });
+    },
+    meter(spent, budget, unit) {
+      deliver({ kind: "meter", spent, budget, unit });
+    },
+    snapshot() {
+      return { ...state, updatedAt: now() };
+    },
+    flush() {
+      if (pendingItems && state.itemsDone !== undefined) {
+        lastItemsDelivery = now();
+        pendingItems = false;
+        deliver({ kind: "items", done: state.itemsDone, total: state.itemsTotal });
+      }
+    },
+  };
+}
+
+/**
+ * Fan one event stream out to several listeners (e.g. the CLI's local renderer
+ * plus the broker heartbeat streamer). Each listener is isolated: one throwing
+ * never starves the others, matching the emitter's own best-effort contract.
+ */
+export function composeListeners(
+  ...listeners: Array<ProgressListener | undefined | null>
+): ProgressListener {
+  const active = listeners.filter((listener): listener is ProgressListener => Boolean(listener));
+  return (event, snapshot) => {
+    for (const listener of active) {
+      try {
+        listener(event, snapshot);
+      } catch {
+        // Listeners are presentation/observability only; never fail the work.
+      }
+    }
+  };
+}
+
+// ── Shared stage registries ─────────────────────────────────────────────────
+// Package constants so the CLI checklist and the app StageTimeline render
+// LITERALLY the same stages in the same order — one mental model everywhere.
+
+export const CRM_SYNC_STAGES = [
+  "owners",
+  "accounts",
+  "contacts",
+  "deals",
+  "counters",
+  "health",
+] as const;
+
+/** A connector snapshot pull — the CRM-sync stages that happen CLI-side. */
+export const SNAPSHOT_PULL_STAGES = ["owners", "accounts", "contacts", "deals"] as const;
+
+/** The Stripe connector's snapshot pull (billing systems have no owners). */
+export const STRIPE_SNAPSHOT_STAGES = ["customers", "subscriptions"] as const;
+
+export const CALL_SYNC_STAGES = ["notes", "transcripts", "insights"] as const;
+
+export const BACKFILL_STRIPE_STAGES = ["invoices", "snapshot", "matching", "plan"] as const;
+
+/** `backfill runs` — replaying local plan runs + health history to the broker. */
+export const RUNS_REPLAY_STAGES = ["runs", "health"] as const;
+
+export const APPLY_STAGES = ["preflight", "operations", "results"] as const;
+
+/** `enrich acquire` — routing sourced candidate rows into a create plan. */
+export const ACQUIRE_STAGES = ["candidates"] as const;
+
+export const MARKET_CAPTURE_STAGES = ["sources", "capture", "classify", "persist"] as const;
+
+/** No-op emitter for callers that don't care (keeps signatures simple). */
+export function nullProgressEmitter(): ProgressEmitter {
+  return createProgressEmitter(() => {});
+}

package/src/runReport.ts

@@ -8,6 +8,7 @@
  * pure added value for users who granted it.
  */
 import { getCredential } from "./credentials.ts";
+import type { ProgressListener, ProgressSnapshot } from "./progress.ts";
 
 type RunEvent = { ts: number; type: string; detail?: string };
 
@@ -61,6 +62,16 @@ export function reportEvent(type: string, detail?: string): void {
   events.push({ ts: Date.now(), type, detail });
 }
 
+// FNV-1a — the stable-id hash used across the package (see backfill.ts).
+function fnv1a(value: string): string {
+  let hash = 0x811c9dc5;
+  for (let i = 0; i < value.length; i += 1) {
+    hash ^= value.charCodeAt(i);
+    hash = Math.imul(hash, 0x01000193);
+  }
+  return (hash >>> 0).toString(16).padStart(8, "0");
+}
+
 // Setup/inspection verbs aren't interesting as "runs" — skip the noise.
 const SKIP_COMMANDS = new Set([
   "login",
@@ -74,6 +85,144 @@ const SKIP_COMMANDS = new Set([
   "-v",
 ]);
 
+/** The stable per-invocation identity flushRunReport and heartbeats share. */
+function runIdentity(
+  args: string[],
+  startedAt: number,
+): { command: string; clientRunId: string } | null {
+  const command = args[0];
+  if (!command || SKIP_COMMANDS.has(command)) return null;
+  const sub = args[1] && !args[1].startsWith("-") ? ` ${args[1]}` : "";
+  const full = `${command}${sub}`;
+  return { command: full, clientRunId: `run_${fnv1a(`${full}:${startedAt}:${process.pid}`)}` };
+}
+
+// ── Live progress streaming ─────────────────────────────────────────────────
+// While a long command runs, a paired CLI streams heartbeats to the hosted
+// app: POST /api/cli/run with the SAME clientRunId the final flushRunReport
+// will use, status "in_progress", and the progress emitter's snapshot. The
+// server upserts by clientRunId, so the terminal flush simply wins. Same
+// invariants as flushRunReport: opt-in (paired only), fire-and-forget,
+// time-capped, never affects the command's outcome or output.
+
+const HEARTBEAT_INTERVAL_MS = 5000;
+/** No heartbeat before the run is ~this old — fast commands never stream. */
+const HEARTBEAT_MIN_RUN_MS = 5000;
+const HEARTBEAT_TIMEOUT_MS = 2000;
+
+type RunContext = { command: string; clientRunId: string; startedAt: number };
+
+let runContext: RunContext | null = null;
+let lastHeartbeatAt = 0;
+// Broker credential resolved once per run (undefined = not looked up yet;
+// null = looked up, not paired).
+let heartbeatBroker: { baseUrl: string; accessToken: string } | null | undefined;
+let inflightHeartbeat: AbortController | null = null;
+
+/**
+ * Arm heartbeat streaming for this invocation. Call once from the entry point
+ * (alongside capturing `startedAt`); without it, progress listeners from
+ * `progressReporter()` are inert. Safe to call for any command — skip-listed
+ * commands simply never stream.
+ */
+export function beginRunReport(args: string[], startedAt: number): void {
+  lastHeartbeatAt = 0;
+  heartbeatBroker = undefined;
+  inflightHeartbeat = null;
+  const identity = runIdentity(args, startedAt);
+  runContext = identity ? { ...identity, startedAt } : null;
+}
+
+/**
+ * A ProgressListener that streams the run's progress snapshot to the paired
+ * hosted app. Verbs compose it with their local renderer:
+ *
+ *   createProgressEmitter(composeListeners(renderer.listener, progressReporter()))
+ *
+ * Throttled to one POST per ~5s, first no earlier than ~5s into the run,
+ * 2s-capped and fire-and-forget. Privacy: stage names and counts only — notes
+ * must stay generic ("batch 4/12"); CRM field values never leave the machine.
+ */
+export function progressReporter(
+  overrides: {
+    now?: () => number;
+    intervalMs?: number;
+    minRunMs?: number;
+    fetchImpl?: typeof fetch;
+  } = {},
+): ProgressListener {
+  const now = overrides.now ?? Date.now;
+  const intervalMs = overrides.intervalMs ?? HEARTBEAT_INTERVAL_MS;
+  const minRunMs = overrides.minRunMs ?? HEARTBEAT_MIN_RUN_MS;
+  const fetchImpl = overrides.fetchImpl ?? fetch;
+  return (_event, snapshot) => {
+    try {
+      if (!runContext) return;
+      const at = now();
+      if (at - runContext.startedAt < minRunMs) return;
+      if (at - lastHeartbeatAt < intervalMs) return;
+      if (heartbeatBroker === undefined) {
+        const broker = getCredential("broker");
+        heartbeatBroker =
+          broker?.baseUrl && broker.accessToken
+            ? { baseUrl: broker.baseUrl.replace(/\/+$/, ""), accessToken: broker.accessToken }
+            : null; // opt-in: only when paired
+      }
+      if (!heartbeatBroker) return;
+      lastHeartbeatAt = at;
+      sendHeartbeat(fetchImpl, heartbeatBroker, runContext, snapshot, at);
+    } catch {
+      // Observability is best-effort; never affect the command.
+    }
+  };
+}
+
+function sendHeartbeat(
+  fetchImpl: typeof fetch,
+  broker: { baseUrl: string; accessToken: string },
+  context: RunContext,
+  snapshot: ProgressSnapshot,
+  at: number,
+): void {
+  // One heartbeat in flight at a time: a stalled POST is superseded, and the
+  // terminal flush aborts any straggler so it can never delay process exit.
+  inflightHeartbeat?.abort();
+  const controller = new AbortController();
+  inflightHeartbeat = controller;
+  const timer = setTimeout(() => controller.abort(), HEARTBEAT_TIMEOUT_MS);
+  (timer as { unref?: () => void }).unref?.();
+  void fetchImpl(`${broker.baseUrl}/api/cli/run`, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${broker.accessToken}`, "Content-Type": "application/json" },
+    body: JSON.stringify({
+      command: context.command,
+      clientRunId: context.clientRunId,
+      status: "in_progress",
+      startedAt: context.startedAt,
+      finishedAt: at,
+      durationMs: at - context.startedAt,
+      progress: {
+        // Stage names + counts only — never CRM field values.
+        stage: snapshot.stage,
+        stageIndex: snapshot.stageIndex,
+        stageCount: snapshot.stageCount,
+        itemsDone: snapshot.itemsDone,
+        itemsTotal: snapshot.itemsTotal,
+        note: snapshot.note,
+        heartbeatAt: at,
+      },
+    }),
+    signal: controller.signal,
+  })
+    .catch(() => {
+      // fire-and-forget
+    })
+    .finally(() => {
+      clearTimeout(timer);
+      if (inflightHeartbeat === controller) inflightHeartbeat = null;
+    });
+}
+
 /**
  * Send the run record if the CLI is paired. Best-effort: a 4s-capped POST that
  * swallows every error. Call once per process from the entry point.
@@ -84,19 +233,25 @@ export async function flushRunReport(
   startedAt: number,
   error?: string,
 ): Promise<void> {
-  const command = args[0];
-  if (!command || SKIP_COMMANDS.has(command)) return;
+  // The terminal upsert supersedes any streaming heartbeat still in flight.
+  inflightHeartbeat?.abort();
+  inflightHeartbeat = null;
+  // Stable per-invocation identity: the server dedupes on clientRunId, so a
+  // retried report, an "in_progress" heartbeat, or a later `backfill runs`
+  // replay of local history never duplicates.
+  const identity = runIdentity(args, startedAt);
+  if (!identity) return;
   const broker = getCredential("broker");
   if (!broker?.baseUrl || !broker.accessToken) return; // opt-in: only when paired
 
-  const sub = args[1] && !args[1].startsWith("-") ? ` ${args[1]}` : "";
   const finishedAt = Date.now();
   try {
     await fetch(`${broker.baseUrl.replace(/\/+$/, "")}/api/cli/run`, {
       method: "POST",
       headers: { Authorization: `Bearer ${broker.accessToken}`, "Content-Type": "application/json" },
       body: JSON.stringify({
-        command: `${command}${sub}`,
+        command: identity.command,
+        clientRunId: identity.clientRunId,
         status,
         startedAt,
         finishedAt,

package/src/types.ts

@@ -48,7 +48,8 @@ export type PatchOperationType =
   // on matchKey at apply time (search is the source of truth; the plan-time
   // snapshot can be stale) and creates only on a confirmed miss, so apply is
   // resolve-first and never double-creates a record a concurrent writer added.
-  // Emitted only by `enrich acquire`, and metered against the acquire budget.
+  // Emitted by `enrich acquire` (metered against the acquire budget) and by
+  // `backfill stripe` (closed-won deals from paid invoices, unmetered).
   | "create_record"
   // Merge a duplicate group into a survivor. beforeValue is the group's
   // record ids; afterValue is the survivor id (requires_human_survivor_selection
@@ -59,7 +60,8 @@ export type PatchOperationType =
  * The afterValue of a `create_record` operation. The connector re-resolves on
  * `matchKey`/`matchValue` at apply time and creates only on a confirmed miss.
  * `estCostUsd` is the acquire meter's per-record charge, recorded against the
- * budget on a successful create.
+ * budget on a successful create. Emitted by `enrich acquire` (metered) and
+ * `backfill stripe` (closed-won deals from paid invoices, unmetered).
  */
 export type CreateRecordPayload = {
   properties: Record<string, string>;
@@ -83,6 +85,22 @@ export type CreateRecordPayload = {
   ownerId?: string;
   /** Audit label for how the owner was chosen (e.g. "fixed", "territory:0"). */
   assignedBy?: string;
+  /**
+   * Deal creates only: provider-neutral stage sentinel. "closed_won" tells the
+   * connector to resolve the target pipeline's REAL closed-won stage id from
+   * the provider's pipeline metadata at apply time (HubSpot: stage with
+   * `metadata.isClosed` true and probability 1) — never by substring-guessing
+   * a stage name. If no such stage is resolvable the operation is skipped.
+   * Plain deal properties (amount, closedate, dealname, …) travel in
+   * `properties` as provider property names, like every other create.
+   */
+  dealStage?: "closed_won";
+  /**
+   * Deal creates only: which pipeline to create the deal in — a pipeline id
+   * or a case-insensitive pipeline label. Absent = the portal's default
+   * pipeline (lowest displayOrder).
+   */
+  dealPipeline?: string;
 };
 
 export type AuditFindingSeverity = "info" | "warning" | "critical";

package/docs/dx-punch-list.md

@@ -1,87 +0,0 @@
-# DX / adoption punch list
-
-Audit date: 2026-06-19. Method: ran the CLI as a new adopter would (bare
-invoke → `doctor` → `audit --demo` → per-command `--help`) and measured the
-friction, rather than reading the source. Companion to the security/strategic
-findings that drove the 0.25.1–0.28.0 hardening releases — same format, same
-intent: a tracked list that drives a release.
-
-## Framing
-
-The **agent path** (SKILL.md verb map, MCP tools, `--json` everywhere,
-gate-shaped exit codes 0/1/2) is cohesive and adopts cleanly. The **human
-path** is the weak side — and the human is a primary user (the consultant-CLI,
-dogfood-on-client-work direction). Every fix below either *deepens an existing
-verb* or *connects two existing verbs*; none adds a new top-level noun. The
-adoption work **is** the "deeper not broader" work.
-
-`doctor` (ends with "Next step") and `fix` (chains audit→suggest→approve→apply)
-already do the right thing. Cohesion here = make every verb behave like
-`doctor` and `fix` already do.
-
-## Findings (evidence-backed)
-
-| # | Finding | Evidence | Impact |
-| --- | --- | --- | --- |
-| F1 | **Front door is a firehose** | bare invoke = 194 lines; 22 verbs / 87 distinct flags; `cli.ts` is 3,615 lines | New user can't find the ~6 things they actually do |
-| F2 | **Credential-free entry point dead-ends** | `audit --demo` (the README's "try it first") dumps **1,467 lines** to stdout and ends on a blank line — no "next step," unlike `doctor` | The single most-run first command strands you |
-| F3 | **No per-command help** | `dedupe --help`, `audit --help` fall through to the same 194-line global wall | A stuck user gets the firehose, not the answer |
-| F4 | **No altitude control on reads** | `audit` has only `--json` / `--out` / `--format`; no `--summary` / `--top` / `--quiet`. Human mode *is* the 1,467-line dump | Default human output is unreadable yet not machine-parseable |
-| F5 | **Tone undercuts positioning** | `audit --demo` footer: "This prototype is dry-run only" | Reads as toy vs. the beta/production-safety-invariant framing |
-| F6 | **No guided mode** | `readline` is used only for the login secret prompt | Every workflow is hand-assembled from flags; nothing scaffolds the loop |
-
-## Fix sequence (highest leverage first)
-
-### 1. Progressive-disclosure help — IN PROGRESS
-Fixes **F1, F3**. The full `usage()` becomes the explicit full reference
-(`fullstackgtm help --full`). Add:
-- `shortUsage()` — a ~30-line map of the verbs grouped by lifecycle phase
-  (Setup · Detect · Prevent · Remediate · Calls · Govern · Market · Schedule),
-  one line each, ending with `<command> --help` and `help --full` pointers.
-  Becomes the default for bare invoke and `--help`.
-- `commandHelp(command)` — focused per-verb help (summary, synopsis, where it
-  sits in the loop, key options, see-also). `<verb> --help` zooms in instead
-  of dumping the surface. `call`/`market`/`enrich`/`bulk-update`/`schedule`
-  keep their existing bespoke help.
-- `help [command] [--full]` command.
-
-### 2. Every verb hands you the next step — DONE (audit keystone)
-Fixes **F2**. `auditNextStep()` gives `audit` context-aware forward guidance on
-stderr (stdout stays clean for pipes/`--out`; suppressed under `--json`):
-- **demo/sample** → "Client-ready writeup: `report --demo`" + "Run on your CRM:
-  `login hubspot` → `audit --provider hubspot --save`"
-- **live + `--save`** → the governed spine: `suggest` → `plans approve
-  --values-from` → `apply`, with the real plan id
-- **0 findings** → "clean — gate new records with `resolve`, schedule a check"
-
-The lifecycle spine made *experiential* — each command knows its phase and
-points forward (`doctor`/`fix` were the templates). `audit` is the keystone
-(the documented dead-end); the helper pattern now generalizes to the other read
-verbs (`report`, `resolve`, `dedupe` previews) in a follow-up.
-
-### 3. Altitude control on the firehose — DONE
-Fixes **F4**. `patchPlanToMarkdown(plan, { summary })` gains a summary view
-(header + Findings-by-Rule table + operation count + pointers to `--full` and
-`report`). `audit` defaults to it — **1,467 → 24 lines** on the demo — with
-`--full` to opt back into the per-operation dump. The default stays *full* for
-every other caller (bulk-update, fix, dedupe, plans show, MCP): write-preview
-verbs keep full operation detail because you approve specific ops there. The
-principled split — audit = "what's wrong" (summary → `report`); write verbs =
-"exactly what I'll change" (full) — is the real win.
-
-### 4. Tone pass — DONE
-Fixes **F5**. The "This prototype is dry-run only" footer is replaced
-everywhere with safety-invariant framing matching README/SKILL: "Dry-run plan —
-read-only. No provider write happens until you approve specific operations and
-run `apply`; placeholder values are refused without an explicit override. These
-safety invariants are not beta." No "prototype" string remains in user output.
-
-## Deferred (bigger bets, gated on 1–4 landing)
-- **Engagement workspace**: per-client `.fullstackgtm/` state that accumulates
-  snapshots, plans, evidence, and a health-over-time score — turns episodic
-  audits into a continuously-operated system.
-- **Journey commands**: 2–3 opinionated composites (onboard-a-client,
-  health-rollup) sequencing existing verbs into revops jobs.
-
-There is no point building the engagement workspace while the front door dumps
-1,467 lines. Land the DX fixes first.