fullstackgtm 0.43.0 → 0.45.0

package/src/connector.ts

@@ -105,6 +105,22 @@ export type ApplyPatchPlanOptions = {
    * edited mid-apply is conflicted out instead of overwritten. Default 25.
    */
   recheckEvery?: number;
+  /**
+   * Per-operation progress as the run executes (presentation only — a
+   * throwing callback never affects the run). `completed` counts every
+   * resolved operation including skips and conflicts; `total` is the plan's
+   * full operation count.
+   */
+  onOperation?: (progress: ApplyProgress) => void;
+};
+
+export type ApplyProgress = {
+  completed: number;
+  total: number;
+  applied: number;
+  failed: number;
+  conflicts: number;
+  skipped: number;
 };
 
 const FIELD_WRITE_OPERATIONS = new Set(["set_field", "clear_field", "link_record"]);
@@ -313,7 +329,36 @@ export async function applyPatchPlan(
     }
   }
 
+  // Progress ticker state: every loop iteration below pushes at most one
+  // result; `lastNotified` guards the rare early-throw path so an iteration
+  // that pushed nothing reports nothing.
+  const progressCounts = { applied: 0, failed: 0, conflicts: 0, skipped: 0 };
+  const resultsBefore = results.length;
+  let lastNotified = results.length;
+  const notifyProgress = () => {
+    if (!options.onOperation || results.length === lastNotified) return;
+    lastNotified = results.length;
+    const last = results[results.length - 1];
+    if (last.status === "applied") progressCounts.applied += 1;
+    else if (last.status === "failed") progressCounts.failed += 1;
+    else if (last.status === "conflict") progressCounts.conflicts += 1;
+    else progressCounts.skipped += 1;
+    try {
+      options.onOperation({
+        completed: results.length - resultsBefore,
+        total: plan.operations.length,
+        ...progressCounts,
+      });
+    } catch {
+      // progress is presentation-only
+    }
+  };
+
   for (const operation of plan.operations) {
+    // Report the previous iteration's result (guarded: no-op if it pushed
+    // nothing). One-iteration lag keeps this a two-line hook instead of a
+    // notify after all ten push sites; the loop-exit call below flushes the last.
+    notifyProgress();
     const batchedResult = batched.get(operation.id);
     if (batchedResult) {
       results.push(batchedResult);
@@ -403,6 +448,7 @@ export async function applyPatchPlan(
       });
     }
   }
+  notifyProgress();
 
   return {
     planId: plan.id,

package/src/connectors/hubspot.ts

@@ -18,6 +18,7 @@ import type {
   GtmObjectType,
   PatchOperation,
   PatchOperationResult,
+  SnapshotProgress,
 } from "../types.ts";
 
 const DEFAULT_API_BASE_URL = "https://api.hubapi.com";
@@ -30,6 +31,8 @@ export type HubspotConnectorOptions = {
   apiBaseUrl?: string;
   /** Injectable fetch for testing. */
   fetchImpl?: typeof fetch;
+  /** Per-page snapshot-pull progress (presentation only — errors are swallowed). */
+  onProgress?: (progress: SnapshotProgress) => void;
 };
 
 const OBJECT_PATHS: Partial<Record<GtmObjectType, string>> = {
@@ -123,7 +126,7 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
     return map;
   }
 
-  async function list(path: string): Promise<any[]> {
+  async function list(path: string, onPage?: (fetched: number) => void): Promise<any[]> {
     const results: any[] = [];
     let after: string | undefined;
     const seen = new Set<string>();
@@ -137,6 +140,11 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
       const separator = path.includes("?") ? "&" : "?";
       const data = await request(`${path}${after ? `${separator}after=${encodeURIComponent(after)}` : ""}`);
       results.push(...(data.results ?? []));
+      try {
+        onPage?.(results.length);
+      } catch {
+        // progress is presentation-only; never let it fail a pull
+      }
       after = data.paging?.next?.after;
     } while (after);
     return results;
@@ -149,7 +157,9 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
       withAssociations: boolean,
     ) => Promise<any[]>,
   ): Promise<CanonicalGtmSnapshot> {
-    const owners = await list("/crm/v3/owners?limit=100");
+    const owners = await list("/crm/v3/owners?limit=100", (fetched) =>
+      options.onProgress?.({ objectType: "user", fetched }),
+    );
     const users: CanonicalUser[] = owners
       .filter((owner) => owner.id)
       .map((owner) => ({
@@ -310,9 +320,11 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
   }
 
   async function fetchSnapshot(): Promise<CanonicalGtmSnapshot> {
+    const canonicalType = { companies: "account", contacts: "contact", deals: "deal" } as const;
     return assembleSnapshot((objectType, properties, withAssociations) =>
       list(
         `/crm/v3/objects/${objectType}?limit=100&properties=${properties}${withAssociations ? "&associations=companies" : ""}`,
+        (fetched) => options.onProgress?.({ objectType: canonicalType[objectType], fetched }),
       ),
     );
   }

package/src/connectors/outboxChannel.ts

@@ -0,0 +1,202 @@
+/**
+ * Outbox channel connector — the governed SEND-side terminus of the outbound
+ * loop (signal → judge → draft → approve → apply → outbox), and the mirror image
+ * of the webhook spool. It is a `GtmConnector` whose `applyOperation` RENDERS an
+ * approved drafted opener to a local outbox artifact instead of writing a CRM —
+ * so it reuses the entire governed apply machinery (approval set, integrity/HMAC
+ * verification, idempotency, run recording) with no change to the apply engine.
+ *
+ * THE INVARIANT IT KEEPS: the CLI **transmits nothing**. Rendering an approved
+ * opener to `<home>/signals/outbox/<channel>.jsonl` is a local file write, never
+ * an SMTP/API connection. A downstream sender (the hosted product, or the
+ * operator's own MTA) drains the outbox and does the actual transmission. This
+ * is the send-side half of the open-core boundary: the governed artifact + its
+ * format are open; always-on transmission infrastructure is hosted/opt-in.
+ *
+ * Governance: it only renders ops that came from `draft` (a `create_task` op
+ * whose policy is `draft:<channel>`); any other op is `skipped` (it is not a
+ * general CRM writer). Idempotent on the operation id — re-applying an approved
+ * plan never duplicates an outbox row. Read paths (`fetchSnapshot`) intentionally
+ * throw: a channel has no snapshot, and `applyPatchPlan` never calls it for a
+ * draft plan (no guards/filter/irreversible ops → no snapshot needed).
+ */
+
+import { appendFileSync, existsSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { ensureSecureHomeDir, writeSecureFile } from "../credentials.ts";
+import { signalsOutboxDir } from "../signals.ts";
+import type { CanonicalGtmSnapshot, GtmConnector, PatchOperation, PatchOperationResult } from "../types.ts";
+
+/** One approved, ready-to-send touch — a governed send INTENT, not a sent message. */
+export type OutboxEntry = {
+  /** Idempotency key = the source operation id (stable per draft op). */
+  id: string;
+  /** email | linkedin | task — from the draft op's `draft:<channel>` policy. */
+  channel: string;
+  /** The CRM target the opener is addressed to (a sender resolves id → address). */
+  objectType: string;
+  objectId: string;
+  /** The APPROVED opener, verbatim as it was signed in the plan op. */
+  body: string;
+  /** The draft op's human-readable reason (carries the account + trigger). */
+  reason?: string;
+  /** Evidence ids the opener was grounded in (the verbatim signal quote). */
+  evidenceIds: string[];
+  /** ISO 8601 — when the CLI rendered this to the outbox (NOT a send time). */
+  renderedAt: string;
+};
+
+/** Channel ids this build can render to. */
+export const CHANNELS = ["outbox"] as const;
+
+export type OutboxChannelOptions = {
+  /** Outbox directory; defaults to the profile-scoped `signalsOutboxDir()`. */
+  outboxDir?: string;
+  /** Injectable clock for deterministic tests. */
+  now?: () => Date;
+};
+
+const DRAFT_POLICY_PREFIX = "draft:";
+
+/**
+ * Build the outbox channel connector. With no `outboxDir`, writes to the
+ * profile-scoped conventional outbox and locks the home down (0700/0600) like
+ * the signal store; with an explicit `outboxDir` (tests), it writes there
+ * without touching the real home.
+ */
+export function createOutboxChannelConnector(options: OutboxChannelOptions = {}): GtmConnector {
+  const usingDefaultHome = options.outboxDir === undefined;
+  const dir = options.outboxDir ?? signalsOutboxDir();
+  const now = options.now ?? (() => new Date());
+
+  function fileFor(channel: string): string {
+    if (!/^[\w.-]+$/.test(channel)) throw new Error(`Invalid outbox channel name: ${channel}`);
+    return join(dir, `${channel}.jsonl`);
+  }
+
+  /** Existing entry ids in a channel file (for idempotent re-apply). */
+  function existingIds(channel: string): Set<string> {
+    const ids = new Set<string>();
+    let raw: string;
+    try {
+      raw = readFileSync(fileFor(channel), "utf8");
+    } catch {
+      return ids; // no file yet
+    }
+    for (const line of raw.split("\n")) {
+      const t = line.trim();
+      if (!t) continue;
+      try {
+        const id = (JSON.parse(t) as { id?: unknown }).id;
+        if (typeof id === "string") ids.add(id);
+      } catch {
+        // Skip a corrupt line rather than fail the whole apply.
+      }
+    }
+    return ids;
+  }
+
+  function append(channel: string, entry: OutboxEntry): void {
+    if (usingDefaultHome) ensureSecureHomeDir();
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+    const path = fileFor(channel);
+    // Owner-only like the signal store: outbox carries opener text + CRM ids.
+    if (!existsSync(path)) writeSecureFile(path, "");
+    appendFileSync(path, `${JSON.stringify(entry)}\n`, { mode: 0o600 });
+  }
+
+  async function applyOperation(operation: PatchOperation): Promise<PatchOperationResult> {
+    const policy = String(operation.sourceRuleOrPolicy ?? "");
+    if (operation.operation !== "create_task" || !policy.startsWith(DRAFT_POLICY_PREFIX)) {
+      return {
+        operationId: operation.id,
+        status: "skipped",
+        detail:
+          "The outbox channel only renders drafted openers (create_task ops from `draft`). " +
+          "Apply other operations through a CRM connector (--provider hubspot|salesforce).",
+      };
+    }
+    const channel = policy.slice(DRAFT_POLICY_PREFIX.length) || "task";
+
+    // Idempotent: a re-applied approved plan must not duplicate the row.
+    if (existingIds(channel).has(operation.id)) {
+      return {
+        operationId: operation.id,
+        status: "applied",
+        detail: `Already in outbox (${channel}.jsonl); idempotent — not duplicated. Nothing transmitted.`,
+      };
+    }
+
+    const entry: OutboxEntry = {
+      id: operation.id,
+      channel,
+      objectType: operation.objectType,
+      objectId: operation.objectId,
+      body: typeof operation.afterValue === "string" ? operation.afterValue : String(operation.afterValue ?? ""),
+      ...(operation.reason ? { reason: operation.reason } : {}),
+      evidenceIds: operation.evidenceIds ?? [],
+      renderedAt: now().toISOString(),
+    };
+    append(channel, entry);
+    return {
+      operationId: operation.id,
+      status: "applied",
+      detail: `Rendered to outbox (${channel}.jsonl) for a downstream sender. Nothing was transmitted.`,
+    };
+  }
+
+  async function fetchSnapshot(): Promise<CanonicalGtmSnapshot> {
+    // A channel has no readable state. apply never calls this for a draft plan
+    // (no guards/filter/irreversible ops); make the misuse explicit if it does.
+    throw new Error(
+      "The outbox channel has no snapshot to read — it is a send-side render target. " +
+        "Use it only to `apply` an approved draft plan (`apply --plan-id <id> --channel outbox`).",
+    );
+  }
+
+  return {
+    provider: "outbox",
+    fetchSnapshot,
+    applyOperation,
+  };
+}
+
+/** Resolve a channel connector by id (mirrors the source-connector registry). */
+export function createChannelConnector(id: string, options: OutboxChannelOptions = {}): GtmConnector {
+  if (id === "outbox") return createOutboxChannelConnector(options);
+  throw new Error(`Unknown channel: ${id} (one of: ${CHANNELS.join(", ")}).`);
+}
+
+/**
+ * Read every outbox entry across all channel files in `dir` (default: the
+ * conventional outbox), newest-appended last. The reader a downstream sender (or
+ * a future `signals outbox` command) uses to drain the queue.
+ */
+export function listOutbox(dir?: string): OutboxEntry[] {
+  const outboxDir = dir ?? signalsOutboxDir();
+  let names: string[];
+  try {
+    names = readdirSync(outboxDir).filter((name) => name.endsWith(".jsonl")).sort();
+  } catch {
+    return [];
+  }
+  const out: OutboxEntry[] = [];
+  for (const name of names) {
+    let raw: string;
+    try {
+      raw = readFileSync(join(outboxDir, name), "utf8");
+    } catch {
+      continue;
+    }
+    for (const line of raw.split("\n")) {
+      const t = line.trim();
+      if (!t) continue;
+      try {
+        out.push(JSON.parse(t) as OutboxEntry);
+      } catch {
+        // Skip corrupt lines.
+      }
+    }
+  }
+  return out;
+}

package/src/connectors/prospectSources.ts

@@ -166,6 +166,63 @@ function strField(field: { value?: unknown } | undefined): string | undefined {
   return typeof v === "string" && v.trim() ? v : undefined;
 }
 
+// ---------------------------------------------------------------------------
+// TAM universe count — "how many ACCOUNTS match this ICP firmographic?"
+//
+// Verified empirically (2026-06-25) against the live providers, because the
+// obvious endpoints lie about totals:
+//   - Explorium /v1/prospects `total_results` == the PAGE size (1→1, 10→10), not
+//     the universe — useless for sizing. So is /v1/businesses for tiny pages? No:
+//   - Explorium /v1/businesses `total_results` IS a real COUNT of matching
+//     companies (US+10000-emp → 9,754; Liechtenstein+10000 → 11), capped at
+//     EXPLORIUM_BUSINESS_COUNT_CAP. At/above the cap it saturates at exactly that
+//     number, so the caller must treat a capped reading as a FLOOR, not a count.
+//   - pipe0/Crustdata people search returns only a pagination cursor, NO total —
+//     it cannot size a universe at all (callers use it for discovery, not counting).
+// So the TAM count source is Explorium /v1/businesses (a company/account count).
+
+export const EXPLORIUM_BUSINESS_COUNT_CAP = 60_000;
+
+export type BusinessCountProbe = {
+  /** matching companies (the account universe); == cap when saturated. */
+  total: number;
+  /** true when total hit EXPLORIUM_BUSINESS_COUNT_CAP — treat total as a lower bound. */
+  capped: boolean;
+};
+
+/**
+ * Count the COMPANIES (accounts) matching an ICP firmographic via Explorium
+ * /v1/businesses. Returns the real total (capped at 60k — `capped` flags the
+ * ceiling) or null if the response carries no `total_results`. Use
+ * `icpToExploriumBusinessFilters` to build `filters` (firmographic field names
+ * differ from /v1/prospects).
+ */
+export async function probeExploriumBusinessCount(opts: {
+  apiKey: string;
+  filters: Record<string, { values?: string[] }>;
+  apiBaseUrl?: string;
+  fetchImpl?: FetchImpl;
+}): Promise<BusinessCountProbe | null> {
+  const fetchImpl = opts.fetchImpl ?? fetch;
+  const base = (opts.apiBaseUrl ?? "https://api.explorium.ai").replace(/\/$/, "");
+  const response = await fetchImpl(`${base}/v1/businesses`, {
+    method: "POST",
+    headers: { "api_key": opts.apiKey, "Content-Type": "application/json" },
+    // page_size 1: we want the envelope's total_results, not the rows. CRITICAL:
+    // do NOT send `size` — Explorium caps total_results to `size` when present
+    // (verified: size:1 → total_results:1; omitted → the real count, e.g. 19,058).
+    body: JSON.stringify({ mode: "full", page_size: 1, page: 1, filters: opts.filters }),
+  });
+  if (!response.ok) {
+    throw new Error(`Explorium /v1/businesses count failed: HTTP ${response.status} ${await safeText(response)}`);
+  }
+  const body = (await response.json()) as { total_results?: unknown };
+  const total = body.total_results;
+  if (typeof total !== "number" || !Number.isFinite(total) || total < 0) return null;
+  const rounded = Math.round(total);
+  return { total: rounded, capped: rounded >= EXPLORIUM_BUSINESS_COUNT_CAP };
+}
+
 function normalizeLinkedin(value: string | undefined): string | undefined {
   if (!value) return undefined;
   const v = value.trim().replace(/^https?:\/\//i, "").replace(/\/$/, "");

package/src/connectors/salesforce.ts

@@ -17,6 +17,7 @@ import type {
   GtmObjectType,
   PatchOperation,
   PatchOperationResult,
+  SnapshotProgress,
 } from "../types.ts";
 
 const DEFAULT_API_VERSION = "v59.0";
@@ -35,6 +36,8 @@ export type SalesforceConnectorOptions = {
   apiVersion?: string;
   /** Injectable fetch for testing. */
   fetchImpl?: typeof fetch;
+  /** Per-page snapshot-pull progress (presentation only — errors are swallowed). */
+  onProgress?: (progress: SnapshotProgress) => void;
 };
 
 const SOBJECT_TYPES: Partial<Record<GtmObjectType, string>> = {
@@ -161,7 +164,7 @@ export function createSalesforceConnector(
     return text ? JSON.parse(text) : null;
   }
 
-  async function query(soql: string): Promise<any[]> {
+  async function query(soql: string, onPage?: (fetched: number) => void): Promise<any[]> {
     const records: any[] = [];
     let next: string | undefined = `/services/data/${apiVersion}/query?q=${encodeURIComponent(soql)}`;
     const seen = new Set<string>();
@@ -171,6 +174,11 @@ export function createSalesforceConnector(
       seen.add(next);
       const data = await request(next);
       records.push(...(data?.records ?? []));
+      try {
+        onPage?.(records.length);
+      } catch {
+        // progress is presentation-only; never let it fail a pull
+      }
       next = data?.nextRecordsUrl ?? undefined;
     }
     return records;
@@ -194,7 +202,12 @@ export function createSalesforceConnector(
   }
 
   async function assembleSnapshot(whereClause: string): Promise<CanonicalGtmSnapshot> {
-    const sfUsers = await query(`SELECT ${selectFields("owners")} FROM User${whereClause}`);
+    const progressFor = (objectType: SnapshotProgress["objectType"]) => (fetched: number) =>
+      options.onProgress?.({ objectType, fetched });
+    const sfUsers = await query(
+      `SELECT ${selectFields("owners")} FROM User${whereClause}`,
+      progressFor("user"),
+    );
     const users: CanonicalUser[] = sfUsers.map((user) => {
       const id = String(readMapped(user, "owners", "id", "Id"));
       const email = stringOrUndefined(readMapped(user, "owners", "email", "Email"));
@@ -212,6 +225,7 @@ export function createSalesforceConnector(
 
     const sfAccounts = await query(
       `SELECT ${selectFields("accounts")} FROM Account${whereClause}`,
+      progressFor("account"),
     );
     const accounts: CanonicalAccount[] = sfAccounts.map((account) => {
       const id = String(readMapped(account, "accounts", "id", "Id"));
@@ -236,6 +250,7 @@ export function createSalesforceConnector(
 
     const sfContacts = await query(
       `SELECT ${selectFields("contacts")} FROM Contact${whereClause}`,
+      progressFor("contact"),
     );
     const contacts: CanonicalContact[] = sfContacts.map((contact) => {
       const id = String(readMapped(contact, "contacts", "id", "Id"));
@@ -257,6 +272,7 @@ export function createSalesforceConnector(
 
     const sfOpportunities = await query(
       `SELECT ${selectFields("deals")} FROM Opportunity${whereClause}`,
+      progressFor("deal"),
     );
     const deals: CanonicalDeal[] = sfOpportunities.map((opportunity) => {
       const id = String(readMapped(opportunity, "deals", "id", "Id"));