fullstackgtm 0.44.0 → 0.45.0

package/src/connector.ts

@@ -105,6 +105,22 @@ export type ApplyPatchPlanOptions = {
    * edited mid-apply is conflicted out instead of overwritten. Default 25.
    */
   recheckEvery?: number;
+  /**
+   * Per-operation progress as the run executes (presentation only — a
+   * throwing callback never affects the run). `completed` counts every
+   * resolved operation including skips and conflicts; `total` is the plan's
+   * full operation count.
+   */
+  onOperation?: (progress: ApplyProgress) => void;
+};
+
+export type ApplyProgress = {
+  completed: number;
+  total: number;
+  applied: number;
+  failed: number;
+  conflicts: number;
+  skipped: number;
 };
 
 const FIELD_WRITE_OPERATIONS = new Set(["set_field", "clear_field", "link_record"]);
@@ -313,7 +329,36 @@ export async function applyPatchPlan(
     }
   }
 
+  // Progress ticker state: every loop iteration below pushes at most one
+  // result; `lastNotified` guards the rare early-throw path so an iteration
+  // that pushed nothing reports nothing.
+  const progressCounts = { applied: 0, failed: 0, conflicts: 0, skipped: 0 };
+  const resultsBefore = results.length;
+  let lastNotified = results.length;
+  const notifyProgress = () => {
+    if (!options.onOperation || results.length === lastNotified) return;
+    lastNotified = results.length;
+    const last = results[results.length - 1];
+    if (last.status === "applied") progressCounts.applied += 1;
+    else if (last.status === "failed") progressCounts.failed += 1;
+    else if (last.status === "conflict") progressCounts.conflicts += 1;
+    else progressCounts.skipped += 1;
+    try {
+      options.onOperation({
+        completed: results.length - resultsBefore,
+        total: plan.operations.length,
+        ...progressCounts,
+      });
+    } catch {
+      // progress is presentation-only
+    }
+  };
+
   for (const operation of plan.operations) {
+    // Report the previous iteration's result (guarded: no-op if it pushed
+    // nothing). One-iteration lag keeps this a two-line hook instead of a
+    // notify after all ten push sites; the loop-exit call below flushes the last.
+    notifyProgress();
     const batchedResult = batched.get(operation.id);
     if (batchedResult) {
       results.push(batchedResult);
@@ -403,6 +448,7 @@ export async function applyPatchPlan(
       });
     }
   }
+  notifyProgress();
 
   return {
     planId: plan.id,

package/src/connectors/hubspot.ts

@@ -18,6 +18,7 @@ import type {
   GtmObjectType,
   PatchOperation,
   PatchOperationResult,
+  SnapshotProgress,
 } from "../types.ts";
 
 const DEFAULT_API_BASE_URL = "https://api.hubapi.com";
@@ -30,6 +31,8 @@ export type HubspotConnectorOptions = {
   apiBaseUrl?: string;
   /** Injectable fetch for testing. */
   fetchImpl?: typeof fetch;
+  /** Per-page snapshot-pull progress (presentation only — errors are swallowed). */
+  onProgress?: (progress: SnapshotProgress) => void;
 };
 
 const OBJECT_PATHS: Partial<Record<GtmObjectType, string>> = {
@@ -123,7 +126,7 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
     return map;
   }
 
-  async function list(path: string): Promise<any[]> {
+  async function list(path: string, onPage?: (fetched: number) => void): Promise<any[]> {
     const results: any[] = [];
     let after: string | undefined;
     const seen = new Set<string>();
@@ -137,6 +140,11 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
       const separator = path.includes("?") ? "&" : "?";
       const data = await request(`${path}${after ? `${separator}after=${encodeURIComponent(after)}` : ""}`);
       results.push(...(data.results ?? []));
+      try {
+        onPage?.(results.length);
+      } catch {
+        // progress is presentation-only; never let it fail a pull
+      }
       after = data.paging?.next?.after;
     } while (after);
     return results;
@@ -149,7 +157,9 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
       withAssociations: boolean,
     ) => Promise<any[]>,
   ): Promise<CanonicalGtmSnapshot> {
-    const owners = await list("/crm/v3/owners?limit=100");
+    const owners = await list("/crm/v3/owners?limit=100", (fetched) =>
+      options.onProgress?.({ objectType: "user", fetched }),
+    );
     const users: CanonicalUser[] = owners
       .filter((owner) => owner.id)
       .map((owner) => ({
@@ -310,9 +320,11 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
   }
 
   async function fetchSnapshot(): Promise<CanonicalGtmSnapshot> {
+    const canonicalType = { companies: "account", contacts: "contact", deals: "deal" } as const;
     return assembleSnapshot((objectType, properties, withAssociations) =>
       list(
         `/crm/v3/objects/${objectType}?limit=100&properties=${properties}${withAssociations ? "&associations=companies" : ""}`,
+        (fetched) => options.onProgress?.({ objectType: canonicalType[objectType], fetched }),
       ),
     );
   }

package/src/connectors/salesforce.ts

@@ -17,6 +17,7 @@ import type {
   GtmObjectType,
   PatchOperation,
   PatchOperationResult,
+  SnapshotProgress,
 } from "../types.ts";
 
 const DEFAULT_API_VERSION = "v59.0";
@@ -35,6 +36,8 @@ export type SalesforceConnectorOptions = {
   apiVersion?: string;
   /** Injectable fetch for testing. */
   fetchImpl?: typeof fetch;
+  /** Per-page snapshot-pull progress (presentation only — errors are swallowed). */
+  onProgress?: (progress: SnapshotProgress) => void;
 };
 
 const SOBJECT_TYPES: Partial<Record<GtmObjectType, string>> = {
@@ -161,7 +164,7 @@ export function createSalesforceConnector(
     return text ? JSON.parse(text) : null;
   }
 
-  async function query(soql: string): Promise<any[]> {
+  async function query(soql: string, onPage?: (fetched: number) => void): Promise<any[]> {
     const records: any[] = [];
     let next: string | undefined = `/services/data/${apiVersion}/query?q=${encodeURIComponent(soql)}`;
     const seen = new Set<string>();
@@ -171,6 +174,11 @@ export function createSalesforceConnector(
       seen.add(next);
       const data = await request(next);
       records.push(...(data?.records ?? []));
+      try {
+        onPage?.(records.length);
+      } catch {
+        // progress is presentation-only; never let it fail a pull
+      }
       next = data?.nextRecordsUrl ?? undefined;
     }
     return records;
@@ -194,7 +202,12 @@ export function createSalesforceConnector(
   }
 
   async function assembleSnapshot(whereClause: string): Promise<CanonicalGtmSnapshot> {
-    const sfUsers = await query(`SELECT ${selectFields("owners")} FROM User${whereClause}`);
+    const progressFor = (objectType: SnapshotProgress["objectType"]) => (fetched: number) =>
+      options.onProgress?.({ objectType, fetched });
+    const sfUsers = await query(
+      `SELECT ${selectFields("owners")} FROM User${whereClause}`,
+      progressFor("user"),
+    );
     const users: CanonicalUser[] = sfUsers.map((user) => {
       const id = String(readMapped(user, "owners", "id", "Id"));
       const email = stringOrUndefined(readMapped(user, "owners", "email", "Email"));
@@ -212,6 +225,7 @@ export function createSalesforceConnector(
 
     const sfAccounts = await query(
       `SELECT ${selectFields("accounts")} FROM Account${whereClause}`,
+      progressFor("account"),
     );
     const accounts: CanonicalAccount[] = sfAccounts.map((account) => {
       const id = String(readMapped(account, "accounts", "id", "Id"));
@@ -236,6 +250,7 @@ export function createSalesforceConnector(
 
     const sfContacts = await query(
       `SELECT ${selectFields("contacts")} FROM Contact${whereClause}`,
+      progressFor("contact"),
     );
     const contacts: CanonicalContact[] = sfContacts.map((contact) => {
       const id = String(readMapped(contact, "contacts", "id", "Id"));
@@ -257,6 +272,7 @@ export function createSalesforceConnector(
 
     const sfOpportunities = await query(
       `SELECT ${selectFields("deals")} FROM Opportunity${whereClause}`,
+      progressFor("deal"),
     );
     const deals: CanonicalDeal[] = sfOpportunities.map((opportunity) => {
       const id = String(readMapped(opportunity, "deals", "id", "Id"));

package/src/connectors/signalSources.ts

@@ -19,10 +19,11 @@
  *     try/catch idiom of atsBoards/prospectSources.
  */
 
-import { readFileSync, readdirSync, statSync } from "node:fs";
-import { join, resolve } from "node:path";
+import { readFileSync, statSync } from "node:fs";
+import { resolve } from "node:path";
 import type { SignalBucket, StagedSignalRow } from "../signals.ts";
 import { SIGNAL_BUCKETS } from "../signals.ts";
+import { parseSpoolText, spoolFilesIn } from "../spoolFiles.ts";
 
 export type SignalSourceShape = "pull" | "push";
 export type SignalSourceAuth = "none" | "api_key" | "oauth";
@@ -99,6 +100,9 @@ export const fileSource: SignalSourceConnector = {
     } catch {
       return []; // missing path: an empty source, not a crash
     }
+    // Container parsing lives in the shared spool reader (src/spoolFiles.ts) so
+    // `signals fetch --from`, `enrich ingest`, and `market observe --from` read
+    // the same convention; the "file source" label keeps error text unchanged.
     const files = isDir ? spoolFilesIn(abs) : [abs];
     const rows: Record<string, unknown>[] = [];
     for (const file of files) {
@@ -108,65 +112,12 @@ export const fileSource: SignalSourceConnector = {
       } catch {
         continue; // one unreadable file in a spool dir must not sink the rest
       }
-      const trimmed = raw.trim();
-      if (!trimmed) continue;
-      const parsed = trimmed.startsWith("[") ? parseJsonArray(trimmed, file) : parseJsonl(trimmed, file);
-      rows.push(...parsed);
+      rows.push(...parseSpoolText(raw, file, "file source"));
     }
     return rows.map((row) => coerceRow(row, this.bucket));
   },
 };
 
-/** Spool files in a landing-zone directory: `*.jsonl` / `*.json`, name-sorted. */
-function spoolFilesIn(dir: string): string[] {
-  let names: string[];
-  try {
-    names = readdirSync(dir);
-  } catch {
-    return [];
-  }
-  return names
-    .filter((name) => name.endsWith(".jsonl") || name.endsWith(".json"))
-    .sort()
-    .map((name) => join(dir, name));
-}
-
-function parseJsonArray(raw: string, path: string): Record<string, unknown>[] {
-  let parsed: unknown;
-  try {
-    parsed = JSON.parse(raw);
-  } catch (error) {
-    throw new Error(`file source ${path}: not valid JSON (${error instanceof Error ? error.message : String(error)}).`);
-  }
-  if (!Array.isArray(parsed)) throw new Error(`file source ${path}: expected a JSON array of staged signal rows.`);
-  return parsed.map((entry, index) => {
-    if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
-      throw new Error(`file source ${path}: row ${index} is not an object.`);
-    }
-    return entry as Record<string, unknown>;
-  });
-}
-
-function parseJsonl(raw: string, path: string): Record<string, unknown>[] {
-  const out: Record<string, unknown>[] = [];
-  const lines = raw.split("\n");
-  lines.forEach((line, index) => {
-    const t = line.trim();
-    if (!t) return;
-    let parsed: unknown;
-    try {
-      parsed = JSON.parse(t);
-    } catch (error) {
-      throw new Error(`file source ${path}: line ${index} is not valid JSON (${error instanceof Error ? error.message : String(error)}).`);
-    }
-    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
-      throw new Error(`file source ${path}: line ${index} is not a JSON object.`);
-    }
-    out.push(parsed as Record<string, unknown>);
-  });
-  return out;
-}
-
 // ---------------------------------------------------------------------------
 // serpapi-news — funding/company signals from a news REST query (API key)
 

package/src/icp.ts

@@ -158,26 +158,10 @@ export function icpToTheirStackFilters(icp: Icp): {
   return f;
 }
 
-/**
- * Crustdata / LinkedIn seniority vocab. ICP job levels are normalized lowercase;
- * Crustdata expects these exact capitalized strings (verified: "CXO",
- * "Vice President", "Director" appear in Crustdata's docs/examples).
- */
-const CRUSTDATA_SENIORITY: Record<string, string> = {
-  cxo: "CXO",
-  "c-suite": "CXO",
-  c_suite: "CXO",
-  founder: "Owner",
-  owner: "Owner",
-  partner: "Partner",
-  vp: "Vice President",
-  "vice president": "Vice President",
-  head: "Director",
-  director: "Director",
-  manager: "Manager",
-  senior: "Senior",
-  entry: "Entry",
-};
+// NOTE: a CRUSTDATA_SENIORITY vocab table lived here until 2026-07-02 — the
+// `current_seniority_levels` filter is broken upstream (see the live findings
+// on icpToCrustdataFilters below), so job levels are no longer sent to the
+// provider at all. Resurrect the table from git history if pipe0 fixes it.
 
 /**
  * ICP industry keyword → LinkedIn industry names Crustdata filters on.
@@ -212,18 +196,22 @@ const CRUSTDATA_INDUSTRY: Record<string, string[]> = {
 /**
  * pipe0 Crustdata people-search config.filters from the ICP. `current_job_titles`
  * matches real LinkedIn title strings (case-sensitive), so keywords are Title
- * Cased. Seniority + industry are mapped to Crustdata's controlled vocab via the
- * tables above.
+ * Cased. Industry is mapped to Crustdata's controlled vocab via the table above.
  *
- * LIVE FINDINGS (2026-06-26): `current_job_titles` is confirmed working (a
- * titles-only RevOps search returns results); `current_title` is rejected (422).
- * The full ICP filter returned 0 — the prime suspect is the industry vocab
- * (LinkedIn v1/v2 taxonomy mismatch), now hedged by sending BOTH generations in
- * CRUSTDATA_INDUSTRY above. NOT yet re-confirmed end-to-end (pipe0 credits were
- * exhausted mid-investigation) — re-run `enrich acquire --source pipe0` once
- * credits refill; if it still returns 0, the next suspects are the
- * `current_seniority_levels` shape/values and `locations`. Note fit-scoring backs
- * up PERSONA precision but NOT industry, so the industry filter is load-bearing.
+ * LIVE FINDINGS (2026-06-26, re-confirmed 2026-07-02 with fresh credits):
+ * `current_job_titles` works; `current_title` is rejected (422). The culprit
+ * that zeroed the full ICP filter is `current_seniority_levels`: ANY non-empty
+ * `include` returns 0 results through pipe0's people:profiles:crustdata@1 —
+ * including Crustdata's own documented values ("CXO", "Director",
+ * "Vice President"), lowercase, and SNAKE_CASE variants — while the identical
+ * search without it returns results. (An array instead of the
+ * {include, exclude} object is a 422, so the shape was right; the filter is
+ * broken upstream.) So job levels are NOT sent to the provider: persona
+ * seniority is enforced by fit scoring (scoreProspectAgainstIcp weights
+ * jobLevels), which was always the precision backstop. `locations` and
+ * `current_employers_linkedin_industries` (both-taxonomy hedge) are live-
+ * verified working in combination with titles. Note fit-scoring backs up
+ * PERSONA precision but NOT industry, so the industry filter is load-bearing.
  */
 export function icpToCrustdataFilters(icp: Icp): Record<string, unknown> {
   const f: Record<string, unknown> = {};
@@ -231,10 +219,6 @@ export function icpToCrustdataFilters(icp: Icp): Record<string, unknown> {
   if (icp.firmographics.geos?.length) {
     f.locations = icp.firmographics.geos.map((g) => COUNTRY_NAMES[g.toLowerCase()] ?? g);
   }
-  if (icp.persona.jobLevels?.length) {
-    const include = [...new Set(icp.persona.jobLevels.map((l) => CRUSTDATA_SENIORITY[l.toLowerCase()]).filter(Boolean))];
-    if (include.length) f.current_seniority_levels = { include, exclude: [] };
-  }
   if (icp.firmographics.industries?.length) {
     const inds = [
       ...new Set(icp.firmographics.industries.flatMap((i) => CRUSTDATA_INDUSTRY[i.toLowerCase()] ?? [titleCase(i)])),

package/src/judge.ts

@@ -612,6 +612,8 @@ export async function judgeSignals(opts: {
   promptTemplate?: string;
   llm?: LlmCallOptions;
   now?: Date;
+  /** Per-account progress (presentation only — a throwing callback never fails the run). */
+  onAccount?: (done: number, total: number, domain: string) => void;
 }): Promise<JudgeDecision[]> {
   const now = opts.now ?? new Date();
   const signalsById = new Map(opts.signals.map((s) => [s.id, s]));
@@ -629,6 +631,11 @@ export async function judgeSignals(opts: {
 
   const decisions: JudgeDecision[] = [];
   for (const [domain, signals] of byAccount) {
+    try {
+      opts.onAccount?.(decisions.length, byAccount.size, domain);
+    } catch {
+      // progress is presentation-only
+    }
     // Memory + fit stay gated on --with-history (scoring semantics unchanged).
     const recentlyTouched =
       opts.withHistory && opts.snapshot ? accountRecentlyTouched(domain, opts.snapshot, now) : false;

package/src/marketClassify.ts

@@ -113,6 +113,8 @@ export type ClassifyMarketOptions = {
   /** Captures directory override (tests); defaults to the profile market home. */
   capturesDir?: string;
   now?: () => Date;
+  /** Per-vendor progress (presentation only — a throwing callback never fails the run). */
+  onVendor?: (done: number, total: number, vendorId: string) => void;
 };
 
 export type ClassifyMarketResult = {
@@ -143,7 +145,12 @@ export async function classifyMarket(
   const observations: MarketObservation[] = [];
   const retriedVendorIds: string[] = [];
 
-  for (const vendorId of vendorIds) {
+  for (const [vendorIndex, vendorId] of vendorIds.entries()) {
+    try {
+      options.onVendor?.(vendorIndex, vendorIds.length, vendorId);
+    } catch {
+      // progress is presentation-only
+    }
     const vendor = config.vendors.find((candidate) => candidate.id === vendorId);
     if (!vendor) throw new Error(`Unknown vendor "${vendorId}"`);
     const vendorEntries = runEntries.filter((entry) => entry.vendorId === vendorId);

package/src/mcp-bin.ts

@@ -9,8 +9,8 @@ const MISSING_PEER_HELP = `The MCP server needs the optional peer dependencies @
 In a project:
   npm install fullstackgtm @modelcontextprotocol/sdk zod
 
-Zero-install:
-  npx -p fullstackgtm -p @modelcontextprotocol/sdk -p zod fullstackgtm-mcp`;
+Zero-install (the fullstackgtm-mcp wrapper package bundles the peers):
+  npx -y fullstackgtm-mcp`;
 
 function isMissingPeerError(error: unknown): boolean {
   if (!(error instanceof Error)) return false;