fullstackgtm 0.21.2 → 0.23.1

package/dist/dedupe.d.ts

@@ -0,0 +1,14 @@
+import type { CanonicalGtmSnapshot, PatchPlan } from "./types.ts";
+export type DedupeOptions = {
+    objectType: "account" | "contact" | "deal";
+    /** identity key records are grouped by (normalized before grouping) */
+    key: "domain" | "email" | "name";
+    /** survivor selection — deterministic either way (default "richest") */
+    keep?: "richest" | "oldest";
+    reason?: string;
+    /** refuse to build plans larger than this (default 500 operations) */
+    maxOperations?: number;
+};
+/** Normalize a record's identity key; undefined when the field is empty. */
+export declare function dedupeKey(record: Record<string, unknown>, key: DedupeOptions["key"]): string | undefined;
+export declare function buildDedupePlan(snapshot: CanonicalGtmSnapshot, options: DedupeOptions): PatchPlan;

package/dist/dedupe.js

@@ -0,0 +1,140 @@
+/**
+ * Governed duplicate cleanup: `dedupe` groups records by a normalized
+ * identity key (account domain, contact email, or name) and builds a
+ * dry-run PatchPlan of merge_records operations — one per duplicate group,
+ * with a DETERMINISTIC survivor. It NEVER writes — the plan flows through
+ * the same plans-approve → apply gate as every other plan.
+ *
+ * The merge contract matches the connectors (see mergeRecords in
+ * connectors/hubspot.ts): afterValue = the survivor id, beforeValue = the
+ * ids of EVERY record in the group (survivor included). Merges are
+ * IRREVERSIBLE on every provider that supports them, so every operation is
+ * riskLevel high and approvalRequired.
+ *
+ * Survivor selection ("--keep"):
+ *   richest (default)  the record with the most non-empty canonical data
+ *                      fields (bookkeeping fields like id/crmId/identities
+ *                      don't count); ties break to the lowest numeric id
+ *   oldest             the lowest numeric id (CRMs assign ids in creation
+ *                      order)
+ */
+import { normalizeDomain } from "./merge.js";
+import { stableHash } from "./rules.js";
+const COLLECTIONS = {
+    account: "accounts",
+    contact: "contacts",
+    deal: "deals",
+};
+/** Which identity keys make sense per object type. */
+const VALID_KEYS = {
+    account: ["domain", "name"],
+    contact: ["email", "name"],
+    deal: ["name"],
+};
+/**
+ * Bookkeeping fields excluded from the richness count: they are populated
+ * (or not) by the sync machinery, not by the quality of the record's data,
+ * so counting them would let plumbing decide which record survives a merge.
+ */
+const NON_DATA_FIELDS = new Set(["id", "provider", "crmId", "identities", "raw", "provenance"]);
+function populatedDataFields(record) {
+    return Object.entries(record).filter(([field, value]) => !NON_DATA_FIELDS.has(field) && value !== undefined && value !== null && value !== "").length;
+}
+/** True when id `a` sorts before id `b` — numeric when both ids are numeric. */
+function idBefore(a, b) {
+    const numericA = Number(a);
+    const numericB = Number(b);
+    if (Number.isFinite(numericA) && Number.isFinite(numericB) && numericA !== numericB) {
+        return numericA < numericB;
+    }
+    return a < b;
+}
+/** Normalize a record's identity key; undefined when the field is empty. */
+export function dedupeKey(record, key) {
+    if (key === "domain")
+        return normalizeDomain(record.domain);
+    const raw = record[key];
+    if (raw === undefined || raw === null)
+        return undefined;
+    const normalized = String(raw).trim().toLowerCase();
+    return normalized || undefined;
+}
+export function buildDedupePlan(snapshot, options) {
+    const keep = options.keep ?? "richest";
+    const maxOperations = options.maxOperations ?? 500;
+    if (!VALID_KEYS[options.objectType].includes(options.key)) {
+        throw new Error(`Cannot dedupe ${COLLECTIONS[options.objectType]} by "${options.key}". Valid keys for ${options.objectType}s: ${VALID_KEYS[options.objectType].join(", ")}.`);
+    }
+    if (keep !== "richest" && keep !== "oldest") {
+        throw new Error(`--keep must be richest or oldest, got "${keep}".`);
+    }
+    const records = snapshot[COLLECTIONS[options.objectType]];
+    const groups = new Map();
+    for (const record of records) {
+        const key = dedupeKey(record, options.key);
+        if (!key)
+            continue; // records without the identity key cannot be duplicates by it
+        const existing = groups.get(key) ?? [];
+        existing.push(record);
+        groups.set(key, existing);
+    }
+    for (const [key, members] of Array.from(groups.entries())) {
+        if (members.length < 2)
+            groups.delete(key);
+    }
+    if (groups.size > maxOperations) {
+        throw new Error(`Found ${groups.size} duplicate groups — above the ${maxOperations}-group safety cap. Raise --max-operations explicitly after reviewing the volume.`);
+    }
+    const operations = [];
+    let duplicateRecordCount = 0;
+    for (const [key, members] of groups) {
+        duplicateRecordCount += members.length;
+        // deterministic survivor: richest data first (ties to lowest id), or
+        // simply the lowest id when keeping the oldest
+        const survivor = [...members].sort((a, b) => {
+            if (keep === "richest") {
+                const richness = populatedDataFields(b) - populatedDataFields(a);
+                if (richness !== 0)
+                    return richness;
+            }
+            return idBefore(String(a.id), String(b.id)) ? -1 : 1;
+        })[0];
+        const groupIds = members
+            .map((member) => String(member.id))
+            .sort((a, b) => (idBefore(a, b) ? -1 : 1));
+        const survivorName = typeof survivor.name === "string" && survivor.name
+            ? survivor.name
+            : typeof survivor.email === "string" && survivor.email
+                ? survivor.email
+                : String(survivor.id);
+        const keepDetail = keep === "richest"
+            ? `${populatedDataFields(survivor)} populated data fields, the most in the group (ties break to the lowest id)`
+            : "the lowest id in the group (oldest record)";
+        operations.push({
+            id: `op_${stableHash(`dedupe:${options.objectType}:${options.key}:${groupIds.join(",")}`)}`,
+            objectType: options.objectType,
+            objectId: String(survivor.id),
+            operation: "merge_records",
+            field: "merge",
+            beforeValue: groupIds,
+            afterValue: String(survivor.id),
+            reason: options.reason ??
+                `${members.length} ${COLLECTIONS[options.objectType]} share ${options.key} "${key}". Merge into "${survivorName}" (${survivor.id}) — survivor has ${keepDetail}.`,
+            riskLevel: "high",
+            approvalRequired: true,
+            sourceRuleOrPolicy: "dedupe",
+            groupId: `grp_${options.objectType}_${String(survivor.id)}`,
+            rollback: "IRREVERSIBLE: provider merges cannot be unmerged. The pre-apply snapshot retains every record's field values; recreate a record manually from it if a merge was wrong.",
+        });
+    }
+    return {
+        id: `patch_plan_${stableHash(`dedupe:${snapshot.provider}:${snapshot.generatedAt}:${options.objectType}:${options.key}:${keep}:${operations.length}`)}`,
+        title: `Dedupe: ${COLLECTIONS[options.objectType]} sharing the same ${options.key}`,
+        createdAt: snapshot.generatedAt,
+        status: operations.length > 0 ? "needs_approval" : "draft",
+        dryRun: true,
+        summary: `${groups.size} duplicate group(s) across ${duplicateRecordCount} ${COLLECTIONS[options.objectType]} (key: ${options.key}, keep: ${keep}); ${operations.length} proposed dry-run merge_records operation(s). Merges are IRREVERSIBLE — review each survivor before approving.`,
+        findings: [],
+        operations,
+    };
+}

package/dist/enrich.d.ts

@@ -0,0 +1,220 @@
+import type { CanonicalGtmSnapshot, PatchPlan } from "./types.ts";
+/**
+ * The enrich layer: governed append/refresh of third-party data into the CRM.
+ *
+ * Every enrichment vendor ships fire-and-forget writeback — data lands without
+ * a diff, without approval, over whatever a human typed. This layer inverts
+ * that: a source (Apollo pull, Clay ingest) feeds a deterministic matcher,
+ * the matcher feeds a fill-blanks-only patch plan, and the plan goes through
+ * the existing dry-run → approval → apply contract. Every proposed value is
+ * traceable to the source payload that produced it (`GtmEvidence` on the
+ * plan), and every write carries a `beforeValue` for apply-time
+ * compare-and-set.
+ *
+ * State lives in a profile-scoped, append-only run store
+ * (`~/.fullstackgtm/profiles/<profile>/enrich/runs/`) that is checkpoint,
+ * staleness ledger, and observability surface in one. The CLI never writes
+ * `fsgtm_enriched_at`-style custom properties into the customer's portal.
+ *
+ * Recurring execution belongs to the horizontal scheduler (docs/schedule.md);
+ * enrich owns no cron logic.
+ */
+export type EnrichObjectType = "company" | "contact";
+export type EnrichSourceKind = "api" | "ingest";
+export type EnrichSourceConfig = {
+    kind: EnrichSourceKind;
+    /** Ingest staging format; csv (column headers) or json (dotted paths). */
+    format?: "csv" | "json";
+};
+export type EnrichAmbiguousPolicy = "skip" | "suggest";
+export type EnrichMatchConfig = {
+    /** Ordered match keys, evaluated against the snapshot. */
+    keys: string[];
+    /** Multi-hit behavior; default skip. */
+    onAmbiguous?: EnrichAmbiguousPolicy;
+};
+export type EnrichFieldConfig = {
+    /** CRM property: canonical field name, or a default HubSpot property name. */
+    crm: string;
+    /** sourceId → dotted JSON path (api/json) or column header (ingest csv). */
+    from: Record<string, string>;
+    /** Opt into `enrich refresh`; fields without it are set once, never revisited. */
+    refresh?: boolean;
+    /** Staleness window for refresh; falls back to policy.defaultStaleDays. */
+    staleDays?: number;
+    /** Per-field conflict policy override. MVP: only "never". */
+    policy?: "never";
+};
+export type EnrichPolicyConfig = {
+    /** Conflict policy ladder. MVP ships "never" (fill blanks only). */
+    overwrite: "never";
+    defaultStaleDays?: number;
+};
+export type EnrichConfig = {
+    sources: Record<string, EnrichSourceConfig>;
+    match: Partial<Record<EnrichObjectType, EnrichMatchConfig>>;
+    fields: Partial<Record<EnrichObjectType, EnrichFieldConfig[]>>;
+    policy: EnrichPolicyConfig;
+};
+export declare const ENRICH_CONFIG_FILE_NAME = "enrich.config.json";
+export declare const DEFAULT_STALE_DAYS = 90;
+/** API source ids the MVP can pull from. */
+export declare const SUPPORTED_API_SOURCES: string[];
+/** Resolve a config `crm` field name to the canonical snapshot field. */
+export declare function resolveCrmField(objectType: EnrichObjectType, name: string): string;
+/**
+ * Strict, up-front validation (the 0.18 lesson: a config crash mid-run is
+ * worse than a refused config). Every problem names the offending entry and
+ * the accepted values.
+ */
+export declare function parseEnrichConfig(raw: string): EnrichConfig;
+export declare function loadEnrichConfig(path: string): EnrichConfig;
+export declare function parseCsv(text: string): Array<Record<string, string>>;
+export type EnrichSourceRecord = {
+    /** e.g. "apollo:org_abc", "clay:row-3". Lands on stamps as sourceRecordId. */
+    id: string;
+    objectType: EnrichObjectType;
+    /** Match-key values (key name → raw value), extracted by the source adapter. */
+    keys: Record<string, string | undefined>;
+    /** Raw source payload; field paths and evidence excerpts read from it. */
+    payload: Record<string, unknown>;
+};
+/** Read a value from a payload: exact key first (CSV headers), then dotted path. */
+export declare function sourceValueAt(payload: Record<string, unknown>, path: string): unknown;
+/** Case-insensitive header lookup for ingest rows ("Email" matches key "email"). */
+export declare function ingestKeyValue(row: Record<string, unknown>, key: string): string | undefined;
+export type MatchOutcome = {
+    status: "matched";
+    recordId: string;
+    matchedKey: string;
+} | {
+    status: "unmatched";
+} | {
+    status: "ambiguous";
+    key: string;
+    candidateIds: string[];
+};
+export declare function matchSourceRecord(snapshot: CanonicalGtmSnapshot, objectType: EnrichObjectType, keys: string[], sourceKeys: Record<string, string | undefined>): MatchOutcome;
+export type EnrichMode = "append" | "refresh";
+export type EnrichCounts = {
+    fetched: number;
+    matched: number;
+    unmatched: number;
+    ambiguous: number;
+    opsEmitted: number;
+};
+export type EnrichStamp = {
+    objectType: EnrichObjectType;
+    objectId: string;
+    /** Canonical field name. */
+    field: string;
+    enrichedAt: string;
+    sourceRecordId: string;
+    /** Source value at stamp time (refresh change-detection observability). */
+    value?: unknown;
+};
+export type EnrichAmbiguity = {
+    sourceRecordId: string;
+    key: string;
+    candidateIds: string[];
+};
+export type EnrichWorkItem = {
+    objectType: EnrichObjectType;
+    objectId: string;
+    /** Canonical field name. */
+    field: string;
+};
+export type BuildEnrichPlanOptions = {
+    config: EnrichConfig;
+    source: string;
+    mode: EnrichMode;
+    snapshot: CanonicalGtmSnapshot;
+    records: EnrichSourceRecord[];
+    /**
+     * Refresh only: the stale (record, field) work set computed from run-store
+     * stamps. Refresh proposes writes ONLY for work-set cells — fields the
+     * ledger proves enrich itself stamped — so policy "never" still never
+     * overwrites a value enrich did not put there.
+     */
+    workSet?: EnrichWorkItem[];
+    now?: () => Date;
+    runLabel: string;
+};
+export type EnrichPlanResult = {
+    plan: PatchPlan;
+    counts: EnrichCounts;
+    stamps: EnrichStamp[];
+    ambiguities: EnrichAmbiguity[];
+    unmatchedSourceIds: string[];
+};
+/**
+ * Match source records against the snapshot and emit a patch plan under the
+ * conflict policy. Append fills blanks only; refresh proposes updates for
+ * stale stamped fields whose source value actually changed (beforeValue =
+ * current CRM value → apply-time compare-and-set rejects drifted records).
+ */
+export declare function buildEnrichPlan(options: BuildEnrichPlanOptions): EnrichPlanResult;
+/** Latest stamp per (objectType, objectId, field) across a source's runs. */
+export declare function latestStamps(runs: EnrichRun[], source: string): Map<string, EnrichStamp>;
+export declare function staleDaysFor(config: EnrichConfig, objectType: EnrichObjectType, field: string): number;
+/**
+ * Stale (record, field) cells: stamped by this source, refresh-eligible in
+ * the config, and older than the staleness window (per-field staleDays →
+ * policy.defaultStaleDays → 90; --stale-days overrides all).
+ */
+export declare function selectStaleWork(config: EnrichConfig, runs: EnrichRun[], source: string, options?: {
+    now?: () => Date;
+    staleDaysOverride?: number;
+}): EnrichWorkItem[];
+export type EnrichRunMode = EnrichMode | "ingest";
+export type EnrichRun = {
+    id: string;
+    runLabel: string;
+    source: string;
+    mode: EnrichRunMode;
+    startedAt: string;
+    /** null while in progress — `status` surfaces it as an interrupted run. */
+    completedAt: string | null;
+    /** Resume point for an interrupted pull (last processed pull key). */
+    cursor: string | null;
+    counts: EnrichCounts;
+    planIds: string[];
+    stamps: EnrichStamp[];
+    /** Staged source rows (ingest mode only), consumed by append/refresh. */
+    staged?: Array<Record<string, unknown>>;
+    /** Object type of the staged rows (ingest mode only). */
+    stagedObjectType?: EnrichObjectType;
+    /**
+     * Source records pulled so far (api pulls with --save). Together with
+     * `cursor` this makes the checkpoint complete: a resumed run replays the
+     * already-paid-for payloads instead of re-fetching them.
+     */
+    pulled?: EnrichSourceRecord[];
+    /** Pull keys the source returned no data for (api pulls with --save). */
+    missedKeys?: string[];
+    /** Match collisions recorded for review (candidate ids included). */
+    ambiguities?: EnrichAmbiguity[];
+};
+export declare function enrichRunId(source: string, runLabel: string): string;
+export declare function enrichRunsDir(baseDir?: string): string;
+export interface EnrichRunStore {
+    /** Append a new run; refuses an existing label (runs are append-only). */
+    append(run: EnrichRun): Promise<EnrichRun>;
+    /** Update an in-progress run (cursor checkpoint, finalization) in place. */
+    update(run: EnrichRun): Promise<EnrichRun>;
+    get(runLabel: string): Promise<EnrichRun | null>;
+    list(): Promise<EnrichRun[]>;
+    latest(filter?: {
+        source?: string;
+        mode?: EnrichRunMode;
+    }): Promise<EnrichRun | null>;
+}
+export declare function createFileEnrichRunStore(directory?: string): EnrichRunStore;
+/**
+ * Infer the object type of staged rows from the configured match keys: the
+ * type whose key columns actually appear on the rows. Exactly one hit wins;
+ * zero or two is an error asking for --objects.
+ */
+export declare function inferIngestObjectType(config: EnrichConfig, source: string, rows: Array<Record<string, unknown>>): EnrichObjectType;
+/** Turn staged ingest rows into source records for the matcher. */
+export declare function stagedSourceRecords(config: EnrichConfig, source: string, run: EnrichRun): EnrichSourceRecord[];