fullstackgtm 0.21.2 → 0.22.0

package/src/cli.ts

@@ -75,6 +75,8 @@ import {
 } from "./llm.ts";
 import { resolveRecord, type ResolveCandidate } from "./resolve.ts";
 import { buildBulkUpdatePlan } from "./bulkUpdate.ts";
+import { buildDedupePlan, type DedupeOptions } from "./dedupe.ts";
+import { buildReassignPlans, type ReassignObjectType } from "./reassign.ts";
 import { suggestValues, type ValueSuggestion } from "./suggest.ts";
 import type { FieldMappings } from "./mappings.ts";
 import type {
@@ -135,7 +137,7 @@ Usage:
                                                against the stored capture it cites before it's accepted — then
                                                compute deterministic front states and drift, render the field
                                                report. refresh = capture → classify → drift → report in one step
-  fullstackgtm bulk-update <account|contact|deal> --where <expr> [--where …] (--set <field>=<value> [--set …] | --archive | --create-task <text>) [--require <field>=<value> …] [--guard <object>:<where>[;<where>]:<none|some> …] [source options] [--save] [--json] [--out <path>]
+  fullstackgtm bulk-update <account|contact|deal> --where <expr> [--where …] (--set <field>=<value> [--set …] | --archive [--force-archive-duplicates] | --create-task <text>) [--require <field>=<value> …] [--guard <object>:<where>[;<where>]:<none|some> …] [source options] [--save] [--json] [--out <path>]
                                                governed generic writes: filter the snapshot
                                                (field=value, field!=value, field~substr, field!~substr,
                                                field:empty, field:notempty, '|' = any-of; canonical fields
@@ -147,6 +149,35 @@ Usage:
                                                apply time (incl. mid-apply rechecks); equality filters
                                                double as preconditions; per-record ops apply
                                                all-or-nothing; guards assert cross-record conditions.
+                                               --set <field>=from:<sourceField> derives the value PER
+                                               RECORD from the snapshot (relational sources like
+                                               account.ownerId included); records whose source is empty
+                                               are skipped and counted, never guessed. --archive refuses
+                                               records that share their identity key (account domain /
+                                               contact email) with another record — merge those with
+                                               \`dedupe\` instead, or --force-archive-duplicates.
+  fullstackgtm dedupe <account|contact|deal> --key <domain|email|name> [--keep richest|oldest] [source options] [--reason <text>] [--max-operations <n>] [--save] [--json] [--out <path>]
+                                               find duplicate groups by normalized identity key and build
+                                               a dry-run plan of merge_records operations — one per group,
+                                               deterministic survivor (richest = most populated data
+                                               fields, ties to lowest id; oldest = lowest id). Approve and
+                                               apply like any plan; merges are IRREVERSIBLE on apply.
+  fullstackgtm reassign --from <ownerId> --to <ownerId> [--objects account,contact,deal] [--where <expr> …] [--except-deal-stage <stage>] [--include-closed-deals] [source options] [--save] [--json] [--out <path>]
+                                               ownership handoff playbook: one bulk-update-style plan per
+                                               object type (ownerId=<from> → <to>). Extra --where scoping
+                                               is account-lifted for deals/contacts (domain~.de becomes
+                                               account.domain~.de); --except-deal-stage <stage> excludes
+                                               deals in that stage AND every record whose account has an
+                                               open deal in it, re-verified per record at apply time.
+                                               Deal plans cover open deals only unless
+                                               --include-closed-deals.
+  fullstackgtm fix --rule <ruleId> --provider <name> [--min-confidence high|low] [--include-creates] [--today <iso>] [--yes]
+                                               one-shot composite: audit ONE rule → save the plan →
+                                               suggest values → approve only suggestion-backed operations
+                                               meeting the confidence bar (plus operations that need no
+                                               value) → with --yes, apply through the provider and print a
+                                               stage-by-stage summary. Without --yes it stops after
+                                               approval and prints the apply command.
   fullstackgtm suggest --plan-id <id> | --plan <path>  [source options] [--json] [--out <path>]
                                                derive values for requires_human_* placeholders
                                                from snapshot evidence, with confidence + reasons
@@ -1257,29 +1288,213 @@ async function bulkUpdateCommand(args: string[]) {
     where,
     set: Object.keys(set).length > 0 ? set : undefined,
     archive: rest.includes("--archive"),
+    forceArchiveDuplicates: rest.includes("--force-archive-duplicates"),
     createTask: option(rest, "--create-task") ?? undefined,
     require: repeatedOption(rest, "--require"),
     guard: repeatedOption(rest, "--guard"),
     reason: option(rest, "--reason") ?? undefined,
     maxOperations: numericOption(rest, "--max-operations"),
   });
-  const out = option(rest, "--out");
+  await emitPlan(plan, rest);
+}
+
+/** Shared plan output plumbing: --out, --save (with the approve/apply hint), --json or markdown. */
+async function emitPlan(plan: PatchPlan, args: string[]) {
+  const out = option(args, "--out");
   if (out) {
     writeFileSync(resolve(process.cwd(), out), `${JSON.stringify(plan, null, 2)}\n`);
   }
-  if (rest.includes("--save")) {
+  if (args.includes("--save")) {
     await createFilePlanStore().save(plan);
     console.error(
       `Saved plan ${plan.id} (${plan.operations.length} operations). Review with \`fullstackgtm plans show ${plan.id}\`, approve with \`fullstackgtm plans approve ${plan.id} --operations <ids|all>\`, then \`fullstackgtm apply --plan-id ${plan.id} --provider <name>\`.`,
     );
   }
-  if (rest.includes("--json")) {
+  if (args.includes("--json")) {
     console.log(JSON.stringify(plan, null, 2));
   } else {
     console.log(patchPlanToMarkdown(plan));
   }
 }
 
+/**
+ * Governed duplicate cleanup: group by a normalized identity key, propose one
+ * merge_records per duplicate group with a deterministic survivor. Never
+ * writes — approve and apply the plan like any audit plan.
+ */
+async function dedupeCommand(args: string[]) {
+  const [objectType, ...rest] = args;
+  if (!objectType || !["account", "contact", "deal"].includes(objectType)) {
+    throw new Error(
+      "Usage: fullstackgtm dedupe <account|contact|deal> --key <domain|email|name> [--keep richest|oldest] [source options] [--reason <text>] [--max-operations <n>] [--save] [--out <path>] [--json]",
+    );
+  }
+  const key = option(rest, "--key");
+  if (!key || !["domain", "email", "name"].includes(key)) {
+    throw new Error("dedupe requires --key <domain|email|name> (the identity field duplicates share).");
+  }
+  const keep = option(rest, "--keep") ?? undefined;
+  const snapshot = await readSnapshot(rest);
+  const plan = buildDedupePlan(snapshot, {
+    objectType: objectType as DedupeOptions["objectType"],
+    key: key as DedupeOptions["key"],
+    keep: (keep ?? undefined) as DedupeOptions["keep"],
+    reason: option(rest, "--reason") ?? undefined,
+    maxOperations: numericOption(rest, "--max-operations"),
+  });
+  await emitPlan(plan, rest);
+}
+
+/**
+ * Ownership handoff playbook: compile one bulk-update-style plan per object
+ * type. Each plan carries its full filter, so eligibility (including the
+ * --except-deal-stage exclusion) is re-verified per record at apply time.
+ */
+async function reassignCommand(args: string[]) {
+  const from = option(args, "--from");
+  const to = option(args, "--to");
+  if (!from || !to) {
+    throw new Error(
+      "Usage: fullstackgtm reassign --from <ownerId> --to <ownerId> [--objects account,contact,deal] [--where <expr> …] [--except-deal-stage <stage>] [--include-closed-deals] [source options] [--reason <text>] [--max-operations <n>] [--save] [--out <path>] [--json]",
+    );
+  }
+  const objects = option(args, "--objects")
+    ?.split(",")
+    .map((value) => value.trim())
+    .filter(Boolean) as ReassignObjectType[] | undefined;
+  const snapshot = await readSnapshot(args);
+  const plans = buildReassignPlans(snapshot, {
+    fromOwnerId: from,
+    toOwnerId: to,
+    objects,
+    where: repeatedOption(args, "--where"),
+    exceptDealStage: option(args, "--except-deal-stage") ?? undefined,
+    includeClosedDeals: args.includes("--include-closed-deals"),
+    reason: option(args, "--reason") ?? undefined,
+    maxOperations: numericOption(args, "--max-operations"),
+  });
+  const out = option(args, "--out");
+  if (out) {
+    writeFileSync(resolve(process.cwd(), out), `${JSON.stringify(plans, null, 2)}\n`);
+  }
+  if (args.includes("--json")) {
+    console.log(JSON.stringify(plans, null, 2));
+    return;
+  }
+  const store = args.includes("--save") ? createFilePlanStore() : null;
+  for (const plan of plans) {
+    if (store) await store.save(plan);
+    console.log(`${plan.id}  ${String(plan.operations.length).padStart(3)} operation(s)  ${plan.title}`);
+    console.log(`    ${plan.summary}`);
+  }
+  if (store) {
+    console.log(
+      `\nSaved ${plans.length} plan(s). For each: \`fullstackgtm plans show <id>\`, \`fullstackgtm plans approve <id> --operations <ids|all>\`, then \`fullstackgtm apply --plan-id <id> --provider <name>\`.`,
+    );
+  } else {
+    console.log("\nDry run only — re-run with --save to store the plans for approval.");
+  }
+}
+
+/**
+ * One-shot composite for a single audit rule: audit → save → suggest →
+ * approve only suggestion-backed operations meeting the confidence bar (plus
+ * operations that carry concrete values and need no human input) → apply
+ * (only with --yes). Every stage goes through the same gates as the manual
+ * chain; placeholder values below the bar stay unapproved.
+ */
+async function fixCommand(args: string[]) {
+  const ruleId = option(args, "--rule");
+  const provider = option(args, "--provider");
+  if (!ruleId || !provider) {
+    throw new Error(
+      "Usage: fullstackgtm fix --rule <ruleId> --provider <name> [--min-confidence high|low] [--include-creates] [--today <iso>] [--yes]",
+    );
+  }
+  const minConfidence = option(args, "--min-confidence") ?? "high";
+  if (!["high", "low"].includes(minConfidence)) {
+    throw new Error("--min-confidence must be high or low");
+  }
+  const includeCreates = args.includes("--include-creates");
+
+  const loaded = loadConfig(option(args, "--config") ?? undefined);
+  const configured = await resolveConfiguredRules(loaded);
+  const rule = configured.find((candidate) => candidate.id === ruleId);
+  if (!rule) {
+    throw new Error(`Unknown rule: ${ruleId}. Available rules: ${configured.map((r) => r.id).join(", ")}`);
+  }
+  const policy = mergePolicy(defaultPolicy(), loaded?.config);
+  const today = option(args, "--today");
+  if (today) policy.today = today;
+
+  const snapshot = await readSnapshot(args);
+  const plan = auditSnapshot(snapshot, policy, [rule]);
+  if (plan.operations.length === 0) {
+    console.log(`fix ${ruleId}: audit proposed 0 operations — nothing to fix.`);
+    return;
+  }
+  const store = createFilePlanStore();
+  await store.save(plan);
+
+  const suggestions = suggestValues(plan, snapshot);
+  const accepted = new Set(minConfidence === "low" ? ["high", "low"] : ["high"]);
+  const overrides: Record<string, string> = {};
+  let belowBar = 0;
+  for (const suggestion of suggestions) {
+    if (
+      suggestion.suggestedValue &&
+      (accepted.has(suggestion.confidence) || (includeCreates && suggestion.confidence === "create"))
+    ) {
+      overrides[suggestion.operationId] = suggestion.suggestedValue;
+    } else {
+      belowBar += 1;
+    }
+  }
+  // Approve operations whose placeholder got a qualifying suggested value,
+  // plus operations that already carry a concrete value (no human input
+  // needed — nothing to guess). Everything else stays unapproved.
+  const placeholderIds = new Set(suggestions.map((suggestion) => suggestion.operationId));
+  const approvedIds = plan.operations
+    .map((operation) => operation.id)
+    .filter((id) => overrides[id] !== undefined || !placeholderIds.has(id));
+
+  const lines = [
+    `fix ${ruleId} via ${provider}:`,
+    `  proposed:  ${plan.operations.length} operation(s) — plan ${plan.id} (saved)`,
+    `  suggested: ${Object.keys(overrides).length} value(s) at ${minConfidence}+ confidence${includeCreates ? " (creates included)" : ""}${belowBar > 0 ? `; ${belowBar} below the bar (left unapproved)` : ""}`,
+    `  approved:  ${approvedIds.length} of ${plan.operations.length}`,
+  ];
+  if (approvedIds.length === 0) {
+    lines.push("  applied:   0 — no operation met the confidence bar");
+    console.log(lines.join("\n"));
+    console.log(
+      `\nWiden with --min-confidence low / --include-creates, or approve manually: \`fullstackgtm plans approve ${plan.id} --operations <ids> --value <opId>=<value>\`.`,
+    );
+    return;
+  }
+  await store.approveOperations(plan.id, approvedIds, overrides);
+
+  if (!args.includes("--yes")) {
+    lines.push("  applied:   0 (stopped before apply — pass --yes to write)");
+    console.log(lines.join("\n"));
+    console.log(`\nApply with:\n  fullstackgtm apply --plan-id ${plan.id} --provider ${provider}`);
+    return;
+  }
+  const connector = await connectorFor(provider, args);
+  const run = await applyPatchPlan(connector, plan, {
+    approvedOperationIds: approvedIds,
+    valueOverrides: overrides,
+  });
+  await store.recordRun(plan.id, run);
+  const counts: Record<string, number> = { applied: 0, conflict: 0, skipped: 0, failed: 0 };
+  for (const result of run.results) counts[result.status] = (counts[result.status] ?? 0) + 1;
+  lines.push(
+    `  applied:   ${counts.applied} · conflicts: ${counts.conflict} · skipped: ${counts.skipped} · failed: ${counts.failed}`,
+  );
+  console.log(lines.join("\n"));
+  if (run.status === "failed") process.exitCode = 1;
+}
+
 async function suggest(args: string[]) {
   const planId = option(args, "--plan-id");
   const planPath = option(args, "--plan");
@@ -2124,6 +2339,18 @@ export async function runCli(argv: string[]) {
     await bulkUpdateCommand(args);
     return;
   }
+  if (command === "dedupe") {
+    await dedupeCommand(args);
+    return;
+  }
+  if (command === "reassign") {
+    await reassignCommand(args);
+    return;
+  }
+  if (command === "fix") {
+    await fixCommand(args);
+    return;
+  }
   if (command === "market") {
     await marketCommand(args);
     return;

package/src/dedupe.ts

@@ -0,0 +1,182 @@
+/**
+ * Governed duplicate cleanup: `dedupe` groups records by a normalized
+ * identity key (account domain, contact email, or name) and builds a
+ * dry-run PatchPlan of merge_records operations — one per duplicate group,
+ * with a DETERMINISTIC survivor. It NEVER writes — the plan flows through
+ * the same plans-approve → apply gate as every other plan.
+ *
+ * The merge contract matches the connectors (see mergeRecords in
+ * connectors/hubspot.ts): afterValue = the survivor id, beforeValue = the
+ * ids of EVERY record in the group (survivor included). Merges are
+ * IRREVERSIBLE on every provider that supports them, so every operation is
+ * riskLevel high and approvalRequired.
+ *
+ * Survivor selection ("--keep"):
+ *   richest (default)  the record with the most non-empty canonical data
+ *                      fields (bookkeeping fields like id/crmId/identities
+ *                      don't count); ties break to the lowest numeric id
+ *   oldest             the lowest numeric id (CRMs assign ids in creation
+ *                      order)
+ */
+import { normalizeDomain } from "./merge.ts";
+import { stableHash } from "./rules.ts";
+import type {
+  CanonicalGtmSnapshot,
+  GtmObjectType,
+  PatchOperation,
+  PatchPlan,
+} from "./types.ts";
+
+export type DedupeOptions = {
+  objectType: "account" | "contact" | "deal";
+  /** identity key records are grouped by (normalized before grouping) */
+  key: "domain" | "email" | "name";
+  /** survivor selection — deterministic either way (default "richest") */
+  keep?: "richest" | "oldest";
+  reason?: string;
+  /** refuse to build plans larger than this (default 500 operations) */
+  maxOperations?: number;
+};
+
+const COLLECTIONS: Record<DedupeOptions["objectType"], "accounts" | "contacts" | "deals"> = {
+  account: "accounts",
+  contact: "contacts",
+  deal: "deals",
+};
+
+/** Which identity keys make sense per object type. */
+const VALID_KEYS: Record<DedupeOptions["objectType"], Array<DedupeOptions["key"]>> = {
+  account: ["domain", "name"],
+  contact: ["email", "name"],
+  deal: ["name"],
+};
+
+/**
+ * Bookkeeping fields excluded from the richness count: they are populated
+ * (or not) by the sync machinery, not by the quality of the record's data,
+ * so counting them would let plumbing decide which record survives a merge.
+ */
+const NON_DATA_FIELDS = new Set(["id", "provider", "crmId", "identities", "raw", "provenance"]);
+
+function populatedDataFields(record: Record<string, unknown>): number {
+  return Object.entries(record).filter(
+    ([field, value]) =>
+      !NON_DATA_FIELDS.has(field) && value !== undefined && value !== null && value !== "",
+  ).length;
+}
+
+/** True when id `a` sorts before id `b` — numeric when both ids are numeric. */
+function idBefore(a: string, b: string): boolean {
+  const numericA = Number(a);
+  const numericB = Number(b);
+  if (Number.isFinite(numericA) && Number.isFinite(numericB) && numericA !== numericB) {
+    return numericA < numericB;
+  }
+  return a < b;
+}
+
+/** Normalize a record's identity key; undefined when the field is empty. */
+export function dedupeKey(
+  record: Record<string, unknown>,
+  key: DedupeOptions["key"],
+): string | undefined {
+  if (key === "domain") return normalizeDomain(record.domain as string | undefined);
+  const raw = record[key];
+  if (raw === undefined || raw === null) return undefined;
+  const normalized = String(raw).trim().toLowerCase();
+  return normalized || undefined;
+}
+
+export function buildDedupePlan(
+  snapshot: CanonicalGtmSnapshot,
+  options: DedupeOptions,
+): PatchPlan {
+  const keep = options.keep ?? "richest";
+  const maxOperations = options.maxOperations ?? 500;
+  if (!VALID_KEYS[options.objectType].includes(options.key)) {
+    throw new Error(
+      `Cannot dedupe ${COLLECTIONS[options.objectType]} by "${options.key}". Valid keys for ${options.objectType}s: ${VALID_KEYS[options.objectType].join(", ")}.`,
+    );
+  }
+  if (keep !== "richest" && keep !== "oldest") {
+    throw new Error(`--keep must be richest or oldest, got "${keep}".`);
+  }
+
+  const records = snapshot[COLLECTIONS[options.objectType]] as Array<Record<string, unknown>>;
+  const groups = new Map<string, Array<Record<string, unknown>>>();
+  for (const record of records) {
+    const key = dedupeKey(record, options.key);
+    if (!key) continue; // records without the identity key cannot be duplicates by it
+    const existing = groups.get(key) ?? [];
+    existing.push(record);
+    groups.set(key, existing);
+  }
+  for (const [key, members] of Array.from(groups.entries())) {
+    if (members.length < 2) groups.delete(key);
+  }
+
+  if (groups.size > maxOperations) {
+    throw new Error(
+      `Found ${groups.size} duplicate groups — above the ${maxOperations}-group safety cap. Raise --max-operations explicitly after reviewing the volume.`,
+    );
+  }
+
+  const operations: PatchOperation[] = [];
+  let duplicateRecordCount = 0;
+  for (const [key, members] of groups) {
+    duplicateRecordCount += members.length;
+    // deterministic survivor: richest data first (ties to lowest id), or
+    // simply the lowest id when keeping the oldest
+    const survivor = [...members].sort((a, b) => {
+      if (keep === "richest") {
+        const richness = populatedDataFields(b) - populatedDataFields(a);
+        if (richness !== 0) return richness;
+      }
+      return idBefore(String(a.id), String(b.id)) ? -1 : 1;
+    })[0];
+    const groupIds = members
+      .map((member) => String(member.id))
+      .sort((a, b) => (idBefore(a, b) ? -1 : 1));
+    const survivorName =
+      typeof survivor.name === "string" && survivor.name
+        ? survivor.name
+        : typeof survivor.email === "string" && survivor.email
+          ? survivor.email
+          : String(survivor.id);
+    const keepDetail =
+      keep === "richest"
+        ? `${populatedDataFields(survivor)} populated data fields, the most in the group (ties break to the lowest id)`
+        : "the lowest id in the group (oldest record)";
+    operations.push({
+      id: `op_${stableHash(`dedupe:${options.objectType}:${options.key}:${groupIds.join(",")}`)}`,
+      objectType: options.objectType as GtmObjectType,
+      objectId: String(survivor.id),
+      operation: "merge_records",
+      field: "merge",
+      beforeValue: groupIds,
+      afterValue: String(survivor.id),
+      reason:
+        options.reason ??
+        `${members.length} ${COLLECTIONS[options.objectType]} share ${options.key} "${key}". Merge into "${survivorName}" (${survivor.id}) — survivor has ${keepDetail}.`,
+      riskLevel: "high",
+      approvalRequired: true,
+      sourceRuleOrPolicy: "dedupe",
+      groupId: `grp_${options.objectType}_${String(survivor.id)}`,
+      rollback:
+        "IRREVERSIBLE: provider merges cannot be unmerged. The pre-apply snapshot retains every record's field values; recreate a record manually from it if a merge was wrong.",
+    });
+  }
+
+  return {
+    id: `patch_plan_${stableHash(
+      `dedupe:${snapshot.provider}:${snapshot.generatedAt}:${options.objectType}:${options.key}:${keep}:${operations.length}`,
+    )}`,
+    title: `Dedupe: ${COLLECTIONS[options.objectType]} sharing the same ${options.key}`,
+    createdAt: snapshot.generatedAt,
+    status: operations.length > 0 ? "needs_approval" : "draft",
+    dryRun: true,
+    summary: `${groups.size} duplicate group(s) across ${duplicateRecordCount} ${COLLECTIONS[options.objectType]} (key: ${options.key}, keep: ${keep}); ${operations.length} proposed dry-run merge_records operation(s). Merges are IRREVERSIBLE — review each survivor before approving.`,
+    findings: [],
+    operations,
+  };
+}

package/src/index.ts

@@ -1,5 +1,7 @@
 export { auditSnapshot, defaultPolicy } from "./audit.ts";
-export { buildBulkUpdatePlan, parseWhere, type BulkUpdateOptions } from "./bulkUpdate.ts";
+export { buildBulkUpdatePlan, isFilterableField, parseWhere, type BulkUpdateOptions } from "./bulkUpdate.ts";
+export { buildDedupePlan, dedupeKey, type DedupeOptions } from "./dedupe.ts";
+export { buildReassignPlans, type ReassignObjectType, type ReassignOptions } from "./reassign.ts";
 export {
   CONFIG_FILE_NAME,
   loadConfig,