fullstackgtm 0.25.2 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/dist/bulkUpdate.js +6 -1
- package/dist/cli.js +50 -0
- package/dist/connector.js +90 -1
- package/dist/dedupe.d.ts +6 -0
- package/dist/dedupe.js +24 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/integrity.d.ts +30 -0
- package/dist/integrity.js +128 -0
- package/dist/marketTaxonomy.d.ts +41 -0
- package/dist/marketTaxonomy.js +193 -0
- package/dist/planStore.d.ts +6 -0
- package/dist/planStore.js +10 -2
- package/dist/schedule.js +4 -0
- package/dist/types.d.ts +16 -0
- package/package.json +1 -1
- package/src/bulkUpdate.ts +6 -1
- package/src/cli.ts +62 -0
- package/src/connector.ts +96 -1
- package/src/dedupe.ts +23 -1
- package/src/index.ts +8 -0
- package/src/integrity.ts +146 -0
- package/src/marketTaxonomy.ts +288 -0
- package/src/planStore.ts +23 -4
- package/src/schedule.ts +6 -0
- package/src/types.ts +16 -0
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import { DEFAULT_MODELS, forcedToolCall, } from "./llm.js";
|
|
2
|
+
import { captureMarket, loadCaptureTexts, } from "./market.js";
|
|
3
|
+
const DEFAULT_MAX_CLAIMS = 16;
|
|
4
|
+
const DEFAULT_PER_VENDOR_CHARS = 6_000;
|
|
5
|
+
/** Stable, human-readable id from a string (claim capability or host). */
|
|
6
|
+
function slugify(value, maxWords = 6) {
|
|
7
|
+
const slug = value
|
|
8
|
+
.toLowerCase()
|
|
9
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
10
|
+
.replace(/^-+|-+$/g, "")
|
|
11
|
+
.split("-")
|
|
12
|
+
.filter(Boolean)
|
|
13
|
+
.slice(0, maxWords)
|
|
14
|
+
.join("-");
|
|
15
|
+
return slug || "item";
|
|
16
|
+
}
|
|
17
|
+
/** Second-level domain as a vendor id seed: https://www.stripe.com/ -> stripe. */
|
|
18
|
+
function vendorIdFromUrl(url) {
|
|
19
|
+
let host;
|
|
20
|
+
try {
|
|
21
|
+
host = new URL(url).hostname;
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
return slugify(url);
|
|
25
|
+
}
|
|
26
|
+
const labels = host.replace(/^www\./, "").split(".");
|
|
27
|
+
const sld = labels.length >= 2 ? labels[labels.length - 2] : labels[0];
|
|
28
|
+
return slugify(sld || host);
|
|
29
|
+
}
|
|
30
|
+
/** Disambiguate repeated ids by suffixing -2, -3, … */
|
|
31
|
+
function uniqueId(base, taken) {
|
|
32
|
+
if (!taken.has(base)) {
|
|
33
|
+
taken.add(base);
|
|
34
|
+
return base;
|
|
35
|
+
}
|
|
36
|
+
for (let n = 2;; n += 1) {
|
|
37
|
+
const candidate = `${base}-${n}`;
|
|
38
|
+
if (!taken.has(candidate)) {
|
|
39
|
+
taken.add(candidate);
|
|
40
|
+
return candidate;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
function provisionalVendors(seeds) {
|
|
45
|
+
const taken = new Set();
|
|
46
|
+
return seeds.map((seed) => {
|
|
47
|
+
const id = uniqueId(vendorIdFromUrl(seed.url), taken);
|
|
48
|
+
const host = (() => {
|
|
49
|
+
try {
|
|
50
|
+
return new URL(seed.url).hostname.replace(/^www\./, "");
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return seed.url;
|
|
54
|
+
}
|
|
55
|
+
})();
|
|
56
|
+
return {
|
|
57
|
+
id,
|
|
58
|
+
name: seed.name?.trim() || host,
|
|
59
|
+
urls: { home: seed.url, pricing: null, product: [] },
|
|
60
|
+
};
|
|
61
|
+
});
|
|
62
|
+
}
|
|
63
|
+
const TAXONOMY_SCHEMA = {
|
|
64
|
+
type: "object",
|
|
65
|
+
required: ["claims"],
|
|
66
|
+
properties: {
|
|
67
|
+
surfaceRule: {
|
|
68
|
+
type: "string",
|
|
69
|
+
description: "One sentence stating how a reader judges LOUD vs QUIET vs ABSENT for this category (e.g. hero/top-nav = LOUD, deeper pages = QUIET, nowhere = ABSENT).",
|
|
70
|
+
},
|
|
71
|
+
claims: {
|
|
72
|
+
type: "array",
|
|
73
|
+
description: "The distinct capability positions vendors in this category compete on. 8-16 of them. Only include claims you can actually see evidence for on the supplied pages.",
|
|
74
|
+
items: {
|
|
75
|
+
type: "object",
|
|
76
|
+
required: ["capability", "icp", "pricingStructure", "definition"],
|
|
77
|
+
properties: {
|
|
78
|
+
capability: {
|
|
79
|
+
type: "string",
|
|
80
|
+
description: "What is being claimed, precise enough to judge loud/quiet/absent. Max ~10 words.",
|
|
81
|
+
},
|
|
82
|
+
icp: { type: "string", description: "Which buyer/ICP this claim cell addresses (category vocabulary)." },
|
|
83
|
+
pricingStructure: {
|
|
84
|
+
type: "string",
|
|
85
|
+
description: "Which pricing structure the claim implies (e.g. per-seat, usage-based, flat, free-tier).",
|
|
86
|
+
},
|
|
87
|
+
definition: {
|
|
88
|
+
type: "string",
|
|
89
|
+
description: "Operational definition a human (or classifier) uses to score any vendor's page LOUD/QUIET/ABSENT on this claim.",
|
|
90
|
+
},
|
|
91
|
+
terms: {
|
|
92
|
+
type: "array",
|
|
93
|
+
items: { type: "string" },
|
|
94
|
+
description: "Exact buyer phrasings for this claim, for deterministic mention matching. 2-5 terms.",
|
|
95
|
+
},
|
|
96
|
+
},
|
|
97
|
+
},
|
|
98
|
+
},
|
|
99
|
+
vendors: {
|
|
100
|
+
type: "array",
|
|
101
|
+
description: "Optional refinements: a clean display name per seed URL, and a pricing-page URL if one is clearly linked.",
|
|
102
|
+
items: {
|
|
103
|
+
type: "object",
|
|
104
|
+
required: ["seedUrl"],
|
|
105
|
+
properties: {
|
|
106
|
+
seedUrl: { type: "string" },
|
|
107
|
+
name: { type: "string" },
|
|
108
|
+
pricingUrl: { type: ["string", "null"] },
|
|
109
|
+
},
|
|
110
|
+
},
|
|
111
|
+
},
|
|
112
|
+
},
|
|
113
|
+
};
|
|
114
|
+
function buildDossier(vendors, capture, perVendorChars) {
|
|
115
|
+
const { entries, textByHash } = capture;
|
|
116
|
+
const unreadable = [];
|
|
117
|
+
const blocks = [];
|
|
118
|
+
for (const vendor of vendors) {
|
|
119
|
+
const hash = entries.find((e) => e.vendorId === vendor.id && e.captureHash)?.captureHash ?? null;
|
|
120
|
+
const text = hash ? textByHash.get(hash) ?? "" : "";
|
|
121
|
+
if (!text.trim()) {
|
|
122
|
+
unreadable.push(vendor.id);
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
blocks.push(`### ${vendor.name} (${vendor.urls.home})\n${text.slice(0, perVendorChars)}`);
|
|
126
|
+
}
|
|
127
|
+
return { dossier: blocks.join("\n\n"), unreadable };
|
|
128
|
+
}
|
|
129
|
+
const INSTRUCTIONS = `You are seeding a competitive "market map" for a category. A market map breaks the category into CLAIMS — the distinct capability positions vendors compete on — so each (vendor x claim) cell can later be scored LOUD / QUIET / ABSENT from that vendor's pages.
|
|
130
|
+
|
|
131
|
+
Propose the claim taxonomy for this category from the competitor homepages below. Rules:
|
|
132
|
+
- Ground every claim in what is actually visible on the supplied pages. Do not invent positions no vendor mentions.
|
|
133
|
+
- Each claim is a cell: a precise capability, the ICP it targets, and the pricing structure it implies.
|
|
134
|
+
- Write each definition so a reader could judge ANY vendor's page LOUD/QUIET/ABSENT against it.
|
|
135
|
+
- Aim for the 8-16 claims that genuinely differentiate vendors. Prefer specific, contested positions over generic table stakes.
|
|
136
|
+
- Provide 2-5 verbatim buyer terms per claim for later mention matching.
|
|
137
|
+
- Optionally return a cleaned display name and a pricing-page URL per seed vendor when evident.`;
|
|
138
|
+
export async function suggestMarketConfig(options) {
|
|
139
|
+
const { category } = options;
|
|
140
|
+
if (options.vendors.length === 0)
|
|
141
|
+
throw new Error("suggestMarketConfig requires at least one seed vendor");
|
|
142
|
+
const maxClaims = options.maxClaims ?? DEFAULT_MAX_CLAIMS;
|
|
143
|
+
const perVendorChars = options.perVendorChars ?? DEFAULT_PER_VENDOR_CHARS;
|
|
144
|
+
const model = options.llm.model ?? DEFAULT_MODELS[options.llm.provider];
|
|
145
|
+
const vendors = provisionalVendors(options.vendors);
|
|
146
|
+
const anchorSeed = options.vendors.find((seed) => seed.anchor);
|
|
147
|
+
const anchorId = anchorSeed ? vendors[options.vendors.indexOf(anchorSeed)]?.id : undefined;
|
|
148
|
+
// Capture the seed homepages so the proposer only sees text we actually
|
|
149
|
+
// fetched (the SSRF guard in captureMarket applies to these user-supplied URLs).
|
|
150
|
+
await captureMarket({ category, vendors, claims: [] }, { dir: options.capturesDir, runLabel: "bootstrap", fetchPage: options.fetchPage, now: options.now });
|
|
151
|
+
const capture = loadCaptureTexts(category, options.capturesDir);
|
|
152
|
+
const { dossier, unreadable } = buildDossier(vendors, capture, perVendorChars);
|
|
153
|
+
if (!dossier.trim()) {
|
|
154
|
+
throw new Error(`market init --auto: none of the ${vendors.length} seed pages returned readable text — check the URLs are public homepages.`);
|
|
155
|
+
}
|
|
156
|
+
const prompt = `${INSTRUCTIONS}\n\nCategory: ${category}\n\nCompetitor homepages:\n${dossier}`;
|
|
157
|
+
const result = (await forcedToolCall(prompt, "propose_market_taxonomy", TAXONOMY_SCHEMA, model, options.llm));
|
|
158
|
+
const takenClaimIds = new Set();
|
|
159
|
+
const claims = (result.claims ?? [])
|
|
160
|
+
.filter((claim) => claim?.capability && claim?.definition)
|
|
161
|
+
.slice(0, maxClaims)
|
|
162
|
+
.map((claim) => ({
|
|
163
|
+
id: uniqueId(slugify(claim.capability), takenClaimIds),
|
|
164
|
+
capability: claim.capability.trim(),
|
|
165
|
+
icp: (claim.icp ?? "").trim() || "general",
|
|
166
|
+
pricingStructure: (claim.pricingStructure ?? "").trim() || "unspecified",
|
|
167
|
+
definition: claim.definition.trim(),
|
|
168
|
+
...(claim.terms?.length ? { terms: claim.terms.map((t) => t.trim()).filter(Boolean) } : {}),
|
|
169
|
+
}));
|
|
170
|
+
if (claims.length === 0) {
|
|
171
|
+
throw new Error("market init --auto: the model proposed no usable claims — try again or seed the taxonomy by hand.");
|
|
172
|
+
}
|
|
173
|
+
// Apply optional vendor refinements (display name + pricing URL), matched by seed URL.
|
|
174
|
+
const refinementByUrl = new Map((result.vendors ?? []).map((v) => [v.seedUrl, v]));
|
|
175
|
+
const refinedVendors = vendors.map((vendor) => {
|
|
176
|
+
const refinement = refinementByUrl.get(vendor.urls.home);
|
|
177
|
+
const pricing = refinement?.pricingUrl && /^https?:\/\//i.test(refinement.pricingUrl) ? refinement.pricingUrl : vendor.urls.pricing;
|
|
178
|
+
return {
|
|
179
|
+
...vendor,
|
|
180
|
+
name: refinement?.name?.trim() || vendor.name,
|
|
181
|
+
urls: { ...vendor.urls, pricing },
|
|
182
|
+
};
|
|
183
|
+
});
|
|
184
|
+
const config = {
|
|
185
|
+
category,
|
|
186
|
+
...(anchorId ? { anchorVendor: anchorId } : {}),
|
|
187
|
+
vendors: refinedVendors,
|
|
188
|
+
claims,
|
|
189
|
+
surfaceRule: result.surfaceRule?.trim() ||
|
|
190
|
+
"LOUD = hero copy OR top-level-nav named product with a dedicated page; QUIET = present on any indexed page below that; ABSENT = nowhere observed; UNOBSERVABLE = capture empty/failed — never score ABSENT from a failed capture.",
|
|
191
|
+
};
|
|
192
|
+
return { config, unreadableVendorIds: unreadable, model };
|
|
193
|
+
}
|
package/dist/planStore.d.ts
CHANGED
|
@@ -11,6 +11,12 @@ export type StoredPlan = {
|
|
|
11
11
|
status: ApprovalStatus;
|
|
12
12
|
approvedOperationIds: string[];
|
|
13
13
|
valueOverrides: Record<string, unknown>;
|
|
14
|
+
/**
|
|
15
|
+
* HMAC of each approved operation's content at approval time (see
|
|
16
|
+
* integrity.ts). Apply re-verifies these so a post-approval edit to the plan
|
|
17
|
+
* file is caught instead of written. Absent on plans approved before 0.26.0.
|
|
18
|
+
*/
|
|
19
|
+
approvalDigests?: Record<string, string>;
|
|
14
20
|
runs: PatchPlanRun[];
|
|
15
21
|
createdAt: string;
|
|
16
22
|
updatedAt: string;
|
package/dist/planStore.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { chmodSync, mkdirSync, readdirSync, readFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { credentialsDir, ensureSecureHomeDir, writeSecureFile } from "./credentials.js";
|
|
4
|
+
import { computeApprovalDigests, loadOrCreateSigningKey } from "./integrity.js";
|
|
4
5
|
/**
|
|
5
6
|
* Plans as JSON files in a directory (default `$FSGTM_HOME/plans`), one file
|
|
6
7
|
* per plan id. Filesystem-shaped on purpose: greppable, diffable, and any
|
|
@@ -90,11 +91,18 @@ export function createFilePlanStore(directory) {
|
|
|
90
91
|
throw new Error(`Plan ${planId} has no operation ${operationId}.`);
|
|
91
92
|
}
|
|
92
93
|
}
|
|
94
|
+
const approvedOperationIds = Array.from(new Set([...stored.approvedOperationIds, ...operationIds]));
|
|
95
|
+
const mergedOverrides = { ...stored.valueOverrides, ...valueOverrides };
|
|
96
|
+
// Bind the approval to the operation content so apply can detect a
|
|
97
|
+
// post-approval edit. Recompute over ALL approved ops (a later approve
|
|
98
|
+
// call may add overrides that change an earlier op's resolved value).
|
|
99
|
+
const approvalDigests = computeApprovalDigests(stored.plan.operations, approvedOperationIds, mergedOverrides, loadOrCreateSigningKey());
|
|
93
100
|
return write({
|
|
94
101
|
...stored,
|
|
95
102
|
status: "approved",
|
|
96
|
-
approvedOperationIds
|
|
97
|
-
valueOverrides:
|
|
103
|
+
approvedOperationIds,
|
|
104
|
+
valueOverrides: mergedOverrides,
|
|
105
|
+
approvalDigests,
|
|
98
106
|
});
|
|
99
107
|
},
|
|
100
108
|
async reject(planId) {
|
package/dist/schedule.js
CHANGED
|
@@ -61,6 +61,10 @@ export function validateSchedulableArgv(argv) {
|
|
|
61
61
|
throw new Error("A scheduled apply cannot take --plan/--approve — file-based approval would bypass the " +
|
|
62
62
|
"plan store's approval state. Use `apply --plan-id <id>` and approve via `plans approve`.");
|
|
63
63
|
}
|
|
64
|
+
if (argv.includes("--value")) {
|
|
65
|
+
throw new Error("A scheduled apply cannot take --value — an unattended run must write exactly the values " +
|
|
66
|
+
"signed at approval. Set the value with `plans approve --value <op>=<v>` and re-approve.");
|
|
67
|
+
}
|
|
64
68
|
return;
|
|
65
69
|
}
|
|
66
70
|
if (!Object.hasOwn(SCHEDULABLE, head)) {
|
package/dist/types.d.ts
CHANGED
|
@@ -239,6 +239,22 @@ export type PatchOperation = {
|
|
|
239
239
|
* member of the group.
|
|
240
240
|
*/
|
|
241
241
|
groupId?: string;
|
|
242
|
+
/**
|
|
243
|
+
* Set only when a human explicitly chose to archive a record that shares an
|
|
244
|
+
* identity key with another (`bulk-update --archive --force-archive-duplicates`).
|
|
245
|
+
* Without it, apply refuses to archive_record a record the live snapshot still
|
|
246
|
+
* sees as a duplicate — archiving a duplicate discards data that merging keeps,
|
|
247
|
+
* and an agent on a dedupe task must not silently substitute archive for merge.
|
|
248
|
+
*/
|
|
249
|
+
forceArchiveDuplicate?: boolean;
|
|
250
|
+
/**
|
|
251
|
+
* For irreversible operations (merge_records, archive_record): the field
|
|
252
|
+
* values of the records that will be destroyed, captured at plan-build time.
|
|
253
|
+
* Merges and archives cannot be undone on any provider, so this is the
|
|
254
|
+
* recovery artifact a human uses to recreate a record by hand if a merge or
|
|
255
|
+
* archive was wrong — the plan file IS the backup.
|
|
256
|
+
*/
|
|
257
|
+
recoverySnapshot?: Record<string, unknown>[];
|
|
242
258
|
};
|
|
243
259
|
/**
|
|
244
260
|
* A patch plan is always a dry-run proposal. Applying a plan never mutates
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "fullstackgtm",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.26.0",
|
|
4
4
|
"description": "Open-source agentic GTM ops framework: canonical GTM data model, pluggable deterministic audits, reviewable dry-run patch plans, approval-gated write-back with conflict detection, and cross-system entity resolution. HubSpot, Salesforce, and Stripe connectors included.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Full Stack GTM",
|
package/src/bulkUpdate.ts
CHANGED
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
* `account.ownerId`, `account.contactCount`; accounts get `contactCount`
|
|
28
28
|
* and `openDealCount`.
|
|
29
29
|
*/
|
|
30
|
+
import { recoverableFields } from "./dedupe.ts";
|
|
30
31
|
import { normalizeDomain } from "./merge.ts";
|
|
31
32
|
import { stableHash } from "./rules.ts";
|
|
32
33
|
import type {
|
|
@@ -399,7 +400,11 @@ export function buildBulkUpdatePlan(
|
|
|
399
400
|
beforeValue: null,
|
|
400
401
|
afterValue: null,
|
|
401
402
|
riskLevel: "high",
|
|
402
|
-
|
|
403
|
+
// Carry the human's explicit force decision to the apply-time guard, and
|
|
404
|
+
// snapshot the record so it can be recreated if the archive was wrong.
|
|
405
|
+
...(options.forceArchiveDuplicates ? { forceArchiveDuplicate: true } : {}),
|
|
406
|
+
recoverySnapshot: [recoverableFields(record)],
|
|
407
|
+
rollback: "Archived records can be restored from the provider's recycle bin within its retention window; recoverySnapshot also retains the field values.",
|
|
403
408
|
});
|
|
404
409
|
continue;
|
|
405
410
|
}
|
package/src/cli.ts
CHANGED
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
import { generateDemoSnapshot } from "./demo.ts";
|
|
35
35
|
import { formatPatchPlanRun, patchPlanToMarkdown } from "./format.ts";
|
|
36
36
|
import { mergeSnapshots } from "./merge.ts";
|
|
37
|
+
import { verifyApprovalDigests } from "./integrity.ts";
|
|
37
38
|
import { createFilePlanStore } from "./planStore.ts";
|
|
38
39
|
import { auditReportToHtml, auditReportToMarkdown, type ReportOptions } from "./report.ts";
|
|
39
40
|
import { builtinAuditRules } from "./rules.ts";
|
|
@@ -60,6 +61,7 @@ import {
|
|
|
60
61
|
type CallDocument,
|
|
61
62
|
} from "./marketOverlay.ts";
|
|
62
63
|
import { computeScaleIndex, scaleReportToText } from "./marketScale.ts";
|
|
64
|
+
import { suggestMarketConfig } from "./marketTaxonomy.ts";
|
|
63
65
|
import { buildWorksheet, classifyMarket } from "./marketClassify.ts";
|
|
64
66
|
import { marketMapToHtml, marketMapToMarkdown } from "./marketReport.ts";
|
|
65
67
|
import {
|
|
@@ -976,6 +978,8 @@ async function marketCommand(args: string[]) {
|
|
|
976
978
|
if (!subcommand || subcommand === "--help" || subcommand === "-h" || rest.includes("--help") || rest.includes("-h")) {
|
|
977
979
|
console.log(`Usage:
|
|
978
980
|
market init --category <name> [--out <path>] write a starter market.config.json
|
|
981
|
+
market init --category <name> --auto --vendor <url> [--vendor <url>...] [--anchor <url>] [--max-claims n]
|
|
982
|
+
LLM-propose vendors + claim taxonomy from seed pages (needs an API key)
|
|
979
983
|
market capture [--config <path>] [--run <label>]
|
|
980
984
|
market classify [--run <label>] [--capture-run <label>] [--vendor <id>] [--model m] [--out <path>]
|
|
981
985
|
market worksheet --vendor <id> [--capture-run <label>] [--out <path>]
|
|
@@ -1025,6 +1029,31 @@ recomputed deterministically on every invocation — never stored.`);
|
|
|
1025
1029
|
if (!category) throw new Error("market init requires --category <name>");
|
|
1026
1030
|
const outPath = resolve(process.cwd(), option(rest, "--out") ?? "market.config.json");
|
|
1027
1031
|
if (existsSync(outPath)) throw new Error(`${outPath} already exists — refusing to overwrite`);
|
|
1032
|
+
|
|
1033
|
+
if (rest.includes("--auto")) {
|
|
1034
|
+
const vendorUrls = repeatedOption(rest, "--vendor");
|
|
1035
|
+
if (vendorUrls.length === 0) {
|
|
1036
|
+
throw new Error("market init --auto requires at least one --vendor <url> (the competitor homepages to seed from)");
|
|
1037
|
+
}
|
|
1038
|
+
const anchorUrl = option(rest, "--anchor");
|
|
1039
|
+
const credential = await requireLlmCredential("market classify");
|
|
1040
|
+
console.error(`Capturing ${vendorUrls.length} seed page(s) and proposing a claim taxonomy with ${credential.provider}…`);
|
|
1041
|
+
const { config, unreadableVendorIds, model } = await suggestMarketConfig({
|
|
1042
|
+
category,
|
|
1043
|
+
vendors: vendorUrls.map((url) => ({ url, anchor: anchorUrl ? url === anchorUrl : false })),
|
|
1044
|
+
llm: { ...credential, model: option(rest, "--model") ?? undefined },
|
|
1045
|
+
maxClaims: numericOption(rest, "--max-claims"),
|
|
1046
|
+
});
|
|
1047
|
+
writeFileSync(outPath, `${JSON.stringify(config, null, 2)}\n`);
|
|
1048
|
+
if (unreadableVendorIds.length > 0) {
|
|
1049
|
+
console.error(`Note: no readable text for ${unreadableVendorIds.join(", ")} — excluded from taxonomy grounding.`);
|
|
1050
|
+
}
|
|
1051
|
+
console.log(
|
|
1052
|
+
`Wrote ${outPath}: ${config.vendors.length} vendors, ${config.claims.length} proposed claims (${model}). Review it, then: fullstackgtm market refresh`,
|
|
1053
|
+
);
|
|
1054
|
+
return;
|
|
1055
|
+
}
|
|
1056
|
+
|
|
1028
1057
|
writeFileSync(outPath, `${JSON.stringify(starterMarketConfig(category), null, 2)}\n`);
|
|
1029
1058
|
console.log(`Wrote ${outPath}. Fill in vendors and claims, then: fullstackgtm market capture`);
|
|
1030
1059
|
return;
|
|
@@ -2552,7 +2581,40 @@ async function apply(args: string[]) {
|
|
|
2552
2581
|
}
|
|
2553
2582
|
plan = stored.plan;
|
|
2554
2583
|
approvedOperationIds = stored.approvedOperationIds;
|
|
2584
|
+
// Downgrade guard: an approved plan with no signatures is either pre-0.26
|
|
2585
|
+
// (re-approve to gain them) or had its approvalDigests stripped to skip the
|
|
2586
|
+
// integrity check. Either way, refuse rather than fall back to trusting the
|
|
2587
|
+
// file. (A plan with zero approved operations has nothing to apply anyway.)
|
|
2588
|
+
if (stored.approvedOperationIds.length > 0 && !stored.approvalDigests) {
|
|
2589
|
+
throw new Error(
|
|
2590
|
+
`Refusing to apply plan ${planId}: it was approved without integrity signatures ` +
|
|
2591
|
+
"(approved before 0.26.0, or its signatures were removed). Re-approve it with " +
|
|
2592
|
+
`\`fullstackgtm plans approve ${planId} --operations <ids|all>\`.`,
|
|
2593
|
+
);
|
|
2594
|
+
}
|
|
2595
|
+
// Integrity gate: the plan file is re-read from disk, so verify each approved
|
|
2596
|
+
// operation still matches what was signed at approval. Verify against the
|
|
2597
|
+
// EFFECTIVE overrides (stored ∪ apply-time --value): the invariant is "what
|
|
2598
|
+
// gets written must equal what was signed", so an apply-time --value that
|
|
2599
|
+
// changes a value the human did not approve is treated as tamper, not a live
|
|
2600
|
+
// override. A mismatch means the plan/overrides were edited after approval —
|
|
2601
|
+
// refuse the whole apply rather than write an unapproved value.
|
|
2555
2602
|
valueOverrides = { ...stored.valueOverrides, ...parseValueOverrides(args) };
|
|
2603
|
+
const verification = verifyApprovalDigests(
|
|
2604
|
+
stored.plan.operations,
|
|
2605
|
+
stored.approvedOperationIds,
|
|
2606
|
+
valueOverrides,
|
|
2607
|
+
stored.approvalDigests,
|
|
2608
|
+
);
|
|
2609
|
+
if (!verification.ok) {
|
|
2610
|
+
const detail =
|
|
2611
|
+
verification.reason === "no_key"
|
|
2612
|
+
? "the plan-signing key is missing (was this plan approved on another machine?). Re-approve it here with `fullstackgtm plans approve`."
|
|
2613
|
+
: `these operations differ from what was approved: ${verification.tampered.join(", ")}. ` +
|
|
2614
|
+
"If you changed a value at apply time, set it at approval instead (`plans approve --value <op>=<v>`) and re-approve; " +
|
|
2615
|
+
"otherwise the plan was edited after approval — review and re-approve.";
|
|
2616
|
+
throw new Error(`Refusing to apply plan ${planId}: ${detail}`);
|
|
2617
|
+
}
|
|
2556
2618
|
} else {
|
|
2557
2619
|
const approve = option(args, "--approve");
|
|
2558
2620
|
if (!approve) {
|
package/src/connector.ts
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
|
+
import { dedupeKey } from "./dedupe.ts";
|
|
1
2
|
import { requiresHumanInput } from "./rules.ts";
|
|
2
3
|
import type {
|
|
4
|
+
CanonicalGtmSnapshot,
|
|
3
5
|
GtmConnector,
|
|
4
6
|
PatchOperation,
|
|
5
7
|
PatchOperationResult,
|
|
@@ -8,6 +10,75 @@ import type {
|
|
|
8
10
|
PatchPlanRunStatus,
|
|
9
11
|
} from "./types.ts";
|
|
10
12
|
|
|
13
|
+
const IRREVERSIBLE_OPERATIONS = new Set(["merge_records", "archive_record"]);
|
|
14
|
+
const IDENTITY_KEY_BY_TYPE: Partial<Record<string, "domain" | "email">> = {
|
|
15
|
+
account: "domain",
|
|
16
|
+
contact: "email",
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
/** snapshot collection for an object type */
|
|
20
|
+
function collectionFor(objectType: string): "accounts" | "contacts" | "deals" | null {
|
|
21
|
+
if (objectType === "account") return "accounts";
|
|
22
|
+
if (objectType === "contact") return "contacts";
|
|
23
|
+
if (objectType === "deal") return "deals";
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Drift/safety check for the two IRREVERSIBLE operations against a fresh
|
|
29
|
+
* snapshot. Returns a conflict detail string, or null if the op is safe to
|
|
30
|
+
* apply. These operations get NO field compare-and-set (there is no single
|
|
31
|
+
* field to compare), so this snapshot check is their only guard.
|
|
32
|
+
*/
|
|
33
|
+
function checkIrreversibleOp(operation: PatchOperation, snapshot: CanonicalGtmSnapshot): string | null {
|
|
34
|
+
const collection = collectionFor(operation.objectType);
|
|
35
|
+
if (!collection) return null;
|
|
36
|
+
const records = snapshot[collection] as Array<Record<string, unknown>>;
|
|
37
|
+
const byId = (id: string) => records.find((record) => String(record.id) === id);
|
|
38
|
+
|
|
39
|
+
if (operation.operation === "archive_record") {
|
|
40
|
+
if (!byId(operation.objectId)) {
|
|
41
|
+
return `Record ${operation.objectType}/${operation.objectId} no longer exists (already archived or merged). Re-plan against current data.`;
|
|
42
|
+
}
|
|
43
|
+
// Archiving a duplicate discards data a merge would keep — refuse unless the
|
|
44
|
+
// human explicitly forced it. This catches every archive_record path (agent,
|
|
45
|
+
// hand-edited plan, audit), not just `bulk-update --archive`.
|
|
46
|
+
if (!operation.forceArchiveDuplicate) {
|
|
47
|
+
const keyName = IDENTITY_KEY_BY_TYPE[operation.objectType];
|
|
48
|
+
if (keyName) {
|
|
49
|
+
const target = byId(operation.objectId)!;
|
|
50
|
+
const key = dedupeKey(target, keyName);
|
|
51
|
+
if (key) {
|
|
52
|
+
const sharers = records.filter(
|
|
53
|
+
(record) => String(record.id) !== operation.objectId && dedupeKey(record, keyName) === key,
|
|
54
|
+
);
|
|
55
|
+
if (sharers.length > 0) {
|
|
56
|
+
return (
|
|
57
|
+
`Refusing to archive ${operation.objectType}/${operation.objectId}: it shares ${keyName} "${key}" with ` +
|
|
58
|
+
`${sharers.length} other record(s) — that's a duplicate, and archiving discards its data where merging keeps it. ` +
|
|
59
|
+
`Merge with \`fullstackgtm dedupe ${operation.objectType} --key ${keyName}\` instead, or rebuild the op with --force-archive-duplicates.`
|
|
60
|
+
);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
if (operation.operation === "merge_records") {
|
|
69
|
+
if (!byId(operation.objectId)) {
|
|
70
|
+
return `Merge survivor ${operation.objectType}/${operation.objectId} no longer exists (archived or merged away since the plan was built). Re-plan — merges are irreversible.`;
|
|
71
|
+
}
|
|
72
|
+
const groupIds = Array.isArray(operation.beforeValue) ? (operation.beforeValue as unknown[]).map(String) : [];
|
|
73
|
+
const losersStillPresent = groupIds.filter((id) => id !== operation.objectId && byId(id));
|
|
74
|
+
if (groupIds.length > 0 && losersStillPresent.length === 0) {
|
|
75
|
+
return `Every record to merge into ${operation.objectType}/${operation.objectId} is already gone (merge already applied?). Nothing to do — re-plan if duplicates remain.`;
|
|
76
|
+
}
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
|
|
11
82
|
export type ApplyPatchPlanOptions = {
|
|
12
83
|
/**
|
|
13
84
|
* Explicit allow-list of operation ids the human approved. Operations not
|
|
@@ -79,10 +150,20 @@ export async function applyPatchPlan(
|
|
|
79
150
|
// closed — but it can be shrunk: re-run the snapshot checks after the
|
|
80
151
|
// first write and every `recheckEvery` writes, conflicting out any
|
|
81
152
|
// operation whose record went stale mid-run.
|
|
153
|
+
// Irreversible ops (merge/archive) need a fresh snapshot too — it is their
|
|
154
|
+
// only drift/safety guard (no field to compare-and-set). Respect a caller's
|
|
155
|
+
// explicit checkConflicts:false opt-out (a stub/known-stale snapshot).
|
|
156
|
+
const hasIrreversibleApproved =
|
|
157
|
+
checkConflicts &&
|
|
158
|
+
plan.operations.some(
|
|
159
|
+
(operation) => approved.has(operation.id) && IRREVERSIBLE_OPERATIONS.has(operation.operation),
|
|
160
|
+
);
|
|
82
161
|
const needsSnapshot =
|
|
83
|
-
((plan.guards && plan.guards.length > 0) || plan.filter) &&
|
|
162
|
+
((plan.guards && plan.guards.length > 0) || plan.filter || hasIrreversibleApproved) &&
|
|
163
|
+
connector.fetchSnapshot;
|
|
84
164
|
const recheckEvery = Math.max(1, options.recheckEvery ?? 25);
|
|
85
165
|
const staleIds = new Set<string>();
|
|
166
|
+
const irreversibleStale = new Map<string, string>();
|
|
86
167
|
let guardFailure: string | null = null;
|
|
87
168
|
const refreshSnapshotChecks = async (): Promise<void> => {
|
|
88
169
|
if (!needsSnapshot) return;
|
|
@@ -95,6 +176,14 @@ export async function applyPatchPlan(
|
|
|
95
176
|
if (!stillEligible.has(operation.objectId)) staleIds.add(operation.objectId);
|
|
96
177
|
}
|
|
97
178
|
}
|
|
179
|
+
irreversibleStale.clear();
|
|
180
|
+
if (checkConflicts) {
|
|
181
|
+
for (const operation of plan.operations) {
|
|
182
|
+
if (!approved.has(operation.id) || !IRREVERSIBLE_OPERATIONS.has(operation.operation)) continue;
|
|
183
|
+
const detail = checkIrreversibleOp(operation, liveSnapshot);
|
|
184
|
+
if (detail) irreversibleStale.set(operation.id, detail);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
98
187
|
for (const guard of plan.guards ?? []) {
|
|
99
188
|
const failure = evaluateGuard(liveSnapshot, guard);
|
|
100
189
|
if (failure) {
|
|
@@ -232,6 +321,12 @@ export async function applyPatchPlan(
|
|
|
232
321
|
if (operation.groupId) poisonedGroups.add(operation.groupId);
|
|
233
322
|
continue;
|
|
234
323
|
}
|
|
324
|
+
const irreversibleConflict = irreversibleStale.get(operation.id);
|
|
325
|
+
if (irreversibleConflict) {
|
|
326
|
+
results.push({ operationId: operation.id, status: "conflict", detail: irreversibleConflict });
|
|
327
|
+
if (operation.groupId) poisonedGroups.add(operation.groupId);
|
|
328
|
+
continue;
|
|
329
|
+
}
|
|
235
330
|
if (operation.groupId && poisonedGroups.has(operation.groupId)) {
|
|
236
331
|
results.push({
|
|
237
332
|
operationId: operation.id,
|
package/src/dedupe.ts
CHANGED
|
@@ -65,6 +65,21 @@ function populatedDataFields(record: Record<string, unknown>): number {
|
|
|
65
65
|
).length;
|
|
66
66
|
}
|
|
67
67
|
|
|
68
|
+
/**
|
|
69
|
+
* The subset of a record worth keeping as a merge-recovery artifact: its id (to
|
|
70
|
+
* reference) plus every populated data field, dropping bulky/plumbing fields
|
|
71
|
+
* (raw, identities, provenance) that aren't needed to recreate it by hand.
|
|
72
|
+
*/
|
|
73
|
+
export function recoverableFields(record: Record<string, unknown>): Record<string, unknown> {
|
|
74
|
+
const out: Record<string, unknown> = { id: String(record.id) };
|
|
75
|
+
for (const [field, value] of Object.entries(record)) {
|
|
76
|
+
if (NON_DATA_FIELDS.has(field)) continue;
|
|
77
|
+
if (value === undefined || value === null || value === "") continue;
|
|
78
|
+
out[field] = value;
|
|
79
|
+
}
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
|
|
68
83
|
/** True when id `a` sorts before id `b` — numeric when both ids are numeric. */
|
|
69
84
|
function idBefore(a: string, b: string): boolean {
|
|
70
85
|
const numericA = Number(a);
|
|
@@ -137,6 +152,12 @@ export function buildDedupePlan(
|
|
|
137
152
|
const groupIds = members
|
|
138
153
|
.map((member) => String(member.id))
|
|
139
154
|
.sort((a, b) => (idBefore(a, b) ? -1 : 1));
|
|
155
|
+
// Recovery artifact: the records that will be merged away (everyone but the
|
|
156
|
+
// survivor), captured with their field values so a human can recreate one by
|
|
157
|
+
// hand if the merge was wrong. Merges are irreversible — the plan is the backup.
|
|
158
|
+
const recoverySnapshot = members
|
|
159
|
+
.filter((member) => String(member.id) !== String(survivor.id))
|
|
160
|
+
.map((member) => recoverableFields(member));
|
|
140
161
|
const survivorName =
|
|
141
162
|
typeof survivor.name === "string" && survivor.name
|
|
142
163
|
? survivor.name
|
|
@@ -162,8 +183,9 @@ export function buildDedupePlan(
|
|
|
162
183
|
approvalRequired: true,
|
|
163
184
|
sourceRuleOrPolicy: "dedupe",
|
|
164
185
|
groupId: `grp_${options.objectType}_${String(survivor.id)}`,
|
|
186
|
+
recoverySnapshot,
|
|
165
187
|
rollback:
|
|
166
|
-
"IRREVERSIBLE: provider merges cannot be unmerged.
|
|
188
|
+
"IRREVERSIBLE: provider merges cannot be unmerged. recoverySnapshot on this operation retains every merged-away record's field values; recreate a record manually from it if a merge was wrong.",
|
|
167
189
|
});
|
|
168
190
|
}
|
|
169
191
|
|
package/src/index.ts
CHANGED
|
@@ -115,6 +115,14 @@ export {
|
|
|
115
115
|
type MergeSuggestion,
|
|
116
116
|
} from "./merge.ts";
|
|
117
117
|
export { createFilePlanStore, type PlanStore, type StoredPlan } from "./planStore.ts";
|
|
118
|
+
export {
|
|
119
|
+
computeApprovalDigests,
|
|
120
|
+
loadOrCreateSigningKey,
|
|
121
|
+
loadSigningKey,
|
|
122
|
+
signApproval,
|
|
123
|
+
verifyApprovalDigests,
|
|
124
|
+
type ApprovalVerification,
|
|
125
|
+
} from "./integrity.ts";
|
|
118
126
|
export { formatPatchPlanRun, patchPlanToMarkdown } from "./format.ts";
|
|
119
127
|
export { auditReportToHtml, auditReportToMarkdown, type ReportOptions } from "./report.ts";
|
|
120
128
|
export {
|