fullstackgtm 0.25.2 → 0.26.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/dist/bulkUpdate.js +6 -1
- package/dist/cli.js +50 -0
- package/dist/connector.js +90 -1
- package/dist/dedupe.d.ts +6 -0
- package/dist/dedupe.js +24 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/integrity.d.ts +30 -0
- package/dist/integrity.js +128 -0
- package/dist/marketTaxonomy.d.ts +41 -0
- package/dist/marketTaxonomy.js +193 -0
- package/dist/planStore.d.ts +6 -0
- package/dist/planStore.js +10 -2
- package/dist/schedule.js +4 -0
- package/dist/types.d.ts +16 -0
- package/package.json +1 -1
- package/src/bulkUpdate.ts +6 -1
- package/src/cli.ts +62 -0
- package/src/connector.ts +96 -1
- package/src/dedupe.ts +23 -1
- package/src/index.ts +8 -0
- package/src/integrity.ts +146 -0
- package/src/marketTaxonomy.ts +288 -0
- package/src/planStore.ts +23 -4
- package/src/schedule.ts +6 -0
- package/src/types.ts +16 -0
package/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,59 @@ The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
|
5
5
|
and the project adheres to [Semantic Versioning](https://semver.org/).
|
|
6
6
|
The path to 1.0 is planned in [docs/roadmap-to-1.0.md](./docs/roadmap-to-1.0.md).
|
|
7
7
|
|
|
8
|
+
## [0.26.0] — 2026-06-15
|
|
9
|
+
|
|
10
|
+
Write-path integrity — the "no write without approval" guarantee now binds to
|
|
11
|
+
operation *content*, and the two irreversible operations finally get a guard.
|
|
12
|
+
Each fix verified by a refute-by-default re-attack; the integrity binding took
|
|
13
|
+
three rounds (the re-attack kept finding unsigned fields that reach a write).
|
|
14
|
+
|
|
15
|
+
### Added
|
|
16
|
+
|
|
17
|
+
- **Plan-approval integrity signatures.** `plans approve` now HMAC-signs each
|
|
18
|
+
approved operation's full apply-relevant content (operation, object, field,
|
|
19
|
+
before/after value, group, preconditions, force flags, the approved value
|
|
20
|
+
override, and reason) with a per-install key (`$FSGTM_HOME/.plan-signing-key`,
|
|
21
|
+
0600). `apply --plan-id` re-verifies and refuses the whole apply if any
|
|
22
|
+
approved operation changed since approval, if the plan was approved without
|
|
23
|
+
signatures (downgrade guard), or if the signing key is absent (a plan
|
|
24
|
+
approved on another machine fails closed). The invariant: **what gets written
|
|
25
|
+
equals what the human signed** — a plan file edited between approval and apply
|
|
26
|
+
(by a synced/backed-up copy, a co-tenant, or a compromised dependency) is
|
|
27
|
+
caught instead of executed. apply-time `--value` is folded into the check, and
|
|
28
|
+
a scheduled `apply` may not take `--value` (it must write exactly the signed
|
|
29
|
+
values).
|
|
30
|
+
- **Recovery snapshots on irreversible operations.** `dedupe` merge ops and
|
|
31
|
+
`bulk-update --archive` ops now carry `recoverySnapshot` — the field values of
|
|
32
|
+
every record that will be destroyed — so the rollback instruction ("recreate
|
|
33
|
+
it by hand") is backed by actual data in the plan, which is the backup.
|
|
34
|
+
|
|
35
|
+
### Security
|
|
36
|
+
|
|
37
|
+
- **Apply-time guard against destroying duplicates (the benchmark self-own).**
|
|
38
|
+
`apply` now refuses any `archive_record` whose target still shares an identity
|
|
39
|
+
key (account domain / contact email) with another live record — unless the
|
|
40
|
+
human explicitly forced it (`--force-archive-duplicates`, which is recorded on
|
|
41
|
+
the operation and signed). This catches every path (agent-driven, hand-edited,
|
|
42
|
+
audit), not just `bulk-update`, so an agent on a dedupe task can no longer
|
|
43
|
+
silently archive a record where it should merge — the rail is safe regardless
|
|
44
|
+
of model strength.
|
|
45
|
+
- **Drift guard for irreversible operations.** `merge_records` and
|
|
46
|
+
`archive_record` got no compare-and-set (there is no single field to compare).
|
|
47
|
+
Apply now checks a fresh snapshot: a merge whose survivor is gone, or whose
|
|
48
|
+
duplicates are already merged, and an archive of a record that no longer
|
|
49
|
+
exists, all conflict out instead of firing an irreversible, replay-unsafe write.
|
|
50
|
+
|
|
51
|
+
### Notes
|
|
52
|
+
|
|
53
|
+
- The CAS empty/null equivalence in field compare-and-set is intentional (CRMs
|
|
54
|
+
normalize `""`↔`null` server-side; distinguishing them would cause false
|
|
55
|
+
conflicts). Known residuals for a follow-up: the archive-duplicate guard keys
|
|
56
|
+
on domain/email only, so `dedupe --key name` (and deal dedupe, which has no
|
|
57
|
+
identity key) are not guarded against destructive archive — use `dedupe`
|
|
58
|
+
(merge) for those; and `marketMapToMarkdown` is not HTML-escaped (the HTML
|
|
59
|
+
report is).
|
|
60
|
+
|
|
8
61
|
## [0.25.2] — 2026-06-15
|
|
9
62
|
|
|
10
63
|
Security hardening I — confirmed fixes from an adversarial audit (each verified
|
package/dist/bulkUpdate.js
CHANGED
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
* `account.ownerId`, `account.contactCount`; accounts get `contactCount`
|
|
28
28
|
* and `openDealCount`.
|
|
29
29
|
*/
|
|
30
|
+
import { recoverableFields } from "./dedupe.js";
|
|
30
31
|
import { normalizeDomain } from "./merge.js";
|
|
31
32
|
import { stableHash } from "./rules.js";
|
|
32
33
|
const FIELD_PATTERN = "[a-zA-Z][a-zA-Z0-9_]*(?:\\.[a-zA-Z][a-zA-Z0-9_]*)?";
|
|
@@ -319,7 +320,11 @@ export function buildBulkUpdatePlan(snapshot, options) {
|
|
|
319
320
|
beforeValue: null,
|
|
320
321
|
afterValue: null,
|
|
321
322
|
riskLevel: "high",
|
|
322
|
-
|
|
323
|
+
// Carry the human's explicit force decision to the apply-time guard, and
|
|
324
|
+
// snapshot the record so it can be recreated if the archive was wrong.
|
|
325
|
+
...(options.forceArchiveDuplicates ? { forceArchiveDuplicate: true } : {}),
|
|
326
|
+
recoverySnapshot: [recoverableFields(record)],
|
|
327
|
+
rollback: "Archived records can be restored from the provider's recycle bin within its retention window; recoverySnapshot also retains the field values.",
|
|
323
328
|
});
|
|
324
329
|
continue;
|
|
325
330
|
}
|
package/dist/cli.js
CHANGED
|
@@ -13,6 +13,7 @@ import { activeProfile, credentialsPath, DEFAULT_PROFILE, deleteCredential, getC
|
|
|
13
13
|
import { generateDemoSnapshot } from "./demo.js";
|
|
14
14
|
import { formatPatchPlanRun, patchPlanToMarkdown } from "./format.js";
|
|
15
15
|
import { mergeSnapshots } from "./merge.js";
|
|
16
|
+
import { verifyApprovalDigests } from "./integrity.js";
|
|
16
17
|
import { createFilePlanStore } from "./planStore.js";
|
|
17
18
|
import { auditReportToHtml, auditReportToMarkdown } from "./report.js";
|
|
18
19
|
import { builtinAuditRules } from "./rules.js";
|
|
@@ -22,6 +23,7 @@ import { captureMarket, computeFrontStates, createFileObservationStore, diffFron
|
|
|
22
23
|
import { assessAxes, axesReportToText } from "./marketAxes.js";
|
|
23
24
|
import { computeDirectives, computeOverlayStats, directivesToPlan, overlayToMarkdown, } from "./marketOverlay.js";
|
|
24
25
|
import { computeScaleIndex, scaleReportToText } from "./marketScale.js";
|
|
26
|
+
import { suggestMarketConfig } from "./marketTaxonomy.js";
|
|
25
27
|
import { buildWorksheet, classifyMarket } from "./marketClassify.js";
|
|
26
28
|
import { marketMapToHtml, marketMapToMarkdown } from "./marketReport.js";
|
|
27
29
|
import { DEFAULT_RUBRIC, detectProviderFromKey, extractInsightsLlm, parseRubric, resolveLlmCredential, scoreCallLlm, validateLlmKey, } from "./llm.js";
|
|
@@ -827,6 +829,8 @@ async function marketCommand(args) {
|
|
|
827
829
|
if (!subcommand || subcommand === "--help" || subcommand === "-h" || rest.includes("--help") || rest.includes("-h")) {
|
|
828
830
|
console.log(`Usage:
|
|
829
831
|
market init --category <name> [--out <path>] write a starter market.config.json
|
|
832
|
+
market init --category <name> --auto --vendor <url> [--vendor <url>...] [--anchor <url>] [--max-claims n]
|
|
833
|
+
LLM-propose vendors + claim taxonomy from seed pages (needs an API key)
|
|
830
834
|
market capture [--config <path>] [--run <label>]
|
|
831
835
|
market classify [--run <label>] [--capture-run <label>] [--vendor <id>] [--model m] [--out <path>]
|
|
832
836
|
market worksheet --vendor <id> [--capture-run <label>] [--out <path>]
|
|
@@ -877,6 +881,27 @@ recomputed deterministically on every invocation — never stored.`);
|
|
|
877
881
|
const outPath = resolve(process.cwd(), option(rest, "--out") ?? "market.config.json");
|
|
878
882
|
if (existsSync(outPath))
|
|
879
883
|
throw new Error(`${outPath} already exists — refusing to overwrite`);
|
|
884
|
+
if (rest.includes("--auto")) {
|
|
885
|
+
const vendorUrls = repeatedOption(rest, "--vendor");
|
|
886
|
+
if (vendorUrls.length === 0) {
|
|
887
|
+
throw new Error("market init --auto requires at least one --vendor <url> (the competitor homepages to seed from)");
|
|
888
|
+
}
|
|
889
|
+
const anchorUrl = option(rest, "--anchor");
|
|
890
|
+
const credential = await requireLlmCredential("market classify");
|
|
891
|
+
console.error(`Capturing ${vendorUrls.length} seed page(s) and proposing a claim taxonomy with ${credential.provider}…`);
|
|
892
|
+
const { config, unreadableVendorIds, model } = await suggestMarketConfig({
|
|
893
|
+
category,
|
|
894
|
+
vendors: vendorUrls.map((url) => ({ url, anchor: anchorUrl ? url === anchorUrl : false })),
|
|
895
|
+
llm: { ...credential, model: option(rest, "--model") ?? undefined },
|
|
896
|
+
maxClaims: numericOption(rest, "--max-claims"),
|
|
897
|
+
});
|
|
898
|
+
writeFileSync(outPath, `${JSON.stringify(config, null, 2)}\n`);
|
|
899
|
+
if (unreadableVendorIds.length > 0) {
|
|
900
|
+
console.error(`Note: no readable text for ${unreadableVendorIds.join(", ")} — excluded from taxonomy grounding.`);
|
|
901
|
+
}
|
|
902
|
+
console.log(`Wrote ${outPath}: ${config.vendors.length} vendors, ${config.claims.length} proposed claims (${model}). Review it, then: fullstackgtm market refresh`);
|
|
903
|
+
return;
|
|
904
|
+
}
|
|
880
905
|
writeFileSync(outPath, `${JSON.stringify(starterMarketConfig(category), null, 2)}\n`);
|
|
881
906
|
console.log(`Wrote ${outPath}. Fill in vendors and claims, then: fullstackgtm market capture`);
|
|
882
907
|
return;
|
|
@@ -2277,7 +2302,32 @@ async function apply(args) {
|
|
|
2277
2302
|
}
|
|
2278
2303
|
plan = stored.plan;
|
|
2279
2304
|
approvedOperationIds = stored.approvedOperationIds;
|
|
2305
|
+
// Downgrade guard: an approved plan with no signatures is either pre-0.26
|
|
2306
|
+
// (re-approve to gain them) or had its approvalDigests stripped to skip the
|
|
2307
|
+
// integrity check. Either way, refuse rather than fall back to trusting the
|
|
2308
|
+
// file. (A plan with zero approved operations has nothing to apply anyway.)
|
|
2309
|
+
if (stored.approvedOperationIds.length > 0 && !stored.approvalDigests) {
|
|
2310
|
+
throw new Error(`Refusing to apply plan ${planId}: it was approved without integrity signatures ` +
|
|
2311
|
+
"(approved before 0.26.0, or its signatures were removed). Re-approve it with " +
|
|
2312
|
+
`\`fullstackgtm plans approve ${planId} --operations <ids|all>\`.`);
|
|
2313
|
+
}
|
|
2314
|
+
// Integrity gate: the plan file is re-read from disk, so verify each approved
|
|
2315
|
+
// operation still matches what was signed at approval. Verify against the
|
|
2316
|
+
// EFFECTIVE overrides (stored ∪ apply-time --value): the invariant is "what
|
|
2317
|
+
// gets written must equal what was signed", so an apply-time --value that
|
|
2318
|
+
// changes a value the human did not approve is treated as tamper, not a live
|
|
2319
|
+
// override. A mismatch means the plan/overrides were edited after approval —
|
|
2320
|
+
// refuse the whole apply rather than write an unapproved value.
|
|
2280
2321
|
valueOverrides = { ...stored.valueOverrides, ...parseValueOverrides(args) };
|
|
2322
|
+
const verification = verifyApprovalDigests(stored.plan.operations, stored.approvedOperationIds, valueOverrides, stored.approvalDigests);
|
|
2323
|
+
if (!verification.ok) {
|
|
2324
|
+
const detail = verification.reason === "no_key"
|
|
2325
|
+
? "the plan-signing key is missing (was this plan approved on another machine?). Re-approve it here with `fullstackgtm plans approve`."
|
|
2326
|
+
: `these operations differ from what was approved: ${verification.tampered.join(", ")}. ` +
|
|
2327
|
+
"If you changed a value at apply time, set it at approval instead (`plans approve --value <op>=<v>`) and re-approve; " +
|
|
2328
|
+
"otherwise the plan was edited after approval — review and re-approve.";
|
|
2329
|
+
throw new Error(`Refusing to apply plan ${planId}: ${detail}`);
|
|
2330
|
+
}
|
|
2281
2331
|
}
|
|
2282
2332
|
else {
|
|
2283
2333
|
const approve = option(args, "--approve");
|
package/dist/connector.js
CHANGED
|
@@ -1,4 +1,69 @@
|
|
|
1
|
+
import { dedupeKey } from "./dedupe.js";
|
|
1
2
|
import { requiresHumanInput } from "./rules.js";
|
|
3
|
+
const IRREVERSIBLE_OPERATIONS = new Set(["merge_records", "archive_record"]);
|
|
4
|
+
const IDENTITY_KEY_BY_TYPE = {
|
|
5
|
+
account: "domain",
|
|
6
|
+
contact: "email",
|
|
7
|
+
};
|
|
8
|
+
/** snapshot collection for an object type */
|
|
9
|
+
function collectionFor(objectType) {
|
|
10
|
+
if (objectType === "account")
|
|
11
|
+
return "accounts";
|
|
12
|
+
if (objectType === "contact")
|
|
13
|
+
return "contacts";
|
|
14
|
+
if (objectType === "deal")
|
|
15
|
+
return "deals";
|
|
16
|
+
return null;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Drift/safety check for the two IRREVERSIBLE operations against a fresh
|
|
20
|
+
* snapshot. Returns a conflict detail string, or null if the op is safe to
|
|
21
|
+
* apply. These operations get NO field compare-and-set (there is no single
|
|
22
|
+
* field to compare), so this snapshot check is their only guard.
|
|
23
|
+
*/
|
|
24
|
+
function checkIrreversibleOp(operation, snapshot) {
|
|
25
|
+
const collection = collectionFor(operation.objectType);
|
|
26
|
+
if (!collection)
|
|
27
|
+
return null;
|
|
28
|
+
const records = snapshot[collection];
|
|
29
|
+
const byId = (id) => records.find((record) => String(record.id) === id);
|
|
30
|
+
if (operation.operation === "archive_record") {
|
|
31
|
+
if (!byId(operation.objectId)) {
|
|
32
|
+
return `Record ${operation.objectType}/${operation.objectId} no longer exists (already archived or merged). Re-plan against current data.`;
|
|
33
|
+
}
|
|
34
|
+
// Archiving a duplicate discards data a merge would keep — refuse unless the
|
|
35
|
+
// human explicitly forced it. This catches every archive_record path (agent,
|
|
36
|
+
// hand-edited plan, audit), not just `bulk-update --archive`.
|
|
37
|
+
if (!operation.forceArchiveDuplicate) {
|
|
38
|
+
const keyName = IDENTITY_KEY_BY_TYPE[operation.objectType];
|
|
39
|
+
if (keyName) {
|
|
40
|
+
const target = byId(operation.objectId);
|
|
41
|
+
const key = dedupeKey(target, keyName);
|
|
42
|
+
if (key) {
|
|
43
|
+
const sharers = records.filter((record) => String(record.id) !== operation.objectId && dedupeKey(record, keyName) === key);
|
|
44
|
+
if (sharers.length > 0) {
|
|
45
|
+
return (`Refusing to archive ${operation.objectType}/${operation.objectId}: it shares ${keyName} "${key}" with ` +
|
|
46
|
+
`${sharers.length} other record(s) — that's a duplicate, and archiving discards its data where merging keeps it. ` +
|
|
47
|
+
`Merge with \`fullstackgtm dedupe ${operation.objectType} --key ${keyName}\` instead, or rebuild the op with --force-archive-duplicates.`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
return null;
|
|
53
|
+
}
|
|
54
|
+
if (operation.operation === "merge_records") {
|
|
55
|
+
if (!byId(operation.objectId)) {
|
|
56
|
+
return `Merge survivor ${operation.objectType}/${operation.objectId} no longer exists (archived or merged away since the plan was built). Re-plan — merges are irreversible.`;
|
|
57
|
+
}
|
|
58
|
+
const groupIds = Array.isArray(operation.beforeValue) ? operation.beforeValue.map(String) : [];
|
|
59
|
+
const losersStillPresent = groupIds.filter((id) => id !== operation.objectId && byId(id));
|
|
60
|
+
if (groupIds.length > 0 && losersStillPresent.length === 0) {
|
|
61
|
+
return `Every record to merge into ${operation.objectType}/${operation.objectId} is already gone (merge already applied?). Nothing to do — re-plan if duplicates remain.`;
|
|
62
|
+
}
|
|
63
|
+
return null;
|
|
64
|
+
}
|
|
65
|
+
return null;
|
|
66
|
+
}
|
|
2
67
|
const FIELD_WRITE_OPERATIONS = new Set(["set_field", "clear_field", "link_record"]);
|
|
3
68
|
function normalizeForComparison(value) {
|
|
4
69
|
if (value === undefined || value === null || value === "")
|
|
@@ -35,9 +100,16 @@ export async function applyPatchPlan(connector, plan, options) {
|
|
|
35
100
|
// closed — but it can be shrunk: re-run the snapshot checks after the
|
|
36
101
|
// first write and every `recheckEvery` writes, conflicting out any
|
|
37
102
|
// operation whose record went stale mid-run.
|
|
38
|
-
|
|
103
|
+
// Irreversible ops (merge/archive) need a fresh snapshot too — it is their
|
|
104
|
+
// only drift/safety guard (no field to compare-and-set). Respect a caller's
|
|
105
|
+
// explicit checkConflicts:false opt-out (a stub/known-stale snapshot).
|
|
106
|
+
const hasIrreversibleApproved = checkConflicts &&
|
|
107
|
+
plan.operations.some((operation) => approved.has(operation.id) && IRREVERSIBLE_OPERATIONS.has(operation.operation));
|
|
108
|
+
const needsSnapshot = ((plan.guards && plan.guards.length > 0) || plan.filter || hasIrreversibleApproved) &&
|
|
109
|
+
connector.fetchSnapshot;
|
|
39
110
|
const recheckEvery = Math.max(1, options.recheckEvery ?? 25);
|
|
40
111
|
const staleIds = new Set();
|
|
112
|
+
const irreversibleStale = new Map();
|
|
41
113
|
let guardFailure = null;
|
|
42
114
|
const refreshSnapshotChecks = async () => {
|
|
43
115
|
if (!needsSnapshot)
|
|
@@ -52,6 +124,16 @@ export async function applyPatchPlan(connector, plan, options) {
|
|
|
52
124
|
staleIds.add(operation.objectId);
|
|
53
125
|
}
|
|
54
126
|
}
|
|
127
|
+
irreversibleStale.clear();
|
|
128
|
+
if (checkConflicts) {
|
|
129
|
+
for (const operation of plan.operations) {
|
|
130
|
+
if (!approved.has(operation.id) || !IRREVERSIBLE_OPERATIONS.has(operation.operation))
|
|
131
|
+
continue;
|
|
132
|
+
const detail = checkIrreversibleOp(operation, liveSnapshot);
|
|
133
|
+
if (detail)
|
|
134
|
+
irreversibleStale.set(operation.id, detail);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
55
137
|
for (const guard of plan.guards ?? []) {
|
|
56
138
|
const failure = evaluateGuard(liveSnapshot, guard);
|
|
57
139
|
if (failure) {
|
|
@@ -182,6 +264,13 @@ export async function applyPatchPlan(connector, plan, options) {
|
|
|
182
264
|
poisonedGroups.add(operation.groupId);
|
|
183
265
|
continue;
|
|
184
266
|
}
|
|
267
|
+
const irreversibleConflict = irreversibleStale.get(operation.id);
|
|
268
|
+
if (irreversibleConflict) {
|
|
269
|
+
results.push({ operationId: operation.id, status: "conflict", detail: irreversibleConflict });
|
|
270
|
+
if (operation.groupId)
|
|
271
|
+
poisonedGroups.add(operation.groupId);
|
|
272
|
+
continue;
|
|
273
|
+
}
|
|
185
274
|
if (operation.groupId && poisonedGroups.has(operation.groupId)) {
|
|
186
275
|
results.push({
|
|
187
276
|
operationId: operation.id,
|
package/dist/dedupe.d.ts
CHANGED
|
@@ -9,6 +9,12 @@ export type DedupeOptions = {
|
|
|
9
9
|
/** refuse to build plans larger than this (default 500 operations) */
|
|
10
10
|
maxOperations?: number;
|
|
11
11
|
};
|
|
12
|
+
/**
|
|
13
|
+
* The subset of a record worth keeping as a merge-recovery artifact: its id (to
|
|
14
|
+
* reference) plus every populated data field, dropping bulky/plumbing fields
|
|
15
|
+
* (raw, identities, provenance) that aren't needed to recreate it by hand.
|
|
16
|
+
*/
|
|
17
|
+
export declare function recoverableFields(record: Record<string, unknown>): Record<string, unknown>;
|
|
12
18
|
/** Normalize a record's identity key; undefined when the field is empty. */
|
|
13
19
|
export declare function dedupeKey(record: Record<string, unknown>, key: DedupeOptions["key"]): string | undefined;
|
|
14
20
|
export declare function buildDedupePlan(snapshot: CanonicalGtmSnapshot, options: DedupeOptions): PatchPlan;
|
package/dist/dedupe.js
CHANGED
|
@@ -40,6 +40,22 @@ const NON_DATA_FIELDS = new Set(["id", "provider", "crmId", "identities", "raw",
|
|
|
40
40
|
function populatedDataFields(record) {
|
|
41
41
|
return Object.entries(record).filter(([field, value]) => !NON_DATA_FIELDS.has(field) && value !== undefined && value !== null && value !== "").length;
|
|
42
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* The subset of a record worth keeping as a merge-recovery artifact: its id (to
|
|
45
|
+
* reference) plus every populated data field, dropping bulky/plumbing fields
|
|
46
|
+
* (raw, identities, provenance) that aren't needed to recreate it by hand.
|
|
47
|
+
*/
|
|
48
|
+
export function recoverableFields(record) {
|
|
49
|
+
const out = { id: String(record.id) };
|
|
50
|
+
for (const [field, value] of Object.entries(record)) {
|
|
51
|
+
if (NON_DATA_FIELDS.has(field))
|
|
52
|
+
continue;
|
|
53
|
+
if (value === undefined || value === null || value === "")
|
|
54
|
+
continue;
|
|
55
|
+
out[field] = value;
|
|
56
|
+
}
|
|
57
|
+
return out;
|
|
58
|
+
}
|
|
43
59
|
/** True when id `a` sorts before id `b` — numeric when both ids are numeric. */
|
|
44
60
|
function idBefore(a, b) {
|
|
45
61
|
const numericA = Number(a);
|
|
@@ -102,6 +118,12 @@ export function buildDedupePlan(snapshot, options) {
|
|
|
102
118
|
const groupIds = members
|
|
103
119
|
.map((member) => String(member.id))
|
|
104
120
|
.sort((a, b) => (idBefore(a, b) ? -1 : 1));
|
|
121
|
+
// Recovery artifact: the records that will be merged away (everyone but the
|
|
122
|
+
// survivor), captured with their field values so a human can recreate one by
|
|
123
|
+
// hand if the merge was wrong. Merges are irreversible — the plan is the backup.
|
|
124
|
+
const recoverySnapshot = members
|
|
125
|
+
.filter((member) => String(member.id) !== String(survivor.id))
|
|
126
|
+
.map((member) => recoverableFields(member));
|
|
105
127
|
const survivorName = typeof survivor.name === "string" && survivor.name
|
|
106
128
|
? survivor.name
|
|
107
129
|
: typeof survivor.email === "string" && survivor.email
|
|
@@ -124,7 +146,8 @@ export function buildDedupePlan(snapshot, options) {
|
|
|
124
146
|
approvalRequired: true,
|
|
125
147
|
sourceRuleOrPolicy: "dedupe",
|
|
126
148
|
groupId: `grp_${options.objectType}_${String(survivor.id)}`,
|
|
127
|
-
|
|
149
|
+
recoverySnapshot,
|
|
150
|
+
rollback: "IRREVERSIBLE: provider merges cannot be unmerged. recoverySnapshot on this operation retains every merged-away record's field values; recreate a record manually from it if a merge was wrong.",
|
|
128
151
|
});
|
|
129
152
|
}
|
|
130
153
|
return {
|
package/dist/index.d.ts
CHANGED
|
@@ -16,6 +16,7 @@ export { apolloPullKeysForAppend, apolloPullKeysForRefresh, createApolloClient,
|
|
|
16
16
|
export { diffFindings, diffSnapshots, diffToMarkdown, type CollectionDiff, type FieldChange, type FindingsDrift, type RecordChange, type SnapshotDiff, } from "./diff.ts";
|
|
17
17
|
export { mergeSnapshots, type MergeConflict, type MergeMatch, type MergeReport, type MergeSuggestion, } from "./merge.ts";
|
|
18
18
|
export { createFilePlanStore, type PlanStore, type StoredPlan } from "./planStore.ts";
|
|
19
|
+
export { computeApprovalDigests, loadOrCreateSigningKey, loadSigningKey, signApproval, verifyApprovalDigests, type ApprovalVerification, } from "./integrity.ts";
|
|
19
20
|
export { formatPatchPlanRun, patchPlanToMarkdown } from "./format.ts";
|
|
20
21
|
export { auditReportToHtml, auditReportToMarkdown, type ReportOptions } from "./report.ts";
|
|
21
22
|
export { HUBSPOT_DEFAULT_FIELD_MAPPINGS, SALESFORCE_DEFAULT_FIELD_MAPPINGS, mappedField, mappedFields, normalizeFieldMappings, readMappedValue, type CrmObjectType, type FieldMappings, } from "./mappings.ts";
|
package/dist/index.js
CHANGED
|
@@ -16,6 +16,7 @@ export { apolloPullKeysForAppend, apolloPullKeysForRefresh, createApolloClient,
|
|
|
16
16
|
export { diffFindings, diffSnapshots, diffToMarkdown, } from "./diff.js";
|
|
17
17
|
export { mergeSnapshots, } from "./merge.js";
|
|
18
18
|
export { createFilePlanStore } from "./planStore.js";
|
|
19
|
+
export { computeApprovalDigests, loadOrCreateSigningKey, loadSigningKey, signApproval, verifyApprovalDigests, } from "./integrity.js";
|
|
19
20
|
export { formatPatchPlanRun, patchPlanToMarkdown } from "./format.js";
|
|
20
21
|
export { auditReportToHtml, auditReportToMarkdown } from "./report.js";
|
|
21
22
|
export { HUBSPOT_DEFAULT_FIELD_MAPPINGS, SALESFORCE_DEFAULT_FIELD_MAPPINGS, mappedField, mappedFields, normalizeFieldMappings, readMappedValue, } from "./mappings.js";
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import type { PatchOperation } from "./types.ts";
|
|
2
|
+
/** Read the signing key, or null if it has not been created yet. */
|
|
3
|
+
export declare function loadSigningKey(): Buffer | null;
|
|
4
|
+
/** Read the signing key, creating a fresh 32-byte one (0600) on first use. */
|
|
5
|
+
export declare function loadOrCreateSigningKey(): Buffer;
|
|
6
|
+
/** HMAC-SHA256 signature of one operation's approved content. */
|
|
7
|
+
export declare function signApproval(operation: PatchOperation, override: unknown, key: Buffer): string;
|
|
8
|
+
/**
|
|
9
|
+
* Compute the approval signature map for a set of approved operation ids,
|
|
10
|
+
* resolving each op from the plan and its (approved) value override.
|
|
11
|
+
*/
|
|
12
|
+
export declare function computeApprovalDigests(operations: PatchOperation[], approvedOperationIds: string[], valueOverrides: Record<string, unknown>, key: Buffer): Record<string, string>;
|
|
13
|
+
export type ApprovalVerification = {
|
|
14
|
+
ok: true;
|
|
15
|
+
} | {
|
|
16
|
+
ok: false;
|
|
17
|
+
reason: "no_key";
|
|
18
|
+
tampered: string[];
|
|
19
|
+
} | {
|
|
20
|
+
ok: false;
|
|
21
|
+
reason: "mismatch";
|
|
22
|
+
tampered: string[];
|
|
23
|
+
};
|
|
24
|
+
/**
|
|
25
|
+
* Verify that every approved operation still matches what was signed. Returns
|
|
26
|
+
* ok:true when there are no stored digests (a pre-integrity plan — nothing to
|
|
27
|
+
* verify), when all match, or fails with the list of operation ids whose
|
|
28
|
+
* content changed since approval.
|
|
29
|
+
*/
|
|
30
|
+
export declare function verifyApprovalDigests(operations: PatchOperation[], approvedOperationIds: string[], valueOverrides: Record<string, unknown>, storedDigests: Record<string, string> | undefined): ApprovalVerification;
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { createHmac, randomBytes } from "node:crypto";
|
|
2
|
+
import { existsSync, readFileSync } from "node:fs";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { credentialsDir, ensureSecureHomeDir, writeSecureFile } from "./credentials.js";
|
|
5
|
+
/**
|
|
6
|
+
* Approval integrity.
|
|
7
|
+
*
|
|
8
|
+
* The plan store records WHICH operation ids a human approved, but the apply
|
|
9
|
+
* path re-reads the operation BODIES fresh from the (user-editable) plan file.
|
|
10
|
+
* Nothing bound the approval to the content: an approved op's afterValue or
|
|
11
|
+
* objectId could be changed on disk between `plans approve` and `apply` — by a
|
|
12
|
+
* compromised dependency, a co-tenant, or a plan file synced/edited on another
|
|
13
|
+
* machine — and the changed value would be written under the prior approval.
|
|
14
|
+
*
|
|
15
|
+
* Fix: at approval time, HMAC-sign each approved operation's security-relevant
|
|
16
|
+
* content (including the approved value override) with a per-install secret key
|
|
17
|
+
* stored 0600 alongside the credentials. At apply time, recompute and verify.
|
|
18
|
+
* Any post-approval edit to the operations or the approved overrides changes the
|
|
19
|
+
* signature; a tamper must now also forge an HMAC it cannot compute without the
|
|
20
|
+
* key. The key never leaves the machine, so a plan approved here and applied
|
|
21
|
+
* elsewhere fails closed ("re-approve on this machine") rather than open.
|
|
22
|
+
*
|
|
23
|
+
* This raises the bar from "trust the plan JSON" to "trust the plan JSON only
|
|
24
|
+
* insofar as it still matches what was signed with the local key." It is not a
|
|
25
|
+
* defense against an attacker who already holds the signing key (same-dir, same
|
|
26
|
+
* permissions as the credential store) — that is the documented boundary.
|
|
27
|
+
*/
|
|
28
|
+
const SIGNING_KEY_FILE = ".plan-signing-key";
|
|
29
|
+
function signingKeyPath() {
|
|
30
|
+
return join(credentialsDir(), SIGNING_KEY_FILE);
|
|
31
|
+
}
|
|
32
|
+
/** Read the signing key, or null if it has not been created yet. */
|
|
33
|
+
export function loadSigningKey() {
|
|
34
|
+
const path = signingKeyPath();
|
|
35
|
+
if (!existsSync(path))
|
|
36
|
+
return null;
|
|
37
|
+
try {
|
|
38
|
+
return Buffer.from(readFileSync(path, "utf8").trim(), "hex");
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return null;
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
/** Read the signing key, creating a fresh 32-byte one (0600) on first use. */
|
|
45
|
+
export function loadOrCreateSigningKey() {
|
|
46
|
+
const existing = loadSigningKey();
|
|
47
|
+
if (existing && existing.length >= 32)
|
|
48
|
+
return existing;
|
|
49
|
+
ensureSecureHomeDir();
|
|
50
|
+
const key = randomBytes(32);
|
|
51
|
+
writeSecureFile(signingKeyPath(), `${key.toString("hex")}\n`);
|
|
52
|
+
return key;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Canonical, stable string of the operation content an approval binds to. Only
|
|
56
|
+
* the fields that determine WHAT gets written: changing any of them must
|
|
57
|
+
* invalidate the approval. `override` is the approved value override for this op
|
|
58
|
+
* (the value actually written when set), so tampering with stored overrides is
|
|
59
|
+
* caught too.
|
|
60
|
+
*/
|
|
61
|
+
function canonicalApprovalContent(operation, override) {
|
|
62
|
+
return JSON.stringify([
|
|
63
|
+
operation.id,
|
|
64
|
+
operation.operation,
|
|
65
|
+
operation.objectType,
|
|
66
|
+
operation.objectId,
|
|
67
|
+
operation.field ?? null,
|
|
68
|
+
operation.beforeValue ?? null,
|
|
69
|
+
operation.afterValue ?? null,
|
|
70
|
+
operation.groupId ?? null,
|
|
71
|
+
// Safety-relevant fields too: editing a precondition could relax a drift
|
|
72
|
+
// guard, and forging forceArchiveDuplicate could suppress the archive-of-
|
|
73
|
+
// duplicate refusal — the signed approval must pin apply BEHAVIOR, not just
|
|
74
|
+
// the written value. `reason` is human-reviewed AND written verbatim into
|
|
75
|
+
// create_task bodies (afterValue ?? reason fallback in the connectors), so a
|
|
76
|
+
// create_task with a null afterValue would otherwise let a disk edit to
|
|
77
|
+
// reason write unapproved text under a still-valid digest.
|
|
78
|
+
operation.preconditions ?? null,
|
|
79
|
+
operation.forceArchiveDuplicate ?? false,
|
|
80
|
+
operation.reason ?? null,
|
|
81
|
+
override === undefined ? null : ["__override__", override],
|
|
82
|
+
]);
|
|
83
|
+
}
|
|
84
|
+
/** HMAC-SHA256 signature of one operation's approved content. */
|
|
85
|
+
export function signApproval(operation, override, key) {
|
|
86
|
+
return createHmac("sha256", key).update(canonicalApprovalContent(operation, override)).digest("hex");
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Compute the approval signature map for a set of approved operation ids,
|
|
90
|
+
* resolving each op from the plan and its (approved) value override.
|
|
91
|
+
*/
|
|
92
|
+
export function computeApprovalDigests(operations, approvedOperationIds, valueOverrides, key) {
|
|
93
|
+
const byId = new Map(operations.map((operation) => [operation.id, operation]));
|
|
94
|
+
const digests = {};
|
|
95
|
+
for (const id of approvedOperationIds) {
|
|
96
|
+
const operation = byId.get(id);
|
|
97
|
+
if (!operation)
|
|
98
|
+
continue;
|
|
99
|
+
digests[id] = signApproval(operation, valueOverrides[id], key);
|
|
100
|
+
}
|
|
101
|
+
return digests;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Verify that every approved operation still matches what was signed. Returns
|
|
105
|
+
* ok:true when there are no stored digests (a pre-integrity plan — nothing to
|
|
106
|
+
* verify), when all match, or fails with the list of operation ids whose
|
|
107
|
+
* content changed since approval.
|
|
108
|
+
*/
|
|
109
|
+
export function verifyApprovalDigests(operations, approvedOperationIds, valueOverrides, storedDigests) {
|
|
110
|
+
if (!storedDigests || Object.keys(storedDigests).length === 0)
|
|
111
|
+
return { ok: true };
|
|
112
|
+
const key = loadSigningKey();
|
|
113
|
+
if (!key)
|
|
114
|
+
return { ok: false, reason: "no_key", tampered: approvedOperationIds };
|
|
115
|
+
const byId = new Map(operations.map((operation) => [operation.id, operation]));
|
|
116
|
+
const tampered = [];
|
|
117
|
+
for (const id of approvedOperationIds) {
|
|
118
|
+
const operation = byId.get(id);
|
|
119
|
+
const expected = storedDigests[id];
|
|
120
|
+
if (!operation || !expected) {
|
|
121
|
+
tampered.push(id);
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
if (signApproval(operation, valueOverrides[id], key) !== expected)
|
|
125
|
+
tampered.push(id);
|
|
126
|
+
}
|
|
127
|
+
return tampered.length === 0 ? { ok: true } : { ok: false, reason: "mismatch", tampered };
|
|
128
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { type LlmCallOptions } from "./llm.ts";
|
|
2
|
+
import { type FetchPage, type MarketConfig } from "./market.ts";
|
|
3
|
+
/**
|
|
4
|
+
* Cold-start taxonomy bootstrap. `market init` writes a stub for a human
|
|
5
|
+
* analyst to fill in; the self-serve hosted map has no analyst in the loop, so
|
|
6
|
+
* this proposes the claim taxonomy automatically from the seed vendors' own
|
|
7
|
+
* pages.
|
|
8
|
+
*
|
|
9
|
+
* Posture matches the rest of the market layer: the LLM is a *proposal* layer
|
|
10
|
+
* grounded in captured evidence (it only sees text we actually fetched), and
|
|
11
|
+
* everything downstream — capture, classify with verbatim-span verification,
|
|
12
|
+
* front states, the report — stays deterministic over the stored observations.
|
|
13
|
+
* The taxonomy it emits is a normal `market.config.json` a human can still edit.
|
|
14
|
+
*/
|
|
15
|
+
export type SeedVendor = {
|
|
16
|
+
url: string;
|
|
17
|
+
/** Display name; derived from the host when omitted. */
|
|
18
|
+
name?: string;
|
|
19
|
+
/** Marks the user's own company as the anchor vendor. */
|
|
20
|
+
anchor?: boolean;
|
|
21
|
+
};
|
|
22
|
+
export type SuggestTaxonomyOptions = {
|
|
23
|
+
category: string;
|
|
24
|
+
vendors: SeedVendor[];
|
|
25
|
+
llm: LlmCallOptions;
|
|
26
|
+
/** Upper bound on proposed claims, to keep classification bounded. */
|
|
27
|
+
maxClaims?: number;
|
|
28
|
+
/** Per-vendor captured-text budget fed to the proposer (chars). */
|
|
29
|
+
perVendorChars?: number;
|
|
30
|
+
/** Test injectables. */
|
|
31
|
+
fetchPage?: FetchPage;
|
|
32
|
+
capturesDir?: string;
|
|
33
|
+
now?: () => Date;
|
|
34
|
+
};
|
|
35
|
+
export type SuggestTaxonomyResult = {
|
|
36
|
+
config: MarketConfig;
|
|
37
|
+
/** Vendors whose homepage capture was empty/failed (excluded from grounding). */
|
|
38
|
+
unreadableVendorIds: string[];
|
|
39
|
+
model: string;
|
|
40
|
+
};
|
|
41
|
+
export declare function suggestMarketConfig(options: SuggestTaxonomyOptions): Promise<SuggestTaxonomyResult>;
|