fullstackgtm 0.41.0 → 0.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/connector.js +34 -0
- package/dist/connectors/hubspot.js +124 -0
- package/dist/connectors/salesforce.js +112 -0
- package/dist/types.d.ts +11 -0
- package/docs/api.md +8 -1
- package/package.json +1 -1
- package/src/connector.ts +37 -0
- package/src/connectors/hubspot.ts +125 -0
- package/src/connectors/salesforce.ts +113 -0
- package/src/types.ts +11 -0
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,23 @@ The path to 1.0 is planned in [docs/roadmap-to-1.0.md](./docs/roadmap-to-1.0.md)
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.42.0] — 2026-06-25
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- **Bulk apply for lead creation (HubSpot + Salesforce).** `applyPatchPlan` now
|
|
15
|
+
routes independent, approved `create_record` **contact** ops through a new
|
|
16
|
+
optional connector capability, `applyCreateContactsBatch`, instead of two API
|
|
17
|
+
round-trips per lead (a resolve-first search + a create). HubSpot batches the
|
|
18
|
+
resolve-first via a `search` `IN` and creates via `batch/create` (100/call);
|
|
19
|
+
Salesforce batches via a SOQL `IN` and the Composite sObject Collections API
|
|
20
|
+
(`allOrNone: false`, 200/call). A batch rejection (e.g. a duplicate that the
|
|
21
|
+
provider 4xxs the whole batch over) falls back to per-record `createRecord`, so
|
|
22
|
+
one bad row never sinks the rest and resolve-first is preserved. Turns ~2N API
|
|
23
|
+
calls into ~N/100; ops that aren't safe to batch (grouped, value-overridden,
|
|
24
|
+
company-associated, conflicted, or non-create) stay on the serial path
|
|
25
|
+
unchanged. Connectors without the capability are unaffected.
|
|
26
|
+
|
|
10
27
|
## [0.41.0] — 2026-06-23
|
|
11
28
|
|
|
12
29
|
### Added
|
package/dist/connector.js
CHANGED
|
@@ -227,7 +227,41 @@ export async function applyPatchPlan(connector, plan, options) {
|
|
|
227
227
|
}
|
|
228
228
|
}
|
|
229
229
|
}
|
|
230
|
+
// Bulk fast-path: route independent, approved create_record CONTACT ops
|
|
231
|
+
// through the connector's batch API (batched resolve-first read + batched
|
|
232
|
+
// writes) instead of one round-trip per record. Only ops that are safe to
|
|
233
|
+
// batch are eligible — every other op (grouped, needs an override, carries a
|
|
234
|
+
// company association, conflicted, or any non-create op) stays on the serial
|
|
235
|
+
// path below, so the safety contract is unchanged. Results are merged in by
|
|
236
|
+
// id; the serial loop short-circuits anything already resolved here.
|
|
237
|
+
const batched = new Map();
|
|
238
|
+
if (connector.applyCreateContactsBatch && !guardFailure) {
|
|
239
|
+
const eligible = plan.operations.filter((operation) => approved.has(operation.id) &&
|
|
240
|
+
operation.operation === "create_record" &&
|
|
241
|
+
operation.objectType === "contact" &&
|
|
242
|
+
options.valueOverrides?.[operation.id] === undefined &&
|
|
243
|
+
!requiresHumanInput(operation.afterValue) &&
|
|
244
|
+
!operation.groupId &&
|
|
245
|
+
!operation.afterValue?.associateCompanyName &&
|
|
246
|
+
!conflicts.has(operation.id) &&
|
|
247
|
+
!irreversibleStale.has(operation.id) &&
|
|
248
|
+
!(staleByFilter && staleByFilter.has(operation.objectId)));
|
|
249
|
+
// Only worth the batch machinery for more than a couple of creates.
|
|
250
|
+
if (eligible.length > 2) {
|
|
251
|
+
const batchResults = await connector.applyCreateContactsBatch(eligible);
|
|
252
|
+
for (const result of batchResults)
|
|
253
|
+
batched.set(result.operationId, result);
|
|
254
|
+
}
|
|
255
|
+
}
|
|
230
256
|
for (const operation of plan.operations) {
|
|
257
|
+
const batchedResult = batched.get(operation.id);
|
|
258
|
+
if (batchedResult) {
|
|
259
|
+
results.push(batchedResult);
|
|
260
|
+
attempted += 1;
|
|
261
|
+
if (batchedResult.status === "applied")
|
|
262
|
+
applied += 1;
|
|
263
|
+
continue;
|
|
264
|
+
}
|
|
231
265
|
if (!approved.has(operation.id)) {
|
|
232
266
|
results.push({
|
|
233
267
|
operationId: operation.id,
|
|
@@ -555,6 +555,129 @@ export function createHubspotConnector(options) {
|
|
|
555
555
|
providerData: { id: contactId, created: true },
|
|
556
556
|
};
|
|
557
557
|
}
|
|
558
|
+
/** Bulk resolve-first lookup: which match values already exist (value→id). */
|
|
559
|
+
async function searchContactsByValues(property, values) {
|
|
560
|
+
const found = new Map();
|
|
561
|
+
for (let i = 0; i < values.length; i += 100) {
|
|
562
|
+
const chunk = values.slice(i, i + 100);
|
|
563
|
+
const data = await request(`/crm/v3/objects/contacts/search`, {
|
|
564
|
+
method: "POST",
|
|
565
|
+
body: JSON.stringify({
|
|
566
|
+
filterGroups: [{ filters: [{ propertyName: property, operator: "IN", values: chunk }] }],
|
|
567
|
+
properties: [property],
|
|
568
|
+
limit: 100,
|
|
569
|
+
}),
|
|
570
|
+
});
|
|
571
|
+
for (const rec of (data?.results ?? [])) {
|
|
572
|
+
const v = rec?.properties?.[property];
|
|
573
|
+
if (typeof v === "string" && rec.id)
|
|
574
|
+
found.set(v.toLowerCase(), String(rec.id));
|
|
575
|
+
}
|
|
576
|
+
}
|
|
577
|
+
return found;
|
|
578
|
+
}
|
|
579
|
+
/**
|
|
580
|
+
* Batch create_record for contacts: bulk resolve-first (search IN), then
|
|
581
|
+
* bulk-create the genuine misses (batch/create, 100/call) — two API calls per
|
|
582
|
+
* ~100 leads instead of two per lead. On a batch-create rejection (e.g. an
|
|
583
|
+
* email collision HubSpot 409s the whole batch over), it falls back to
|
|
584
|
+
* per-record createRecord for that chunk, so one bad row never sinks the rest
|
|
585
|
+
* and resolve-first is preserved. Returns one result per input op.
|
|
586
|
+
*/
|
|
587
|
+
async function applyCreateContactsBatch(operations) {
|
|
588
|
+
const byId = new Map();
|
|
589
|
+
// Group by the HubSpot search property (matchKey is usually uniform across a
|
|
590
|
+
// plan, but a mixed plan stays correct this way).
|
|
591
|
+
const groups = new Map();
|
|
592
|
+
for (const op of operations) {
|
|
593
|
+
const matchKey = op.afterValue?.matchKey || "email";
|
|
594
|
+
const searchProperty = HUBSPOT_DEFAULT_FIELD_MAPPINGS.contacts[matchKey] ?? matchKey;
|
|
595
|
+
let arr = groups.get(searchProperty);
|
|
596
|
+
if (!arr)
|
|
597
|
+
groups.set(searchProperty, (arr = []));
|
|
598
|
+
arr.push(op);
|
|
599
|
+
}
|
|
600
|
+
for (const [searchProperty, ops] of groups) {
|
|
601
|
+
const wanted = ops
|
|
602
|
+
.map((op) => String(op.afterValue.matchValue ?? "").trim())
|
|
603
|
+
.filter(Boolean);
|
|
604
|
+
const existing = await searchContactsByValues(searchProperty, [...new Set(wanted.map((v) => v.toLowerCase()))]);
|
|
605
|
+
const toCreate = [];
|
|
606
|
+
const seenInBatch = new Set();
|
|
607
|
+
for (const op of ops) {
|
|
608
|
+
const payload = op.afterValue;
|
|
609
|
+
const matchKey = payload.matchKey || "email";
|
|
610
|
+
const matchValue = String(payload.matchValue ?? "").trim();
|
|
611
|
+
if (!matchValue) {
|
|
612
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: "create_record needs a non-empty matchValue to resolve-first." });
|
|
613
|
+
continue;
|
|
614
|
+
}
|
|
615
|
+
const lower = matchValue.toLowerCase();
|
|
616
|
+
const dedupeKey = `${matchKey}:${lower}`;
|
|
617
|
+
const already = createdContactsByMatch.get(dedupeKey);
|
|
618
|
+
if (already) {
|
|
619
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already created earlier in this run; not duplicating.`, providerData: { id: already, existing: true } });
|
|
620
|
+
continue;
|
|
621
|
+
}
|
|
622
|
+
const existingId = existing.get(lower);
|
|
623
|
+
if (existingId) {
|
|
624
|
+
createdContactsByMatch.set(dedupeKey, existingId);
|
|
625
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already exists (${existingId}); resolve-first declined to create.`, providerData: { id: existingId, existing: true } });
|
|
626
|
+
continue;
|
|
627
|
+
}
|
|
628
|
+
if (seenInBatch.has(lower)) {
|
|
629
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} duplicated within this batch; not duplicating.` });
|
|
630
|
+
continue;
|
|
631
|
+
}
|
|
632
|
+
seenInBatch.add(lower);
|
|
633
|
+
toCreate.push(op);
|
|
634
|
+
}
|
|
635
|
+
for (let i = 0; i < toCreate.length; i += 100) {
|
|
636
|
+
const chunk = toCreate.slice(i, i + 100);
|
|
637
|
+
const inputs = chunk.map((op) => {
|
|
638
|
+
const payload = op.afterValue;
|
|
639
|
+
const properties = { ...payload.properties };
|
|
640
|
+
if (payload.ownerId && !properties.hubspot_owner_id)
|
|
641
|
+
properties.hubspot_owner_id = String(payload.ownerId);
|
|
642
|
+
properties.hs_object_source_detail_2 = `fullstackgtm acquire (${op.id})`;
|
|
643
|
+
return { properties };
|
|
644
|
+
});
|
|
645
|
+
try {
|
|
646
|
+
const data = await request(`/crm/v3/objects/contacts/batch/create`, {
|
|
647
|
+
method: "POST",
|
|
648
|
+
body: JSON.stringify({ inputs }),
|
|
649
|
+
});
|
|
650
|
+
const created = (data?.results ?? []);
|
|
651
|
+
const idByValue = new Map();
|
|
652
|
+
for (const rec of created) {
|
|
653
|
+
const v = rec?.properties?.[searchProperty];
|
|
654
|
+
if (typeof v === "string" && rec.id)
|
|
655
|
+
idByValue.set(v.toLowerCase(), String(rec.id));
|
|
656
|
+
}
|
|
657
|
+
for (const op of chunk) {
|
|
658
|
+
const payload = op.afterValue;
|
|
659
|
+
const matchKey = payload.matchKey || "email";
|
|
660
|
+
const matchValue = String(payload.matchValue).trim();
|
|
661
|
+
const id = idByValue.get(matchValue.toLowerCase());
|
|
662
|
+
if (id) {
|
|
663
|
+
createdContactsByMatch.set(`${matchKey}:${matchValue.toLowerCase()}`, id);
|
|
664
|
+
byId.set(op.id, { operationId: op.id, status: "applied", detail: `Created contact ${matchKey}=${matchValue} (${id}).`, providerData: { id, created: true } });
|
|
665
|
+
}
|
|
666
|
+
else {
|
|
667
|
+
// Result didn't echo this value — resolve it per-record to be safe.
|
|
668
|
+
byId.set(op.id, await createRecord(op));
|
|
669
|
+
}
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
catch {
|
|
673
|
+
// Whole-chunk rejection (e.g. a collision). Isolate per record.
|
|
674
|
+
for (const op of chunk)
|
|
675
|
+
byId.set(op.id, await createRecord(op));
|
|
676
|
+
}
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
return operations.map((op) => byId.get(op.id) ?? { operationId: op.id, status: "skipped", detail: "create_record was not processed." });
|
|
680
|
+
}
|
|
558
681
|
async function createTask(operation) {
|
|
559
682
|
const associationTypeId = TASK_ASSOCIATION_TYPE_IDS[operation.objectType];
|
|
560
683
|
if (associationTypeId === undefined) {
|
|
@@ -801,6 +924,7 @@ export function createHubspotConnector(options) {
|
|
|
801
924
|
fetchSnapshot,
|
|
802
925
|
fetchChanges,
|
|
803
926
|
applyOperation,
|
|
927
|
+
applyCreateContactsBatch,
|
|
804
928
|
readField,
|
|
805
929
|
};
|
|
806
930
|
}
|
|
@@ -489,6 +489,117 @@ export function createSalesforceConnector(options) {
|
|
|
489
489
|
providerData: { id: contactId, created: true },
|
|
490
490
|
};
|
|
491
491
|
}
|
|
492
|
+
/**
|
|
493
|
+
* Batch create_record for contacts: bulk resolve-first (SOQL IN), then create
|
|
494
|
+
* the misses via the Composite sObject Collections API (allOrNone:false,
|
|
495
|
+
* 200/call), instead of two round-trips per lead. A record-level failure in
|
|
496
|
+
* the collection (e.g. a duplicate rule) falls back to per-record createRecord
|
|
497
|
+
* so one bad row never sinks the rest. Returns one result per input op.
|
|
498
|
+
*/
|
|
499
|
+
async function applyCreateContactsBatch(operations) {
|
|
500
|
+
const byId = new Map();
|
|
501
|
+
const groups = new Map();
|
|
502
|
+
for (const op of operations) {
|
|
503
|
+
const matchKey = op.afterValue?.matchKey || "email";
|
|
504
|
+
const searchField = mappedField(mappings, "contacts", matchKey, SALESFORCE_DEFAULT_FIELD_MAPPINGS.contacts[matchKey] ?? matchKey);
|
|
505
|
+
let arr = groups.get(searchField);
|
|
506
|
+
if (!arr)
|
|
507
|
+
groups.set(searchField, (arr = []));
|
|
508
|
+
arr.push(op);
|
|
509
|
+
}
|
|
510
|
+
for (const [searchField, ops] of groups) {
|
|
511
|
+
// 1. Bulk resolve-first via SOQL IN (chunked; values SOQL-escaped).
|
|
512
|
+
const existing = new Map();
|
|
513
|
+
const uniqueValues = [
|
|
514
|
+
...new Set(ops.map((op) => String(op.afterValue.matchValue ?? "").trim()).filter(Boolean)),
|
|
515
|
+
];
|
|
516
|
+
for (let i = 0; i < uniqueValues.length; i += 200) {
|
|
517
|
+
const inList = uniqueValues
|
|
518
|
+
.slice(i, i + 200)
|
|
519
|
+
.map((v) => `'${v.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`)
|
|
520
|
+
.join(",");
|
|
521
|
+
const rows = await query(`SELECT Id, ${searchField} FROM Contact WHERE ${searchField} IN (${inList})`);
|
|
522
|
+
for (const row of rows) {
|
|
523
|
+
const v = row?.[searchField];
|
|
524
|
+
if (typeof v === "string" && row.Id)
|
|
525
|
+
existing.set(v.toLowerCase(), String(row.Id));
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
// 2. Partition: existing / already-this-run / dup-in-batch / to-create.
|
|
529
|
+
const toCreate = [];
|
|
530
|
+
const seenInBatch = new Set();
|
|
531
|
+
for (const op of ops) {
|
|
532
|
+
const payload = op.afterValue;
|
|
533
|
+
const matchKey = payload.matchKey || "email";
|
|
534
|
+
const matchValue = String(payload.matchValue ?? "").trim();
|
|
535
|
+
if (!matchValue) {
|
|
536
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: "create_record needs a non-empty matchValue to resolve-first." });
|
|
537
|
+
continue;
|
|
538
|
+
}
|
|
539
|
+
const lower = matchValue.toLowerCase();
|
|
540
|
+
const dedupeKey = `${matchKey}:${lower}`;
|
|
541
|
+
const already = createdContactsByMatch.get(dedupeKey);
|
|
542
|
+
if (already) {
|
|
543
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already created earlier in this run; not duplicating.`, providerData: { id: already, existing: true } });
|
|
544
|
+
continue;
|
|
545
|
+
}
|
|
546
|
+
const existingId = existing.get(lower);
|
|
547
|
+
if (existingId) {
|
|
548
|
+
createdContactsByMatch.set(dedupeKey, existingId);
|
|
549
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already exists (${existingId}); resolve-first declined to create.`, providerData: { id: existingId, existing: true } });
|
|
550
|
+
continue;
|
|
551
|
+
}
|
|
552
|
+
if (seenInBatch.has(lower)) {
|
|
553
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} duplicated within this batch; not duplicating.` });
|
|
554
|
+
continue;
|
|
555
|
+
}
|
|
556
|
+
seenInBatch.add(lower);
|
|
557
|
+
toCreate.push(op);
|
|
558
|
+
}
|
|
559
|
+
// 3. Bulk create via Composite sObject Collections (allOrNone:false →
|
|
560
|
+
// per-record results in request order).
|
|
561
|
+
for (let i = 0; i < toCreate.length; i += 200) {
|
|
562
|
+
const chunk = toCreate.slice(i, i + 200);
|
|
563
|
+
const records = chunk.map((op) => {
|
|
564
|
+
const payload = op.afterValue;
|
|
565
|
+
const fields = { ...payload.properties };
|
|
566
|
+
if (payload.ownerId) {
|
|
567
|
+
const ownerField = mappedField(mappings, "contacts", "ownerId", SALESFORCE_DEFAULT_FIELD_MAPPINGS.contacts.ownerId ?? "OwnerId");
|
|
568
|
+
if (!fields[ownerField])
|
|
569
|
+
fields[ownerField] = String(payload.ownerId);
|
|
570
|
+
}
|
|
571
|
+
return { attributes: { type: "Contact" }, ...fields };
|
|
572
|
+
});
|
|
573
|
+
let results;
|
|
574
|
+
try {
|
|
575
|
+
results = (await request(`/services/data/${apiVersion}/composite/sobjects`, {
|
|
576
|
+
method: "POST",
|
|
577
|
+
body: JSON.stringify({ allOrNone: false, records }),
|
|
578
|
+
}));
|
|
579
|
+
}
|
|
580
|
+
catch {
|
|
581
|
+
for (const op of chunk)
|
|
582
|
+
byId.set(op.id, await createRecord(op));
|
|
583
|
+
continue;
|
|
584
|
+
}
|
|
585
|
+
for (let j = 0; j < chunk.length; j++) {
|
|
586
|
+
const op = chunk[j];
|
|
587
|
+
const res = Array.isArray(results) ? results[j] : undefined;
|
|
588
|
+
const payload = op.afterValue;
|
|
589
|
+
const matchKey = payload.matchKey || "email";
|
|
590
|
+
const matchValue = String(payload.matchValue).trim();
|
|
591
|
+
if (res?.success && res.id) {
|
|
592
|
+
createdContactsByMatch.set(`${matchKey}:${matchValue.toLowerCase()}`, String(res.id));
|
|
593
|
+
byId.set(op.id, { operationId: op.id, status: "applied", detail: `Created contact ${matchKey}=${matchValue} (${res.id}).`, providerData: { id: String(res.id), created: true } });
|
|
594
|
+
}
|
|
595
|
+
else {
|
|
596
|
+
byId.set(op.id, await createRecord(op)); // record-level failure → isolate per record
|
|
597
|
+
}
|
|
598
|
+
}
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
return operations.map((op) => byId.get(op.id) ?? { operationId: op.id, status: "skipped", detail: "create_record was not processed." });
|
|
602
|
+
}
|
|
492
603
|
async function applyOperation(operation) {
|
|
493
604
|
try {
|
|
494
605
|
switch (operation.operation) {
|
|
@@ -568,6 +679,7 @@ export function createSalesforceConnector(options) {
|
|
|
568
679
|
fetchSnapshot,
|
|
569
680
|
fetchChanges,
|
|
570
681
|
applyOperation,
|
|
682
|
+
applyCreateContactsBatch,
|
|
571
683
|
readField,
|
|
572
684
|
};
|
|
573
685
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -384,6 +384,17 @@ export type GtmConnector = {
|
|
|
384
384
|
provider: CrmProvider;
|
|
385
385
|
fetchSnapshot: () => Promise<CanonicalGtmSnapshot>;
|
|
386
386
|
applyOperation?: (operation: PatchOperation) => Promise<PatchOperationResult>;
|
|
387
|
+
/**
|
|
388
|
+
* Bulk fast-path for independent `create_record` contact ops: batch the
|
|
389
|
+
* resolve-first read and the create writes instead of one round-trip per
|
|
390
|
+
* record (orders of magnitude fewer API calls for lead acquisition). The
|
|
391
|
+
* caller (`applyPatchPlan`) only routes ops here that are safe to batch —
|
|
392
|
+
* approved, no group, no value override, no company association — and falls
|
|
393
|
+
* back to `applyOperation` for the rest. Implementations MUST preserve
|
|
394
|
+
* resolve-first (never create over an existing match) and return exactly one
|
|
395
|
+
* result per input op. Optional: connectors without it use `applyOperation`.
|
|
396
|
+
*/
|
|
397
|
+
applyCreateContactsBatch?: (operations: PatchOperation[]) => Promise<PatchOperationResult[]>;
|
|
387
398
|
/**
|
|
388
399
|
* Read the live value of one canonical field, used for compare-and-set:
|
|
389
400
|
* apply orchestration refuses to write over values that drifted since the
|
package/docs/api.md
CHANGED
|
@@ -37,9 +37,16 @@ release.
|
|
|
37
37
|
|
|
38
38
|
## Connectors
|
|
39
39
|
|
|
40
|
-
- `GtmConnector` — `{ provider, fetchSnapshot(), applyOperation?, readField?, fetchChanges? }`.
|
|
40
|
+
- `GtmConnector` — `{ provider, fetchSnapshot(), applyOperation?, applyCreateContactsBatch?, readField?, fetchChanges? }`.
|
|
41
41
|
- Connectors never silently drop unresolvable records; audits surface them.
|
|
42
42
|
- `fetchChanges(sinceIso)` returns a partial snapshot; change feeds may omit associations.
|
|
43
|
+
- `applyCreateContactsBatch?(operations)` — optional bulk fast-path for
|
|
44
|
+
independent `create_record` contact ops. `applyPatchPlan` routes safe-to-batch
|
|
45
|
+
creates here (batched resolve-first + batched create — ~N/100 calls instead of
|
|
46
|
+
~2N), and falls back to `applyOperation` per record on a batch rejection and
|
|
47
|
+
for any op that isn't batch-safe (grouped, value-overridden, company-
|
|
48
|
+
associated, conflicted). HubSpot: `search IN` + `batch/create`; Salesforce:
|
|
49
|
+
SOQL `IN` + Composite sObject Collections (`allOrNone:false`).
|
|
43
50
|
- `createHubspotConnector(options)` — read/write/readField/fetchChanges. `applyOperation` implements every `PatchOperationType`: `set_field`, `clear_field`, `link_record`, `create_task`, `create_record` (resolve-first net-new contact/company create — re-checks the dedupe key at apply, never double-creates), `archive_record`, `merge_records` (HubSpot v3 merge — pairwise, irreversible; survivor must belong to the duplicate group). (The Salesforce connector implements the same operation set, with the platform-specific constraints noted below.)
|
|
44
51
|
- `createSalesforceConnector(options)` — read/write/readField/fetchChanges; probabilities normalized to 0..1. `applyOperation` implements every operation type, with two platform constraints: `merge_records` covers **Accounts and Contacts** only (Salesforce exposes no Opportunity merge), and `create_record` resolve-first creates **contacts/accounts** (SOQL search on the match key, create only on a confirmed miss; stamps the canonical `ownerId` onto `OwnerId`).
|
|
45
52
|
- `createStripeConnector(options)` — read-only billing by design (`applyOperation` returns `skipped`); email domains are the cross-system merge keys. Implements `fetchChanges` (incremental via `created[gte]`).
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "fullstackgtm",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.42.0",
|
|
4
4
|
"description": "Open-source agentic GTM ops framework: canonical GTM data model, pluggable deterministic audits, reviewable dry-run patch plans, approval-gated write-back with conflict detection, and cross-system entity resolution. HubSpot, Salesforce, and Stripe connectors included.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Full Stack GTM LLC <ryan@fullstackgtm.com> (https://fullstackgtm.com)",
|
package/src/connector.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { dedupeKey } from "./dedupe.ts";
|
|
|
2
2
|
import { requiresHumanInput } from "./rules.ts";
|
|
3
3
|
import type {
|
|
4
4
|
CanonicalGtmSnapshot,
|
|
5
|
+
CreateRecordPayload,
|
|
5
6
|
GtmConnector,
|
|
6
7
|
PatchOperation,
|
|
7
8
|
PatchOperationResult,
|
|
@@ -283,7 +284,43 @@ export async function applyPatchPlan(
|
|
|
283
284
|
}
|
|
284
285
|
}
|
|
285
286
|
|
|
287
|
+
// Bulk fast-path: route independent, approved create_record CONTACT ops
|
|
288
|
+
// through the connector's batch API (batched resolve-first read + batched
|
|
289
|
+
// writes) instead of one round-trip per record. Only ops that are safe to
|
|
290
|
+
// batch are eligible — every other op (grouped, needs an override, carries a
|
|
291
|
+
// company association, conflicted, or any non-create op) stays on the serial
|
|
292
|
+
// path below, so the safety contract is unchanged. Results are merged in by
|
|
293
|
+
// id; the serial loop short-circuits anything already resolved here.
|
|
294
|
+
const batched = new Map<string, PatchOperationResult>();
|
|
295
|
+
if (connector.applyCreateContactsBatch && !guardFailure) {
|
|
296
|
+
const eligible = plan.operations.filter(
|
|
297
|
+
(operation) =>
|
|
298
|
+
approved.has(operation.id) &&
|
|
299
|
+
operation.operation === "create_record" &&
|
|
300
|
+
operation.objectType === "contact" &&
|
|
301
|
+
options.valueOverrides?.[operation.id] === undefined &&
|
|
302
|
+
!requiresHumanInput(operation.afterValue) &&
|
|
303
|
+
!operation.groupId &&
|
|
304
|
+
!(operation.afterValue as CreateRecordPayload | undefined)?.associateCompanyName &&
|
|
305
|
+
!conflicts.has(operation.id) &&
|
|
306
|
+
!irreversibleStale.has(operation.id) &&
|
|
307
|
+
!(staleByFilter && staleByFilter.has(operation.objectId)),
|
|
308
|
+
);
|
|
309
|
+
// Only worth the batch machinery for more than a couple of creates.
|
|
310
|
+
if (eligible.length > 2) {
|
|
311
|
+
const batchResults = await connector.applyCreateContactsBatch(eligible);
|
|
312
|
+
for (const result of batchResults) batched.set(result.operationId, result);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
286
316
|
for (const operation of plan.operations) {
|
|
317
|
+
const batchedResult = batched.get(operation.id);
|
|
318
|
+
if (batchedResult) {
|
|
319
|
+
results.push(batchedResult);
|
|
320
|
+
attempted += 1;
|
|
321
|
+
if (batchedResult.status === "applied") applied += 1;
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
287
324
|
if (!approved.has(operation.id)) {
|
|
288
325
|
results.push({
|
|
289
326
|
operationId: operation.id,
|
|
@@ -660,6 +660,130 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
|
|
|
660
660
|
};
|
|
661
661
|
}
|
|
662
662
|
|
|
663
|
+
/** Bulk resolve-first lookup: which match values already exist (value→id). */
|
|
664
|
+
async function searchContactsByValues(property: string, values: string[]): Promise<Map<string, string>> {
|
|
665
|
+
const found = new Map<string, string>();
|
|
666
|
+
for (let i = 0; i < values.length; i += 100) {
|
|
667
|
+
const chunk = values.slice(i, i + 100);
|
|
668
|
+
const data = await request(`/crm/v3/objects/contacts/search`, {
|
|
669
|
+
method: "POST",
|
|
670
|
+
body: JSON.stringify({
|
|
671
|
+
filterGroups: [{ filters: [{ propertyName: property, operator: "IN", values: chunk }] }],
|
|
672
|
+
properties: [property],
|
|
673
|
+
limit: 100,
|
|
674
|
+
}),
|
|
675
|
+
});
|
|
676
|
+
for (const rec of (data?.results ?? []) as Array<{ id?: string; properties?: Record<string, string> }>) {
|
|
677
|
+
const v = rec?.properties?.[property];
|
|
678
|
+
if (typeof v === "string" && rec.id) found.set(v.toLowerCase(), String(rec.id));
|
|
679
|
+
}
|
|
680
|
+
}
|
|
681
|
+
return found;
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
/**
|
|
685
|
+
* Batch create_record for contacts: bulk resolve-first (search IN), then
|
|
686
|
+
* bulk-create the genuine misses (batch/create, 100/call) — two API calls per
|
|
687
|
+
* ~100 leads instead of two per lead. On a batch-create rejection (e.g. an
|
|
688
|
+
* email collision HubSpot 409s the whole batch over), it falls back to
|
|
689
|
+
* per-record createRecord for that chunk, so one bad row never sinks the rest
|
|
690
|
+
* and resolve-first is preserved. Returns one result per input op.
|
|
691
|
+
*/
|
|
692
|
+
async function applyCreateContactsBatch(operations: PatchOperation[]): Promise<PatchOperationResult[]> {
|
|
693
|
+
const byId = new Map<string, PatchOperationResult>();
|
|
694
|
+
// Group by the HubSpot search property (matchKey is usually uniform across a
|
|
695
|
+
// plan, but a mixed plan stays correct this way).
|
|
696
|
+
const groups = new Map<string, PatchOperation[]>();
|
|
697
|
+
for (const op of operations) {
|
|
698
|
+
const matchKey = (op.afterValue as CreateRecordPayload | undefined)?.matchKey || "email";
|
|
699
|
+
const searchProperty = HUBSPOT_DEFAULT_FIELD_MAPPINGS.contacts[matchKey] ?? matchKey;
|
|
700
|
+
let arr = groups.get(searchProperty);
|
|
701
|
+
if (!arr) groups.set(searchProperty, (arr = []));
|
|
702
|
+
arr.push(op);
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
for (const [searchProperty, ops] of groups) {
|
|
706
|
+
const wanted = ops
|
|
707
|
+
.map((op) => String((op.afterValue as CreateRecordPayload).matchValue ?? "").trim())
|
|
708
|
+
.filter(Boolean);
|
|
709
|
+
const existing = await searchContactsByValues(searchProperty, [...new Set(wanted.map((v) => v.toLowerCase()))]);
|
|
710
|
+
|
|
711
|
+
const toCreate: PatchOperation[] = [];
|
|
712
|
+
const seenInBatch = new Set<string>();
|
|
713
|
+
for (const op of ops) {
|
|
714
|
+
const payload = op.afterValue as CreateRecordPayload;
|
|
715
|
+
const matchKey = payload.matchKey || "email";
|
|
716
|
+
const matchValue = String(payload.matchValue ?? "").trim();
|
|
717
|
+
if (!matchValue) {
|
|
718
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: "create_record needs a non-empty matchValue to resolve-first." });
|
|
719
|
+
continue;
|
|
720
|
+
}
|
|
721
|
+
const lower = matchValue.toLowerCase();
|
|
722
|
+
const dedupeKey = `${matchKey}:${lower}`;
|
|
723
|
+
const already = createdContactsByMatch.get(dedupeKey);
|
|
724
|
+
if (already) {
|
|
725
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already created earlier in this run; not duplicating.`, providerData: { id: already, existing: true } });
|
|
726
|
+
continue;
|
|
727
|
+
}
|
|
728
|
+
const existingId = existing.get(lower);
|
|
729
|
+
if (existingId) {
|
|
730
|
+
createdContactsByMatch.set(dedupeKey, existingId);
|
|
731
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already exists (${existingId}); resolve-first declined to create.`, providerData: { id: existingId, existing: true } });
|
|
732
|
+
continue;
|
|
733
|
+
}
|
|
734
|
+
if (seenInBatch.has(lower)) {
|
|
735
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} duplicated within this batch; not duplicating.` });
|
|
736
|
+
continue;
|
|
737
|
+
}
|
|
738
|
+
seenInBatch.add(lower);
|
|
739
|
+
toCreate.push(op);
|
|
740
|
+
}
|
|
741
|
+
|
|
742
|
+
for (let i = 0; i < toCreate.length; i += 100) {
|
|
743
|
+
const chunk = toCreate.slice(i, i + 100);
|
|
744
|
+
const inputs = chunk.map((op) => {
|
|
745
|
+
const payload = op.afterValue as CreateRecordPayload;
|
|
746
|
+
const properties: Record<string, string> = { ...payload.properties };
|
|
747
|
+
if (payload.ownerId && !properties.hubspot_owner_id) properties.hubspot_owner_id = String(payload.ownerId);
|
|
748
|
+
properties.hs_object_source_detail_2 = `fullstackgtm acquire (${op.id})`;
|
|
749
|
+
return { properties };
|
|
750
|
+
});
|
|
751
|
+
try {
|
|
752
|
+
const data = await request(`/crm/v3/objects/contacts/batch/create`, {
|
|
753
|
+
method: "POST",
|
|
754
|
+
body: JSON.stringify({ inputs }),
|
|
755
|
+
});
|
|
756
|
+
const created = (data?.results ?? []) as Array<{ id?: string; properties?: Record<string, string> }>;
|
|
757
|
+
const idByValue = new Map<string, string>();
|
|
758
|
+
for (const rec of created) {
|
|
759
|
+
const v = rec?.properties?.[searchProperty];
|
|
760
|
+
if (typeof v === "string" && rec.id) idByValue.set(v.toLowerCase(), String(rec.id));
|
|
761
|
+
}
|
|
762
|
+
for (const op of chunk) {
|
|
763
|
+
const payload = op.afterValue as CreateRecordPayload;
|
|
764
|
+
const matchKey = payload.matchKey || "email";
|
|
765
|
+
const matchValue = String(payload.matchValue).trim();
|
|
766
|
+
const id = idByValue.get(matchValue.toLowerCase());
|
|
767
|
+
if (id) {
|
|
768
|
+
createdContactsByMatch.set(`${matchKey}:${matchValue.toLowerCase()}`, id);
|
|
769
|
+
byId.set(op.id, { operationId: op.id, status: "applied", detail: `Created contact ${matchKey}=${matchValue} (${id}).`, providerData: { id, created: true } });
|
|
770
|
+
} else {
|
|
771
|
+
// Result didn't echo this value — resolve it per-record to be safe.
|
|
772
|
+
byId.set(op.id, await createRecord(op));
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
} catch {
|
|
776
|
+
// Whole-chunk rejection (e.g. a collision). Isolate per record.
|
|
777
|
+
for (const op of chunk) byId.set(op.id, await createRecord(op));
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
return operations.map(
|
|
783
|
+
(op) => byId.get(op.id) ?? { operationId: op.id, status: "skipped", detail: "create_record was not processed." },
|
|
784
|
+
);
|
|
785
|
+
}
|
|
786
|
+
|
|
663
787
|
async function createTask(operation: PatchOperation): Promise<PatchOperationResult> {
|
|
664
788
|
const associationTypeId = TASK_ASSOCIATION_TYPE_IDS[operation.objectType];
|
|
665
789
|
if (associationTypeId === undefined) {
|
|
@@ -926,6 +1050,7 @@ export function createHubspotConnector(options: HubspotConnectorOptions): Requir
|
|
|
926
1050
|
fetchSnapshot,
|
|
927
1051
|
fetchChanges,
|
|
928
1052
|
applyOperation,
|
|
1053
|
+
applyCreateContactsBatch,
|
|
929
1054
|
readField,
|
|
930
1055
|
};
|
|
931
1056
|
}
|
|
@@ -597,6 +597,118 @@ export function createSalesforceConnector(
|
|
|
597
597
|
};
|
|
598
598
|
}
|
|
599
599
|
|
|
600
|
+
/**
|
|
601
|
+
* Batch create_record for contacts: bulk resolve-first (SOQL IN), then create
|
|
602
|
+
* the misses via the Composite sObject Collections API (allOrNone:false,
|
|
603
|
+
* 200/call), instead of two round-trips per lead. A record-level failure in
|
|
604
|
+
* the collection (e.g. a duplicate rule) falls back to per-record createRecord
|
|
605
|
+
* so one bad row never sinks the rest. Returns one result per input op.
|
|
606
|
+
*/
|
|
607
|
+
async function applyCreateContactsBatch(operations: PatchOperation[]): Promise<PatchOperationResult[]> {
|
|
608
|
+
const byId = new Map<string, PatchOperationResult>();
|
|
609
|
+
const groups = new Map<string, PatchOperation[]>();
|
|
610
|
+
for (const op of operations) {
|
|
611
|
+
const matchKey = (op.afterValue as CreateRecordPayload | undefined)?.matchKey || "email";
|
|
612
|
+
const searchField = mappedField(mappings, "contacts", matchKey, SALESFORCE_DEFAULT_FIELD_MAPPINGS.contacts[matchKey] ?? matchKey);
|
|
613
|
+
let arr = groups.get(searchField);
|
|
614
|
+
if (!arr) groups.set(searchField, (arr = []));
|
|
615
|
+
arr.push(op);
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
for (const [searchField, ops] of groups) {
|
|
619
|
+
// 1. Bulk resolve-first via SOQL IN (chunked; values SOQL-escaped).
|
|
620
|
+
const existing = new Map<string, string>();
|
|
621
|
+
const uniqueValues = [
|
|
622
|
+
...new Set(ops.map((op) => String((op.afterValue as CreateRecordPayload).matchValue ?? "").trim()).filter(Boolean)),
|
|
623
|
+
];
|
|
624
|
+
for (let i = 0; i < uniqueValues.length; i += 200) {
|
|
625
|
+
const inList = uniqueValues
|
|
626
|
+
.slice(i, i + 200)
|
|
627
|
+
.map((v) => `'${v.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`)
|
|
628
|
+
.join(",");
|
|
629
|
+
const rows = await query(`SELECT Id, ${searchField} FROM Contact WHERE ${searchField} IN (${inList})`);
|
|
630
|
+
for (const row of rows) {
|
|
631
|
+
const v = row?.[searchField];
|
|
632
|
+
if (typeof v === "string" && row.Id) existing.set(v.toLowerCase(), String(row.Id));
|
|
633
|
+
}
|
|
634
|
+
}
|
|
635
|
+
|
|
636
|
+
// 2. Partition: existing / already-this-run / dup-in-batch / to-create.
|
|
637
|
+
const toCreate: PatchOperation[] = [];
|
|
638
|
+
const seenInBatch = new Set<string>();
|
|
639
|
+
for (const op of ops) {
|
|
640
|
+
const payload = op.afterValue as CreateRecordPayload;
|
|
641
|
+
const matchKey = payload.matchKey || "email";
|
|
642
|
+
const matchValue = String(payload.matchValue ?? "").trim();
|
|
643
|
+
if (!matchValue) {
|
|
644
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: "create_record needs a non-empty matchValue to resolve-first." });
|
|
645
|
+
continue;
|
|
646
|
+
}
|
|
647
|
+
const lower = matchValue.toLowerCase();
|
|
648
|
+
const dedupeKey = `${matchKey}:${lower}`;
|
|
649
|
+
const already = createdContactsByMatch.get(dedupeKey);
|
|
650
|
+
if (already) {
|
|
651
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already created earlier in this run; not duplicating.`, providerData: { id: already, existing: true } });
|
|
652
|
+
continue;
|
|
653
|
+
}
|
|
654
|
+
const existingId = existing.get(lower);
|
|
655
|
+
if (existingId) {
|
|
656
|
+
createdContactsByMatch.set(dedupeKey, existingId);
|
|
657
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} already exists (${existingId}); resolve-first declined to create.`, providerData: { id: existingId, existing: true } });
|
|
658
|
+
continue;
|
|
659
|
+
}
|
|
660
|
+
if (seenInBatch.has(lower)) {
|
|
661
|
+
byId.set(op.id, { operationId: op.id, status: "skipped", detail: `Contact ${matchKey}=${matchValue} duplicated within this batch; not duplicating.` });
|
|
662
|
+
continue;
|
|
663
|
+
}
|
|
664
|
+
seenInBatch.add(lower);
|
|
665
|
+
toCreate.push(op);
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
// 3. Bulk create via Composite sObject Collections (allOrNone:false →
|
|
669
|
+
// per-record results in request order).
|
|
670
|
+
for (let i = 0; i < toCreate.length; i += 200) {
|
|
671
|
+
const chunk = toCreate.slice(i, i + 200);
|
|
672
|
+
const records = chunk.map((op) => {
|
|
673
|
+
const payload = op.afterValue as CreateRecordPayload;
|
|
674
|
+
const fields: Record<string, string> = { ...payload.properties };
|
|
675
|
+
if (payload.ownerId) {
|
|
676
|
+
const ownerField = mappedField(mappings, "contacts", "ownerId", SALESFORCE_DEFAULT_FIELD_MAPPINGS.contacts.ownerId ?? "OwnerId");
|
|
677
|
+
if (!fields[ownerField]) fields[ownerField] = String(payload.ownerId);
|
|
678
|
+
}
|
|
679
|
+
return { attributes: { type: "Contact" }, ...fields };
|
|
680
|
+
});
|
|
681
|
+
let results: Array<{ id?: string; success?: boolean }>;
|
|
682
|
+
try {
|
|
683
|
+
results = (await request(`/services/data/${apiVersion}/composite/sobjects`, {
|
|
684
|
+
method: "POST",
|
|
685
|
+
body: JSON.stringify({ allOrNone: false, records }),
|
|
686
|
+
})) as Array<{ id?: string; success?: boolean }>;
|
|
687
|
+
} catch {
|
|
688
|
+
for (const op of chunk) byId.set(op.id, await createRecord(op));
|
|
689
|
+
continue;
|
|
690
|
+
}
|
|
691
|
+
for (let j = 0; j < chunk.length; j++) {
|
|
692
|
+
const op = chunk[j];
|
|
693
|
+
const res = Array.isArray(results) ? results[j] : undefined;
|
|
694
|
+
const payload = op.afterValue as CreateRecordPayload;
|
|
695
|
+
const matchKey = payload.matchKey || "email";
|
|
696
|
+
const matchValue = String(payload.matchValue).trim();
|
|
697
|
+
if (res?.success && res.id) {
|
|
698
|
+
createdContactsByMatch.set(`${matchKey}:${matchValue.toLowerCase()}`, String(res.id));
|
|
699
|
+
byId.set(op.id, { operationId: op.id, status: "applied", detail: `Created contact ${matchKey}=${matchValue} (${res.id}).`, providerData: { id: String(res.id), created: true } });
|
|
700
|
+
} else {
|
|
701
|
+
byId.set(op.id, await createRecord(op)); // record-level failure → isolate per record
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
return operations.map(
|
|
708
|
+
(op) => byId.get(op.id) ?? { operationId: op.id, status: "skipped", detail: "create_record was not processed." },
|
|
709
|
+
);
|
|
710
|
+
}
|
|
711
|
+
|
|
600
712
|
async function applyOperation(operation: PatchOperation): Promise<PatchOperationResult> {
|
|
601
713
|
try {
|
|
602
714
|
switch (operation.operation) {
|
|
@@ -683,6 +795,7 @@ export function createSalesforceConnector(
|
|
|
683
795
|
fetchSnapshot,
|
|
684
796
|
fetchChanges,
|
|
685
797
|
applyOperation,
|
|
798
|
+
applyCreateContactsBatch,
|
|
686
799
|
readField,
|
|
687
800
|
};
|
|
688
801
|
}
|
package/src/types.ts
CHANGED
|
@@ -470,6 +470,17 @@ export type GtmConnector = {
|
|
|
470
470
|
provider: CrmProvider;
|
|
471
471
|
fetchSnapshot: () => Promise<CanonicalGtmSnapshot>;
|
|
472
472
|
applyOperation?: (operation: PatchOperation) => Promise<PatchOperationResult>;
|
|
473
|
+
/**
|
|
474
|
+
* Bulk fast-path for independent `create_record` contact ops: batch the
|
|
475
|
+
* resolve-first read and the create writes instead of one round-trip per
|
|
476
|
+
* record (orders of magnitude fewer API calls for lead acquisition). The
|
|
477
|
+
* caller (`applyPatchPlan`) only routes ops here that are safe to batch —
|
|
478
|
+
* approved, no group, no value override, no company association — and falls
|
|
479
|
+
* back to `applyOperation` for the rest. Implementations MUST preserve
|
|
480
|
+
* resolve-first (never create over an existing match) and return exactly one
|
|
481
|
+
* result per input op. Optional: connectors without it use `applyOperation`.
|
|
482
|
+
*/
|
|
483
|
+
applyCreateContactsBatch?: (operations: PatchOperation[]) => Promise<PatchOperationResult[]>;
|
|
473
484
|
/**
|
|
474
485
|
* Read the live value of one canonical field, used for compare-and-set:
|
|
475
486
|
* apply orchestration refuses to write over values that drifted since the
|