@dhyasama/totem-merge 7.1.4 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/transactional-lead-merges.md +77 -0
- package/index.js +9 -0
- package/lib/atomicMerge.js +53 -0
- package/lib/attachmentMerge.js +65 -0
- package/lib/controls/customer-settings-gate.js +79 -0
- package/lib/leadMerge.js +100 -0
- package/lib/organizations.js +30 -9
- package/lib/people.js +36 -14
- package/lib/privateMergeChecks.js +27 -0
- package/lib/privateRecordMerge.js +133 -0
- package/lib/transactionContext.js +29 -0
- package/package.json +2 -2
- package/test/atomic-lead-merge.mongo.test.js +311 -0
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# Transactional company and person merges
|
|
2
|
+
|
|
3
|
+
Status: 7.2.0 release candidate, 2026-10-24. No consumer enables this path yet.
|
|
4
|
+
Publishing this package does not activate its transactional merge option.
|
|
5
|
+
|
|
6
|
+
## First-rollout scope
|
|
7
|
+
|
|
8
|
+
The owner's first rollout covers Customer settings (already completed), Lead,
|
|
9
|
+
Tag and Valuation. Notes, Documents and custom fields remain on their existing
|
|
10
|
+
records. `transactionalPrivateData: true` opts into the transaction and native
|
|
11
|
+
Lead/Tag/Valuation handling only; it does not activate the deferred migrations.
|
|
12
|
+
|
|
13
|
+
Both document and ID entry points reload the source and target on every retry.
|
|
14
|
+
Source deletion, target save, existing reference updates, Lead/Tag/Valuation
|
|
15
|
+
updates and their legacy mirrors commit together. Company merges union distinct
|
|
16
|
+
leads per customer with no count limit. Person merges replace the source Person
|
|
17
|
+
and deduplicate assignments. Tags combine groups/items per owner. Every valuation
|
|
18
|
+
occurrence and database ID survives; conflicting legacy IDs block the merge.
|
|
19
|
+
|
|
20
|
+
The path requires a replica set, models on one connection, uniqueness indexes on
|
|
21
|
+
Lead/Tag/Valuation, and the existing open customer-settings control record. Its
|
|
22
|
+
transaction fence coordinates with lifecycle admission and private-data writers.
|
|
23
|
+
No write pause or queue/schedule hold is introduced. Missing additive Leads can
|
|
24
|
+
be initialized from chairs; Tags and Valuations must already be reconciled.
|
|
25
|
+
Invalid owners, blocking lifecycle jobs, missing people and destination drift
|
|
26
|
+
fail the whole transaction. A committed retry is recognized by merged history.
|
|
27
|
+
|
|
28
|
+
## Deferred fields stay in legacy storage
|
|
29
|
+
|
|
30
|
+
The CustomField collection is neither read nor written, and its index is not a
|
|
31
|
+
merge prerequisite. Embedded Organization custom fields remain embedded. Their
|
|
32
|
+
values are combined per customer, preserving zero/false/null and disjoint keys;
|
|
33
|
+
conflicts stop the merge for review. This prevents the old generic array merger
|
|
34
|
+
from dropping entries without subdocument IDs. Fund/LP/Deal values are unchanged.
|
|
35
|
+
|
|
36
|
+
Note/Document legacy parent links are combined on the surviving Organization or
|
|
37
|
+
Person. Existing Document.organization references to a merged company are
|
|
38
|
+
retargeted in the same transaction, preserving extraction behavior. Attachment
|
|
39
|
+
ownership, lifecycle and cross-tenant LP checks still apply to these legacy links.
|
|
40
|
+
Content, IDs and storage keys remain intact. Hidden conflicting legacy Document
|
|
41
|
+
company links block the merge.
|
|
42
|
+
|
|
43
|
+
No Note.entities or Document.entities arrays are created, consumed, updated or
|
|
44
|
+
used to synthesize parent links. Existing deferred native data is left intact;
|
|
45
|
+
future attachment/CustomField reconciliation must account for merged identities
|
|
46
|
+
before any later activation. It must not replay an obsolete source plan.
|
|
47
|
+
|
|
48
|
+
Reference queues run serially in the transaction. The draft also propagates queue
|
|
49
|
+
errors, scopes Deal/Document/Note updates to each record's owner, and preserves
|
|
50
|
+
Person source/referrer metadata on the default path. Those changes still require
|
|
51
|
+
release review; they are separate from enabling new-store authority.
|
|
52
|
+
|
|
53
|
+
## Validation and release gates
|
|
54
|
+
|
|
55
|
+
42 targeted tests pass, including synthetic replica-set merges against the web
|
|
56
|
+
consumer's published models 17.0.0. Coverage includes unlimited Leads, tenant
|
|
57
|
+
isolation, transaction rollback/retries, concurrent fenced edits, committed replay,
|
|
58
|
+
Tag drift, valuation identity, embedded custom-field preservation/conflicts,
|
|
59
|
+
untouched native CustomFields/entities, legacy attachment links and lifecycle
|
|
60
|
+
checks. Merges work without a CustomField uniqueness index and never initialize
|
|
61
|
+
attachment entities. Tests use a uniquely named disposable local database.
|
|
62
|
+
|
|
63
|
+
Before adoption: review/release the package, complete Lead/Tag/Valuation writer
|
|
64
|
+
coverage and reconciliation, test production-sized transaction cost, and verify
|
|
65
|
+
all consumers, caches and Pentest behavior. The narrowed preparation completed a clean 423-Customer production comparison
|
|
66
|
+
on 2026-10-24; repeat live parity checks before activation. Native attachment and CustomField consumers
|
|
67
|
+
must remain disabled. This draft is not readiness to cut over or remove fields.
|
|
68
|
+
|
|
69
|
+
## 7.2.0 release review
|
|
70
|
+
|
|
71
|
+
The current consumer baseline is models 17.0.0, including Account.customer and
|
|
72
|
+
retired Interaction/activity-list models. The 42 focused merge/security tests
|
|
73
|
+
pass against that exact dependency. The default constructor remains opt-out.
|
|
74
|
+
Adoption also includes the reviewed default-path fixes above; it is not a claim
|
|
75
|
+
that every legacy merge behavior is identical. Before transactional activation,
|
|
76
|
+
finish writer deployment and production-scale cost checks. Once enabled, rollback
|
|
77
|
+
must retain native/legacy mirroring rather than reverting to an unfenced merger.
|
package/index.js
CHANGED
|
@@ -7,6 +7,7 @@ var Logger = require('@dhyasama/totem-logger');
|
|
|
7
7
|
// Constructor
|
|
8
8
|
var Merge = function(mongoose, config) {
|
|
9
9
|
|
|
10
|
+
this.transactionalPrivateData = config.transactionalPrivateData === true;
|
|
10
11
|
this.env = process.env.NODE_ENV || 'development';
|
|
11
12
|
this.logger = new Logger(config.logentries);
|
|
12
13
|
this.CapTable = mongoose.model('CapTable');
|
|
@@ -30,11 +31,15 @@ var Merge = function(mongoose, config) {
|
|
|
30
31
|
//////////////////////////////////////////////
|
|
31
32
|
|
|
32
33
|
Merge.prototype.organizations = function(target, source, user, customerId, cb) {
|
|
34
|
+
|
|
35
|
+
if (this.transactionalPrivateData) return require('./lib/atomicMerge')(this, 'Organization', target, source, user, customerId).then(result => cb(null, result), error => cb(error, null));
|
|
33
36
|
organizations.merge(this, target, source, user, customerId, cb);
|
|
34
37
|
};
|
|
35
38
|
|
|
36
39
|
Merge.prototype.organizationsById = function(targetId, sourceId, user, customerId, cb) {
|
|
37
40
|
|
|
41
|
+
if (this.transactionalPrivateData) return require('./lib/atomicMerge')(this, 'Organization', targetId, sourceId, user, customerId).then(result => cb(null, result), error => cb(error, null));
|
|
42
|
+
|
|
38
43
|
var self = this;
|
|
39
44
|
|
|
40
45
|
self.logger.writeToInfoLog('Merge org ' + sourceId + ' into ' + targetId);
|
|
@@ -63,11 +68,15 @@ Merge.prototype.organizationsById = function(targetId, sourceId, user, customerI
|
|
|
63
68
|
};
|
|
64
69
|
|
|
65
70
|
Merge.prototype.people = function(target, source, user, customerId, cb) {
|
|
71
|
+
|
|
72
|
+
if (this.transactionalPrivateData) return require('./lib/atomicMerge')(this, 'Person', target, source, user, customerId).then(result => cb(null, result), error => cb(error, null));
|
|
66
73
|
people.merge(this, target, source, user, customerId, cb);
|
|
67
74
|
};
|
|
68
75
|
|
|
69
76
|
Merge.prototype.peopleById = function(targetId, sourceId, user, customerId, cb) {
|
|
70
77
|
|
|
78
|
+
if (this.transactionalPrivateData) return require('./lib/atomicMerge')(this, 'Person', targetId, sourceId, user, customerId).then(result => cb(null, result), error => cb(error, null));
|
|
79
|
+
|
|
71
80
|
var self = this;
|
|
72
81
|
|
|
73
82
|
self.logger.writeToInfoLog('Merge person ' + sourceId + ' into ' + targetId);
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
const scopedContext = require('./transactionContext');
|
|
3
|
+
const leads = require('./leadMerge');
|
|
4
|
+
const privateRecords = require('./privateRecordMerge');
|
|
5
|
+
const attachments = require('./attachmentMerge');
|
|
6
|
+
const gate = require('./controls/customer-settings-gate');
|
|
7
|
+
const implementations = { Organization: require('./organizations'), Person: require('./people') };
|
|
8
|
+
module.exports = async function atomicMerge(context, kind, targetId, sourceId, user, customerId) {
|
|
9
|
+
const Model = context[kind], connection = Model.db, db = connection.db;
|
|
10
|
+
if (Object.values(context).some(value => value?.modelName && value.db !== connection)) throw Error('Merge models must share one connection');
|
|
11
|
+
const id = value => {
|
|
12
|
+
const text = String(value?._id || value || '');
|
|
13
|
+
if (!/^[a-f0-9]{24}$/i.test(text)) throw Error('Invalid merge ID');
|
|
14
|
+
return new Model.base.Types.ObjectId(text);
|
|
15
|
+
};
|
|
16
|
+
targetId = id(targetId); sourceId = id(sourceId);
|
|
17
|
+
if (String(targetId) === String(sourceId)) throw Error('Cannot merge an entity into itself');
|
|
18
|
+
for (const [collection, key, partial] of [
|
|
19
|
+
['leads', { customer: 1, organization: 1 }],
|
|
20
|
+
['tags', { customer: 1, 'entity.model': 1, 'entity.id': 1 }],
|
|
21
|
+
['valuations', { customer: 1, organization: 1, legacyId: 1 }, { legacyId: { $type: 'objectId' } }]
|
|
22
|
+
]) {
|
|
23
|
+
const indexes = await db.collection(collection).indexes();
|
|
24
|
+
if (!indexes.some(index => index.unique && !index.sparse &&
|
|
25
|
+
JSON.stringify(index.partialFilterExpression) === JSON.stringify(partial) &&
|
|
26
|
+
JSON.stringify(index.key) === JSON.stringify(key))) throw Error(collection + ' unique index is required');
|
|
27
|
+
}
|
|
28
|
+
const session = await connection.startSession();
|
|
29
|
+
try {
|
|
30
|
+
return await session.withTransaction(async () => {
|
|
31
|
+
await gate.writer(db, session, 'customer');
|
|
32
|
+
// Fresh documents on every retry. Never merge a stale, tenant-filtered
|
|
33
|
+
// caller object or retain mutated documents across transaction attempts.
|
|
34
|
+
const target = await Model.findById(targetId).session(session);
|
|
35
|
+
const source = await Model.findById(sourceId).session(session);
|
|
36
|
+
if (!target) throw Error('Merge target not found');
|
|
37
|
+
if (!source) {
|
|
38
|
+
if ((target.merged || []).some(value => String(value) === String(sourceId))) return target;
|
|
39
|
+
throw Error('Merge source not found');
|
|
40
|
+
}
|
|
41
|
+
const plan = await leads.prepare(db, session, kind, target, source);
|
|
42
|
+
const privatePlan = await privateRecords.prepare(db, session, kind, target, source);
|
|
43
|
+
const attachmentPlan = await attachments.prepare(db, session, kind, target, source);
|
|
44
|
+
const contextInTransaction = scopedContext(context, session);
|
|
45
|
+
await new Promise((resolve, reject) => implementations[kind].merge(contextInTransaction, target, source, user, customerId,
|
|
46
|
+
(error, result) => error ? reject(error) : resolve(result)));
|
|
47
|
+
await leads.apply(db, session, plan);
|
|
48
|
+
await privateRecords.apply(db, session, privatePlan);
|
|
49
|
+
await attachments.apply(db, session, attachmentPlan);
|
|
50
|
+
return Model.findById(targetId).session(session);
|
|
51
|
+
}, { readConcern: { level: 'snapshot' }, writeConcern: { w: 'majority' } });
|
|
52
|
+
} finally { await session.endSession(); }
|
|
53
|
+
};
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
const { id, oid, array, owners } = require('./privateMergeChecks');
|
|
3
|
+
const parentCollections = { Organization: 'organizations', Person: 'people', LimitedPartner: 'limitedpartners' };
|
|
4
|
+
async function prepare(db, session, kind, target, source) {
|
|
5
|
+
const targetId = id(target._id), sourceId = id(source._id), ids = [target._id, source._id];
|
|
6
|
+
const parents = await db.collection(parentCollections[kind]).find({ _id: { $in: ids } },
|
|
7
|
+
{ session, projection: { notes: 1, documents: 1 } }).toArray();
|
|
8
|
+
const records = [], mirrors = {}, ownerIds = [];
|
|
9
|
+
for (const collection of ['notes', 'documents']) {
|
|
10
|
+
const parentIds = [...new Set(parents.flatMap(parent => array(parent[collection]).map(id)))];
|
|
11
|
+
// First rollout preserves legacy links only; native entities are deferred.
|
|
12
|
+
const predicates = [{ _id: { $in: parentIds.map(oid) } }];
|
|
13
|
+
if (kind === 'Organization' && collection === 'documents') predicates.push({ organization: { $in: ids } });
|
|
14
|
+
const attachments = await db.collection(collection).find({ $or: predicates }, { session }).toArray();
|
|
15
|
+
if (parentIds.some(value => !attachments.some(attachment => id(attachment._id) === value))) throw Error('Missing attachment on merge parent');
|
|
16
|
+
const attachmentIds = attachments.map(attachment => attachment._id);
|
|
17
|
+
// Recover every legacy parent, not just the merging pair. This catches a
|
|
18
|
+
// second company hidden in a legacy array and preserves all Note links.
|
|
19
|
+
const reverse = [];
|
|
20
|
+
for (const [model, name] of Object.entries(parentCollections)) {
|
|
21
|
+
for (const parent of await db.collection(name).find({ [collection]: { $in: attachmentIds } },
|
|
22
|
+
{ session, projection: { [collection]: 1, customer: 1 } }).toArray()) reverse.push({ model, parent });
|
|
23
|
+
}
|
|
24
|
+
mirrors[collection] = [];
|
|
25
|
+
for (const attachment of attachments) {
|
|
26
|
+
const customer = id(attachment.customer), entities = new Map();
|
|
27
|
+
ownerIds.push(customer);
|
|
28
|
+
function add(model, ref) {
|
|
29
|
+
if (!parentCollections[model]) throw Error('Invalid attachment entity model');
|
|
30
|
+
let value = id(ref);
|
|
31
|
+
if (model === kind && value === sourceId) value = targetId;
|
|
32
|
+
entities.set(model + '/' + value, { model, id: oid(value) });
|
|
33
|
+
}
|
|
34
|
+
for (const { model, parent } of reverse) if (array(parent[collection]).some(ref => id(ref) === id(attachment._id))) {
|
|
35
|
+
if (model === 'LimitedPartner' && id(parent.customer) !== customer) throw Error('Cross-customer attachment association');
|
|
36
|
+
add(model, parent._id);
|
|
37
|
+
}
|
|
38
|
+
if (collection === 'documents' && attachment.organization != null) add('Organization', attachment.organization);
|
|
39
|
+
const next = [...entities.values()], companies = next.filter(entity => entity.model === 'Organization');
|
|
40
|
+
if (collection === 'documents' && companies.length > 1) throw Error('Document has conflicting company associations');
|
|
41
|
+
for (const entity of next) {
|
|
42
|
+
const parent = await db.collection(parentCollections[entity.model]).findOne({ _id: entity.id }, { session, projection: { customer: 1 } });
|
|
43
|
+
if (!parent) throw Error('Missing attachment entity');
|
|
44
|
+
if (entity.model === 'LimitedPartner' && id(parent.customer) !== customer) throw Error('Cross-customer attachment association');
|
|
45
|
+
}
|
|
46
|
+
// Keep only existing legacy ownership references current during a merge.
|
|
47
|
+
// Do not initialize entities or infer a new Document.organization value.
|
|
48
|
+
if (kind === 'Organization' && collection === 'documents' && attachment.organization != null && id(attachment.organization) === sourceId) {
|
|
49
|
+
records.push({ collection, attachment, set: { organization: oid(targetId) } });
|
|
50
|
+
}
|
|
51
|
+
if (parentIds.includes(id(attachment._id))) mirrors[collection].push(attachment._id);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
await owners(db, session, ownerIds);
|
|
55
|
+
return { records, mirrors, kind, targetId };
|
|
56
|
+
}
|
|
57
|
+
async function apply(db, session, plan) {
|
|
58
|
+
for (const { collection, attachment, set } of plan.records) {
|
|
59
|
+
const result = await db.collection(collection).updateOne({ _id: attachment._id, customer: attachment.customer }, { $set: set, $inc: { __v: 1 } }, { session });
|
|
60
|
+
if (result.matchedCount !== 1) throw Error('Attachment changed during merge');
|
|
61
|
+
}
|
|
62
|
+
const result = await db.collection(parentCollections[plan.kind]).updateOne({ _id: oid(plan.targetId) }, { $set: plan.mirrors }, { session });
|
|
63
|
+
if (result.matchedCount !== 1) throw Error('Attachment merge target changed');
|
|
64
|
+
}
|
|
65
|
+
module.exports = { prepare, apply };
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// Customer-only runtime fencing. Missing or obsolete controls fail closed.
|
|
4
|
+
const KEY = 'customer-settings-v1';
|
|
5
|
+
const collection = db => db.collection('migrationcontrols');
|
|
6
|
+
const sources = new Set(['customer']);
|
|
7
|
+
const drained = { $or: [{ activeLifecycle: { $exists: false } }, { activeLifecycle: { $size: 0 } }] };
|
|
8
|
+
function check(expected) {
|
|
9
|
+
if (!expected || !Number.isSafeInteger(expected.epoch) || expected.epoch < 0 ||
|
|
10
|
+
expected.rolling === true || !sources.has(expected.source) || !['open', 'paused'].includes(expected.state)) {
|
|
11
|
+
throw Error('Invalid settings gate expectation');
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
async function initialize(db) {
|
|
15
|
+
await collection(db).updateOne({ _id: KEY }, { $setOnInsert: {
|
|
16
|
+
state: 'paused', source: 'customer', epoch: 0, fence: 0, activeLifecycle: []
|
|
17
|
+
} }, { upsert: true, writeConcern: { w: 'majority' } });
|
|
18
|
+
return collection(db).findOne({ _id: KEY });
|
|
19
|
+
}
|
|
20
|
+
async function transition(db, expected, next) {
|
|
21
|
+
check(expected);
|
|
22
|
+
if (!next || !sources.has(next.source) || !['open', 'paused'].includes(next.state) ||
|
|
23
|
+
(next.source !== expected.source && (expected.state !== 'paused' || next.state !== 'paused'))) {
|
|
24
|
+
throw Error('Source changes require a paused settings gate');
|
|
25
|
+
}
|
|
26
|
+
const result = await collection(db).updateOne({ _id: KEY, epoch: expected.epoch,
|
|
27
|
+
source: expected.source, state: expected.state,
|
|
28
|
+
...(expected.state === 'paused' && (next.state === 'open' || next.source !== expected.source) ? drained : {}) }, {
|
|
29
|
+
$set: { state: next.state, source: next.source }, $inc: { epoch: 1, fence: 1 }
|
|
30
|
+
}, { writeConcern: { w: 'majority' } });
|
|
31
|
+
if (result.matchedCount !== 1) throw Error('Settings gate changed or lifecycle work has not drained');
|
|
32
|
+
return { ...next, epoch: expected.epoch + 1 };
|
|
33
|
+
}
|
|
34
|
+
async function fence(db, session, expected) {
|
|
35
|
+
check(expected);
|
|
36
|
+
if (!session?.inTransaction()) throw Error('Settings gate requires a transaction');
|
|
37
|
+
const result = await collection(db).updateOne({ _id: KEY, epoch: expected.epoch,
|
|
38
|
+
state: expected.state, source: expected.source, ...(expected.state === 'paused' ? drained : {}) }, { $inc: { fence: 1 } }, { session });
|
|
39
|
+
if (result.matchedCount !== 1) throw Error('Settings writes paused, gate changed or lifecycle work has not drained');
|
|
40
|
+
}
|
|
41
|
+
// Writers read the current epoch inside their transaction, then write the row.
|
|
42
|
+
// A simultaneous pause either waits for this transaction or forces its retry.
|
|
43
|
+
async function writer(db, session, source) {
|
|
44
|
+
if (!sources.has(source)) throw Error('Invalid settings source');
|
|
45
|
+
const current = await collection(db).findOne({ _id: KEY }, { session });
|
|
46
|
+
if (!current || current.state !== 'open' || current.source !== source) throw Error('Settings writes paused or source mismatch');
|
|
47
|
+
await fence(db, session, current);
|
|
48
|
+
}
|
|
49
|
+
// Lifecycle operations share the Customer settings transaction fence.
|
|
50
|
+
// Admission and the job insert/claim must commit in the same transaction.
|
|
51
|
+
async function admit(db, session) {
|
|
52
|
+
if (!session?.inTransaction()) throw Error('Lifecycle admission requires a transaction');
|
|
53
|
+
const current = await collection(db).findOne({ _id: KEY }, { session });
|
|
54
|
+
if (!current) throw Error('Settings gate is not installed');
|
|
55
|
+
check(current);
|
|
56
|
+
if (current.activeLifecycle !== undefined && !Array.isArray(current.activeLifecycle)) throw Error('Invalid lifecycle drain state');
|
|
57
|
+
if (current.state !== 'open') throw Object.assign(Error('Lifecycle admission paused'), { code: 'SETTINGS_PAUSED' });
|
|
58
|
+
await fence(db, session, current);
|
|
59
|
+
return current;
|
|
60
|
+
}
|
|
61
|
+
function token(value) {
|
|
62
|
+
if (typeof value !== 'string' || !/^[a-zA-Z0-9-]{1,100}$/.test(value)) throw Error('Invalid lifecycle token');
|
|
63
|
+
}
|
|
64
|
+
async function holdLifecycle(db, session, owner) {
|
|
65
|
+
token(owner);
|
|
66
|
+
await admit(db, session);
|
|
67
|
+
await collection(db).updateOne({ _id: KEY }, { $addToSet: { activeLifecycle: owner } }, { session });
|
|
68
|
+
}
|
|
69
|
+
// No timeout expiry: a stale heartbeat is not proof an external operation ended.
|
|
70
|
+
// Worker calls this only after a clean completion/yield. Failure/crash tokens
|
|
71
|
+
// require operator evidence that the invocation and outstanding work have ended.
|
|
72
|
+
async function releaseLifecycle(db, owner) {
|
|
73
|
+
token(owner);
|
|
74
|
+
const result = await collection(db).updateOne({ _id: KEY }, {
|
|
75
|
+
$pull: { activeLifecycle: owner }, $inc: { fence: 1 }
|
|
76
|
+
}, { writeConcern: { w: 'majority' } });
|
|
77
|
+
if (result.matchedCount !== 1) throw Error('Settings gate is not installed');
|
|
78
|
+
}
|
|
79
|
+
module.exports = { KEY, initialize, transition, fence, writer, admit, holdLifecycle, releaseLifecycle };
|
package/lib/leadMerge.js
ADDED
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
const crypto = require('node:crypto');
|
|
3
|
+
const { Types: { ObjectId } } = require('mongoose');
|
|
4
|
+
const text = value => String(value?._id || value || '').toLowerCase();
|
|
5
|
+
function id(value) {
|
|
6
|
+
const result = text(value);
|
|
7
|
+
if (!/^[a-f0-9]{24}$/.test(result)) throw Error('Invalid lead merge reference');
|
|
8
|
+
return result;
|
|
9
|
+
}
|
|
10
|
+
const oid = value => new ObjectId(id(value));
|
|
11
|
+
const unique = values => [...new Set(values.map(id))];
|
|
12
|
+
const equal = (a, b) => JSON.stringify([...a].sort()) === JSON.stringify([...b].sort());
|
|
13
|
+
const key = (customer, organization) => customer + '/' + organization;
|
|
14
|
+
|
|
15
|
+
// Capture/validate before source deletion. The caller holds the same transaction
|
|
16
|
+
// fence as Lead writers and customer deletion/downgrade admission throughout.
|
|
17
|
+
async function prepare(db, session, kind, target, source) {
|
|
18
|
+
const targetId = id(target), sourceId = id(source);
|
|
19
|
+
const companyFilter = kind === 'Organization' ? { _id: { $in: [oid(targetId), oid(sourceId)] } } :
|
|
20
|
+
{ $or: ['first', 'second'].map(slot => ({ ['chairs.' + slot]: { $in: [oid(targetId), oid(sourceId)] } })) };
|
|
21
|
+
const leadFilter = kind === 'Organization' ? { organization: { $in: [oid(targetId), oid(sourceId)] } } :
|
|
22
|
+
{ people: { $in: [oid(targetId), oid(sourceId)] } };
|
|
23
|
+
const leads = await db.collection('leads').find(leadFilter, { session }).toArray();
|
|
24
|
+
const companyIds = leads.map(lead => oid(lead.organization));
|
|
25
|
+
const companies = await db.collection('organizations').find({ $or: [companyFilter, { _id: { $in: companyIds } }] },
|
|
26
|
+
{ session, projection: { chairs: 1 } }).toArray();
|
|
27
|
+
const groups = new Map(), owners = new Set();
|
|
28
|
+
for (const company of companies) {
|
|
29
|
+
if (company.chairs != null && !Array.isArray(company.chairs)) throw Error('Malformed legacy lead array');
|
|
30
|
+
for (const chair of company.chairs || []) {
|
|
31
|
+
const customer = id(chair.customer), organization = id(company);
|
|
32
|
+
if (Object.keys(chair).some(field => !['_id', 'customer', 'first', 'second'].includes(field))) throw Error('Unknown legacy lead fields');
|
|
33
|
+
const groupKey = key(customer, organization);
|
|
34
|
+
if (!groups.has(groupKey)) groups.set(groupKey, { customer, organization, people: [], lead: null });
|
|
35
|
+
const group = groups.get(groupKey);
|
|
36
|
+
group.people = unique([...group.people, ...[chair.first, chair.second].filter(value => value != null)]);
|
|
37
|
+
owners.add(customer);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
// Read all groups on affected companies, including empty/new Lead records.
|
|
41
|
+
const existing = await db.collection('leads').find({ organization: { $in: companies.map(company => company._id) } }, { session }).toArray();
|
|
42
|
+
for (const lead of existing) {
|
|
43
|
+
const customer = id(lead.customer), organization = id(lead.organization), groupKey = key(customer, organization);
|
|
44
|
+
if (typeof lead.customer === 'string' || typeof lead.organization === 'string' || !Array.isArray(lead.people)) throw Error('Malformed Lead storage');
|
|
45
|
+
if (!groups.has(groupKey)) groups.set(groupKey, { customer, organization, people: [], lead: null });
|
|
46
|
+
const group = groups.get(groupKey), people = unique(lead.people);
|
|
47
|
+
if (group.lead || people.length !== lead.people.length || !equal(group.people, people)) throw Error('Lead merge requires reconciled stores');
|
|
48
|
+
group.lead = lead; owners.add(customer);
|
|
49
|
+
}
|
|
50
|
+
if (leads.some(lead => !companies.some(company => id(company) === id(lead.organization)))) throw Error('Lead company is missing');
|
|
51
|
+
for (const customer of owners) {
|
|
52
|
+
const tenant = await db.collection('customers').findOne({ _id: oid(customer) }, { session, projection: { organization: 1 } });
|
|
53
|
+
if (!tenant || id(tenant.organization) !== customer) throw Error('Customer identity mismatch');
|
|
54
|
+
const variants = [oid(customer), customer];
|
|
55
|
+
const jobs = await db.collection('deletions').find({ kind: 'job', $or: [{ _id: { $in: variants } }, { customerId: { $in: variants } }] }, { session }).toArray();
|
|
56
|
+
if (jobs.some(job => job.operation === 'delete' || !['completed', 'scheduled'].includes(job.status))) throw Error('Customer lifecycle blocks lead merge');
|
|
57
|
+
}
|
|
58
|
+
return { kind, targetId, sourceId, groups, companies };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
async function apply(db, session, plan) {
|
|
62
|
+
const output = new Map();
|
|
63
|
+
for (const group of plan.groups.values()) {
|
|
64
|
+
const organization = plan.kind === 'Organization' ? plan.targetId : group.organization;
|
|
65
|
+
const people = plan.kind === 'Person' ? unique(group.people.map(person => person === plan.sourceId ? plan.targetId : person)) : group.people;
|
|
66
|
+
const groupKey = key(group.customer, organization);
|
|
67
|
+
if (!output.has(groupKey)) output.set(groupKey, { customer: group.customer, organization, people: [], originals: [] });
|
|
68
|
+
const next = output.get(groupKey);
|
|
69
|
+
next.people = unique([...next.people, ...people]); next.originals.push(group);
|
|
70
|
+
}
|
|
71
|
+
for (const group of output.values()) {
|
|
72
|
+
const peopleFound = await db.collection('people').countDocuments({ _id: { $in: group.people.map(oid) } }, { session });
|
|
73
|
+
if (peopleFound !== group.people.length) throw Error('Lead Person is missing after merge');
|
|
74
|
+
const keep = group.originals.find(original => original.organization === group.organization && original.lead)?.lead || group.originals.find(original => original.lead)?.lead;
|
|
75
|
+
for (const original of group.originals) {
|
|
76
|
+
if (original.lead && original.lead !== keep) await db.collection('leads').deleteOne({ _id: original.lead._id }, { session });
|
|
77
|
+
}
|
|
78
|
+
if (keep) {
|
|
79
|
+
const result = await db.collection('leads').updateOne({ _id: keep._id }, { $set: {
|
|
80
|
+
organization: oid(group.organization), people: group.people.map(oid)
|
|
81
|
+
}, $inc: { __v: 1 } }, { session });
|
|
82
|
+
if (result.matchedCount !== 1) throw Error('Lead changed during merge');
|
|
83
|
+
} else {
|
|
84
|
+
const stable = crypto.createHash('sha256').update(`customer-data-v1:${group.customer}/organizations/${group.organization}/chairs`).digest('hex').slice(0, 24);
|
|
85
|
+
await db.collection('leads').insertOne({ _id: new ObjectId(stable), customer: oid(group.customer), organization: oid(group.organization),
|
|
86
|
+
people: group.people.map(oid), __v: 0 }, { session });
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
const organizations = plan.kind === 'Organization' ? [plan.targetId] : plan.companies.map(id);
|
|
90
|
+
for (const organization of organizations) {
|
|
91
|
+
const chairs = [];
|
|
92
|
+
for (const group of output.values()) if (group.organization === organization) {
|
|
93
|
+
for (let i = 0; i < group.people.length; i += 2) chairs.push({ _id: new ObjectId(), customer: oid(group.customer), first: oid(group.people[i]),
|
|
94
|
+
...(group.people[i + 1] ? { second: oid(group.people[i + 1]) } : {}) });
|
|
95
|
+
}
|
|
96
|
+
const result = await db.collection('organizations').updateOne({ _id: oid(organization) }, { $set: { chairs } }, { session });
|
|
97
|
+
if (result.matchedCount !== 1) throw Error('Lead company changed during merge');
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
module.exports = { prepare, apply };
|
package/lib/organizations.js
CHANGED
|
@@ -136,7 +136,7 @@ var updateReferences = function updateReferences(context, target, source, merged
|
|
|
136
136
|
// Do not update cap tables, financials or rounds as these are complicated
|
|
137
137
|
// and get cleaned and updated on each pull anyway
|
|
138
138
|
|
|
139
|
-
async.parallel([
|
|
139
|
+
(context.mergeSession ? async.series : async.parallel)([
|
|
140
140
|
updateCapTables.bind(context, newOrgId, oldOrgId),
|
|
141
141
|
updateDeals.bind(context, newOrgId, oldOrgId, customerId),
|
|
142
142
|
updateFinancials.bind(context, newOrgId, oldOrgId),
|
|
@@ -153,6 +153,7 @@ var updateCapTables = function updateCapTables(newOrgId, oldOrgId, cb) {
|
|
|
153
153
|
logger.writeToInfoLog(' Updating cap tables...');
|
|
154
154
|
|
|
155
155
|
var self = this;
|
|
156
|
+
cb = _.once(cb);
|
|
156
157
|
|
|
157
158
|
// Get all cap tables for this org
|
|
158
159
|
self.CapTable.collection.find({ 'organization': oldOrgId }).toArray().then(function(items) {
|
|
@@ -165,6 +166,7 @@ var updateCapTables = function updateCapTables(newOrgId, oldOrgId, cb) {
|
|
|
165
166
|
_.each(items, function(item) {
|
|
166
167
|
tasks.push({
|
|
167
168
|
id: item._id,
|
|
169
|
+
customer: item.customer,
|
|
168
170
|
update: { $set: { organization: newOrgId } }
|
|
169
171
|
});
|
|
170
172
|
});
|
|
@@ -177,8 +179,9 @@ var updateCapTables = function updateCapTables(newOrgId, oldOrgId, cb) {
|
|
|
177
179
|
if (result) { updated.push(result); }
|
|
178
180
|
return callback(null, result);
|
|
179
181
|
}, function(err) { return callback(err, null); });
|
|
180
|
-
}, 8);
|
|
182
|
+
}, self.mergeSession ? 1 : 8);
|
|
181
183
|
|
|
184
|
+
queue.error = function(err) { queue.kill(); cb(err, null); };
|
|
182
185
|
queue.drain = function() { return cb(null, updated); };
|
|
183
186
|
|
|
184
187
|
queue.push(tasks);
|
|
@@ -192,6 +195,7 @@ var updateDeals = function updateDeals(newOrgId, oldOrgId, customerId, cb) {
|
|
|
192
195
|
logger.writeToInfoLog(' Updating deals...');
|
|
193
196
|
|
|
194
197
|
var self = this;
|
|
198
|
+
cb = _.once(cb);
|
|
195
199
|
|
|
196
200
|
// Get all deals for this org
|
|
197
201
|
self.Deal.collection.find({ 'organization': oldOrgId }).toArray().then(function(items) {
|
|
@@ -204,6 +208,7 @@ var updateDeals = function updateDeals(newOrgId, oldOrgId, customerId, cb) {
|
|
|
204
208
|
_.each(items, function(item) {
|
|
205
209
|
tasks.push({
|
|
206
210
|
id: item._id,
|
|
211
|
+
customer: item.customer,
|
|
207
212
|
update: { $set: { organization: newOrgId } }
|
|
208
213
|
});
|
|
209
214
|
});
|
|
@@ -212,13 +217,14 @@ var updateDeals = function updateDeals(newOrgId, oldOrgId, customerId, cb) {
|
|
|
212
217
|
|
|
213
218
|
var queue = async.queue(function(task, callback) {
|
|
214
219
|
|
|
215
|
-
self.Deal.modifyById(task.id, task.update, { CUSTOMER_ID:
|
|
220
|
+
self.Deal.modifyById(task.id, task.update, { CUSTOMER_ID: task.customer }, function(err, result) {
|
|
216
221
|
if (err) { return callback(err, null); }
|
|
217
222
|
else if (result) { updated.push(result); }
|
|
218
223
|
return callback(null, result);
|
|
219
224
|
});
|
|
220
|
-
}, 8);
|
|
225
|
+
}, self.mergeSession ? 1 : 8);
|
|
221
226
|
|
|
227
|
+
queue.error = function(err) { queue.kill(); cb(err, null); };
|
|
222
228
|
queue.drain = function() { return cb(null, updated); };
|
|
223
229
|
|
|
224
230
|
queue.push(tasks);
|
|
@@ -232,6 +238,7 @@ var updateFinancials = function updateFinancials(newOrgId, oldOrgId, cb) {
|
|
|
232
238
|
logger.writeToInfoLog(' Updating financials...');
|
|
233
239
|
|
|
234
240
|
var self = this;
|
|
241
|
+
cb = _.once(cb);
|
|
235
242
|
|
|
236
243
|
// Get all financials for this org
|
|
237
244
|
self.Financials.collection.find({ 'organization': oldOrgId }).toArray().then(function(items) {
|
|
@@ -244,6 +251,7 @@ var updateFinancials = function updateFinancials(newOrgId, oldOrgId, cb) {
|
|
|
244
251
|
_.each(items, function(item) {
|
|
245
252
|
tasks.push({
|
|
246
253
|
id: item._id,
|
|
254
|
+
customer: item.customer,
|
|
247
255
|
update: { $set: { organization: newOrgId } }
|
|
248
256
|
});
|
|
249
257
|
});
|
|
@@ -256,8 +264,9 @@ var updateFinancials = function updateFinancials(newOrgId, oldOrgId, cb) {
|
|
|
256
264
|
if (result) { updated.push(result); }
|
|
257
265
|
return callback(null, result);
|
|
258
266
|
}, function(err) { return callback(err, null); });
|
|
259
|
-
}, 8);
|
|
267
|
+
}, self.mergeSession ? 1 : 8);
|
|
260
268
|
|
|
269
|
+
queue.error = function(err) { queue.kill(); cb(err, null); };
|
|
261
270
|
queue.drain = function() { return cb(null, updated); };
|
|
262
271
|
|
|
263
272
|
queue.push(tasks);
|
|
@@ -271,6 +280,7 @@ var updateMessages = function updateMessages(newOrgId, oldOrgId, cb) {
|
|
|
271
280
|
logger.writeToInfoLog(' Updating messages...');
|
|
272
281
|
|
|
273
282
|
var self = this;
|
|
283
|
+
cb = _.once(cb);
|
|
274
284
|
|
|
275
285
|
self.Message.collection.find({ 'organization': oldOrgId }).toArray().then(function(items) {
|
|
276
286
|
|
|
@@ -282,6 +292,7 @@ var updateMessages = function updateMessages(newOrgId, oldOrgId, cb) {
|
|
|
282
292
|
_.each(items, function(item) {
|
|
283
293
|
tasks.push({
|
|
284
294
|
id: item._id,
|
|
295
|
+
customer: item.customer,
|
|
285
296
|
update: { $set: { organization: newOrgId } }
|
|
286
297
|
});
|
|
287
298
|
});
|
|
@@ -294,8 +305,9 @@ var updateMessages = function updateMessages(newOrgId, oldOrgId, cb) {
|
|
|
294
305
|
else if (result) { updated.push(result); }
|
|
295
306
|
return callback(null, result);
|
|
296
307
|
});
|
|
297
|
-
}, 8);
|
|
308
|
+
}, self.mergeSession ? 1 : 8);
|
|
298
309
|
|
|
310
|
+
queue.error = function(err) { queue.kill(); cb(err, null); };
|
|
299
311
|
queue.drain = function() { return cb(null, updated); };
|
|
300
312
|
|
|
301
313
|
queue.push(tasks);
|
|
@@ -309,6 +321,7 @@ var updateNews = function updateNews(newOrgId, oldOrgId, cb) {
|
|
|
309
321
|
logger.writeToInfoLog(' Updating news...');
|
|
310
322
|
|
|
311
323
|
var self = this;
|
|
324
|
+
cb = _.once(cb);
|
|
312
325
|
|
|
313
326
|
self.News.collection.find({ 'org': oldOrgId }).toArray().then(function(items) {
|
|
314
327
|
|
|
@@ -320,6 +333,7 @@ var updateNews = function updateNews(newOrgId, oldOrgId, cb) {
|
|
|
320
333
|
_.each(items, function(item) {
|
|
321
334
|
tasks.push({
|
|
322
335
|
id: item._id,
|
|
336
|
+
customer: item.customer,
|
|
323
337
|
update: { $set: { org: newOrgId } }
|
|
324
338
|
});
|
|
325
339
|
});
|
|
@@ -332,8 +346,9 @@ var updateNews = function updateNews(newOrgId, oldOrgId, cb) {
|
|
|
332
346
|
else if (result) { updated.push(result); }
|
|
333
347
|
return callback(null, result);
|
|
334
348
|
});
|
|
335
|
-
}, 8);
|
|
349
|
+
}, self.mergeSession ? 1 : 8);
|
|
336
350
|
|
|
351
|
+
queue.error = function(err) { queue.kill(); cb(err, null); };
|
|
337
352
|
queue.drain = function() { return cb(null, updated); };
|
|
338
353
|
|
|
339
354
|
queue.push(tasks);
|
|
@@ -347,6 +362,7 @@ var updateOrganizations = function updateOrganizations(newOrgId, oldOrgId, cb) {
|
|
|
347
362
|
logger.writeToInfoLog(' Updating orgs...');
|
|
348
363
|
|
|
349
364
|
var self = this;
|
|
365
|
+
cb = _.once(cb);
|
|
350
366
|
|
|
351
367
|
self.Organization.collection.find({ $or: [{ 'merged': oldOrgId }, { 'operating.acquired.public.by': oldOrgId }, { 'operating.acquired.private.by': oldOrgId }, { 'related': oldOrgId }] }).toArray().then(function(items) {
|
|
352
368
|
|
|
@@ -377,6 +393,7 @@ var updateOrganizations = function updateOrganizations(newOrgId, oldOrgId, cb) {
|
|
|
377
393
|
|
|
378
394
|
tasks.push({
|
|
379
395
|
id: item._id,
|
|
396
|
+
customer: item.customer,
|
|
380
397
|
update: {
|
|
381
398
|
$set: setObj
|
|
382
399
|
}
|
|
@@ -392,8 +409,9 @@ var updateOrganizations = function updateOrganizations(newOrgId, oldOrgId, cb) {
|
|
|
392
409
|
else if (result) { updated.push(result); }
|
|
393
410
|
return callback(null, result);
|
|
394
411
|
});
|
|
395
|
-
}, 8);
|
|
412
|
+
}, self.mergeSession ? 1 : 8);
|
|
396
413
|
|
|
414
|
+
queue.error = function(err) { queue.kill(); cb(err, null); };
|
|
397
415
|
queue.drain = function() { return cb(null, updated); };
|
|
398
416
|
|
|
399
417
|
queue.push(tasks);
|
|
@@ -407,6 +425,7 @@ var updateRounds = function updateRounds(newOrgId, oldOrgId, cb) {
|
|
|
407
425
|
logger.writeToInfoLog(' Updating rounds...');
|
|
408
426
|
|
|
409
427
|
var self = this;
|
|
428
|
+
cb = _.once(cb);
|
|
410
429
|
|
|
411
430
|
// Get all rounds for this org
|
|
412
431
|
self.Round.collection.find({ 'organization': oldOrgId }).toArray().then(function(items) {
|
|
@@ -419,6 +438,7 @@ var updateRounds = function updateRounds(newOrgId, oldOrgId, cb) {
|
|
|
419
438
|
_.each(items, function(item) {
|
|
420
439
|
tasks.push({
|
|
421
440
|
id: item._id,
|
|
441
|
+
customer: item.customer,
|
|
422
442
|
update: { $set: { organization: newOrgId } }
|
|
423
443
|
});
|
|
424
444
|
});
|
|
@@ -431,8 +451,9 @@ var updateRounds = function updateRounds(newOrgId, oldOrgId, cb) {
|
|
|
431
451
|
if (result) { updated.push(result); }
|
|
432
452
|
return callback(null, result);
|
|
433
453
|
}, function(err) { return callback(err, null); });
|
|
434
|
-
}, 8);
|
|
454
|
+
}, self.mergeSession ? 1 : 8);
|
|
435
455
|
|
|
456
|
+
queue.error = function(err) { queue.kill(); cb(err, null); };
|
|
436
457
|
queue.drain = function() { return cb(null, updated); };
|
|
437
458
|
|
|
438
459
|
queue.push(tasks);
|