@dhyasama/totem-merge 7.1.3 → 7.2.0

package/lib/people.js

@@ -142,7 +142,7 @@ var updateReferences = function updateReferences(context, target, source, merged
   // Do not update cap tables or rounds as these are complicated
   // and get cleaned and updated on each pull anyway
 
-  async.parallel([
+  (context.mergeSession ? async.series : async.parallel)([
 
     updateDeals.bind(context, newPersonId, oldPersonId, mergedBy),
 
@@ -169,6 +169,7 @@ var updateDeals = function updateDeals(newPersonId, oldPersonId, user, cb) {
   console.log('  Updating deals...');
 
   var self = this;
+  cb = _.once(cb);
 
   // Get all deals for this person
   self.Deal.collection.find({ $or: [{ 'source.person': oldPersonId }, { 'referrer.person': oldPersonId }]}).toArray().then(function(items) {
@@ -182,11 +183,12 @@ var updateDeals = function updateDeals(newPersonId, oldPersonId, user, cb) {
 
       var setObj = {};
 
-      if (item.source && item.source.person && item.source.person.toString() === oldPersonId.toString()) { setObj.source = { person: newPersonId }; }
-      if (item.referrer && item.referrer.person && item.referrer.person.toString() === oldPersonId.toString()) { setObj.referrer = { person: newPersonId }; }
+      if (item.source && item.source.person && item.source.person.toString() === oldPersonId.toString()) { setObj['source.person'] = newPersonId; }
+      if (item.referrer && item.referrer.person && item.referrer.person.toString() === oldPersonId.toString()) { setObj['referrer.person'] = newPersonId; }
 
       tasks.push({
         id: item._id,
+        customer: item.customer,
         update: { $set: setObj }
       });
 
@@ -195,13 +197,14 @@ var updateDeals = function updateDeals(newPersonId, oldPersonId, user, cb) {
     console.log('    Found ' + tasks.length + ' deal references to update');
 
     var queue = async.queue(function(task, callback) {
-      self.Deal.modifyById(task.id, task.update, {}, function(err, result) {
+      self.Deal.modifyById(task.id, task.update, { CUSTOMER_ID: task.customer }, function(err, result) {
         if (err) { return callback(err, null); }
         else if (result) { updated.push(result); }
         return callback(null, result);
       });
-    }, 8);
+    }, self.mergeSession ? 1 : 8);
 
+    queue.error = function(err) { queue.kill(); cb(err, null); };
     queue.drain = function() { return cb(null, updated); };
 
     queue.push(tasks);
@@ -215,6 +218,7 @@ var updateDocuments = function updateDocuments(newPersonId, oldPersonId, user, c
   console.log('  Updating docs...');
 
   var self = this;
+  cb = _.once(cb);
 
   if (!self.Document || !self.Document.collection) { return cb(null, []); }
 
@@ -235,6 +239,7 @@ var updateDocuments = function updateDocuments(newPersonId, oldPersonId, user, c
       if (Object.keys(setObj).length > 0) {
         tasks.push({
           id: item._id,
+          customer: item.customer,
           update: { $set: setObj }
         });
       }
@@ -244,13 +249,14 @@ var updateDocuments = function updateDocuments(newPersonId, oldPersonId, user, c
     console.log('    Found ' + tasks.length + ' doc references to update');
 
     var queue = async.queue(function(task, callback) {
-      self.Document.modifyById(task.id, task.update, {}, function(err, result) {
+      self.Document.modifyById(task.id, task.update, { CUSTOMER_ID: task.customer }, function(err, result) {
         if (err) { return callback(err, null); }
         else if (result) { updated.push(result); }
         return callback(null, result);
       });
-    }, 8);
+    }, self.mergeSession ? 1 : 8);
 
+    queue.error = function(err) { queue.kill(); cb(err, null); };
     queue.drain = function() { return cb(null, updated); };
 
     queue.push(tasks);
@@ -264,6 +270,7 @@ var updateMeetings = function updateMeetings(newPersonId, oldPersonId, user, cb)
   console.log('  Updating meetings...');
 
   var self = this;
+  cb = _.once(cb);
 
   if (!self.Meeting || !self.Meeting.collection) {
     return cb(null, []);
@@ -286,6 +293,7 @@ var updateMeetings = function updateMeetings(newPersonId, oldPersonId, user, cb)
 
       tasks.push({
         id: item._id,
+        customer: item.customer,
         update: { $set: { 'people': item.people } }
       });
 
@@ -298,8 +306,9 @@ var updateMeetings = function updateMeetings(newPersonId, oldPersonId, user, cb)
         if (result) { updated.push(result); }
         return callback(null, result);
       }, function(err) { return callback(err, null); });
-    }, 8);
+    }, self.mergeSession ? 1 : 8);
 
+    queue.error = function(err) { queue.kill(); cb(err, null); };
     queue.drain = function() { return cb(null, updated); };
 
     queue.push(tasks);
@@ -313,6 +322,7 @@ var updateLimitedPartners = function updateLimitedPartners(newPersonId, oldPerso
   console.log('  Updating lps...');
 
   var self = this;
+  cb = _.once(cb);
 
   // Get all lps for this person
   self.LimitedPartner.collection.find({ 'people': oldPersonId }).toArray().then(function(items) {
@@ -331,6 +341,7 @@ var updateLimitedPartners = function updateLimitedPartners(newPersonId, oldPerso
 
       tasks.push({
         id: item._id,
+        customer: item.customer,
         update: { $set: {
             'people': item.people
           }
@@ -347,8 +358,9 @@ var updateLimitedPartners = function updateLimitedPartners(newPersonId, oldPerso
         else if (result) { updated.push(result); }
         return callback(null, result);
       });
-    }, 8);
+    }, self.mergeSession ? 1 : 8);
 
+    queue.error = function(err) { queue.kill(); cb(err, null); };
     queue.drain = function() { return cb(null, updated); };
 
     queue.push(tasks);
@@ -362,6 +374,7 @@ var updateMessages = function updateMessages(newPersonId, oldPersonId, user, cb)
   console.log('  Updating messages...');
 
   var self = this;
+  cb = _.once(cb);
 
   // Get all messages for this person
   self.Message.collection.find({ $or: [{ 'recipients.external': oldPersonId }, { 'recipients.internal': oldPersonId }] }).toArray().then(function(items) {
@@ -385,6 +398,7 @@ var updateMessages = function updateMessages(newPersonId, oldPersonId, user, cb)
 
       tasks.push({
         id: item._id,
+        customer: item.customer,
         update: { $set: {
             'recipients.internal': item.recipients.internal,
             'recipients.external': item.recipients.external
@@ -402,8 +416,9 @@ var updateMessages = function updateMessages(newPersonId, oldPersonId, user, cb)
         else if (result) { updated.push(result); }
         return callback(null, result);
       });
-    }, 8);
+    }, self.mergeSession ? 1 : 8);
 
+    queue.error = function(err) { queue.kill(); cb(err, null); };
     queue.drain = function() { return cb(null, updated); };
 
     queue.push(tasks);
@@ -417,6 +432,7 @@ var updateNotes = function updateNotes(newPersonId, oldPersonId, user, cb) {
   console.log('  Updating notes...');
 
   var self = this;
+  cb = _.once(cb);
 
   // Get all notes for this person
   self.Note.collection.find({ 'createdBy': oldPersonId }).toArray().then(function(items) {
@@ -434,6 +450,7 @@ var updateNotes = function updateNotes(newPersonId, oldPersonId, user, cb) {
 
       tasks.push({
         id: item._id,
+        customer: item.customer,
         update: { $set: setObj }
       });
 
@@ -442,13 +459,14 @@ var updateNotes = function updateNotes(newPersonId, oldPersonId, user, cb) {
     console.log('    Found ' + tasks.length + ' note references to update');
 
     var queue = async.queue(function(task, callback) {
-      self.Note.modifyById(task.id, task.update, {}, function(err, result) {
+      self.Note.modifyById(task.id, task.update, { CUSTOMER_ID: task.customer }, function(err, result) {
         if (err) { return callback(err, null); }
         else if (result) { updated.push(result); }
         return callback(null, result);
       });
-    }, 8);
+    }, self.mergeSession ? 1 : 8);
 
+    queue.error = function(err) { queue.kill(); cb(err, null); };
     queue.drain = function() { return cb(null, updated); };
 
     queue.push(tasks);
@@ -462,6 +480,7 @@ var updateOrganizations = function updateOrganizations(newPersonId, oldPersonId,
   console.log('  Updating orgs...');
 
   var self = this;
+  cb = _.once(cb);
 
   // Get all orgs for this person
   self.Organization.collection.find({ $or: [{ 'people.person': oldPersonId }, { 'chairs.first': oldPersonId }, { 'chairs.second': oldPersonId }] }).toArray().then(function(orgs) {
@@ -544,8 +563,9 @@ var updateOrganizations = function updateOrganizations(newPersonId, oldPersonId,
         else if (result) { updated.push(result); }
         return callback(null, result);
       });
-    }, 8);
+    }, self.mergeSession ? 1 : 8);
 
+    queue.error = function(err) { queue.kill(); cb(err, null); };
     queue.drain = function() { return cb(null, updated); };
 
     queue.push(tasks);
@@ -559,6 +579,7 @@ var updatePeople = function updatePeople(newPersonId, oldPersonId, user, cb) {
   console.log('  Updating people...');
 
   var self = this;
+  cb = _.once(cb);
 
   // Update Person.related where oldPersonId appears
   self.Person.collection.find({ 'related': oldPersonId }).toArray().then(function(people) {
@@ -592,8 +613,9 @@ var updatePeople = function updatePeople(newPersonId, oldPersonId, user, cb) {
         else if (result) { updated.push(result); }
         return callback(null, result);
       });
-    }, 8);
+    }, self.mergeSession ? 1 : 8);
 
+    queue.error = function(err) { queue.kill(); cb(err, null); };
     queue.drain = function() { return cb(null, updated); };
 
     queue.push(tasks);

package/lib/privateMergeChecks.js

@@ -0,0 +1,27 @@
+'use strict';
+const { Types: { ObjectId } } = require('mongoose');
+const { isDeepStrictEqual } = require('node:util');
+function id(value) {
+  if (value?._bsontype !== 'ObjectId' || !/^[a-f0-9]{24}$/i.test(String(value))) throw Error('Malformed private merge reference');
+  return String(value);
+}
+const oid = value => new ObjectId(value);
+function array(value) {
+  if (value == null) return [];
+  if (!Array.isArray(value)) throw Error('Malformed private merge array');
+  return value;
+}
+function object(value) {
+  if (!value || ![Object.prototype, null].includes(Object.getPrototypeOf(value))) throw Error('Malformed custom fields');
+  return value;
+}
+async function owners(db, session, values) {
+  for (const customer of new Set(values)) {
+    const tenant = await db.collection('customers').findOne({ _id: oid(customer) }, { session, projection: { organization: 1 } });
+    if (!tenant || id(tenant.organization) !== customer) throw Error('Customer identity mismatch');
+    const variants = [customer, oid(customer)];
+    const jobs = await db.collection('deletions').find({ kind: 'job', $or: [{ _id: { $in: variants } }, { customerId: { $in: variants } }] }, { session }).toArray();
+    if (jobs.some(job => job.operation === 'delete' || !['completed', 'scheduled'].includes(job.status))) throw Error('Customer lifecycle blocks private merge');
+  }
+}
+module.exports = { id, oid, array, object, owners, equal: isDeepStrictEqual };

package/lib/privateRecordMerge.js

@@ -0,0 +1,133 @@
+'use strict';
+const { id, oid, array, object, owners, equal } = require('./privateMergeChecks');
+const own = (object, key) => Object.prototype.hasOwnProperty.call(object, key);
+function groups(entries) {
+  const result = new Map();
+  for (const entry of array(entries)) {
+    if (typeof entry.group !== 'string' || !entry.group.trim() || entry.group !== entry.group.trim() || result.has(entry.group)) throw Error('Malformed tag groups');
+    const items = array(entry.items);
+    if (items.some(item => typeof item !== 'string' || !item.trim() || item !== item.trim()) || new Set(items).size !== items.length) throw Error('Malformed tag items');
+    result.set(entry.group, { ...entry, items: [...items] });
+  }
+  return result;
+}
+function tagValue(entries) {
+  return [...groups(entries)].map(([name, entry]) => [name, [...entry.items].sort()]).sort((a, b) => a[0].localeCompare(b[0]));
+}
+function valuationValue(entry) {
+  // Stored values only. Do not fill absent values using schema defaults.
+  const result = {};
+  for (const key of ['date', 'valuation', 'pricePerShare', 'method']) if (own(entry, key)) result[key] = entry[key];
+  return result;
+}
+function sameEntries(actual, expected) {
+  const unmatched = [...expected];
+  for (const entry of actual) {
+    const position = unmatched.findIndex(candidate => equal(candidate, entry));
+    if (position < 0) return false;
+    unmatched.splice(position, 1);
+  }
+  return unmatched.length === 0;
+}
+async function prepare(db, session, kind, target, source) {
+  const ids = [target._id, source._id], targetId = id(target._id), ownerIds = [];
+  const plan = { targetId, kind, records: [], mirrors: {} };
+  // Raw parent reads prevent Mongoose defaults from becoming stored values.
+  const parents = await db.collection(kind === 'Organization' ? 'organizations' : 'people').find({ _id: { $in: ids } }, { session }).toArray();
+  for (const [collection, field] of [['tags', 'filters'], ['valuations', 'valuations']]) {
+    if (kind === 'Person' && collection !== 'tags') continue;
+    const filter = collection === 'valuations' ? { organization: { $in: ids } } :
+      { 'entity.model': kind, 'entity.id': { $in: ids } };
+    const rows = await db.collection(collection).find(filter, { session }).toArray();
+    const buckets = new Map();
+    for (const row of rows) {
+      const customer = id(row.customer), entity = id(collection === 'valuations' ? row.organization : row.entity.id);
+      ownerIds.push(customer);
+      if (!buckets.has(customer)) buckets.set(customer, new Map());
+      const entities = buckets.get(customer);
+      if (!entities.has(entity)) entities.set(entity, []);
+      entities.get(entity).push(row);
+    }
+    if (kind === 'Organization') {
+      for (const parent of parents) {
+        const legacy = array(parent[field]), entity = id(parent._id);
+        const customers = new Set(legacy.map(entry => id(entry.customer)));
+        for (const [customer, entities] of buckets) if (entities.has(entity)) customers.add(customer);
+        for (const customer of customers) {
+          const old = legacy.filter(entry => id(entry.customer) === customer);
+          const current = buckets.get(customer)?.get(entity) || [];
+          let matches;
+          if (collection === 'tags') matches = current.length === 1 && equal(tagValue(current[0].groups), tagValue(old));
+          else matches = sameEntries(current.map(valuationValue), old.map(valuationValue));
+          if (!matches) throw Error('Private merge requires reconciled ' + collection);
+          // A legacy valuation ID is part of its identity, not just its value.
+          if (collection === 'valuations' && current.some(row => row.legacyId &&
+              !old.some(entry => entry._id && id(entry._id) === id(row.legacyId) && equal(valuationValue(entry), valuationValue(row))))) {
+            throw Error('Valuation legacy identity mismatch');
+          }
+        }
+      }
+    }
+    const mirror = [];
+    for (const [customer, entities] of buckets) {
+      const targetRows = entities.get(targetId) || [], sourceRows = entities.get(id(source._id)) || [];
+      if (collection !== 'valuations' && (targetRows.length > 1 || sourceRows.length > 1)) throw Error('Duplicate private records');
+      const all = [...targetRows, ...sourceRows];
+      if (collection === 'valuations') {
+        const seen = new Set();
+        for (const row of all) {
+          if (row.legacyId) {
+            const legacyId = id(row.legacyId);
+            if (seen.has(legacyId)) throw Error('Valuation legacy ID collision requires resolution');
+            seen.add(legacyId);
+          }
+          plan.records.push({ collection, keep: row, set: { organization: oid(targetId) }, remove: [] });
+          mirror.push({ _id: row.legacyId || row._id, customer: oid(customer), ...valuationValue(row) });
+        }
+      } else if (collection === 'tags') {
+        const combined = new Map();
+        for (const row of all) for (const [name, entry] of groups(row.groups)) {
+          if (!combined.has(name)) combined.set(name, { ...entry, items: [] });
+          const group = combined.get(name);
+          group.items = [...new Set([...group.items, ...entry.items])];
+        }
+        const next = [...combined.values()];
+        plan.records.push({ collection, keep: all[0], remove: all.slice(1), set: { entity: { model: kind, id: oid(targetId) }, groups: next } });
+        mirror.push(...next.map(group => ({ ...group, customer: oid(customer) })));
+      }
+    }
+    if (kind === 'Organization') plan.mirrors[field] = mirror;
+  }
+  // Custom fields stay embedded. Preserve both parents' values without consulting
+  // or updating the deferred CustomField collection. The old generic array
+  // merger can discard entries that have no subdocument ID.
+  if (kind === 'Organization' && parents.some(parent => own(parent, 'customFields'))) {
+    const embedded = new Map();
+    const ordered = [targetId, id(source._id)].map(value => parents.find(parent => id(parent._id) === value));
+    for (const parent of ordered) for (const entry of array(parent.customFields)) {
+      const customer = id(entry.customer); ownerIds.push(customer);
+      if (!embedded.has(customer)) embedded.set(customer, { ...entry, customer: oid(customer), fields: Object.create(null) });
+      const fields = embedded.get(customer).fields;
+      for (const [key, value] of Object.entries(object(entry.fields))) {
+        if (own(fields, key) && !equal(fields[key], value)) throw Error('Conflicting embedded custom fields require resolution');
+        fields[key] = value;
+      }
+    }
+    plan.mirrors.customFields = [...embedded.values()];
+  }
+  await owners(db, session, ownerIds);
+  return plan;
+}
+async function apply(db, session, plan) {
+  for (const item of plan.records) {
+    for (const row of item.remove) await db.collection(item.collection).deleteOne({ _id: row._id, customer: row.customer }, { session });
+    const result = await db.collection(item.collection).updateOne({ _id: item.keep._id, customer: item.keep.customer },
+      { $set: item.set, $inc: { __v: 1 } }, { session });
+    if (result.matchedCount !== 1) throw Error('Private record changed during merge');
+  }
+  if (plan.kind === 'Organization') {
+    const result = await db.collection('organizations').updateOne({ _id: oid(plan.targetId) }, { $set: plan.mirrors }, { session });
+    if (result.matchedCount !== 1) throw Error('Private merge target changed');
+  }
+}
+module.exports = { prepare, apply };

package/lib/transactionContext.js

@@ -0,0 +1,29 @@
+'use strict';
+// Per-merge facades, never mutations of global models. Only operations used by
+// the merge package are exposed, with the same session on every query/write.
+function collection(raw, session) {
+  return {
+    find: (filter, options) => raw.find(filter, { ...options, session }),
+    findOne: (filter, options) => raw.findOne(filter, { ...options, session }),
+    updateOne: (filter, update, options) => raw.updateOne(filter, update, { ...options, session })
+  };
+}
+module.exports = function transactionContext(context, session) {
+  const result = { ...context, mergeSession: session };
+  for (const [name, Model] of Object.entries(context)) {
+    if (!Model?.modelName) continue;
+    const scoped = {
+      modelName: Model.modelName,
+      collection: collection(Model.collection, session),
+      db: { collection: name => collection(Model.db.collection(name), session) },
+      findByIdAndUpdate: (...args) => Model.findByIdAndUpdate(...args).session(session),
+      findOneAndUpdate: (...args) => Model.findOneAndUpdate(...args).session(session),
+      deleteOne: (...args) => Model.deleteOne(...args).session(session)
+    };
+    for (const method of ['upsert', 'modifyById']) {
+      if (Model[method]) scoped[method] = (...args) => Model[method].apply(scoped, args);
+    }
+    result[name] = scoped;
+  }
+  return result;
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dhyasama/totem-merge",
-  "version": "7.1.3",
+  "version": "7.2.0",
   "description": "Org and person merging package",
   "main": "index.js",
   "scripts": {
@@ -27,7 +27,7 @@
     "mongoose": "^8.24.3"
   },
   "devDependencies": {
-    "@dhyasama/totem-models": "15.1.0",
+    "@dhyasama/totem-models": "17.0.0",
     "mocha": "^12.0.2",
     "mocha-mongoose": "^1.1.1",
     "should": "^8.2.2"