@dhyasama/totem-merge 6.0.1

package/README.md

@@ -0,0 +1,25 @@
+## Merge two organizations or two people
+
+* copies source properties to target
+* will not overwrite properties on target, i.e., first-in wins
+* if a single value property, copies over if not value in target, ignores otherwise
+* if an array, concats and dedupes
+* creates property on target if it exists on source but not target
+* recursive to unlimited depth
+* source is marked as deleted, i.e., soft deleted
+* option to just flag rather than merge which is handy for automated use
+
+``` js
+npm install --save @dhyasama/ff-merge
+```
+
+``` js
+var Merge = require('@dhyasama/ff-merge');
+var merge = new Merge(mongoose, config);
+```
+
+``` js
+merge.organizations(targetOrg, sourceOrg, 'some username', function(err, result) {});
+
+merge.people(targetPerson, sourcePerson, 'some username', function(err, result) {});
+```

package/cli.js

@@ -0,0 +1,154 @@
+"use strict";
+
+var mongoose = require('mongoose');
+var config = require('./config');
+var models = require('@dhyasama/totem-models')(mongoose, config);
+var Merge = require('./index.js');
+var merge = new Merge(mongoose, config);
+var program = require('commander');
+var async = require('async');
+var Organization = mongoose.model('Organization');
+var Person = mongoose.model('Person');
+
+mongoose.connection.on('error', function(err) {
+  done(err);
+});
+
+mongoose.connect(config.db.uri, function (err, res) {
+
+  if (err) { done(err); }
+  else {
+
+    console.log('Connected to ' + config.db.uri.substring(config.db.uri.indexOf('@') + 1) + '\n');
+
+    buildProgram();
+
+  }
+
+});
+
+// npm start p 5637a8c0281347cc0bbfca30 5637a8c0281347cc0bbfca3e jason
+// npm start o 5627c370edff044dd5c129ef 5a81b38ceba193f2e7adbc7d jason
+
+var buildProgram = function buildProgram() {
+
+  program
+    .version('0.0.1')
+    .description('Merge people and organizations');
+
+  program
+    .command('people <target> <source> <username>')
+    .alias('p')
+    .description('Merge two people. The source will be merged into the target')
+    .action(function (target, source, username) {
+
+      username = username ? username.toLowerCase() : '';
+
+      if (!validate(target, source, username)) { done(); }
+
+      console.log('Getting people...');
+
+      async.parallel([
+        Person.getById.bind(Person, target),
+        Person.getById.bind(Person, source)
+      ], function(err, results) {
+
+        if (err) { done(err); }
+        else if (!results || results.length != 2 || !results[0] || !results[1]) {
+          console.log('  Unable to get people. Perhaps one doesn\'t exist?');
+          done();
+        }
+
+        merge.people(results[0], results[1], username, function(err, result) {
+
+          if (err) { done(err); }
+
+          console.log('**********');
+          console.log('RESULT');
+          console.log(JSON.stringify(result, null, 2));
+
+          done();
+
+        });
+
+      });
+
+    });
+
+  program
+    .command('organizations <target> <source> <username>')
+    .alias('o')
+    .description('Merge two organizations. The source will be merged into the target')
+    .action(function (target, source, username) {
+
+      username = username ? username.toLowerCase() : '';
+
+      if (!validate(target, source, username)) { done(); }
+
+      console.log('Getting orgs...');
+
+      async.parallel([
+        Organization.getById.bind(Organization, target, { role: 'none' }),
+        Organization.getById.bind(Organization, source, { role: 'none' })
+      ], function(err, results) {
+
+        if (err) { done(err); }
+        else if (!results || results.length != 2 || !results[0] || !results[1]) {
+          console.log('Unable to get orgs');
+          done();
+        }
+
+        console.log('Merging orgs...')
+
+        merge.organizations(results[0], results[1], username, function(err, result) {
+
+          if (err) { done(err); }
+
+          console.log('**********');
+          console.log('RESULT');
+          console.log(JSON.stringify(result, null, 2));
+
+          done();
+
+        });
+
+      });
+
+    });
+
+  program.parse(process.argv);
+
+};
+
+var done = function done(err) {
+
+  if (err) { console.error(err); }
+
+  console.log('DONE');
+
+  process.exit();
+
+};
+
+var validate = function validate(target, source, username) {
+
+  var valid = true;
+
+  if (!mongoose.Types.ObjectId.isValid(target)) {
+    console.log('target is not a valid objectid');
+    valid = false;
+  }
+
+  if (!mongoose.Types.ObjectId.isValid(source)) {
+    console.log('source is not a valid objectid');
+    valid = false;
+  }
+
+  if (username != 'steven' && username != 'jason') {
+    console.log('username is not valid');
+    valid = false;
+  }
+
+  return valid;
+
+};

package/config.js

@@ -0,0 +1,23 @@
+module.exports = {
+
+  USERNAME: 'jason',
+
+  CUSTOMER_ID: "5a95ac2ec991f858321c19d6",
+
+  LPS_ENABLED: false,
+
+  suppressDebugLog: true,
+
+  // db: {
+  //   uri: 'mongodb://ffvc:ytnn5m7A9snIJSOLRqwo7Bpo2QkyYx6q@ds043952.mongolab.com:43952/ff-platform-test'
+  // },
+
+  db: {
+    uri: 'mongodb://jason:T576G150HPXLA5q4BJ2o2zj747B5030x@development-shard-00-00-gv0f3.mongodb.net:27017,development-shard-00-01-gv0f3.mongodb.net:27017,development-shard-00-02-gv0f3.mongodb.net:27017/development?ssl=true&replicaSet=Development-shard-0&authSource=admin'
+  },
+
+  crypto: {
+    key: 'DUM'
+  }
+
+};

package/index.js

@@ -0,0 +1,109 @@
+"use strict";
+
+var organizations = require('./lib/organizations');
+var people = require('./lib/people');
+var Logger = require('@dhyasama/totem-logger');
+
+// Constructor
+var Merge = function(mongoose, config) {
+
+  this.env = process.env.NODE_ENV || 'development';
+  this.logger = new Logger(config.logentries);
+  this.CapTable = mongoose.model('CapTable');
+  this.Deal = mongoose.model('Deal');
+  this.Document = mongoose.model('Document');
+  this.Financials = mongoose.model('Financials');
+  this.Flag = mongoose.model('Flag');
+  this.Interaction = mongoose.model('Interaction');
+  this.LimitedPartner = mongoose.model('LimitedPartner');
+  this.List = mongoose.model('List');
+  this.Message = mongoose.model('Message');
+  this.News = mongoose.model('News');
+  this.Note = mongoose.model('Note');
+  this.Organization = mongoose.model('Organization');
+  this.Person = mongoose.model('Person');
+  this.Round = mongoose.model('Round');
+  this.Snapshot = mongoose.model('Snapshot');
+
+  // can choose to flag an entity as needing to be merged rather than actually merging it
+  // particularly useful for automated processes, e.g., a batch import that detects dupes
+  // but you don't want to auto-merge 1000 records
+  // defaults to false
+  this.flagOnly = config.flag == true ? true : false;
+
+  // turn on debug if in development and not explicitly turned off
+  mongoose.set('debug', (!config.suppressDebugLog && this.env == 'development'));
+
+};
+
+//////////////////////////////////////////////
+
+Merge.prototype.organizations = function(target, source, user, customerId, cb) {
+  organizations.merge(this, target, source, user, customerId, cb);
+};
+
+Merge.prototype.organizationsById = function(targetId, sourceId, user, customerId, cb) {
+
+  var self = this;
+
+  self.logger.writeToInfoLog('Merge org ' + sourceId + ' into ' + targetId);
+
+  self.Organization.getById(targetId, { CUSTOMER_ID: customerId, isWorkerProcess: true }, function(err, target) {
+
+    if (err) { return cb(err, null); }
+    else if (!target) { return cb(new Error('target not found'), null); }
+
+    self.logger.writeToInfoLog('  Found target ' + target.name);
+
+    self.Organization.getById(sourceId, { CUSTOMER_ID: customerId, isWorkerProcess: true }, function(err, source) {
+
+      if (err) { return cb(err, null); }
+      else if (!source) { return cb(new Error('source not found'), null); }
+
+      self.logger.writeToInfoLog('  Found source ' + source.name);
+      self.logger.writeToInfoLog('  Merging...');
+
+      organizations.merge(self, target, source, user, customerId, cb);
+
+    });
+
+  });
+
+};
+
+Merge.prototype.people = function(target, source, user, customerId, cb) {
+  people.merge(this, target, source, user, customerId, cb);
+};
+
+Merge.prototype.peopleById = function(targetId, sourceId, user, customerId, cb) {
+
+  var self = this;
+
+  self.logger.writeToInfoLog('Merge person ' + sourceId + ' into ' + targetId);
+
+  self.Person.getById(targetId, { CUSTOMER_ID: customerId, isWorkerProcess: true }, function(err, target) {
+
+    if (err) { return cb(err, null); }
+    else if (!target) { return cb(new Error('target not found'), null); }
+
+    console.log('  Found target', target.name.full);
+    console.log('  Target sources:', target.sources.length);
+
+    self.Person.getById(sourceId, { CUSTOMER_ID: customerId, isWorkerProcess: true }, function(err, source) {
+
+      if (err) { return cb(err, null); }
+      else if (!source) { return cb(new Error('source not found'), null); }
+
+      self.logger.writeToInfoLog('  Found source ' + source.name.full);
+      self.logger.writeToInfoLog('  Source sources ' + source.sources.length);
+      self.logger.writeToInfoLog('  Merging...');
+
+      people.merge(self, target, source, user, customerId, cb);
+
+    });
+
+  });
+
+};
+
+module.exports = Merge;

package/lib/changeBSON.js

@@ -0,0 +1,22 @@
+"use strict";
+
+var fs = require('fs');
+var filePath = './node_modules/mongoose-post-find/node_modules/bson/ext/index.js';
+var config = require('./../config');
+var Logger = require('@dhyasama/totem-logger');
+var logger = new Logger(config.logentries);
+
+fs.readFile(filePath, 'utf8', function (err, data) {
+
+  if (err) { logger.handleErr(err); }
+  else {
+
+    var result = data.replace("bson = require('../build/Release/bson');", "bson = require('bson');");
+
+    fs.writeFile(filePath, result, 'utf8', function (err) {
+      if (err) { logger.handleErr(err); }
+    });
+
+  }
+
+});

package/lib/helpers.js

@@ -0,0 +1,117 @@
+"use strict";
+
+var _ = require('underscore');
+var utilities = require('@dhyasama/ffvc-utilities');
+var phoneFormatter = require('phone-formatter');
+
+var createSnapshot = module.exports.createSnapshot = function createSnapshot(context, collectionName, doc, createdBy, cb) {
+
+  context.logger.writeToInfoLog('Creating snapshot - ' + collectionName + ' - ' + createdBy + ' - ' + doc._id);
+
+  var snapshot = new context.Snapshot({
+    createdBy: createdBy,
+    collectionName: collectionName,
+    doc: doc
+  });
+
+  context.Snapshot.upsert(snapshot, cb);
+
+};
+
+// Generic, recursive object merging
+var mergeObjects = function mergeObjects(target, source, doNotMergeKeys) {
+
+  if (source == 'undefined') return;
+
+  var jsonSource = JSON.parse(JSON.stringify(source));
+
+  for (var key in jsonSource) {
+
+    if (doNotMergeKeys.indexOf(key) >= 0) continue;
+
+    var sourceItem = source[key];
+    if (!sourceItem) continue;
+
+    var type = typeOf(target[key]);
+    if (type == 'null') type = typeOf(source[key]);
+
+    if (type == 'object') {
+      mergeObjects(target[key], sourceItem, doNotMergeKeys);
+    }
+    else if (type === 'array') {
+      target[key] = target[key] || [];
+      target[key] = unique(target[key].concat(sourceItem), key);
+    }
+    else {
+      if (target[key] === null) {
+        target[key] = sourceItem;
+      }
+    }
+
+  }
+
+  return target;
+
+};
+
+var removeEntity = module.exports.removeEntity = function removeEntity(context, entity, cb) {
+
+  context.logger.writeToInfoLog('Removing entity - ' + entity._id.toString());
+
+  entity.remove(cb);
+
+};
+
+// Used for determining if need to dive in recursively
+var typeOf = function typeOf(input) {
+  return ({}).toString.call(input).slice(8, -1).toLowerCase();
+};
+
+// Deduping with some type support
+var unique = function unique(array, key) {
+
+  // translate each of these to "company"
+  var companyKeys = [
+    'first',
+    'second',
+    'boards',
+    'leadership',
+    'pcs',
+    'investments'
+  ];
+
+  // for arrays of objects, specify the object key to use for comparison
+  if (key == 'people') key = 'person';
+  else if (key == 'phone') key = 'number';
+  else if (companyKeys.indexOf(key) >= 0) key = 'company';
+  else if (key == 'orgs') key = 'org';
+  else if (key == 'investedIn') key = 'id';
+
+  array = _.compact(array);
+
+  var result = _.uniq(array, false, function(item) {
+
+    var val;
+
+    if (utilities.isValidObjectId(item.toString())) {
+      val = item;
+    }
+    else if (typeOf(item) == 'object') {
+      val = key == 'number' ? phoneFormatter.normalize(item[key]) : item[key];
+    }
+    else {
+      val = key == 'number' ? phoneFormatter.normalize(item) : item;
+    }
+
+    // take one last shot
+    if (typeof val == 'undefined') val = item['_id'];
+
+    return (val || '').toString();
+
+  });
+
+  return result;
+
+};
+
+module.exports.mergeObjects = mergeObjects;