@dhyasama/totem-merge 7.1.0 → 7.1.1

package/bin/bootstrap

@@ -0,0 +1,11 @@
+#!/usr/bin/env node
+"use strict";
+
+var runtimeSecrets = require('../lib/runtimeSecrets');
+
+runtimeSecrets.loadRuntimeSecrets()
+  .then(function() { require('../cli'); })
+  .catch(function(error) {
+    console.error('runtime_secret_bootstrap_failed', { code: error.name || error.code || 'UnknownError' });
+    process.exit(1);
+  });

package/config.js

@@ -8,12 +8,8 @@ module.exports = {
 
   suppressDebugLog: true,
 
-  // db: {
-  //   uri: 'mongodb://ffvc:ytnn5m7A9snIJSOLRqwo7Bpo2QkyYx6q@ds043952.mongolab.com:43952/ff-platform-test'
-  // },
-
   db: {
-    uri: 'mongodb://jason:T576G150HPXLA5q4BJ2o2zj747B5030x@development-shard-00-00-gv0f3.mongodb.net:27017,development-shard-00-01-gv0f3.mongodb.net:27017,development-shard-00-02-gv0f3.mongodb.net:27017/development?ssl=true&replicaSet=Development-shard-0&authSource=admin'
+    uri: process.env.DB_URI || ''
   },
 
   crypto: {

package/infrastructure/security-checks/runtime-secret-contract.cjs

@@ -0,0 +1,91 @@
+'use strict';
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+const vm = require('node:vm');
+const { test } = require('node:test');
+const root = path.resolve(__dirname, '../..');
+const files = fs.existsSync(path.join(root, 'lib/runtimeSecrets.js'))
+  ? ['lib/runtimeSecrets.js']
+  : fs.readdirSync(root).flatMap(dir => ['runtimeSecrets.js', 'runtime_secrets.js']
+      .map(name => path.join(dir, 'src/libs', name)).filter(file => fs.existsSync(path.join(root, file))));
+assert(files.length > 0, 'No runtime loaders found');
+
+function fixture(file, env = {}) {
+  const process = { env: { NODE_ENV: 'test', ...env } };
+  const module = { exports: {} };
+  const warnings = [];
+  class Client { constructor(options) { this.options = options; } }
+  class Command { constructor(input) { this.input = input; } }
+  const sdk = { SecretsManager: Client, SecretsManagerClient: Client, GetSecretValueCommand: Command };
+  vm.runInNewContext(fs.readFileSync(path.join(root, file), 'utf8'), {
+    module, exports: module.exports, process, Buffer,
+    console: { warn: (...args) => warnings.push(args), info: () => {} },
+    require: name => {
+      assert(['aws-sdk', '@aws-sdk/client-secrets-manager'].includes(name));
+      return sdk;
+    }
+  }, { filename: file });
+  const loader = module.exports;
+  const values = Object.fromEntries((loader.REQUIRED_KEYS || ['DB_URI']).map(key => [key, 'synthetic-' + key]));
+  const response = () => ({ SecretString: JSON.stringify(values) });
+  const client = fn => ({ send: fn, getSecretValue: () => ({ promise: fn }) });
+  return { loader, env: process.env, values, response, client, warnings };
+}
+
+for (const file of files) {
+  test(file + ': explicit secret ID required in deployed runtimes', async () => {
+    for (const env of [
+      { NODE_ENV: 'production', RUNTIME_SECRET_REQUIRED: 'false' },
+      { AWS_LAMBDA_FUNCTION_NAME: 'synthetic-function', RUNTIME_SECRET_REQUIRED: 'false' },
+      { RUNTIME_SECRET_REQUIRED: 'true' },
+      { NODE_ENV: 'production', RUNTIME_SECRET_ID: '   ' }
+    ]) {
+      const f = fixture(file, env);
+      let called = false;
+      await assert.rejects(f.loader.loadRuntimeSecrets({ client: f.client(async () => { called = true; }) }), /RUNTIME_SECRET_ID is required/);
+      assert.equal(called, false);
+    }
+  });
+  test(file + ': failed production fetch can recover without restarting', async () => {
+    const f = fixture(file, { NODE_ENV: 'production', RUNTIME_SECRET_ID: '/synthetic/secret', RUNTIME_SECRET_REQUIRED: 'false' });
+    let calls = 0;
+    const client = f.client(async () => { if (++calls === 1) throw new Error('synthetic transient failure'); return f.response(); });
+    await assert.rejects(f.loader.loadRuntimeSecrets({ client }), /synthetic transient failure/);
+    assert.equal((await f.loader.loadRuntimeSecrets({ client })).source, 'secrets_manager');
+    assert.equal(calls, 2);
+    assert.equal(f.warnings.length, 0);
+  });
+  test(file + ': concurrent calls share one fetch and successful values stay cached', async () => {
+    const f = fixture(file, { RUNTIME_SECRET_ID: '/synthetic/secret' });
+    let calls = 0, resolve;
+    const client = f.client(() => { calls++; return new Promise(done => { resolve = done; }); });
+    const first = f.loader.loadRuntimeSecrets({ client });
+    const second = f.loader.loadRuntimeSecrets({ client });
+    assert.equal(first, second);
+    resolve(f.response());
+    await Promise.all([first, second]);
+    await f.loader.loadRuntimeSecrets({ client });
+    assert.equal(calls, 1);
+    for (const key of Object.keys(f.values)) assert.equal(f.env[key], f.values[key]);
+  });
+  test(file + ': invalid secret never partially overwrites configuration and can retry', async () => {
+    const f = fixture(file, { NODE_ENV: 'production', RUNTIME_SECRET_ID: '/synthetic/secret' });
+    const keys = Object.keys(f.values);
+    for (const key of keys) f.env[key] = 'old-synthetic-value';
+    let calls = 0;
+    const incomplete = { ...f.values, [keys.at(-1)]: '' };
+    const client = f.client(async () => ++calls === 1 ? { SecretString: JSON.stringify(incomplete) } : f.response());
+    await assert.rejects(f.loader.loadRuntimeSecrets({ client }));
+    for (const key of keys) assert.equal(f.env[key], 'old-synthetic-value');
+    assert.equal((await f.loader.loadRuntimeSecrets({ client })).source, 'secrets_manager');
+  });
+  test(file + ': development fallback does not prevent a later successful fetch', async () => {
+    const f = fixture(file, { RUNTIME_SECRET_ID: '/synthetic/secret' });
+    let calls = 0;
+    const client = f.client(async () => { if (++calls === 1) throw new Error('synthetic denied'); return f.response(); });
+    try { await f.loader.loadRuntimeSecrets({ client }); } catch (error) { assert.match(error.message, /synthetic denied/); }
+    assert.equal((await f.loader.loadRuntimeSecrets({ client })).source, 'secrets_manager');
+    assert.equal(calls, 2);
+  });
+}

package/infrastructure/security-checks/runtime-secrets.md

@@ -0,0 +1,16 @@
+# Runtime secret loading and credential cleanup
+
+Successful runtime secrets are loaded once, before configuration, clients, and
+handlers initialize. Values remain cached until process replacement. Failed
+fetches and development fallback results are not cached, so a later call can
+retry; concurrent calls share one pending fetch. AWS SDK retries are bounded
+(three attempts). Invalid secret contents never partially update configuration.
+
+Production (`NODE_ENV=production`), deployed Lambda runtimes, and any runtime
+with `RUNTIME_SECRET_REQUIRED=true` require an explicit, nonblank
+`RUNTIME_SECRET_ID`. Setting the required flag to false cannot enable legacy
+fallback in production or Lambda. Explicit development defaults remain unchanged.
+
+See the [shared restart and cleanup runbook](https://github.com/Totem-VC/totem-services/blob/main/infrastructure/security-checks/runtime-secrets.md) for release verification, credential cleanup, and remaining limitations.
+
+Run `node --test infrastructure/security-checks/runtime-secret-contract.cjs` for isolated loader checks. These source changes require the normal application release to affect running processes. No credential was rotated.

package/lib/helpers.js

@@ -43,6 +43,10 @@ var mergeObjects = function mergeObjects(target, source, doNotMergeKeys) {
 
   for (var key in jsonSource) {
 
+    // Never follow prototype paths or merge inherited source properties.
+    // Check before reading target[key], which may resolve to a prototype.
+    if (!Object.prototype.hasOwnProperty.call(jsonSource, key)) continue;
+    if (key === '__proto__' || key === 'constructor' || key === 'prototype') continue;
     if (doNotMergeKeys.indexOf(key) >= 0) continue;
 
     var sourceItem = jsonSource[key];

package/lib/runtimeSecrets.js

@@ -0,0 +1,44 @@
+"use strict";
+
+var SecretsManager = require('@aws-sdk/client-secrets-manager');
+var SECRET_ID = '/totem/dev/totem-merge';
+var loadPromise;
+
+// Production and deployed Lambda runtimes must never use legacy credentials.
+const secretsRequired = () => process.env.RUNTIME_SECRET_REQUIRED === 'true' ||
+  process.env.NODE_ENV === 'production' || Boolean(process.env.AWS_LAMBDA_FUNCTION_NAME);
+
+function applySecret(response) {
+  var values = JSON.parse(response.SecretString || Buffer.from(response.SecretBinary).toString('utf8'));
+  if (!values.DB_URI || typeof values.DB_URI !== 'string') throw new Error('Runtime secret missing DB_URI');
+  process.env.DB_URI = values.DB_URI;
+  return { source: 'secrets_manager' };
+}
+
+async function load(client) {
+  var secretId = (process.env.RUNTIME_SECRET_ID || '').trim();
+  if (!secretId && secretsRequired()) throw new Error('RUNTIME_SECRET_ID is required for this runtime');
+  secretId = secretId || SECRET_ID;
+  return applySecret(await client.send(new SecretsManager.GetSecretValueCommand({ SecretId: secretId })));
+}
+
+function loadRuntimeSecrets(options) {
+  if (!loadPromise) {
+    const client = (options && options.client) || new SecretsManager.SecretsManagerClient({ region: process.env.AWS_REGION || 'us-east-1', maxAttempts: 3 });
+    // Cache successful loads until process replacement. Rejected/legacy loads
+    // are retried on the next call; concurrent callers share this attempt.
+    loadPromise = load(client).then(result => {
+      if (result.source !== 'secrets_manager') loadPromise = undefined;
+      return result;
+    }, error => {
+      loadPromise = undefined;
+      throw error;
+    });
+  }
+  return loadPromise;
+}
+
+function resetForTest() { loadPromise = undefined; }
+
+module.exports = { SECRET_ID: SECRET_ID, applySecret: applySecret,
+  loadRuntimeSecrets: loadRuntimeSecrets, resetForTest: resetForTest };

package/package.json

@@ -1,11 +1,10 @@
 {
   "name": "@dhyasama/totem-merge",
-  "version": "7.1.0",
+  "version": "7.1.1",
   "description": "Org and person merging package",
   "main": "index.js",
   "scripts": {
-    "prestart": "node ./lib/changeBSON",
-    "start": "NODE_ENV=test node ./cli",
+    "start": "node ./bin/bootstrap",
     "test": "NODE_ENV=test mocha -R spec -t 10000"
   },
   "publishConfig": {
@@ -16,7 +15,8 @@
     "node": ">=24 <25"
   },
   "dependencies": {
-    "@dhyasama/totem-logger": "^2.1.0",
+    "@aws-sdk/client-secrets-manager": "^3.883.0",
+    "@dhyasama/totem-logger": "3.0.0",
     "@dhyasama/totem-utilities": "^4.0.0",
     "async": "^2.6.4",
     "commander": "^2.20.3",
@@ -24,12 +24,21 @@
     "underscore": "^1.13.6"
   },
   "peerDependencies": {
-    "mongoose": "^8.23.0"
+    "mongoose": "^8.24.3"
   },
   "devDependencies": {
     "@dhyasama/totem-models": "^12.14.0",
-    "mocha": "^5.2.0",
+    "mocha": "^12.0.2",
     "mocha-mongoose": "^1.1.1",
     "should": "^8.2.2"
+  },
+  "overrides": {
+    "r7insight_node": {
+      "lodash": "4.18.1",
+      "semver": "5.7.2",
+      "codependency": {
+        "semver": "5.7.2"
+      }
+    }
   }
 }

package/test/config.js

@@ -7,7 +7,7 @@ module.exports = {
   LPS_ENABLED: false,
 
   db: {
-    uri: 'mongodb://gZvwpF5aLFvdc7KFa7qLLiZYv6CTLdfs:TKfdi3LD4583f8EL11k5Y0uukZ151d9n@test-shard-00-00-gv0f3.mongodb.net:27017,test-shard-00-01-gv0f3.mongodb.net:27017,test-shard-00-02-gv0f3.mongodb.net:27017/test?ssl=true&replicaSet=Test-shard-0&authSource=admin'
+    uri: process.env.TEST_DB_URI || 'mongodb://127.0.0.1:27017/totem-merge-test'
   },
 
   crypto: {

package/test/index.js

@@ -2,11 +2,7 @@
 
 var config = require('./config');
 var dbUri = config.db.uri;
-
-if (dbUri.indexOf('@test-shard-') === -1) {
-  console.log('Not a test db. Quitting...');
-  process.exit();
-}
+var describeMerge = process.env.TEST_DB_URI ? describe : describe.skip;
 
 var should = require("should");
 var mongoose = require('mongoose');
@@ -26,7 +22,7 @@ var customer;
 var org1, org2, org3, org4, org5, org6;
 var person1, person2, person3;
 
-describe('Merge', function() {
+describeMerge('Merge', function() {
 
   // connect to test db
   before(async function() {

package/test/merge-security.test.js

@@ -0,0 +1,119 @@
+'use strict';
+
+var assert = require('node:assert/strict');
+var mongoose = require('mongoose');
+var mergeObjects = require('../lib/helpers').mergeObjects;
+
+describe('mergeObjects security and compatibility', function() {
+  it('does not traverse __proto__ at the root or inside nested objects', function() {
+    [false, true].forEach(function(nested) {
+      var prototype = { mergeSecurityMarker: null };
+      var object = Object.create(prototype);
+      var payload = JSON.parse('{"__proto__":{"mergeSecurityMarker":"changed"}}');
+      mergeObjects(nested ? { profile: object } : object,
+        nested ? { profile: payload } : payload, []);
+      assert.equal(prototype.mergeSecurityMarker, null);
+      assert.equal(Object.getPrototypeOf(object), prototype);
+    });
+  });
+
+  it('does not write through Object.prototype', function() {
+    var key = '__totemMergeSecurityTest';
+    assert.equal(Object.hasOwn(Object.prototype, key), false);
+    Object.defineProperty(Object.prototype, key, {
+      value: null, writable: true, configurable: true
+    });
+    try {
+      mergeObjects({}, JSON.parse('{"__proto__":{"__totemMergeSecurityTest":true}}'), []);
+      assert.equal(Object.prototype[key], null);
+      assert.equal(({})[key], null);
+    } finally {
+      delete Object.prototype[key];
+    }
+  });
+
+  it('ignores constructor and prototype keys at each recursive level', function() {
+    var target = {
+      constructor: { prototype: { marker: null } },
+      prototype: { marker: null },
+      profile: { constructor: null, prototype: null, label: null }
+    };
+    var source = JSON.parse('{"constructor":{"prototype":{"marker":"bad"}},' +
+      '"prototype":{"marker":"bad"},' +
+      '"profile":{"constructor":"bad","prototype":"bad","label":"allowed"}}');
+    mergeObjects(target, source, []);
+    assert.equal(target.constructor.prototype.marker, null);
+    assert.equal(target.prototype.marker, null);
+    assert.deepEqual(target.profile, { constructor: null, prototype: null, label: 'allowed' });
+  });
+
+  it('ignores inherited source properties returned by a document conversion', function() {
+    var plain = Object.create({ inherited: 'not source data' });
+    plain.name = 'Allowed';
+    var target = { inherited: null, name: null };
+    mergeObjects(target, { toObject: function() { return plain; } }, []);
+    assert.deepEqual(target, { inherited: null, name: 'Allowed' });
+  });
+
+  it('preserves target precedence, exclusions, falsy values and source data', function() {
+    var target = {
+      name: 'Keep', description: null, enabled: false, count: 0,
+      empty: '', omitted: null, zero: null, disabled: null,
+      profile: { label: null, retained: 'Keep nested', omitted: null }
+    };
+    var source = {
+      name: 'Replace', description: 'Fill', enabled: true, count: 10,
+      empty: 'Replace', omitted: 'Excluded', zero: 0, disabled: false,
+      missing: 'Do not introduce',
+      profile: { label: 'Fill nested', retained: 'Replace', omitted: 'Excluded' }
+    };
+    var before = JSON.stringify(source);
+    assert.equal(mergeObjects(target, source, ['omitted']), target);
+    assert.deepEqual(target, {
+      name: 'Keep', description: 'Fill', enabled: false, count: 0,
+      empty: '', omitted: null, zero: null, disabled: null,
+      profile: { label: 'Fill nested', retained: 'Keep nested', omitted: null }
+    });
+    assert.equal(JSON.stringify(source), before);
+  });
+
+  it('preserves array ordering, ObjectId deduplication and people/phone keys', function() {
+    var first = new mongoose.Types.ObjectId();
+    var second = new mongoose.Types.ObjectId();
+    var target = {
+      aliases: ['First'], refs: [first],
+      people: [{ person: first, title: 'Keep' }],
+      phone: [{ number: '(212) 555-0100', label: 'Keep' }]
+    };
+    mergeObjects(target, {
+      aliases: ['First', 'Second'], refs: [first, second],
+      people: [{ person: first, title: 'Replace' }, { person: second, title: 'New' }],
+      phone: [{ number: '2125550100', label: 'Replace' }, { number: '2125550101', label: 'New' }]
+    }, []);
+    assert.deepEqual(target.aliases, ['First', 'Second']);
+    assert.deepEqual(target.refs.map(String), [String(first), String(second)]);
+    assert.equal(target.people.length, 2);
+    assert.equal(target.people[0].title, 'Keep');
+    assert.equal(target.people[1].title, 'New');
+    assert.equal(target.phone.length, 2);
+    assert.equal(target.phone[0].label, 'Keep');
+  });
+
+  it('merges real Mongoose documents without invoking virtual getters', function() {
+    var isolated = new mongoose.Mongoose();
+    var schema = new isolated.Schema({
+      name: String, description: { type: String, default: null }, aliases: [String],
+      profile: { label: { type: String, default: null } }
+    });
+    schema.virtual('unsafeVirtual').get(function() { throw new Error('Virtual invoked'); });
+    var Model = isolated.model('MergeSecurityFixture', schema);
+    var target = new Model({ name: 'Keep', aliases: ['First'] });
+    var source = new Model({ name: 'Replace', description: 'Fill', aliases: ['First', 'Second'], profile: { label: 'Nested' } });
+    mergeObjects(target, source, ['_id', '__v']);
+    assert.equal(target.name, 'Keep');
+    assert.equal(target.description, 'Fill');
+    assert.deepEqual(Array.from(target.aliases), ['First', 'Second']);
+    assert.equal(target.profile.label, 'Nested');
+    assert.equal(source.name, 'Replace');
+  });
+});

package/test/runtime-secrets.test.js

@@ -0,0 +1,27 @@
+"use strict";
+
+var assert = require('node:assert/strict');
+var fs = require('node:fs');
+var path = require('node:path');
+var runtimeSecrets = require('../lib/runtimeSecrets');
+
+describe('totem-merge runtime secrets', function() {
+  afterEach(function() { runtimeSecrets.resetForTest(); });
+
+  it('loads DB_URI from the dedicated development secret', async function() {
+    var client = { send: async function(command) {
+      assert.equal(command.input.SecretId, runtimeSecrets.SECRET_ID);
+      return { SecretString: JSON.stringify({ DB_URI: 'mongodb://test-host/test' }) };
+    } };
+    delete process.env.RUNTIME_SECRET_ID;
+    await runtimeSecrets.loadRuntimeSecrets({ client: client });
+    assert.equal(process.env.DB_URI, 'mongodb://test-host/test');
+  });
+
+  it('contains no credentialed MongoDB URI in source configuration', function() {
+    var root = path.join(__dirname, '..');
+    ['config.js', 'test/config.js'].forEach(function(file) {
+      assert.doesNotMatch(fs.readFileSync(path.join(root, file), 'utf8'), /mongodb(?:\+srv)?:\/\/[^\s]+:[^\s]+@/);
+    });
+  });
+});

package/lib/changeBSON.js

@@ -1,22 +0,0 @@
-"use strict";
-
-var fs = require('fs');
-var filePath = './node_modules/mongoose-post-find/node_modules/bson/ext/index.js';
-var config = require('./../config');
-var Logger = require('@dhyasama/totem-logger');
-var logger = new Logger(config.logentries);
-
-fs.readFile(filePath, 'utf8', function (err, data) {
-
-  if (err) { logger.handleErr(err); }
-  else {
-
-    var result = data.replace("bson = require('../build/Release/bson');", "bson = require('bson');");
-
-    fs.writeFile(filePath, result, 'utf8', function (err) {
-      if (err) { logger.handleErr(err); }
-    });
-
-  }
-
-});