ep_webrtc 2.1.1 → 2.2.2

package/README.md

@@ -185,6 +185,31 @@ Example:
   },
 ```
 
+#### Horizontally scaled TURN servers
+
+To spread load across multiple TURN services, you can enable sharding:
+
+```json
+  "ep_webrtc": {
+    "iceServers": [
+      {"urls": ["stun:shard0.example.com", "turn:shard0.example.com"]},
+      {"urls": ["stun:shard1.example.com", "turn:shard1.example.com"]},
+      {"urls": ["stun:shard2.example.com", "turn:shard2.example.com"]},
+      {"urls": ["stun:shard3.example.com", "turn:shard3.example.com"]},
+    ],
+    "shardIceServers": true
+  },
+```
+
+When `shardIceServers` is `false` (the default), all clients receive all
+RTCIceServer objects in the `iceServers` list and it's up to the browser to
+figure out how to use them to connect with peers. When `true`, this plugin
+assigns a single entry from `iceServers` to each pad and gives out only that
+assigned entry to users that connect to the pad. The intention is to provide a
+better guarantee of load distribution across a set of TURN servers, and to avoid
+an unnecessary network hop when both peers are configured to force the use of
+TURN.
+
 ### Microphone Settings
 
 The microphone can be configured by setting `audio.constraints` to any [audio

package/index.js

@@ -49,8 +49,10 @@ const defaultSettings = {
   iceServers: [{urls: ['stun:stun.l.google.com:19302']}],
   listenClass: null,
   moreInfoUrl: {},
+  shardIceServers: false,
 };
 let settings = null;
+let shardIceServersHmacSecret;
 let socketio;
 
 const addContextToError = (err, pfx) => {
@@ -139,54 +141,90 @@ const fetchJson = async (url, opts = {}) => {
   return await res.json();
 };
 
-exports.clientVars = async (hookName, {clientVars: {userId: authorId}}) => ({ep_webrtc: {
-  ...settings,
-  iceServers: await Promise.all(settings.iceServers.map(async (server) => {
-    switch (server.credentialType) {
-      case 'coturn ephemeral password': {
-        const {lifetime = 60 * 60 * 12 /* seconds */} = server;
-        const username = `${Math.floor(Date.now() / 1000) + lifetime}:${authorId}`;
-        const hmac = crypto.createHmac('sha1', server.credential);
-        hmac.update(username);
-        const credential = hmac.digest('base64');
-        return {urls: server.urls, username, credential};
-      }
-      case 'xirsys ephemeral credentials': {
-        const {
-          url,
-          username,
-          credential,
-          lifetime: expire = 12 * 60 * 60, // seconds
-          method = 'PUT',
-          headers: h = {},
-          jsonBody: b = {},
-        } = server;
-        // Can't set default values for the Content-Type and Authorization headers by using an
-        // object literal with spread (e.g., `{'content-type': 'foo', ...h}`) because the Headers
-        // constructor uses `.append()` internally instead of `.set()`. This matters if a header is
-        // repeated multiple times by using different mixes of upper- and lower-case letters.
-        const headers = new globalThis.Headers(h);
-        if (!headers.has('content-type')) headers.set('content-type', 'application/json');
-        if (username && !headers.has('authorization')) {
-          headers.set('authorization',
-              `Basic ${Buffer.from(`${username}:${credential}`).toString('base64')}`);
+exports.clientVars = async (hookName, {clientVars: {userId: authorId}, pad: {id: padId}}) => {
+  let iceServers = settings.iceServers;
+  if (settings.shardIceServers && iceServers.length > 1) {
+    // We could simply hash the pad ID, but we include some randomness to make it slightly harder
+    // for a malicious user to overload a particular shard by picking pad IDs that all use the same
+    // shard. (The randomness forces malicious users to try multiple pad IDs and keep the ones that
+    // use the same shard.) The randomness also helps avoid chronic imbalance due to unlucky
+    // assignments; generating a new secret will reassign the shards.
+    //
+    // The secret is generated at startup, so all users visiting the same pad will get the same HMAC
+    // value (and thus the same shard) until Etherpad is restarted. Users that connect after
+    // Etherpad restarts might be assigned a different shard from the users on the pad that received
+    // their clientVars before Etherpad restarted. This doesn't affect protocol correctness, but it
+    // might result in three network hops instead of two: client A sends to TURN A which relays to
+    // TURN B which relays to client B, instead of client A sends to TURN AB which relays to
+    // localhost (TURN AB) which relays to client B. This should be rare because it will only happen
+    // if all of the following are true:
+    //
+    //   * Both users have configured their browsers to force relay.
+    //   * One user loaded the pad before Etherpad restarted and the other loaded after.
+    //   * The new random value caused the pad to be assigned to a different shard.
+    //
+    // TODO: Convey ICE servers via a message that is sent every time a user connects. (CLIENT_VARS
+    // is only sent on initial connection, so if a client reconnects due to Etherpad restarting, a
+    // new CLIENT_VARS is not sent.) This will allow the server to select a different shard for a
+    // pad when it restarts, and all clients (old and new) will use the new shard for new sessions.
+    //
+    // TODO: Select the shard for the pad when the first user joins the pad and forget that
+    // selection once all users have left. This would enable alternative load balancing schemes such
+    // as true random or least loaded.
+    const hmac = crypto.createHmac('sha256', shardIceServersHmacSecret);
+    hmac.update(padId);
+    const i = Number(BigInt(`0x${hmac.digest('hex')}`) % BigInt(iceServers.length));
+    iceServers = iceServers.slice(i, i + 1);
+  }
+  return {ep_webrtc: {
+    ...settings,
+    iceServers: await Promise.all(iceServers.map(async (server) => {
+      switch (server.credentialType) {
+        case 'coturn ephemeral password': {
+          const {lifetime = 60 * 60 * 12 /* seconds */} = server;
+          const username = `${Math.floor(Date.now() / 1000) + lifetime}:${authorId}`;
+          const hmac = crypto.createHmac('sha1', server.credential);
+          hmac.update(username);
+          const credential = hmac.digest('base64');
+          return {urls: server.urls, username, credential};
         }
-        const body =
-            JSON.stringify(b && typeof b === 'object' ? {format: 'urls', expire, ...b} : b);
-        try {
-          const {v, s} = await fetchJson(url, {method, headers, body});
-          if (s !== 'ok') throw new Error(`API error: ${v}`);
-          return v.iceServers;
-        } catch (err) {
-          const newErr = addContextToError(err, 'failed to get TURN credentials: ');
-          logger.error(newErr.stack || newErr.toString());
-          throw newErr;
+        case 'xirsys ephemeral credentials': {
+          const {
+            url,
+            username,
+            credential,
+            lifetime: expire = 12 * 60 * 60, // seconds
+            method = 'PUT',
+            headers: h = {},
+            jsonBody: b = {},
+          } = server;
+          // Can't set default values for the Content-Type and Authorization headers by using an
+          // object literal with spread (e.g., `{'content-type': 'foo', ...h}`) because the Headers
+          // constructor uses `.append()` internally instead of `.set()`. This matters if a header
+          // is repeated multiple times by using different mixes of upper- and lower-case letters.
+          const headers = new globalThis.Headers(h);
+          if (!headers.has('content-type')) headers.set('content-type', 'application/json');
+          if (username && !headers.has('authorization')) {
+            headers.set('authorization',
+                `Basic ${Buffer.from(`${username}:${credential}`).toString('base64')}`);
+          }
+          const body =
+              JSON.stringify(b && typeof b === 'object' ? {format: 'urls', expire, ...b} : b);
+          try {
+            const {v, s} = await fetchJson(url, {method, headers, body});
+            if (s !== 'ok') throw new Error(`API error: ${v}`);
+            return v.iceServers;
+          } catch (err) {
+            const newErr = addContextToError(err, 'failed to get TURN credentials: ');
+            logger.error(newErr.stack || newErr.toString());
+            throw newErr;
+          }
         }
+        default: return server;
       }
-      default: return server;
-    }
-  })),
-}});
+    })),
+  }};
+};
 
 exports.handleMessage = async (hookName, {message, socket}) => {
   if (message.type === 'COLLABROOM' && message.data.type === 'RTC_MESSAGE') {
@@ -243,6 +281,9 @@ exports.loadSettings = async (hookName, {settings: {ep_webrtc: s = {}}}) => {
     }
     return false;
   })();
+  if (settings.shardIceServers && settings.iceServers.length > 1) {
+    shardIceServersHmacSecret = await util.promisify(crypto.randomBytes.bind(crypto))(16);
+  }
   logger.info('configured:', util.inspect({
     ...settings,
     iceServers: settings.iceServers.map((s) => s.credential ? {...s, credential: '*****'} : s),

package/package.json

@@ -5,7 +5,7 @@
     "url": "git@github.com:ether/ep_webrtc.git",
     "type": "git"
   },
-  "version": "2.1.1",
+  "version": "2.2.2",
   "description": "WebRTC based audio/video chat to Etherpad",
   "author": "John McLear <john@mclear.co.uk>",
   "contributors": [],
@@ -22,13 +22,15 @@
     "url": "https://etherpad.org/"
   },
   "devDependencies": {
-    "eslint": "^8.10.0",
-    "eslint-config-etherpad": "^3.0.5",
+    "eslint": "^8.11.0",
+    "eslint-config-etherpad": "^3.0.9",
+    "mocha": "^9.2.2",
     "typescript": "^4.6.2"
   },
   "scripts": {
     "lint": "eslint .",
-    "lint:fix": "eslint --fix ."
+    "lint:fix": "eslint --fix .",
+    "test": "mocha --recursive static/tests/backend/specs"
   },
   "peerDependencies": {
     "ep_etherpad-lite": ">=1.8.7"

package/static/js/index.js

@@ -87,7 +87,7 @@ class LocalTracks extends EventTargetPolyfill {
     this.videoIsScreenshare = false;
   }
 
-  _getTracks(kind) {
+  getTracks(kind) {
     return kind === 'audio' ? this.stream.getAudioTracks()
       : kind === 'video' ? this.stream.getVideoTracks()
       : this.stream.getTracks();
@@ -96,7 +96,7 @@ class LocalTracks extends EventTargetPolyfill {
   setTrack(kind, newTrack) {
     newTrack = newTrack || null; // Convert undefined to null.
     let oldTrack = null;
-    const tracks = this._getTracks(kind);
+    const tracks = this.getTracks(kind);
     for (const track of tracks) {
       if (track.kind !== kind) continue;
       if (track === newTrack) return; // No change.
@@ -109,7 +109,7 @@ class LocalTracks extends EventTargetPolyfill {
       debug(`adding ${kind} track ${newTrack.id} to local stream`);
       newTrack.addEventListener('ended', () => {
         debug(`local ${kind} track ${newTrack.id} ended`);
-        if (!this._getTracks(kind).includes(newTrack)) return;
+        if (!this.getTracks(kind).includes(newTrack)) return;
         this.setTrack(kind, null);
       });
       this.stream.addTrack(newTrack);
@@ -163,45 +163,42 @@ class PeerState extends EventTargetPolyfill {
     };
     this._closed = false;
     this._pc = null;
+    this._xcvrs = {};
     this._remoteStream = null;
-    this._onremovetrack =
-        () => { if (this._remoteStream.getTracks().length === 0) this._setRemoteStream(null); };
     this._ontrackchanged = async ({oldTrack, newTrack}) => {
-      if (this._pc == null || this._pc.connectionState === 'closed') return;
-      this._debug(`replacing ${oldTrack ? oldTrack.kind : newTrack.kind} track ` +
-                  `${oldTrack ? oldTrack.id : '(null)'} with ` +
-                  `${newTrack ? newTrack.id : '(null)'}`);
-      if (oldTrack != null) {
-        for (const sender of this._pc.getSenders()) {
-          if (sender.track !== oldTrack) continue;
-          if (newTrack != null) {
-            try {
-              return await sender.replaceTrack(newTrack);
-            } catch (err) {
-              this._debug('renegotiation is required');
-              logErrorToServer(err);
-            }
-          }
-          this._pc.removeTrack(sender);
-          break;
-        }
-      }
-      if (newTrack != null) this._pc.addTrack(newTrack, this._localTracks.stream);
+      const {kind} = oldTrack || newTrack;
+      await this._sendLocalTrack(kind, newTrack);
     };
     this._localTracks.addEventListener('trackchanged', this._ontrackchanged);
   }
 
+  async _sendLocalTrack(kind, track) {
+    if (this._pc == null || this._pc.connectionState === 'closed') return;
+    const {sender: {track: old} = {}} = this._xcvrs[kind] || {};
+    if ((old == null && track == null) || old === track) return;
+    this._debug(
+        `replacing ${kind} track ${old ? old.id : '(null)'} with ${track ? track.id : '(null)'}`);
+    const xcvr = this._xcvrs[kind];
+    if (!xcvr) return;
+    try {
+      if (track != null) xcvr.direction = 'sendrecv'; // Will throw if not already bidirectional.
+      await xcvr.sender.replaceTrack(track);
+    } catch (err) {
+      this._debug('renegotiation is required');
+      this._resetConnection(err);
+      return;
+    }
+  }
+
   _setRemoteStream(stream) {
     if (stream === this._remoteStream) return;
     if (this._remoteStream != null) {
       const oldStream = this._remoteStream;
-      oldStream.removeEventListener('removetrack', this._onremovetrack);
       this._remoteStream = null;
       this.dispatchEvent(new StreamEvent('streamgone', oldStream));
     }
     if (stream != null) {
       this._remoteStream = stream;
-      stream.addEventListener('removetrack', this._onremovetrack);
       this.dispatchEvent(new StreamEvent('stream', stream));
     }
   }
@@ -221,20 +218,19 @@ class PeerState extends EventTargetPolyfill {
     this._ids.from.instance = ++nextInstanceId;
     this._ids.to = peerIds;
     const pc = new RTCPeerConnection(this._pcConfig);
-    pc.addEventListener('track', ({track, streams}) => {
-      if (streams.length !== 1) throw new Error('Expected track to be in exactly one stream');
-      this._setRemoteStream(streams[0]);
+    pc.addEventListener('track', async ({track, transceiver}) => {
+      this._debug(`Received ${track.kind} track from peer, ID: ${track.id}`);
+      const stream = this._remoteStream || new MediaStream();
+      stream.addTrack(track);
+      this._setRemoteStream(stream);
+      if (!this._caller) {
+        this._xcvrs[track.kind] = transceiver;
+        // _sendLocalTrack might call _resetConnection, so it should be called last.
+        await this._sendLocalTrack(track.kind, this._localTracks.getTracks(track.kind)[0]);
+      }
     });
     pc.addEventListener('icecandidate', ({candidate}) => this._sendMessage({candidate}));
     pc.addEventListener('negotiationneeded', async () => {
-      if (!this._caller) {
-        this._debug('Waiting for peer to call');
-        // It is possible that the last invite sent to the peer was sent before the peer was ready
-        // to accept invites, so the peer might not know that it should call now. Send another
-        // invite just in case. The peer will ignore any superfluous invites.
-        this._sendMessage({invite: 'invite'});
-        return;
-      }
       try {
         await pc.setLocalDescription();
         this._failedSLDAttempts = 0;
@@ -291,11 +287,32 @@ class PeerState extends EventTargetPolyfill {
 
     if (this._pc != null) this._pc.close();
     this._pc = pc;
-
-    const tracks = this._localTracks.stream.getTracks();
-    for (const track of tracks) {
-      this._debug(`start streaming ${track.kind} track ${track.id}`);
-      pc.addTrack(track, this._localTracks.stream);
+    this._xcvrs = {};
+
+    if (this._caller) {
+      // Adding transceivers triggers negotiation with the peer, which we want to do as soon as
+      // possible to warm up the peer connection (ICE takes a while). Adding a track implicitly adds
+      // a transceiver so we could do that instead, but:
+      //
+      //   * We might not have a local track yet (maybe the user hasn't yet allowed access to the
+      //     camera/mic).
+      //   * Using a dummy silence track until a real track is ready might result in two
+      //     unidirectional SDP m-lines (one in each direction) instead of a single bidirectional
+      //     m-line. This seems to trigger an echo bug in Safari.
+      //
+      // Using transceivers to warm up the connection is the approach taken in:
+      // https://www.w3.org/TR/2021/REC-webrtc-20210126/#advanced-peer-to-peer-example-with-warm-up
+      // Also see: https://webrtcstandards.info/ice-warm-up-m-line-webrtc-transceivers/
+      Promise.all(['audio', 'video'].map(async (t) => {
+        this._debug(`adding ${t} transceiver`);
+        this._xcvrs[t] = this._pc.addTransceiver(t);
+        await this._sendLocalTrack(t, this._localTracks.getTracks(t)[0]);
+      }));
+    } else {
+      // It is possible that the last invite sent to the peer was sent before the peer was ready to
+      // accept invites, so the peer might not know that it should call now. Send another invite
+      // just in case. The peer will ignore any superfluous invites.
+      this._sendMessage({invite: 'invite'});
     }
   }
 
@@ -385,17 +402,7 @@ exports.rtc = new class {
   constructor() {
     this._activated = null;
     this._settings = null;
-    this._disabledSilence = (() => {
-      const ctx = new AudioContext();
-      const gain = ctx.createGain();
-      const dst = gain.connect(ctx.createMediaStreamDestination());
-      const track = dst.stream.getAudioTracks()[0];
-      track.enabled = false;
-      return track;
-    })();
-    this._blankStream = new MediaStream([this._disabledSilence]);
     this._localTracks = new LocalTracks();
-    this._localTracks.setTrack(this._disabledSilence.kind, this._disabledSilence);
     this._localTracks.addEventListener('trackchanged', ({oldTrack, newTrack}) => {
       // Normally the self-view UI only needs to be updated if the user clicks on something, but it
       // also needs to be updated if the browser decides to end the local stream for whatever
@@ -620,8 +627,7 @@ exports.rtc = new class {
     try {
       const devices = [];
       const addAudioTrack = updateAudio && this._selfViewButtons.audio.enabled &&
-          !this._localTracks.stream.getAudioTracks().some(
-              (t) => t !== this._disabledSilence && t.readyState === 'live');
+          !this._localTracks.stream.getAudioTracks().some((t) => t.readyState === 'live');
       if (addAudioTrack) devices.push('mic');
       const addVideoTrack = updateVideo && this._selfViewButtons.video.enabled &&
           (!this._localTracks.stream.getVideoTracks().some((t) => t.readyState === 'live') ||
@@ -674,7 +680,7 @@ exports.rtc = new class {
         for (const track of this._localTracks.stream.getAudioTracks()) {
           // Re-check the state of the button because the user might have clicked it while
           // getUserMedia() was running.
-          track.enabled = track !== this._disabledSilence && this._selfViewButtons.audio.enabled;
+          track.enabled = this._selfViewButtons.audio.enabled;
         }
         const hasAudio = this._localTracks.stream.getAudioTracks().some(
             (t) => t.enabled && t.readyState === 'live');
@@ -880,8 +886,9 @@ exports.rtc = new class {
             minWidth: iw > width ? `${iw}px` : '',
             minHeight: nh + ih > height ? `${nh + ih}px` : '',
           });
-        })
-        .appendTo($('#rtcbox'));
+        });
+    if (isLocal) $videoContainer.prependTo($('#rtcbox'));
+    else $videoContainer.appendTo($('#rtcbox'));
     this.updatePeerNameAndColor(this.getUserFromId(userId));
 
     // For tests it is important to know when an asynchronous event handler has finishing handling
@@ -1278,7 +1285,7 @@ exports.rtc = new class {
         logDisconnectErrorTimeout =
             setTimeout(() => logErrorToServer(new Error('RTC connection lost'), 0), 10000);
         ($videoContainer.data('updateMinSize') || (() => {}))();
-        await this.setStream(userId, this._blankStream);
+        await this.setStream(userId, new MediaStream());
       });
       peer.addEventListener('closed', () => {
         _debug('PeerState closed');

package/static/tests/backend/init.js

@@ -0,0 +1,24 @@
+'use strict';
+
+const common = require('ep_etherpad-lite/tests/backend/common');
+const fsp = require('fs').promises;
+const path = require('path');
+const pluginDefs = require('ep_etherpad-lite/static/js/pluginfw/plugin_defs');
+const plugins = require('ep_etherpad-lite/static/js/pluginfw/plugins');
+
+module.exports = async () => {
+  const agent = await common.init();
+  if (pluginDefs.plugins.ep_webrtc == null) {
+    const packagePath = path.dirname(require.resolve('../../../package.json'));
+    plugins.getPackages = async () => ({
+      'ep_etherpad-lite': pluginDefs.plugins['ep_etherpad-lite'].package,
+      'ep_webrtc': {
+        ...require('../../../package.json'),
+        path: packagePath,
+        realPath: await fsp.realpath(packagePath),
+      },
+    });
+    await plugins.update();
+  }
+  return agent;
+};

package/static/tests/backend/specs/sharding.js

@@ -0,0 +1,130 @@
+'use strict';
+
+const assert = require('assert').strict;
+const common = require('ep_etherpad-lite/tests/backend/common');
+const init = require('../init');
+const plugin = require('../../../../index');
+const settings = require('ep_etherpad-lite/node/utils/Settings');
+
+describe(__filename, function () {
+  let agent;
+  const backup = {settings: {...settings}};
+  const iceServers = [...Array(1000).keys()].map((i) => ({urls: [`turn:turn${i}.example.com`]}));
+
+  const reload = async (settings = {}) => {
+    await plugin.loadSettings('loadSettings', {settings: {ep_webrtc: {iceServers, ...settings}}});
+  };
+
+  const getIceServers = async (padId = common.randomString()) => {
+    while (getIceServers._busy != null) {
+      await getIceServers._busy;
+    }
+    if (++getIceServers._active >= getIceServers._limit) {
+      getIceServers._busy = new Promise((resolve) => getIceServers._resolve = resolve);
+    }
+    try {
+      const res = await agent.get(`/p/${padId}`).expect(200);
+      const socket = await common.connect(res);
+      try {
+        const {type, data: clientVars} = await common.handshake(socket, padId);
+        assert.equal(type, 'CLIENT_VARS');
+        return clientVars.ep_webrtc.iceServers;
+      } finally {
+        socket.close();
+      }
+    } finally {
+      if (--getIceServers._active < getIceServers._limit && getIceServers._busy != null) {
+        getIceServers._resolve();
+        getIceServers._busy = null;
+      }
+    }
+  };
+  getIceServers._limit = 5; // Avoid timeouts caused by overload.
+  getIceServers._active = 0;
+  getIceServers._resolve = () => {};
+
+  before(async function () {
+    settings.requireAuthentication = false;
+    agent = await init();
+  });
+
+  after(async function () {
+    Object.assign(settings, backup.settings);
+    await plugin.loadSettings('loadSettings', {settings});
+  });
+
+  it('defaults to disabled', async function () {
+    await reload();
+    const got = await getIceServers();
+    assert.deepEqual(got, iceServers);
+  });
+
+  it('explicitly disabled', async function () {
+    await reload({shardIceServers: false});
+    const got = await getIceServers();
+    assert.deepEqual(got, iceServers);
+  });
+
+  it('enabled, zero entries', async function () {
+    await reload({iceServers: [], shardIceServers: true});
+    assert.deepEqual(await getIceServers(), []);
+  });
+
+  it('enabled, one entry', async function () {
+    const entries = [{urls: ['turn:turn.example.com']}];
+    await reload({iceServers: entries, shardIceServers: true});
+    assert.deepEqual(await getIceServers(), entries);
+  });
+
+  describe('enabled, multiple entries', function () {
+    beforeEach(async function () {
+      await reload({shardIceServers: true});
+    });
+
+    it('only gives one entry to each client', async function () {
+      const got = await getIceServers();
+      assert.equal(got.length, 1);
+      assert(iceServers.some((s) => {
+        try {
+          assert.deepEqual(got[0], s);
+          return true;
+        } catch (err) {
+          return false;
+        }
+      }));
+    });
+
+    it('same pad gets same entry', async function () {
+      this.timeout(60000);
+      const assignments = new Map(await Promise.all([...Array(10).keys()].map(async () => {
+        const padId = common.randomString();
+        return [padId, await getIceServers(padId)];
+      })));
+      await Promise.all([...assignments].map(async ([padId, want]) => {
+        const got = await getIceServers(padId);
+        assert.deepEqual(got, want);
+      }));
+    });
+
+    it('randomizes assignments on reload', async function () {
+      this.timeout(60000);
+      const oldAssignments = new Map(await Promise.all([...Array(10).keys()].map(async () => {
+        const padId = common.randomString();
+        return [padId, await getIceServers(padId)];
+      })));
+      await reload({shardIceServers: true});
+      const newAssignments = new Map(await Promise.all(
+          [...oldAssignments.keys()].map(async (padId) => [padId, await getIceServers(padId)])));
+      // With 10 pad IDs and 1000 ICE servers, the probability that every new assignment exactly
+      // matches the old assignment is effectively zero.
+      assert([...newAssignments].some(([padId, newAssignment]) => {
+        try {
+          assert.deepEqual(newAssignment, oldAssignments.get(padId));
+          return false;
+        } catch (err) {
+          return true;
+        }
+      }));
+    });
+  });
+});