mixpanel-browser 2.53.0 → 2.54.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mixpanel-browser",
-  "version": "2.53.0",
+  "version": "2.54.1",
   "description": "The official Mixpanel JavaScript browser client library",
   "main": "dist/mixpanel.cjs.js",
   "directories": {
@@ -54,7 +54,6 @@
     "request": "2.88.0",
     "rollup": "2.79.1",
     "rollup-plugin-esbuild": "4.10.3",
-    "rollup-plugin-npm": "1.4.0",
     "sinon": "8.1.1",
     "sinon-chai": "3.5.0",
     "webpack": "1.12.2"

package/rollup.config.js

@@ -1,11 +1,11 @@
-import npm from 'rollup-plugin-npm';
+import nodeResolve from '@rollup/plugin-node-resolve';
 
 export default {
     plugins: [
-        npm({
+        nodeResolve({
             browser: true,
             main: true,
             jsnext: true,
         })
     ]
-}
+};

package/src/config.js

@@ -1,6 +1,6 @@
 var Config = {
     DEBUG: false,
-    LIB_VERSION: '2.53.0'
+    LIB_VERSION: '2.54.1'
 };
 
 export default Config;

package/src/loaders/bundle-loaders.js

@@ -0,0 +1,22 @@
+// For loading separate bundles asynchronously via script tag
+// so that we don't load them until they are needed at runtime.
+export function loadAsync (src, onload) {
+    var scriptEl = document.createElement('script');
+    scriptEl.type = 'text/javascript';
+    scriptEl.async = true;
+    scriptEl.onload = onload;
+    scriptEl.src = src;
+    document.head.appendChild(scriptEl);
+}
+
+// For builds that have everything in one bundle, no extra work.
+export function loadNoop (_src, onload) {
+    onload();
+}
+
+// For builds that do NOT want any extra bundles (e.g. session recorder)
+// and just the main SDK, throw an error when trying to load a separate bundle.
+// eslint-disable-next-line no-unused-vars
+export function loadThrowError (src, _onload) {
+    throw new Error('This build of Mixpanel only includes core SDK functionality, could not load ' + src);
+}

package/src/loaders/loader-globals.js

@@ -1,4 +1,5 @@
 /* eslint camelcase: "off" */
 import { init_from_snippet } from '../mixpanel-core';
+import {loadAsync} from './bundle-loaders';
 
-init_from_snippet();
+init_from_snippet(loadAsync);

package/src/loaders/loader-module-core.js

@@ -0,0 +1,7 @@
+/* eslint camelcase: "off" */
+import {init_as_module} from '../mixpanel-core.js';
+import {loadThrowError} from './bundle-loaders.js';
+
+var mixpanel = init_as_module(loadThrowError);
+
+export default mixpanel;

package/src/loaders/loader-module-with-async-recorder.js

@@ -0,0 +1,7 @@
+/* eslint camelcase: "off" */
+import {init_as_module} from '../mixpanel-core.js';
+import {loadAsync} from './bundle-loaders.js';
+
+var mixpanel = init_as_module(loadAsync);
+
+export default mixpanel;

package/src/loaders/loader-module.js

@@ -1,6 +1,9 @@
 /* eslint camelcase: "off" */
+import '../recorder';
+
 import { init_as_module } from '../mixpanel-core';
+import { loadNoop } from './bundle-loaders';
 
-var mixpanel = init_as_module();
+var mixpanel = init_as_module(loadNoop);
 
 export default mixpanel;

package/src/mixpanel-core.js

@@ -47,6 +47,12 @@ Globals should be all caps
 */
 
 var init_type;       // MODULE or SNIPPET loader
+// allow bundlers to specify how extra code (recorder bundle) should be loaded
+// eslint-disable-next-line no-unused-vars
+var load_extra_bundle = function(src, _onload) {
+    throw new Error(src + ' not available in this build.');
+};
+
 var mixpanel_master; // main mixpanel instance / object
 var INIT_MODULE  = 0;
 var INIT_SNIPPET = 1;
@@ -140,7 +146,9 @@ var DEFAULT_CONFIG = {
     'hooks':                             {},
     'record_block_class':                new RegExp('^(mp-block|fs-exclude|amp-block|rr-block|ph-no-capture)$'),
     'record_block_selector':             'img, video',
+    'record_collect_fonts':              false,
     'record_idle_timeout_ms':            30 * 60 * 1000, // 30 minutes
+    'record_inline_images':              false,
     'record_mask_text_class':            new RegExp('^(mp-mask|fs-mask|amp-mask|rr-mask|ph-mask)$'),
     'record_mask_text_selector':         '*',
     'record_max_ms':                     MAX_RECORDING_MS,
@@ -376,12 +384,7 @@ MixpanelLib.prototype.start_session_recording = addOptOutCheckMixpanelLib(functi
     }, this);
 
     if (_.isUndefined(window['__mp_recorder'])) {
-        var scriptEl = document.createElement('script');
-        scriptEl.type = 'text/javascript';
-        scriptEl.async = true;
-        scriptEl.onload = handleLoadedRecorder;
-        scriptEl.src = this.get_config('recorder_src');
-        document.head.appendChild(scriptEl);
+        load_extra_bundle(this.get_config('recorder_src'), handleLoadedRecorder);
     } else {
         handleLoadedRecorder();
     }
@@ -680,7 +683,8 @@ MixpanelLib.prototype._send_request = function(url, data, options, callback) {
                         lib.report_error(error);
                         if (callback) {
                             if (verbose_mode) {
-                                callback({status: 0, error: error, xhr_req: req});
+                                var response_headers = req['responseHeaders'] || {};
+                                callback({status: 0, httpStatusCode: req['status'], error: error, retryAfter: response_headers['Retry-After']});
                             } else {
                                 callback(0);
                             }
@@ -780,6 +784,7 @@ MixpanelLib.prototype.init_batchers = function() {
                 attrs.queue_key,
                 {
                     libConfig: this['config'],
+                    errorReporter: this.get_config('error_reporter'),
                     sendRequestFunc: _.bind(function(data, options, cb) {
                         this._send_request(
                             this.get_config('api_host') + attrs.endpoint,
@@ -791,8 +796,8 @@ MixpanelLib.prototype.init_batchers = function() {
                     beforeSendHook: _.bind(function(item) {
                         return this._run_hook('before_send_' + attrs.type, item);
                     }, this),
-                    errorReporter: this.get_config('error_reporter'),
-                    stopAllBatchingFunc: _.bind(this.stop_batch_senders, this)
+                    stopAllBatchingFunc: _.bind(this.stop_batch_senders, this),
+                    usePersistence: true
                 }
             );
         }, this);
@@ -2241,7 +2246,8 @@ var add_dom_loaded_handler = function() {
     _.register_event(window, 'load', dom_loaded_handler, true);
 };
 
-export function init_from_snippet() {
+export function init_from_snippet(bundle_loader) {
+    load_extra_bundle = bundle_loader;
     init_type = INIT_SNIPPET;
     mixpanel_master = window[PRIMARY_INSTANCE_NAME];
 
@@ -2281,7 +2287,8 @@ export function init_from_snippet() {
     add_dom_loaded_handler();
 }
 
-export function init_as_module() {
+export function init_as_module(bundle_loader) {
+    load_extra_bundle = bundle_loader;
     init_type = INIT_MODULE;
     mixpanel_master = new MixpanelLib();
 

package/src/recorder/index.js

@@ -1,11 +1,36 @@
-import {default as record} from 'rrweb/es/rrweb/packages/rrweb/src/record/index.js';
+import { record } from 'rrweb';
+import { IncrementalSource, EventType } from '@rrweb/types';
 
-import { MAX_RECORDING_MS, console_with_prefix, _ } from '../utils'; // eslint-disable-line camelcase
+import { MAX_RECORDING_MS, console_with_prefix, _, window} from '../utils'; // eslint-disable-line camelcase
 import { addOptOutCheckMixpanelLib } from '../gdpr-utils';
+import { RequestBatcher } from '../request-batcher';
 
 var logger = console_with_prefix('recorder');
 var CompressionStream = window['CompressionStream'];
 
+var RECORDER_BATCHER_LIB_CONFIG = {
+    'batch_size': 1000,
+    'batch_flush_interval_ms': 10 * 1000,
+    'batch_request_timeout_ms': 90 * 1000,
+    'batch_autostart': true
+};
+
+var ACTIVE_SOURCES = new Set([
+    IncrementalSource.MouseMove,
+    IncrementalSource.MouseInteraction,
+    IncrementalSource.Scroll,
+    IncrementalSource.ViewportResize,
+    IncrementalSource.Input,
+    IncrementalSource.TouchMove,
+    IncrementalSource.MediaInteraction,
+    IncrementalSource.Drag,
+    IncrementalSource.Selection,
+]);
+
+function isUserEvent(ev) {
+    return ev.type === EventType.IncrementalSnapshot && ACTIVE_SOURCES.has(ev.data.source);
+}
+
 var MixpanelRecorder = function(mixpanelInstance) {
     this._mixpanel = mixpanelInstance;
 
@@ -16,14 +41,24 @@ var MixpanelRecorder = function(mixpanelInstance) {
     this.seqNo = 0;
     this.replayId = null;
     this.replayStartTime = null;
-    this.batchStartTime = null;
-    this.replayLengthMs = 0;
     this.sendBatchId = null;
 
     this.idleTimeoutId = null;
     this.maxTimeoutId = null;
 
     this.recordMaxMs = MAX_RECORDING_MS;
+    this._initBatcher();
+};
+
+
+MixpanelRecorder.prototype._initBatcher = function () {
+    this.batcher = new RequestBatcher('__mprec', {
+        libConfig: RECORDER_BATCHER_LIB_CONFIG,
+        sendRequestFunc: _.bind(this.flushEventsWithOptOut, this),
+        errorReporter: _.bind(this.reportError, this),
+        flushOnlyOnInterval: true,
+        usePersistence: false
+    });
 };
 
 // eslint-disable-next-line camelcase
@@ -31,7 +66,7 @@ MixpanelRecorder.prototype.get_config = function(configVar) {
     return this._mixpanel.get_config(configVar);
 };
 
-MixpanelRecorder.prototype.startRecording = function () {
+MixpanelRecorder.prototype.startRecording = function (shouldStopBatcher) {
     if (this._stopRecording !== null) {
         logger.log('Recording already in progress, skipping startRecording.');
         return;
@@ -45,12 +80,18 @@ MixpanelRecorder.prototype.startRecording = function () {
 
     this.recEvents = [];
     this.seqNo = 0;
-    this.startDate = new Date();
-    this.replayStartTime = this.startDate.getTime();
-    this.batchStartTime = this.replayStartTime;
+    this.replayStartTime = null;
 
     this.replayId = _.UUID();
-    this.replayLengthMs = 0;
+
+    if (shouldStopBatcher) {
+        // this is the case when we're starting recording after a reset
+        // and don't want to send anything over the network until there's
+        // actual user activity
+        this.batcher.stop();
+    } else {
+        this.batcher.start();
+    }
 
     var resetIdleTimeout = _.bind(function () {
         clearTimeout(this.idleTimeoutId);
@@ -62,26 +103,32 @@ MixpanelRecorder.prototype.startRecording = function () {
 
     this._stopRecording = record({
         'emit': _.bind(function (ev) {
-            this.recEvents.push(ev);
-            this.replayLengthMs = new Date().getTime() - this.replayStartTime;
-            resetIdleTimeout();
+            this.batcher.enqueue(ev);
+            if (isUserEvent(ev)) {
+                if (this.batcher.stopped) {
+                    // start flushing again after user activity
+                    this.batcher.start();
+                }
+                resetIdleTimeout();
+            }
         }, this),
-        'maskAllInputs': true,
-        'maskTextSelector': this.get_config('record_mask_text_selector'),
+        'blockClass': this.get_config('record_block_class'),
         'blockSelector': this.get_config('record_block_selector'),
+        'collectFonts': this.get_config('record_collect_fonts'),
+        'inlineImages': this.get_config('record_inline_images'),
+        'maskAllInputs': true,
         'maskTextClass': this.get_config('record_mask_text_class'),
-        'blockClass': this.get_config('record_block_class'),
+        'maskTextSelector': this.get_config('record_mask_text_selector')
     });
 
     resetIdleTimeout();
 
-    this.sendBatchId = setInterval(_.bind(this.flushEventsWithOptOut, this), 10000);
     this.maxTimeoutId = setTimeout(_.bind(this.resetRecording, this), this.recordMaxMs);
 };
 
 MixpanelRecorder.prototype.resetRecording = function () {
     this.stopRecording();
-    this.startRecording();
+    this.startRecording(true);
 };
 
 MixpanelRecorder.prototype.stopRecording = function () {
@@ -90,10 +137,16 @@ MixpanelRecorder.prototype.stopRecording = function () {
         this._stopRecording = null;
     }
 
-    this._flushEvents(); // flush any remaining events
+    if (this.batcher.stopped) {
+        // never got user activity to flush after reset, so just clear the batcher
+        this.batcher.clear();
+    } else {
+        // flush any remaining events from running batcher
+        this.batcher.flush();
+        this.batcher.stop();
+    }
     this.replayId = null;
 
-    clearInterval(this.sendBatchId);
     clearTimeout(this.idleTimeoutId);
     clearTimeout(this.maxTimeoutId);
 };
@@ -102,8 +155,8 @@ MixpanelRecorder.prototype.stopRecording = function () {
  * Flushes the current batch of events to the server, but passes an opt-out callback to make sure
  * we stop recording and dump any queued events if the user has opted out.
  */
-MixpanelRecorder.prototype.flushEventsWithOptOut = function () {
-    this._flushEvents(_.bind(this._onOptOut, this));
+MixpanelRecorder.prototype.flushEventsWithOptOut = function (data, options, cb) {
+    this._flushEvents(data, options, cb, _.bind(this._onOptOut, this));
 };
 
 MixpanelRecorder.prototype._onOptOut = function (code) {
@@ -114,33 +167,60 @@ MixpanelRecorder.prototype._onOptOut = function (code) {
     }
 };
 
-MixpanelRecorder.prototype._sendRequest = function(reqParams, reqBody) {
+MixpanelRecorder.prototype._sendRequest = function(reqParams, reqBody, callback) {
+    var onSuccess = _.bind(function (response, responseBody) {
+        // Increment sequence counter only if the request was successful to guarantee ordering.
+        // RequestBatcher will always flush the next batch after the previous one succeeds.
+        if (response.status === 200) {
+            this.seqNo++;
+        }
+
+        callback({
+            status: 0,
+            httpStatusCode: response.status,
+            responseBody: responseBody,
+            retryAfter: response.headers.get('Retry-After')
+        });
+    }, this);
+
     window['fetch'](this.get_config('api_host') + '/' + this.get_config('api_routes')['record'] + '?' + new URLSearchParams(reqParams), {
         'method': 'POST',
         'headers': {
             'Authorization': 'Basic ' + btoa(this.get_config('token') + ':'),
             'Content-Type': 'application/octet-stream'
         },
-        'body': reqBody
+        'body': reqBody,
+    }).then(function (response) {
+        response.json().then(function (responseBody) {
+            onSuccess(response, responseBody);
+        }).catch(function (error) {
+            callback({error: error});
+        });
+    }).catch(function (error) {
+        callback({error: error});
     });
 };
 
-/**
- * @api private
- * Private method, flushes the current batch of events to the server.
- */
-MixpanelRecorder.prototype._flushEvents = addOptOutCheckMixpanelLib(function() {
-    var numEvents = this.recEvents.length;
+MixpanelRecorder.prototype._flushEvents = addOptOutCheckMixpanelLib(function (data, options, callback) {
+    const numEvents = data.length;
+
     if (numEvents > 0) {
+        // each rrweb event has a timestamp - leverage those to get time properties
+        var batchStartTime = data[0].timestamp;
+        if (this.seqNo === 0) {
+            this.replayStartTime = batchStartTime;
+        }
+        var replayLengthMs = data[numEvents - 1].timestamp - this.replayStartTime;
+
         var reqParams = {
             'distinct_id': String(this._mixpanel.get_distinct_id()),
-            'seq': this.seqNo++,
-            'batch_start_time': this.batchStartTime / 1000,
+            'seq': this.seqNo,
+            'batch_start_time': batchStartTime / 1000,
             'replay_id': this.replayId,
-            'replay_length_ms': this.replayLengthMs,
+            'replay_length_ms': replayLengthMs,
             'replay_start_time': this.replayStartTime / 1000
         };
-        var eventsJson = _.JSONEncode(this.recEvents);
+        var eventsJson = _.JSONEncode(data);
 
         // send ID management props if they exist
         var deviceId = this._mixpanel.get_property('$device_id');
@@ -152,8 +232,6 @@ MixpanelRecorder.prototype._flushEvents = addOptOutCheckMixpanelLib(function() {
             reqParams['$user_id'] = userId;
         }
 
-        this.recEvents = this.recEvents.slice(numEvents);
-        this.batchStartTime = new Date().getTime();
         if (CompressionStream) {
             var jsonStream = new Blob([eventsJson], {type: 'application/json'}).stream();
             var gzipStream = jsonStream.pipeThrough(new CompressionStream('gzip'));
@@ -161,13 +239,27 @@ MixpanelRecorder.prototype._flushEvents = addOptOutCheckMixpanelLib(function() {
                 .blob()
                 .then(_.bind(function(compressedBlob) {
                     reqParams['format'] = 'gzip';
-                    this._sendRequest(reqParams, compressedBlob);
+                    this._sendRequest(reqParams, compressedBlob, callback);
                 }, this));
         } else {
             reqParams['format'] = 'body';
-            this._sendRequest(reqParams, eventsJson);
+            this._sendRequest(reqParams, eventsJson, callback);
         }
     }
 });
 
+
+MixpanelRecorder.prototype.reportError = function(msg, err) {
+    logger.error.apply(logger.error, arguments);
+    try {
+        if (!err && !(msg instanceof Error)) {
+            msg = new Error(msg);
+        }
+        this.get_config('error_reporter')(msg, err);
+    } catch(err) {
+        logger.error(err);
+    }
+};
+
+
 window['__mp_recorder'] = MixpanelRecorder;

package/src/request-batcher.js

@@ -17,7 +17,8 @@ var RequestBatcher = function(storageKey, options) {
     this.errorReporter = options.errorReporter;
     this.queue = new RequestQueue(storageKey, {
         errorReporter: _.bind(this.reportError, this),
-        storage: options.storage
+        storage: options.storage,
+        usePersistence: options.usePersistence
     });
 
     this.libConfig = options.libConfig;
@@ -34,6 +35,11 @@ var RequestBatcher = function(storageKey, options) {
 
     // extra client-side dedupe
     this.itemIdsSentSuccessfully = {};
+
+    // Make the flush occur at the interval specified by flushIntervalMs, default behavior will attempt consecutive flushes
+    // as long as the queue is not empty. This is useful for high-frequency events like Session Replay where we might end up
+    // in a request loop and get ratelimited by the server.
+    this.flushOnlyOnInterval = options.flushOnlyOnInterval || false;
 };
 
 /**
@@ -91,7 +97,11 @@ RequestBatcher.prototype.resetFlush = function() {
 RequestBatcher.prototype.scheduleFlush = function(flushMS) {
     this.flushInterval = flushMS;
     if (!this.stopped) { // don't schedule anymore if batching has been stopped
-        this.timeoutID = setTimeout(_.bind(this.flush, this), this.flushInterval);
+        this.timeoutID = setTimeout(_.bind(function() {
+            if (!this.stopped) {
+                this.flush();
+            }
+        }, this), this.flushInterval);
     }
 };
 
@@ -118,6 +128,9 @@ RequestBatcher.prototype.flush = function(options) {
         var startTime = new Date().getTime();
         var currentBatchSize = this.batchSize;
         var batch = this.queue.fillBatch(currentBatchSize);
+        // if there's more items in the queue than the batch size, attempt
+        // to flush again after the current batch is done.
+        var attemptSecondaryFlush = batch.length === currentBatchSize;
         var dataForRequest = [];
         var transformedItems = {};
         _.each(batch, function(item) {
@@ -185,22 +198,17 @@ RequestBatcher.prototype.flush = function(options) {
                     this.flush();
                 } else if (
                     _.isObject(res) &&
-                    res.xhr_req &&
-                    (res.xhr_req['status'] >= 500 || res.xhr_req['status'] === 429 || res.error === 'timeout')
+                    (res.httpStatusCode >= 500 || res.httpStatusCode === 429 || res.error === 'timeout')
                 ) {
                     // network or API error, or 429 Too Many Requests, retry
                     var retryMS = this.flushInterval * 2;
-                    var headers = res.xhr_req['responseHeaders'];
-                    if (headers) {
-                        var retryAfter = headers['Retry-After'];
-                        if (retryAfter) {
-                            retryMS = (parseInt(retryAfter, 10) * 1000) || retryMS;
-                        }
+                    if (res.retryAfter) {
+                        retryMS = (parseInt(res.retryAfter, 10) * 1000) || retryMS;
                     }
                     retryMS = Math.min(MAX_RETRY_INTERVAL_MS, retryMS);
                     this.reportError('Error; retry in ' + retryMS + ' ms');
                     this.scheduleFlush(retryMS);
-                } else if (_.isObject(res) && res.xhr_req && res.xhr_req['status'] === 413) {
+                } else if (_.isObject(res) && res.httpStatusCode === 413) {
                     // 413 Payload Too Large
                     if (batch.length > 1) {
                         var halvedBatchSize = Math.max(1, Math.floor(currentBatchSize / 2));
@@ -224,7 +232,11 @@ RequestBatcher.prototype.flush = function(options) {
                         _.bind(function(succeeded) {
                             if (succeeded) {
                                 this.consecutiveRemovalFailures = 0;
-                                this.flush(); // handle next batch if the queue isn't empty
+                                if (this.flushOnlyOnInterval && !attemptSecondaryFlush) {
+                                    this.resetFlush(); // schedule next batch with a delay
+                                } else {
+                                    this.flush(); // handle next batch if the queue isn't empty
+                                }
                             } else {
                                 this.reportError('Failed to remove items from queue');
                                 if (++this.consecutiveRemovalFailures > 5) {
@@ -272,7 +284,6 @@ RequestBatcher.prototype.flush = function(options) {
         }
         logger.log('MIXPANEL REQUEST:', dataForRequest);
         this.sendRequest(dataForRequest, requestOptions, batchSendCallback);
-
     } catch(err) {
         this.reportError('Error flushing request queue', err);
         this.resetFlush();