mixpanel-browser 2.43.0 → 2.46.0

package/src/request-batcher.js

@@ -1,3 +1,4 @@
+import Config from './config';
 import { RequestQueue } from './request-queue';
 import { console_with_prefix, _ } from './utils'; // eslint-disable-line camelcase
 
@@ -13,17 +14,26 @@ var logger = console_with_prefix('batch');
  * @constructor
  */
 var RequestBatcher = function(storageKey, options) {
-    this.queue = new RequestQueue(storageKey, {storage: options.storage});
+    this.errorReporter = options.errorReporter;
+    this.queue = new RequestQueue(storageKey, {
+        errorReporter: _.bind(this.reportError, this),
+        storage: options.storage
+    });
 
     this.libConfig = options.libConfig;
     this.sendRequest = options.sendRequestFunc;
     this.beforeSendHook = options.beforeSendHook;
+    this.stopAllBatching = options.stopAllBatchingFunc;
 
     // seed variable batch size + flush interval with configured values
     this.batchSize = this.libConfig['batch_size'];
     this.flushInterval = this.libConfig['batch_flush_interval_ms'];
 
     this.stopped = !this.libConfig['batch_autostart'];
+    this.consecutiveRemovalFailures = 0;
+
+    // extra client-side dedupe
+    this.itemIdsSentSuccessfully = {};
 };
 
 /**
@@ -39,6 +49,7 @@ RequestBatcher.prototype.enqueue = function(item, cb) {
  */
 RequestBatcher.prototype.start = function() {
     this.stopped = false;
+    this.consecutiveRemovalFailures = 0;
     this.flush();
 };
 
@@ -115,7 +126,34 @@ RequestBatcher.prototype.flush = function(options) {
                 payload = this.beforeSendHook(payload);
             }
             if (payload) {
-                dataForRequest.push(payload);
+                // mp_sent_by_lib_version prop captures which lib version actually
+                // sends each event (regardless of which version originally queued
+                // it for sending)
+                if (payload['event'] && payload['properties']) {
+                    payload['properties'] = _.extend(
+                        {},
+                        payload['properties'],
+                        {'mp_sent_by_lib_version': Config.LIB_VERSION}
+                    );
+                }
+                var addPayload = true;
+                var itemId = item['id'];
+                if (itemId) {
+                    if ((this.itemIdsSentSuccessfully[itemId] || 0) > 5) {
+                        this.reportError('[dupe] item ID sent too many times, not sending', {
+                            item: item,
+                            batchSize: batch.length,
+                            timesSent: this.itemIdsSentSuccessfully[itemId]
+                        });
+                        addPayload = false;
+                    }
+                } else {
+                    this.reportError('[dupe] found item with no ID', {item: item});
+                }
+
+                if (addPayload) {
+                    dataForRequest.push(payload);
+                }
             }
             transformedItems[item['id']] = payload;
         }, this);
@@ -143,7 +181,7 @@ RequestBatcher.prototype.flush = function(options) {
                     res.error === 'timeout' &&
                     new Date().getTime() - startTime >= timeoutMS
                 ) {
-                    logger.error('Network timeout; retrying');
+                    this.reportError('Network timeout; retrying');
                     this.flush();
                 } else if (
                     _.isObject(res) &&
@@ -160,17 +198,17 @@ RequestBatcher.prototype.flush = function(options) {
                         }
                     }
                     retryMS = Math.min(MAX_RETRY_INTERVAL_MS, retryMS);
-                    logger.error('Error; retry in ' + retryMS + ' ms');
+                    this.reportError('Error; retry in ' + retryMS + ' ms');
                     this.scheduleFlush(retryMS);
                 } else if (_.isObject(res) && res.xhr_req && res.xhr_req['status'] === 413) {
                     // 413 Payload Too Large
                     if (batch.length > 1) {
                         var halvedBatchSize = Math.max(1, Math.floor(currentBatchSize / 2));
                         this.batchSize = Math.min(this.batchSize, halvedBatchSize, batch.length - 1);
-                        logger.error('413 response; reducing batch size to ' + this.batchSize);
+                        this.reportError('413 response; reducing batch size to ' + this.batchSize);
                         this.resetFlush();
                     } else {
-                        logger.error('Single-event request too large; dropping', batch);
+                        this.reportError('Single-event request too large; dropping', batch);
                         this.resetBatchSize();
                         removeItemsFromQueue = true;
                     }
@@ -183,12 +221,43 @@ RequestBatcher.prototype.flush = function(options) {
                 if (removeItemsFromQueue) {
                     this.queue.removeItemsByID(
                         _.map(batch, function(item) { return item['id']; }),
-                        _.bind(this.flush, this) // handle next batch if the queue isn't empty
+                        _.bind(function(succeeded) {
+                            if (succeeded) {
+                                this.consecutiveRemovalFailures = 0;
+                                this.flush(); // handle next batch if the queue isn't empty
+                            } else {
+                                this.reportError('Failed to remove items from queue');
+                                if (++this.consecutiveRemovalFailures > 5) {
+                                    this.reportError('Too many queue failures; disabling batching system.');
+                                    this.stopAllBatching();
+                                } else {
+                                    this.resetFlush();
+                                }
+                            }
+                        }, this)
                     );
+
+                    // client-side dedupe
+                    _.each(batch, _.bind(function(item) {
+                        var itemId = item['id'];
+                        if (itemId) {
+                            this.itemIdsSentSuccessfully[itemId] = this.itemIdsSentSuccessfully[itemId] || 0;
+                            this.itemIdsSentSuccessfully[itemId]++;
+                            if (this.itemIdsSentSuccessfully[itemId] > 5) {
+                                this.reportError('[dupe] item ID sent too many times', {
+                                    item: item,
+                                    batchSize: batch.length,
+                                    timesSent: this.itemIdsSentSuccessfully[itemId]
+                                });
+                            }
+                        } else {
+                            this.reportError('[dupe] found item with no ID while removing', {item: item});
+                        }
+                    }, this));
                 }
 
             } catch(err) {
-                logger.error('Error handling API response', err);
+                this.reportError('Error handling API response', err);
                 this.resetFlush();
             }
         }, this);
@@ -205,9 +274,26 @@ RequestBatcher.prototype.flush = function(options) {
         this.sendRequest(dataForRequest, requestOptions, batchSendCallback);
 
     } catch(err) {
-        logger.error('Error flushing request queue', err);
+        this.reportError('Error flushing request queue', err);
         this.resetFlush();
     }
 };
 
+/**
+ * Log error to global logger and optional user-defined logger.
+ */
+RequestBatcher.prototype.reportError = function(msg, err) {
+    logger.error.apply(logger.error, arguments);
+    if (this.errorReporter) {
+        try {
+            if (!(err instanceof Error)) {
+                err = new Error(msg);
+            }
+            this.errorReporter(msg, err);
+        } catch(err) {
+            logger.error(err);
+        }
+    }
+};
+
 export { RequestBatcher };

package/src/request-queue.js

@@ -1,5 +1,5 @@
 import { SharedLock } from './shared-lock';
-import { cheap_guid, console_with_prefix, JSONParse, JSONStringify, _ } from './utils'; // eslint-disable-line camelcase
+import { cheap_guid, console_with_prefix, localStorageSupported, JSONParse, JSONStringify, _ } from './utils'; // eslint-disable-line camelcase
 
 var logger = console_with_prefix('batch');
 
@@ -23,6 +23,7 @@ var RequestQueue = function(storageKey, options) {
     options = options || {};
     this.storageKey = storageKey;
     this.storage = options.storage || window.localStorage;
+    this.reportError = options.errorReporter || _.bind(logger.error, logger);
     this.lock = new SharedLock(storageKey, {storage: this.storage});
 
     this.pid = options.pid || null; // pass pid to test out storage lock contention scenarios
@@ -60,18 +61,18 @@ RequestQueue.prototype.enqueue = function(item, flushInterval, cb) {
                 this.memQueue.push(queueEntry);
             }
         } catch(err) {
-            logger.error('Error enqueueing item', item);
+            this.reportError('Error enqueueing item', item);
             succeeded = false;
         }
         if (cb) {
             cb(succeeded);
         }
-    }, this), function lockFailure(err) {
-        logger.error('Error acquiring storage lock', err);
+    }, this), _.bind(function lockFailure(err) {
+        this.reportError('Error acquiring storage lock', err);
         if (cb) {
             cb(false);
         }
-    }, this.pid);
+    }, this), this.pid);
 };
 
 /**
@@ -131,25 +132,61 @@ RequestQueue.prototype.removeItemsByID = function(ids, cb) {
     _.each(ids, function(id) { idSet[id] = true; });
 
     this.memQueue = filterOutIDsAndInvalid(this.memQueue, idSet);
-    this.lock.withLock(_.bind(function lockAcquired() {
+
+    var removeFromStorage = _.bind(function() {
         var succeeded;
         try {
             var storedQueue = this.readFromStorage();
             storedQueue = filterOutIDsAndInvalid(storedQueue, idSet);
             succeeded = this.saveToStorage(storedQueue);
+
+            // an extra check: did storage report success but somehow
+            // the items are still there?
+            if (succeeded) {
+                storedQueue = this.readFromStorage();
+                for (var i = 0; i < storedQueue.length; i++) {
+                    var item = storedQueue[i];
+                    if (item['id'] && !!idSet[item['id']]) {
+                        this.reportError('Item not removed from storage');
+                        return false;
+                    }
+                }
+            }
         } catch(err) {
-            logger.error('Error removing items', ids);
+            this.reportError('Error removing items', ids);
             succeeded = false;
         }
+        return succeeded;
+    }, this);
+
+    this.lock.withLock(function lockAcquired() {
+        var succeeded = removeFromStorage();
         if (cb) {
             cb(succeeded);
         }
-    }, this), function lockFailure(err) {
-        logger.error('Error acquiring storage lock', err);
+    }, _.bind(function lockFailure(err) {
+        var succeeded = false;
+        this.reportError('Error acquiring storage lock', err);
+        if (!localStorageSupported(this.storage, true)) {
+            // Looks like localStorage writes have stopped working sometime after
+            // initialization (probably full), and so nobody can acquire locks
+            // anymore. Consider it temporarily safe to remove items without the
+            // lock, since nobody's writing successfully anyway.
+            succeeded = removeFromStorage();
+            if (!succeeded) {
+                // OK, we couldn't even write out the smaller queue. Try clearing it
+                // entirely.
+                try {
+                    this.storage.removeItem(this.storageKey);
+                } catch(err) {
+                    this.reportError('Error clearing queue', err);
+                }
+            }
+        }
         if (cb) {
-            cb(false);
+            cb(succeeded);
         }
-    }, this.pid);
+    }, this), this.pid);
 };
 
 // internal helper for RequestQueue.updatePayloads
@@ -184,18 +221,18 @@ RequestQueue.prototype.updatePayloads = function(itemsToUpdate, cb) {
             storedQueue = updatePayloads(storedQueue, itemsToUpdate);
             succeeded = this.saveToStorage(storedQueue);
         } catch(err) {
-            logger.error('Error updating items', itemsToUpdate);
+            this.reportError('Error updating items', itemsToUpdate);
             succeeded = false;
         }
         if (cb) {
             cb(succeeded);
         }
-    }, this), function lockFailure(err) {
-        logger.error('Error acquiring storage lock', err);
+    }, this), _.bind(function lockFailure(err) {
+        this.reportError('Error acquiring storage lock', err);
         if (cb) {
             cb(false);
         }
-    }, this.pid);
+    }, this), this.pid);
 };
 
 /**
@@ -209,12 +246,12 @@ RequestQueue.prototype.readFromStorage = function() {
         if (storageEntry) {
             storageEntry = JSONParse(storageEntry);
             if (!_.isArray(storageEntry)) {
-                logger.error('Invalid storage entry:', storageEntry);
+                this.reportError('Invalid storage entry:', storageEntry);
                 storageEntry = null;
             }
         }
     } catch (err) {
-        logger.error('Error retrieving queue', err);
+        this.reportError('Error retrieving queue', err);
         storageEntry = null;
     }
     return storageEntry || [];
@@ -228,7 +265,7 @@ RequestQueue.prototype.saveToStorage = function(queue) {
         this.storage.setItem(this.storageKey, JSONStringify(queue));
         return true;
     } catch (err) {
-        logger.error('Error saving queue', err);
+        this.reportError('Error saving queue', err);
         return false;
     }
 };

package/src/utils.js

@@ -150,14 +150,6 @@ _.bind = function(func, context) {
     return bound;
 };
 
-_.bind_instance_methods = function(obj) {
-    for (var func in obj) {
-        if (typeof(obj[func]) === 'function') {
-            obj[func] = _.bind(obj[func], obj);
-        }
-    }
-};
-
 /**
  * @param {*=} obj
  * @param {function(...*)=} iterator
@@ -186,19 +178,6 @@ _.each = function(obj, iterator, context) {
     }
 };
 
-_.escapeHTML = function(s) {
-    var escaped = s;
-    if (escaped && _.isString(escaped)) {
-        escaped = escaped
-            .replace(/&/g, '&')
-            .replace(/</g, '&lt;')
-            .replace(/>/g, '&gt;')
-            .replace(/"/g, '&quot;')
-            .replace(/'/g, '&#039;');
-    }
-    return escaped;
-};
-
 _.extend = function(obj) {
     _.each(slice.call(arguments, 1), function(source) {
         for (var prop in source) {
@@ -374,33 +353,6 @@ _.formatDate = function(d) {
         pad(d.getUTCSeconds());
 };
 
-_.safewrap = function(f) {
-    return function() {
-        try {
-            return f.apply(this, arguments);
-        } catch (e) {
-            console.critical('Implementation error. Please turn on debug and contact support@mixpanel.com.');
-            if (Config.DEBUG){
-                console.critical(e);
-            }
-        }
-    };
-};
-
-_.safewrap_class = function(klass, functions) {
-    for (var i = 0; i < functions.length; i++) {
-        klass.prototype[functions[i]] = _.safewrap(klass.prototype[functions[i]]);
-    }
-};
-
-_.safewrap_instance_methods = function(obj) {
-    for (var func in obj) {
-        if (typeof(obj[func]) === 'function') {
-            obj[func] = _.safewrap(obj[func]);
-        }
-    }
-};
-
 _.strip_empty_properties = function(p) {
     var ret = {};
     _.each(p, function(v, k) {