@herdwatch/lokijs 1.5.8-dev.6 → 1.5.12-dev.0

package/src/incremental-indexeddb-adapter.js

@@ -13,6 +13,9 @@
   return (function() {
     "use strict";
 
+    /* jshint -W030 */
+    var DEBUG = typeof window !== 'undefined' && !!window.__loki_incremental_idb_debug;
+
     /**
      * An improved Loki persistence adapter for IndexedDB (not compatible with LokiIndexedAdapter)
      *     Unlike LokiIndexedAdapter, the database is saved not as one big JSON blob, but split into
@@ -30,14 +33,37 @@
      * @constructor IncrementalIndexedDBAdapter
      *
      * @param {object=} options Configuration options for the adapter
-     * @param {boolean} options.onversionchange Function to call on `IDBDatabase.onversionchange` event
+     * @param {function} options.onversionchange Function to call on `IDBDatabase.onversionchange` event
      *     (most likely database deleted from another browser tab)
+     * @param {function} options.onFetchStart Function to call once IDB load has begun.
+     *     Use this as an opportunity to execute code concurrently while IDB does work on a separate thread
+     * @param {function} options.onDidOverwrite Called when this adapter is forced to overwrite contents
+     *     of IndexedDB. This happens if there's another open tab of the same app that's making changes.
+     *     You might use it as an opportunity to alert user to the potential loss of data
+     * @param {function} options.serializeChunk Called with a chunk (array of Loki documents) before
+     *     it's saved to IndexedDB. You can use it to manually compress on-disk representation
+     *     for faster database loads. Hint: Hand-written conversion of objects to arrays is very
+     *     profitable for performance. If you use this, you must also pass options.deserializeChunk.
+     * @param {function} options.deserializeChunk Called with a chunk serialized with options.serializeChunk
+     *     Expects an array of Loki documents as the return value
+     * @param {number} options.megachunkCount Number of parallel requests for data when loading database.
+     *     Can be tuned for a specific application
+     * @param {array} options.lazyCollections Names of collections that should be deserialized lazily
+     *     Only use this for collections that aren't used at launch
      */
     function IncrementalIndexedDBAdapter(options) {
       this.mode = "incremental";
       this.options = options || {};
       this.chunkSize = 100;
+      this.megachunkCount = this.options.megachunkCount || 24;
+      this.lazyCollections = this.options.lazyCollections || [];
       this.idb = null; // will be lazily loaded on first operation that needs it
+      this._prevLokiVersionId = null;
+      this._prevCollectionVersionIds = {};
+
+      if (!(this.megachunkCount >= 4 && this.megachunkCount % 2 === 0)) {
+        throw new Error('megachunkCount must be >=4 and divisible by 2');
+      }
     }
 
     // chunkId - index of the data chunk - e.g. chunk 0 will be lokiIds 0-99
@@ -47,6 +73,7 @@
       var maxId = minId + this.chunkSize - 1;
 
       // use idIndex to find first collection.data position within the $loki range
+      collection.ensureId();
       var idIndex = collection.idIndex;
 
       var firstDataPosition = null;
@@ -86,7 +113,7 @@
         }
       }
 
-      // TODO: remove sanity checks when everything is fully tested
+      // verify
       var firstElement = collection.data[firstDataPosition];
       if (!(firstElement && firstElement.$loki >= minId && firstElement.$loki <= maxId)) {
         throw new Error("broken invariant firstelement");
@@ -101,7 +128,6 @@
       // will have holes when data is deleted)
       var chunkData = collection.data.slice(firstDataPosition, lastDataPosition + 1);
 
-      // TODO: remove sanity checks when everything is fully tested
       if (chunkData.length > this.chunkSize) {
         throw new Error("broken invariant - chunk size");
       }
@@ -120,55 +146,241 @@
      * db.saveDatabase();
      *
      * @param {string} dbname - the name to give the serialized database
-     * @param {object} dbcopy - copy of the Loki database
+     * @param {function} getLokiCopy - returns copy of the Loki database
      * @param {function} callback - (Optional) callback passed obj.success with true or false
      * @memberof IncrementalIndexedDBAdapter
      */
-    IncrementalIndexedDBAdapter.prototype.saveDatabase = function(dbname, loki, callback) {
+    IncrementalIndexedDBAdapter.prototype.saveDatabase = function(dbname, getLokiCopy, callback) {
       var that = this;
-      console.log("exportDatabase - begin");
-      console.time("exportDatabase");
 
-      var chunksToSave = [];
+      if (!this.idb) {
+        this._initializeIDB(dbname, callback, function() {
+          that.saveDatabase(dbname, getLokiCopy, callback);
+        });
+        return;
+      }
+
+      if (this.operationInProgress) {
+        throw new Error("Error while saving to database - another operation is already in progress. Please use throttledSaves=true option on Loki object");
+      }
+      this.operationInProgress = true;
+
+      DEBUG && console.log("saveDatabase - begin");
+      DEBUG && console.time("saveDatabase");
+      function finish(e) {
+        DEBUG && e && console.error(e);
+        DEBUG && console.timeEnd("saveDatabase");
+        that.operationInProgress = false;
+        callback(e);
+      }
+
+      // try..catch is required, e.g.:
+      // InvalidStateError: Failed to execute 'transaction' on 'IDBDatabase': The database connection is closing.
+      // (this may happen if another tab has called deleteDatabase)
+      try {
+        var updatePrevVersionIds = function () {
+          console.error('Unexpected successful tx - cannot update previous version ids');
+        };
+        var didOverwrite = false;
+
+        var tx = this.idb.transaction(['LokiIncrementalData'], "readwrite");
+        tx.oncomplete = function() {
+          updatePrevVersionIds();
+          finish();
+          if (didOverwrite && that.options.onDidOverwrite) {
+            that.options.onDidOverwrite();
+          }
+        };
+
+        tx.onerror = function(e) {
+          finish(e);
+        };
+
+        tx.onabort = function(e) {
+          finish(e);
+        };
+
+        var store = tx.objectStore('LokiIncrementalData');
+
+        var performSave = function (maxChunkIds) {
+          try {
+            var incremental = !maxChunkIds;
+            var chunkInfo = that._putInChunks(store, getLokiCopy(), incremental, maxChunkIds);
+            // Update last seen version IDs, but only after the transaction is successful
+            updatePrevVersionIds = function() {
+              that._prevLokiVersionId = chunkInfo.lokiVersionId;
+              chunkInfo.collectionVersionIds.forEach(function (collectionInfo) {
+                that._prevCollectionVersionIds[collectionInfo.name] = collectionInfo.versionId;
+              });
+            };
+            tx.commit && tx.commit();
+          } catch (error) {
+            console.error('idb performSave failed: ', error);
+            tx.abort();
+          }
+        };
+
+        // Incrementally saving changed chunks breaks down if there is more than one writer to IDB
+        // (multiple tabs of the same web app), leading to data corruption. To fix that, we save all
+        // metadata chunks (loki + collections) with a unique ID on each save and remember it. Before
+        // the subsequent save, we read loki from IDB to check if its version ID changed. If not, we're
+        // guaranteed that persisted DB is consistent with our diff. Otherwise, we fall back to the slow
+        // path and overwrite *all* database chunks with our version. Both reading and writing must
+        // happen in the same IDB transaction for this to work.
+        // TODO: We can optimize the slow path by fetching collection metadata chunks and comparing their
+        // version IDs with those last seen by us. Since any change in collection data requires a metadata
+        // chunk save, we're guaranteed that if the IDs match, we don't need to overwrite chukns of this collection
+        var getAllKeysThenSave = function() {
+          // NOTE: We must fetch all keys to protect against a case where another tab has wrote more
+          // chunks whan we did -- if so, we must delete them.
+          idbReq(store.getAllKeys(), function(e) {
+            var maxChunkIds = getMaxChunkIds(e.target.result);
+            performSave(maxChunkIds);
+          }, function(e) {
+            console.error('Getting all keys failed: ', e);
+            tx.abort();
+          });
+        };
+
+        var getLokiThenSave = function() {
+          idbReq(store.get('loki'), function(e) {
+            if (lokiChunkVersionId(e.target.result) === that._prevLokiVersionId) {
+              performSave();
+            } else {
+              DEBUG && console.warn('Another writer changed Loki IDB, using slow path...');
+              didOverwrite = true;
+              getAllKeysThenSave();
+            }
+          }, function(e) {
+            console.error('Getting loki chunk failed: ', e);
+            tx.abort();
+          });
+        };
+
+        getLokiThenSave();
+      } catch (error) {
+        finish(error);
+      }
+    };
+
+    // gets current largest chunk ID for each collection
+    function getMaxChunkIds(allKeys) {
+      var maxChunkIds = {};
+
+      allKeys.forEach(function (key) {
+        var keySegments = key.split(".");
+        // table.chunk.2317
+        if (keySegments.length === 3 && keySegments[1] === "chunk") {
+          var collection = keySegments[0];
+          var chunkId = parseInt(keySegments[2]) || 0;
+          var currentMax = maxChunkIds[collection];
+
+          if (!currentMax || chunkId > currentMax) {
+            maxChunkIds[collection] = chunkId;
+          }
+        }
+      });
+      return maxChunkIds;
+    }
 
-      loki.collections.forEach(function(collection, i) {
+    function lokiChunkVersionId(chunk) {
+      try {
+        if (chunk) {
+          var loki = JSON.parse(chunk.value);
+          return loki.idbVersionId || null;
+        } else {
+          return null;
+        }
+      } catch (e) {
+        console.error('Error while parsing loki chunk', e);
+        return null;
+      }
+    }
+
+    IncrementalIndexedDBAdapter.prototype._putInChunks = function(idbStore, loki, incremental, maxChunkIds) {
+      var that = this;
+      var collectionVersionIds = [];
+      var savedSize = 0;
+
+      var prepareCollection = function (collection, i) {
         // Find dirty chunk ids
         var dirtyChunks = new Set();
-        collection.dirtyIds.forEach(function(lokiId) {
+        incremental && collection.dirtyIds.forEach(function(lokiId) {
           var chunkId = (lokiId / that.chunkSize) | 0;
           dirtyChunks.add(chunkId);
         });
         collection.dirtyIds = [];
 
         // Serialize chunks to save
-        dirtyChunks.forEach(function(chunkId) {
+        var prepareChunk = function (chunkId) {
           var chunkData = that._getChunk(collection, chunkId);
+          if (that.options.serializeChunk) {
+            chunkData = that.options.serializeChunk(collection.name, chunkData);
+          }
           // we must stringify now, because IDB is asynchronous, and underlying objects are mutable
-          chunksToSave.push({
+          // In general, it's also faster to stringify, because we need serialization anyway, and
+          // JSON.stringify is much better optimized than IDB's structured clone
+          chunkData = JSON.stringify(chunkData);
+          savedSize += chunkData.length;
+          DEBUG && incremental && console.log('Saving: ' + collection.name + ".chunk." + chunkId);
+          idbStore.put({
             key: collection.name + ".chunk." + chunkId,
-            value: JSON.stringify(chunkData),
+            value: chunkData,
           });
-        });
+        };
+        if (incremental) {
+          dirtyChunks.forEach(prepareChunk);
+        } else {
+          // add all chunks
+          var maxChunkId = (collection.maxId / that.chunkSize) | 0;
+          for (var j = 0; j <= maxChunkId; j += 1) {
+            prepareChunk(j);
+          }
 
-        collection.data = [];
-        // this is recreated on load anyway, so we can make metadata smaller
-        collection.isIndex = [];
+          // delete chunks with larger ids than what we have
+          // NOTE: we don't have to delete metadata chunks as they will be absent from loki anyway
+          // NOTE: failures are silently ignored, so we don't have to worry about holes
+          var persistedMaxChunkId = maxChunkIds[collection.name] || 0;
+          for (var k = maxChunkId + 1; k <= persistedMaxChunkId; k += 1) {
+            var deletedChunkName = collection.name + ".chunk." + k;
+            idbStore.delete(deletedChunkName);
+            DEBUG && console.warn('Deleted chunk: ' + deletedChunkName);
+          }
+        }
 
-        // save collection metadata as separate chunk, leave only names in loki
-        // TODO: To reduce IO, we should only save this chunk when it has changed
-        chunksToSave.push({
-          key: collection.name + ".metadata",
-          value: JSON.stringify(collection),
-        });
+        // save collection metadata as separate chunk (but only if changed)
+        if (collection.dirty || dirtyChunks.size || !incremental) {
+          collection.idIndex = []; // this is recreated lazily
+          collection.data = [];
+          collection.idbVersionId = randomVersionId();
+          collectionVersionIds.push({ name: collection.name, versionId: collection.idbVersionId });
+
+          var metadataChunk = JSON.stringify(collection);
+          savedSize += metadataChunk.length;
+          DEBUG && incremental && console.log('Saving: ' + collection.name + ".metadata");
+          idbStore.put({
+            key: collection.name + ".metadata",
+            value: metadataChunk,
+          });
+        }
+
+        // leave only names in the loki chunk
         loki.collections[i] = { name: collection.name };
-      });
+      };
+      loki.collections.forEach(prepareCollection);
 
+      loki.idbVersionId = randomVersionId();
       var serializedMetadata = JSON.stringify(loki);
-      loki = null; // allow GC of the DB copy
+      savedSize += serializedMetadata.length;
 
-      chunksToSave.push({ key: "loki", value: serializedMetadata });
+      DEBUG && incremental && console.log('Saving: loki');
+      idbStore.put({ key: "loki", value: serializedMetadata });
 
-      that._saveChunks(dbname, chunksToSave, callback);
+      DEBUG && console.log("saved size: " + savedSize);
+      return {
+        lokiVersionId: loki.idbVersionId,
+        collectionVersionIds: collectionVersionIds,
+      };
     };
 
     /**
@@ -188,131 +400,137 @@
      */
     IncrementalIndexedDBAdapter.prototype.loadDatabase = function(dbname, callback) {
       var that = this;
-      console.log("loadDatabase - begin");
-      console.time("loadDatabase");
-      this._getAllChunks(dbname, function(chunks) {
-        if (!Array.isArray(chunks)) {
-          // we got an error
-          console.timeEnd("loadDatabase");
-          callback(chunks);
-        }
 
-        if (!chunks.length) {
-          console.timeEnd("loadDatabase");
-          callback(null);
-          return;
-        }
+      if (this.operationInProgress) {
+        throw new Error("Error while loading database - another operation is already in progress. Please use throttledSaves=true option on Loki object");
+      }
 
-        console.log("Found chunks:", chunks.length);
+      this.operationInProgress = true;
 
-        that._sortChunksInPlace(chunks);
+      DEBUG && console.log("loadDatabase - begin");
+      DEBUG && console.time("loadDatabase");
 
-        // repack chunks into a map
-        var loki;
-        var chunkCollections = {};
+      var finish = function (value) {
+        DEBUG && console.timeEnd("loadDatabase");
+        that.operationInProgress = false;
+        callback(value);
+      };
 
-        chunks.forEach(function(object) {
-          var key = object.key;
-          var value = object.value;
-          if (key === "loki") {
-            loki = value;
-            return;
-          } else if (key.includes(".")) {
-            var keySegments = key.split(".");
-            if (keySegments.length === 3 && keySegments[1] === "chunk") {
-              var colName = keySegments[0];
-              if (chunkCollections[colName]) {
-                chunkCollections[colName].dataChunks.push(value);
-              } else {
-                chunkCollections[colName] = {
-                  metadata: null,
-                  dataChunks: [value],
-                };
-              }
-              return;
-            } else if (keySegments.length === 2 && keySegments[1] === "metadata") {
-              var name = keySegments[0];
-              if (chunkCollections[name]) {
-                chunkCollections[name].metadata = value;
-              } else {
-                chunkCollections[name] = { metadata: value, dataChunks: [] };
-              }
-              return;
-            }
+      this._getAllChunks(dbname, function(chunks) {
+        try {
+          if (!Array.isArray(chunks)) {
+            throw chunks; // we have an error
           }
 
-          console.error("Unknown chunk " + key);
-          callback(new Error("Invalid database - unknown chunk found"));
-        });
-        chunks = null;
+          if (!chunks.length) {
+            return finish(null);
+          }
 
-        if (!loki) {
-          callback(new Error("Invalid database - missing database metadata"));
-        }
+          DEBUG && console.log("Found chunks:", chunks.length);
 
-        // parse Loki object
-        loki = JSON.parse(loki);
+          // repack chunks into a map
+          chunks = chunksToMap(chunks);
+          var loki = chunks.loki;
+          chunks.loki = null; // gc
 
-        // populate collections with data
-        that._populate(loki, chunkCollections);
-        chunkCollections = null;
+          // populate collections with data
+          populateLoki(loki, chunks.chunkMap, that.options.deserializeChunk, that.lazyCollections);
+          chunks = null; // gc
 
-        console.timeEnd("loadDatabase");
-        callback(loki);
+          // remember previous version IDs
+          that._prevLokiVersionId = loki.idbVersionId || null;
+          that._prevCollectionVersionIds = {};
+          loki.collections.forEach(function (collection) {
+            that._prevCollectionVersionIds[collection.name] = collection.idbVersionId || null;
+          });
+
+          return finish(loki);
+        } catch (error) {
+          that._prevLokiVersionId = null;
+          that._prevCollectionVersionIds = {};
+          return finish(error);
+        }
       });
     };
 
-    IncrementalIndexedDBAdapter.prototype._sortChunksInPlace = function(chunks) {
-      // sort chunks in place to load data in the right order (ascending loki ids)
-      // on both Safari and Chrome, we'll get chunks in order like this: 0, 1, 10, 100...
-      var getSortKey = function(object) {
-        var key = object.key;
-        if (key.includes(".")) {
-          var segments = key.split(".");
-          if (segments.length === 3 && segments[1] === "chunk") {
-            return parseInt(segments[2], 10);
+    function chunksToMap(chunks) {
+      var loki;
+      var chunkMap = {};
+
+      sortChunksInPlace(chunks);
+
+      chunks.forEach(function(chunk) {
+        var type = chunk.type;
+        var value = chunk.value;
+        var name = chunk.collectionName;
+        if (type === "loki") {
+          loki = value;
+        } else if (type === "data") {
+          if (chunkMap[name]) {
+            chunkMap[name].dataChunks.push(value);
+          } else {
+            chunkMap[name] = {
+              metadata: null,
+              dataChunks: [value],
+            };
+          }
+        } else if (type === "metadata") {
+          if (chunkMap[name]) {
+            chunkMap[name].metadata = value;
+          } else {
+            chunkMap[name] = { metadata: value, dataChunks: [] };
           }
+        } else {
+          throw new Error("unreachable");
         }
-
-        return -1; // consistent type must be returned
-      };
-      chunks.sort(function(a, b) {
-        var aKey = getSortKey(a),
-          bKey = getSortKey(b);
-        if (aKey < bKey) return -1;
-        if (aKey > bKey) return 1;
-        return 0;
       });
-    };
 
-    IncrementalIndexedDBAdapter.prototype._populate = function(loki, chunkCollections) {
-      loki.collections.forEach(function(collectionStub, i) {
-        var chunkCollection = chunkCollections[collectionStub.name];
+      if (!loki) {
+        throw new Error("Corrupted database - missing database metadata");
+      }
+
+      return { loki: loki, chunkMap: chunkMap };
+    }
 
+    function populateLoki(loki, chunkMap, deserializeChunk, lazyCollections) {
+      loki.collections.forEach(function populateCollection(collectionStub, i) {
+        var name = collectionStub.name;
+        var chunkCollection = chunkMap[name];
         if (chunkCollection) {
-          // TODO: What if metadata is missing?
-          var collection = JSON.parse(chunkCollection.metadata);
+          if (!chunkCollection.metadata) {
+            throw new Error("Corrupted database - missing metadata chunk for " + name);
+          }
+          var collection = chunkCollection.metadata;
           chunkCollection.metadata = null;
-
           loki.collections[i] = collection;
 
-          var dataChunks = chunkCollection.dataChunks;
-          dataChunks.forEach(function(chunkObj, i) {
-            var chunk = JSON.parse(chunkObj);
-            chunkObj = null; // make string available for GC
-            dataChunks[i] = null;
-
-            chunk.forEach(function(doc) {
-              collection.data.push(doc);
+          var isLazy = lazyCollections.includes(name);
+          var lokiDeserializeCollectionChunks = function () {
+            DEBUG && isLazy && console.log("lazy loading " + name);
+            var data = [];
+            var dataChunks = chunkCollection.dataChunks;
+            dataChunks.forEach(function populateChunk(chunk, i) {
+              if (isLazy) {
+                chunk = JSON.parse(chunk);
+                if (deserializeChunk) {
+                  chunk = deserializeChunk(name, chunk);
+                }
+              }
+              chunk.forEach(function(doc) {
+                data.push(doc);
+              });
+              dataChunks[i] = null;
             });
-          });
+            return data;
+          };
+          collection.getData = lokiDeserializeCollectionChunks;
         }
       });
-    };
+    }
 
     IncrementalIndexedDBAdapter.prototype._initializeIDB = function(dbname, onError, onSuccess) {
       var that = this;
-      console.log("initializing idb");
+      DEBUG && console.log("initializing idb");
 
       if (this.idbInitInProgress) {
         throw new Error("Cannot open IndexedDB because open is already in progress");
@@ -323,7 +541,7 @@
 
       openRequest.onupgradeneeded = function(e) {
         var db = e.target.result;
-        console.log('onupgradeneeded, old version: ' + e.oldVersion);
+        DEBUG && console.log('onupgradeneeded, old version: ' + e.oldVersion);
 
         if (e.oldVersion < 1) {
           // Version 1 - Initial - Create database
@@ -336,19 +554,25 @@
 
       openRequest.onsuccess = function(e) {
         that.idbInitInProgress = false;
-        that.idb = e.target.result;
+        var db = e.target.result;
+        that.idb = db;
 
-        if (!that.idb.objectStoreNames.contains('LokiIncrementalData')) {
+        if (!db.objectStoreNames.contains('LokiIncrementalData')) {
           onError(new Error("Missing LokiIncrementalData"));
           // Attempt to recover (after reload) by deleting database, since it's damaged anyway
           that.deleteDatabase(dbname);
           return;
         }
 
-        console.log("init success");
+        DEBUG && console.log("init success");
+
+        db.onversionchange = function(versionChangeEvent) {
+          // Ignore if database was deleted and recreated in the meantime
+          if (that.idb !== db) {
+            return;
+          }
 
-        that.idb.onversionchange = function(versionChangeEvent) {
-          console.log('IDB version change', versionChangeEvent);
+          DEBUG && console.log('IDB version change', versionChangeEvent);
           // This function will be called if another connection changed DB version
           // (Most likely database was deleted from another browser tab, unless there's a new version
           // of this adapter, or someone makes a connection to IDB outside of this adapter)
@@ -356,6 +580,7 @@
           // The database will be unusable after this. Be sure to supply `onversionchange` option
           // to force logout
           that.idb.close();
+          that.idb = null;
           if (that.options.onversionchange) {
             that.options.onversionchange(versionChangeEvent);
           }
@@ -371,79 +596,148 @@
 
       openRequest.onerror = function(e) {
         that.idbInitInProgress = false;
-        console.error("IndexeddB open error", e);
+        console.error("IndexedDB open error", e);
         onError(e);
       };
     };
 
-    IncrementalIndexedDBAdapter.prototype._saveChunks = function(dbname, chunks, callback) {
+    IncrementalIndexedDBAdapter.prototype._getAllChunks = function(dbname, callback) {
       var that = this;
       if (!this.idb) {
         this._initializeIDB(dbname, callback, function() {
-          that._saveChunks(dbname, chunks, callback);
+          that._getAllChunks(dbname, callback);
         });
         return;
       }
 
-      if (this.operationInProgress) {
-        throw new Error("Error while saving to database - another operation is already in progress. Please use throttledSaves=true option on Loki object");
-      }
+      var tx = this.idb.transaction(['LokiIncrementalData'], "readonly");
+      var store = tx.objectStore('LokiIncrementalData');
 
-      this.operationInProgress = true;
+      var deserializeChunk = this.options.deserializeChunk;
+      var lazyCollections = this.lazyCollections;
+
+      // If there are a lot of chunks (>100), don't request them all in one go, but in multiple
+      // "megachunks" (chunks of chunks). This improves concurrency, as main thread is already busy
+      // while IDB process is still fetching data. Details: https://github.com/techfort/LokiJS/pull/874
+      function getMegachunks(keys) {
+        var megachunkCount = that.megachunkCount;
+        var keyRanges = createKeyRanges(keys, megachunkCount);
+
+        var allChunks = [];
+        var megachunksReceived = 0;
+
+        function processMegachunk(e, megachunkIndex, keyRange) {
+          // var debugMsg = 'processing chunk ' + megachunkIndex + ' (' + keyRange.lower + ' -- ' + keyRange.upper + ')'
+          // DEBUG && console.time(debugMsg);
+          var megachunk = e.target.result;
+          megachunk.forEach(function (chunk, i) {
+            parseChunk(chunk, deserializeChunk, lazyCollections);
+            allChunks.push(chunk);
+            megachunk[i] = null; // gc
+          });
+          // DEBUG && console.timeEnd(debugMsg);
 
-      var tx = this.idb.transaction(['LokiIncrementalData'], "readwrite");
-      tx.oncomplete = function() {
-        that.operationInProgress = false;
-        console.timeEnd("exportDatabase");
-        callback();
-      };
+          megachunksReceived += 1;
+          if (megachunksReceived === megachunkCount) {
+            callback(allChunks);
+          }
+        }
 
-      tx.onerror = function(e) {
-        that.operationInProgress = false;
-        callback(e);
-      };
+        // Stagger megachunk requests - first one half, then request the second when first one comes
+        // back. This further improves concurrency.
+        var megachunkWaves = 2;
+        var megachunksPerWave = megachunkCount / megachunkWaves;
+        function requestMegachunk(index, wave) {
+          var keyRange = keyRanges[index];
+          idbReq(store.getAll(keyRange), function(e) {
+            if (wave < megachunkWaves) {
+              requestMegachunk(index + megachunksPerWave, wave + 1);
+            }
 
-      tx.onabort = function(e) {
-        that.operationInProgress = false;
-        callback(e);
-      };
+            processMegachunk(e, index, keyRange);
+          }, function(e) {
+            callback(e);
+          });
+        }
 
-      var store = tx.objectStore('LokiIncrementalData');
+        for (var i = 0; i < megachunksPerWave; i += 1) {
+          requestMegachunk(i, 1);
+        }
+      }
 
-      chunks.forEach(function(object) {
-        store.put(object);
-      });
-    };
+      function getAllChunks() {
+        idbReq(store.getAll(), function(e) {
+          var allChunks = e.target.result;
+          allChunks.forEach(function (chunk) {
+            parseChunk(chunk, deserializeChunk, lazyCollections);
+          });
+          callback(allChunks);
+        }, function(e) {
+          callback(e);
+        });
+      }
 
-    IncrementalIndexedDBAdapter.prototype._getAllChunks = function(dbname, callback) {
-      var that = this;
-      if (!this.idb) {
-        this._initializeIDB(dbname, callback, function() {
-          that._getAllChunks(dbname, callback);
+      function getAllKeys() {
+        function onDidGetKeys(keys) {
+          keys.sort();
+          if (keys.length > 100) {
+            getMegachunks(keys);
+          } else {
+            getAllChunks();
+          }
+        }
+
+        idbReq(store.getAllKeys(), function(e) {
+          onDidGetKeys(e.target.result);
+        }, function(e) {
+          callback(e);
         });
-        return;
+
+        if (that.options.onFetchStart) {
+          that.options.onFetchStart();
+        }
       }
 
-      if (this.operationInProgress) {
-        throw new Error("Error while loading database - another operation is already in progress. Please use throttledSaves=true option on Loki object");
+      getAllKeys();
+    };
+
+    function classifyChunk(chunk) {
+      var key = chunk.key;
+
+      if (key === 'loki') {
+        chunk.type = 'loki';
+        return;
+      } else if (key.includes('.')) {
+        var keySegments = key.split(".");
+        if (keySegments.length === 3 && keySegments[1] === "chunk") {
+          chunk.type = 'data';
+          chunk.collectionName = keySegments[0];
+          chunk.index = parseInt(keySegments[2], 10);
+          return;
+        } else if (keySegments.length === 2 && keySegments[1] === "metadata") {
+          chunk.type = 'metadata';
+          chunk.collectionName = keySegments[0];
+          return;
+        }
       }
 
-      this.operationInProgress = true;
+      console.error("Unknown chunk " + key);
+      throw new Error("Corrupted database - unknown chunk found");
+    }
 
-      var tx = this.idb.transaction(['LokiIncrementalData'], "readonly");
+    function parseChunk(chunk, deserializeChunk, lazyCollections) {
+      classifyChunk(chunk);
 
-      var request = tx.objectStore('LokiIncrementalData').getAll();
-      request.onsuccess = function(e) {
-        that.operationInProgress = false;
-        var chunks = e.target.result;
-        callback(chunks);
-      };
+      var isData = chunk.type === 'data';
+      var isLazy = lazyCollections.includes(chunk.collectionName);
 
-      request.onerror = function(e) {
-        that.operationInProgress = false;
-        callback(e);
-      };
-    };
+      if (!(isData && isLazy)) {
+        chunk.value = JSON.parse(chunk.value);
+      }
+      if (deserializeChunk && isData && !isLazy) {
+        chunk.value = deserializeChunk(chunk.collectionName, chunk.value);
+      }
+    }
 
     /**
      * Deletes a database from IndexedDB
@@ -467,8 +761,11 @@
       this.operationInProgress = true;
 
       var that = this;
-      console.log("deleteDatabase - begin");
-      console.time("deleteDatabase");
+      DEBUG && console.log("deleteDatabase - begin");
+      DEBUG && console.time("deleteDatabase");
+
+      this._prevLokiVersionId = null;
+      this._prevCollectionVersionIds = {};
 
       if (this.idb) {
         this.idb.close();
@@ -479,7 +776,7 @@
 
       request.onsuccess = function() {
         that.operationInProgress = false;
-        console.timeEnd("deleteDatabase");
+        DEBUG && console.timeEnd("deleteDatabase");
         callback({ success: true });
       };
 
@@ -496,6 +793,54 @@
       };
     };
 
+    function randomVersionId() {
+      // Appears to have enough entropy for chunk version IDs
+      // (Only has to be different than enough of its own previous versions that there's no writer
+      // that thinks a new version is the same as an earlier one, not globally unique)
+      return Math.random().toString(36).substring(2);
+    }
+
+    function sortChunksInPlace(chunks) {
+      // sort chunks in place to load data in the right order (ascending loki ids)
+      // on both Safari and Chrome, we'll get chunks in order like this: 0, 1, 10, 100...
+      chunks.sort(function(a, b) {
+        return (a.index || 0) - (b.index || 0);
+      });
+    }
+
+    function createKeyRanges(keys, count) {
+      var countPerRange = Math.floor(keys.length / count);
+      var keyRanges = [];
+      var minKey, maxKey;
+      for (var i = 0; i < count; i += 1) {
+        minKey = keys[countPerRange * i];
+        maxKey = keys[countPerRange * (i + 1)];
+        if (i === 0) {
+          // ... < maxKey
+          keyRanges.push(IDBKeyRange.upperBound(maxKey, true));
+        } else if (i === count - 1) {
+          // >= minKey
+          keyRanges.push(IDBKeyRange.lowerBound(minKey));
+        } else {
+          // >= minKey && < maxKey
+          keyRanges.push(IDBKeyRange.bound(minKey, maxKey, false, true));
+        }
+      }
+      return keyRanges;
+    }
+
+    function idbReq(request, onsuccess, onerror) {
+      request.onsuccess = function (e) {
+        try {
+          return onsuccess(e);
+        } catch (error) {
+          onerror(error);
+        }
+      };
+      request.onerror = onerror;
+      return request;
+    }
+
     return IncrementalIndexedDBAdapter;
   })();
 });