@aws-amplify/datastore 3.14.1-unstable.2 → 3.14.1

package/lib/sync/index.js

@@ -100,7 +100,7 @@ var ControlMessage;
     ControlMessage["SYNC_ENGINE_READY"] = "ready";
 })(ControlMessage = exports.ControlMessage || (exports.ControlMessage = {}));
 var SyncEngine = /** @class */ (function () {
-    function SyncEngine(schema, namespaceResolver, modelClasses, userModelClasses, storage, modelInstanceCreator, conflictHandler, errorHandler, syncPredicates, amplifyConfig, authModeStrategy, amplifyContext) {
+    function SyncEngine(schema, namespaceResolver, modelClasses, userModelClasses, storage, modelInstanceCreator, conflictHandler, errorHandler, syncPredicates, amplifyConfig, authModeStrategy, amplifyContext, connectivityMonitor) {
         if (amplifyConfig === void 0) { amplifyConfig = {}; }
         this.schema = schema;
         this.namespaceResolver = namespaceResolver;
@@ -112,15 +112,18 @@ var SyncEngine = /** @class */ (function () {
         this.amplifyConfig = amplifyConfig;
         this.authModeStrategy = authModeStrategy;
         this.amplifyContext = amplifyContext;
+        this.connectivityMonitor = connectivityMonitor;
         this.online = false;
         this.modelSyncedStatus = new WeakMap();
+        this.runningProcesses = new core_1.BackgroundProcessManager();
         var MutationEvent = this.modelClasses['MutationEvent'];
         this.outbox = new outbox_1.MutationEventOutbox(this.schema, MutationEvent, modelInstanceCreator, ownSymbol);
         this.modelMerger = new merger_1.ModelMerger(this.outbox, ownSymbol);
         this.syncQueriesProcessor = new sync_1.SyncProcessor(this.schema, this.syncPredicates, this.amplifyConfig, this.authModeStrategy, errorHandler, this.amplifyContext);
         this.subscriptionsProcessor = new subscription_1.SubscriptionProcessor(this.schema, this.syncPredicates, this.amplifyConfig, this.authModeStrategy, errorHandler, this.amplifyContext);
         this.mutationsProcessor = new mutation_1.MutationProcessor(this.schema, this.storage, this.userModelClasses, this.outbox, this.modelInstanceCreator, MutationEvent, this.amplifyConfig, this.authModeStrategy, errorHandler, conflictHandler, this.amplifyContext);
-        this.datastoreConnectivity = new datastoreConnectivity_1.default();
+        this.datastoreConnectivity =
+            this.connectivityMonitor || new datastoreConnectivity_1.default();
     }
     SyncEngine.prototype.getModelSyncedStatus = function (modelConstructor) {
         return this.modelSyncedStatus.get(modelConstructor);
@@ -130,7 +133,7 @@ var SyncEngine = /** @class */ (function () {
         return new zen_observable_ts_1.default(function (observer) {
             logger.log('starting sync engine...');
             var subscriptions = [];
-            (function () { return __awaiter(_this, void 0, void 0, function () {
+            _this.runningProcesses.add(function () { return __awaiter(_this, void 0, void 0, function () {
                 var err_1, startPromise, hasMutationsInOutbox;
                 var _this = this;
                 return __generator(this, function (_a) {
@@ -146,154 +149,188 @@ var SyncEngine = /** @class */ (function () {
                             observer.error(err_1);
                             return [2 /*return*/];
                         case 3:
-                            startPromise = new Promise(function (resolve) {
+                            startPromise = new Promise(function (doneStarting, failedStarting) {
                                 _this.datastoreConnectivity.status().subscribe(function (_a) {
                                     var online = _a.online;
                                     return __awaiter(_this, void 0, void 0, function () {
-                                        var ctlSubsObservable_1, dataSubsObservable, err_2, error_1;
-                                        var _b;
                                         var _this = this;
-                                        return __generator(this, function (_c) {
-                                            switch (_c.label) {
-                                                case 0:
-                                                    if (!(online && !this.online)) return [3 /*break*/, 10];
-                                                    this.online = online;
-                                                    observer.next({
-                                                        type: ControlMessage.SYNC_ENGINE_NETWORK_STATUS,
-                                                        data: {
-                                                            active: this.online,
-                                                        },
-                                                    });
-                                                    dataSubsObservable = void 0;
-                                                    if (!isNode) return [3 /*break*/, 1];
-                                                    logger.warn('Realtime disabled when in a server-side environment');
-                                                    return [3 /*break*/, 6];
-                                                case 1:
-                                                    //#region GraphQL Subscriptions
-                                                    _b = __read(this.subscriptionsProcessor.start(), 2), 
-                                                    // const ctlObservable: Observable<CONTROL_MSG>
-                                                    ctlSubsObservable_1 = _b[0], 
-                                                    // const dataObservable: Observable<[TransformerMutationType, SchemaModel, Readonly<{
-                                                    // id: string;
-                                                    // } & Record<string, any>>]>
-                                                    dataSubsObservable = _b[1];
-                                                    _c.label = 2;
-                                                case 2:
-                                                    _c.trys.push([2, 4, , 5]);
-                                                    return [4 /*yield*/, new Promise(function (resolve, reject) {
-                                                            var ctlSubsSubscription = ctlSubsObservable_1.subscribe({
-                                                                next: function (msg) {
-                                                                    if (msg === subscription_1.CONTROL_MSG.CONNECTED) {
-                                                                        resolve();
+                                        return __generator(this, function (_b) {
+                                            return [2 /*return*/, this.runningProcesses.isOpen &&
+                                                    this.runningProcesses.add(function (onTerminate) { return __awaiter(_this, void 0, void 0, function () {
+                                                        var ctlSubsObservable_1, dataSubsObservable, err_2, error_1;
+                                                        var _a;
+                                                        var _this = this;
+                                                        return __generator(this, function (_b) {
+                                                            switch (_b.label) {
+                                                                case 0:
+                                                                    if (!(online && !this.online)) return [3 /*break*/, 10];
+                                                                    this.online = online;
+                                                                    observer.next({
+                                                                        type: ControlMessage.SYNC_ENGINE_NETWORK_STATUS,
+                                                                        data: {
+                                                                            active: this.online,
+                                                                        },
+                                                                    });
+                                                                    dataSubsObservable = void 0;
+                                                                    if (!isNode) return [3 /*break*/, 1];
+                                                                    logger.warn('Realtime disabled when in a server-side environment');
+                                                                    return [3 /*break*/, 6];
+                                                                case 1:
+                                                                    //#region GraphQL Subscriptions
+                                                                    _a = __read(this.subscriptionsProcessor.start(), 2), 
+                                                                    // const ctlObservable: Observable<CONTROL_MSG>
+                                                                    ctlSubsObservable_1 = _a[0], 
+                                                                    // const dataObservable: Observable<[TransformerMutationType, SchemaModel, Readonly<{
+                                                                    // id: string;
+                                                                    // } & Record<string, any>>]>
+                                                                    dataSubsObservable = _a[1];
+                                                                    _b.label = 2;
+                                                                case 2:
+                                                                    _b.trys.push([2, 4, , 5]);
+                                                                    return [4 /*yield*/, new Promise(function (resolve, reject) {
+                                                                            onTerminate.then(reject);
+                                                                            var ctlSubsSubscription = ctlSubsObservable_1.subscribe({
+                                                                                next: function (msg) {
+                                                                                    if (msg === subscription_1.CONTROL_MSG.CONNECTED) {
+                                                                                        resolve();
+                                                                                    }
+                                                                                },
+                                                                                error: function (err) {
+                                                                                    reject(err);
+                                                                                    var handleDisconnect = _this.disconnectionHandler();
+                                                                                    handleDisconnect(err);
+                                                                                },
+                                                                            });
+                                                                            subscriptions.push(ctlSubsSubscription);
+                                                                        })];
+                                                                case 3:
+                                                                    _b.sent();
+                                                                    return [3 /*break*/, 5];
+                                                                case 4:
+                                                                    err_2 = _b.sent();
+                                                                    observer.error(err_2);
+                                                                    failedStarting();
+                                                                    return [2 /*return*/];
+                                                                case 5:
+                                                                    logger.log('Realtime ready');
+                                                                    observer.next({
+                                                                        type: ControlMessage.SYNC_ENGINE_SUBSCRIPTIONS_ESTABLISHED,
+                                                                    });
+                                                                    _b.label = 6;
+                                                                case 6:
+                                                                    _b.trys.push([6, 8, , 9]);
+                                                                    return [4 /*yield*/, new Promise(function (resolve, reject) {
+                                                                            var syncQuerySubscription = _this.syncQueriesObservable().subscribe({
+                                                                                next: function (message) {
+                                                                                    var type = message.type;
+                                                                                    if (type ===
+                                                                                        ControlMessage.SYNC_ENGINE_SYNC_QUERIES_READY) {
+                                                                                        resolve();
+                                                                                    }
+                                                                                    observer.next(message);
+                                                                                },
+                                                                                complete: function () {
+                                                                                    resolve();
+                                                                                },
+                                                                                error: function (error) {
+                                                                                    reject(error);
+                                                                                },
+                                                                            });
+                                                                            if (syncQuerySubscription) {
+                                                                                subscriptions.push(syncQuerySubscription);
+                                                                            }
+                                                                        })];
+                                                                case 7:
+                                                                    _b.sent();
+                                                                    return [3 /*break*/, 9];
+                                                                case 8:
+                                                                    error_1 = _b.sent();
+                                                                    observer.error(error_1);
+                                                                    failedStarting();
+                                                                    return [2 /*return*/];
+                                                                case 9:
+                                                                    //#endregion
+                                                                    //#region process mutations (outbox)
+                                                                    subscriptions.push(this.mutationsProcessor
+                                                                        .start()
+                                                                        .subscribe(function (_a) {
+                                                                        var modelDefinition = _a.modelDefinition, item = _a.model, hasMore = _a.hasMore;
+                                                                        return _this.runningProcesses.add(function () { return __awaiter(_this, void 0, void 0, function () {
+                                                                            var modelConstructor, model;
+                                                                            var _this = this;
+                                                                            return __generator(this, function (_a) {
+                                                                                switch (_a.label) {
+                                                                                    case 0:
+                                                                                        modelConstructor = this.userModelClasses[modelDefinition.name];
+                                                                                        model = this.modelInstanceCreator(modelConstructor, item);
+                                                                                        return [4 /*yield*/, this.storage.runExclusive(function (storage) {
+                                                                                                return _this.modelMerger.merge(storage, model, modelDefinition);
+                                                                                            })];
+                                                                                    case 1:
+                                                                                        _a.sent();
+                                                                                        observer.next({
+                                                                                            type: ControlMessage.SYNC_ENGINE_OUTBOX_MUTATION_PROCESSED,
+                                                                                            data: {
+                                                                                                model: modelConstructor,
+                                                                                                element: model,
+                                                                                            },
+                                                                                        });
+                                                                                        observer.next({
+                                                                                            type: ControlMessage.SYNC_ENGINE_OUTBOX_STATUS,
+                                                                                            data: {
+                                                                                                isEmpty: !hasMore,
+                                                                                            },
+                                                                                        });
+                                                                                        return [2 /*return*/];
+                                                                                }
+                                                                            });
+                                                                        }); }, 'mutation processor event');
+                                                                    }));
+                                                                    //#endregion
+                                                                    //#region Merge subscriptions buffer
+                                                                    // TODO: extract to function
+                                                                    if (!isNode) {
+                                                                        subscriptions.push(dataSubsObservable.subscribe(function (_a) {
+                                                                            var _b = __read(_a, 3), _transformerMutationType = _b[0], modelDefinition = _b[1], item = _b[2];
+                                                                            return _this.runningProcesses.add(function () { return __awaiter(_this, void 0, void 0, function () {
+                                                                                var modelConstructor, model;
+                                                                                var _this = this;
+                                                                                return __generator(this, function (_a) {
+                                                                                    switch (_a.label) {
+                                                                                        case 0:
+                                                                                            modelConstructor = this.userModelClasses[modelDefinition.name];
+                                                                                            model = this.modelInstanceCreator(modelConstructor, item);
+                                                                                            return [4 /*yield*/, this.storage.runExclusive(function (storage) {
+                                                                                                    return _this.modelMerger.merge(storage, model, modelDefinition);
+                                                                                                })];
+                                                                                        case 1:
+                                                                                            _a.sent();
+                                                                                            return [2 /*return*/];
+                                                                                    }
+                                                                                });
+                                                                            }); }, 'subscription dataSubsObservable event');
+                                                                        }));
                                                                     }
-                                                                },
-                                                                error: function (err) {
-                                                                    reject(err);
-                                                                    var handleDisconnect = _this.disconnectionHandler();
-                                                                    handleDisconnect(err);
-                                                                },
-                                                            });
-                                                            subscriptions.push(ctlSubsSubscription);
-                                                        })];
-                                                case 3:
-                                                    _c.sent();
-                                                    return [3 /*break*/, 5];
-                                                case 4:
-                                                    err_2 = _c.sent();
-                                                    observer.error(err_2);
-                                                    return [2 /*return*/];
-                                                case 5:
-                                                    logger.log('Realtime ready');
-                                                    observer.next({
-                                                        type: ControlMessage.SYNC_ENGINE_SUBSCRIPTIONS_ESTABLISHED,
-                                                    });
-                                                    _c.label = 6;
-                                                case 6:
-                                                    _c.trys.push([6, 8, , 9]);
-                                                    return [4 /*yield*/, new Promise(function (resolve, reject) {
-                                                            var syncQuerySubscription = _this.syncQueriesObservable().subscribe({
-                                                                next: function (message) {
-                                                                    var type = message.type;
-                                                                    if (type === ControlMessage.SYNC_ENGINE_SYNC_QUERIES_READY) {
-                                                                        resolve();
+                                                                    return [3 /*break*/, 11];
+                                                                case 10:
+                                                                    if (!online) {
+                                                                        this.online = online;
+                                                                        observer.next({
+                                                                            type: ControlMessage.SYNC_ENGINE_NETWORK_STATUS,
+                                                                            data: {
+                                                                                active: this.online,
+                                                                            },
+                                                                        });
+                                                                        subscriptions.forEach(function (sub) { return sub.unsubscribe(); });
+                                                                        subscriptions = [];
                                                                     }
-                                                                    observer.next(message);
-                                                                },
-                                                                complete: function () {
-                                                                    resolve();
-                                                                },
-                                                                error: function (error) {
-                                                                    reject(error);
-                                                                },
-                                                            });
-                                                            if (syncQuerySubscription) {
-                                                                subscriptions.push(syncQuerySubscription);
+                                                                    _b.label = 11;
+                                                                case 11:
+                                                                    doneStarting();
+                                                                    return [2 /*return*/];
                                                             }
-                                                        })];
-                                                case 7:
-                                                    _c.sent();
-                                                    return [3 /*break*/, 9];
-                                                case 8:
-                                                    error_1 = _c.sent();
-                                                    observer.error(error_1);
-                                                    return [2 /*return*/];
-                                                case 9:
-                                                    //#endregion
-                                                    //#region process mutations
-                                                    subscriptions.push(this.mutationsProcessor
-                                                        .start()
-                                                        .subscribe(function (_a) {
-                                                        var modelDefinition = _a.modelDefinition, item = _a.model, hasMore = _a.hasMore;
-                                                        var modelConstructor = _this.userModelClasses[modelDefinition.name];
-                                                        var model = _this.modelInstanceCreator(modelConstructor, item);
-                                                        _this.storage.runExclusive(function (storage) {
-                                                            return _this.modelMerger.merge(storage, model, modelDefinition);
-                                                        });
-                                                        observer.next({
-                                                            type: ControlMessage.SYNC_ENGINE_OUTBOX_MUTATION_PROCESSED,
-                                                            data: {
-                                                                model: modelConstructor,
-                                                                element: model,
-                                                            },
-                                                        });
-                                                        observer.next({
-                                                            type: ControlMessage.SYNC_ENGINE_OUTBOX_STATUS,
-                                                            data: {
-                                                                isEmpty: !hasMore,
-                                                            },
                                                         });
-                                                    }));
-                                                    //#endregion
-                                                    //#region Merge subscriptions buffer
-                                                    // TODO: extract to function
-                                                    if (!isNode) {
-                                                        subscriptions.push(dataSubsObservable.subscribe(function (_a) {
-                                                            var _b = __read(_a, 3), _transformerMutationType = _b[0], modelDefinition = _b[1], item = _b[2];
-                                                            var modelConstructor = _this.userModelClasses[modelDefinition.name];
-                                                            var model = _this.modelInstanceCreator(modelConstructor, item);
-                                                            _this.storage.runExclusive(function (storage) {
-                                                                return _this.modelMerger.merge(storage, model, modelDefinition);
-                                                            });
-                                                        }));
-                                                    }
-                                                    return [3 /*break*/, 11];
-                                                case 10:
-                                                    if (!online) {
-                                                        this.online = online;
-                                                        observer.next({
-                                                            type: ControlMessage.SYNC_ENGINE_NETWORK_STATUS,
-                                                            data: {
-                                                                active: this.online,
-                                                            },
-                                                        });
-                                                        subscriptions.forEach(function (sub) { return sub.unsubscribe(); });
-                                                        subscriptions = [];
-                                                    }
-                                                    _c.label = 11;
-                                                case 11:
-                                                    resolve();
-                                                    return [2 /*return*/];
-                                            }
+                                                    }); }, 'datastore connectivity event')];
                                         });
                                     });
                                 });
@@ -309,39 +346,44 @@ var SyncEngine = /** @class */ (function () {
                                 next: function (_a) {
                                     var opType = _a.opType, model = _a.model, element = _a.element, condition = _a.condition;
                                     return __awaiter(_this, void 0, void 0, function () {
-                                        var namespace, MutationEventConstructor, modelDefinition, graphQLCondition, mutationEvent;
+                                        var _this = this;
                                         return __generator(this, function (_b) {
-                                            switch (_b.label) {
-                                                case 0:
-                                                    namespace = this.schema.namespaces[this.namespaceResolver(model)];
-                                                    MutationEventConstructor = this.modelClasses['MutationEvent'];
-                                                    modelDefinition = this.getModelDefinition(model);
-                                                    graphQLCondition = utils_1.predicateToGraphQLCondition(condition, modelDefinition);
-                                                    mutationEvent = utils_1.createMutationInstanceFromModelOperation(namespace.relationships, this.getModelDefinition(model), opType, model, element, graphQLCondition, MutationEventConstructor, this.modelInstanceCreator);
-                                                    return [4 /*yield*/, this.outbox.enqueue(this.storage, mutationEvent)];
-                                                case 1:
-                                                    _b.sent();
-                                                    observer.next({
-                                                        type: ControlMessage.SYNC_ENGINE_OUTBOX_MUTATION_ENQUEUED,
-                                                        data: {
-                                                            model: model,
-                                                            element: element,
-                                                        },
-                                                    });
-                                                    observer.next({
-                                                        type: ControlMessage.SYNC_ENGINE_OUTBOX_STATUS,
-                                                        data: {
-                                                            isEmpty: false,
-                                                        },
+                                            return [2 /*return*/, this.runningProcesses.add(function () { return __awaiter(_this, void 0, void 0, function () {
+                                                    var namespace, MutationEventConstructor, modelDefinition, graphQLCondition, mutationEvent;
+                                                    return __generator(this, function (_a) {
+                                                        switch (_a.label) {
+                                                            case 0:
+                                                                namespace = this.schema.namespaces[this.namespaceResolver(model)];
+                                                                MutationEventConstructor = this.modelClasses['MutationEvent'];
+                                                                modelDefinition = this.getModelDefinition(model);
+                                                                graphQLCondition = utils_1.predicateToGraphQLCondition(condition, modelDefinition);
+                                                                mutationEvent = utils_1.createMutationInstanceFromModelOperation(namespace.relationships, this.getModelDefinition(model), opType, model, element, graphQLCondition, MutationEventConstructor, this.modelInstanceCreator);
+                                                                return [4 /*yield*/, this.outbox.enqueue(this.storage, mutationEvent)];
+                                                            case 1:
+                                                                _a.sent();
+                                                                observer.next({
+                                                                    type: ControlMessage.SYNC_ENGINE_OUTBOX_MUTATION_ENQUEUED,
+                                                                    data: {
+                                                                        model: model,
+                                                                        element: element,
+                                                                    },
+                                                                });
+                                                                observer.next({
+                                                                    type: ControlMessage.SYNC_ENGINE_OUTBOX_STATUS,
+                                                                    data: {
+                                                                        isEmpty: false,
+                                                                    },
+                                                                });
+                                                                return [4 /*yield*/, startPromise];
+                                                            case 2:
+                                                                _a.sent();
+                                                                if (this.online) {
+                                                                    this.mutationsProcessor.resume();
+                                                                }
+                                                                return [2 /*return*/];
+                                                        }
                                                     });
-                                                    return [4 /*yield*/, startPromise];
-                                                case 2:
-                                                    _b.sent();
-                                                    if (this.online) {
-                                                        this.mutationsProcessor.resume();
-                                                    }
-                                                    return [2 /*return*/];
-                                            }
+                                                }); }, 'storage event')];
                                         });
                                     });
                                 },
@@ -367,10 +409,7 @@ var SyncEngine = /** @class */ (function () {
                             return [2 /*return*/];
                     }
                 });
-            }); })();
-            return function () {
-                subscriptions.forEach(function (sub) { return sub.unsubscribe(); });
-            };
+            }); }, 'sync start');
         });
     };
     SyncEngine.prototype.getModelsMetadataWithNextFullSync = function (currentTimeStamp) {
@@ -381,7 +420,7 @@ var SyncEngine = /** @class */ (function () {
                 switch (_b.label) {
                     case 0:
                         _a = Map.bind;
-                        return [4 /*yield*/, this.getModelsMetadata()];
+                        return [4 /*yield*/, this.runningProcesses.add(function () { return _this.getModelsMetadata(); }, 'sync/index getModelsMetadataWithNextFullSync')];
                     case 1:
                         modelLastSync = new (_a.apply(Map, [void 0, (_b.sent()).map(function (_a) {
                                 var namespace = _a.namespace, model = _a.model, lastSync = _a.lastSync, lastFullSync = _a.lastFullSync, fullSyncInterval = _a.fullSyncInterval, lastSyncPredicate = _a.lastSyncPredicate;
@@ -406,228 +445,261 @@ var SyncEngine = /** @class */ (function () {
         }
         return new zen_observable_ts_1.default(function (observer) {
             var syncQueriesSubscription;
-            var waitTimeoutId;
-            (function () { return __awaiter(_this, void 0, void 0, function () {
-                var _loop_1, this_1;
-                var _this = this;
-                return __generator(this, function (_a) {
-                    switch (_a.label) {
-                        case 0:
-                            _loop_1 = function () {
-                                var count, modelLastSync, paginatingModels, newestFullSyncStartedAt, theInterval, start, duration, newestStartedAt, msNextFullSync;
-                                return __generator(this, function (_a) {
-                                    switch (_a.label) {
-                                        case 0:
-                                            count = new WeakMap();
-                                            return [4 /*yield*/, this_1.getModelsMetadataWithNextFullSync(Date.now())];
-                                        case 1:
-                                            modelLastSync = _a.sent();
-                                            paginatingModels = new Set(modelLastSync.keys());
-                                            return [4 /*yield*/, new Promise(function (resolve) {
-                                                    syncQueriesSubscription = _this.syncQueriesProcessor
-                                                        .start(modelLastSync)
-                                                        .subscribe({
-                                                        next: function (_a) {
-                                                            var namespace = _a.namespace, modelDefinition = _a.modelDefinition, items = _a.items, done = _a.done, startedAt = _a.startedAt, isFullSync = _a.isFullSync;
-                                                            return __awaiter(_this, void 0, void 0, function () {
-                                                                var modelConstructor, modelName, modelMetadata_1, lastFullSync, fullSyncInterval, counts;
-                                                                var _this = this;
-                                                                return __generator(this, function (_b) {
-                                                                    switch (_b.label) {
-                                                                        case 0:
-                                                                            modelConstructor = this.userModelClasses[modelDefinition.name];
-                                                                            if (!count.has(modelConstructor)) {
-                                                                                count.set(modelConstructor, {
-                                                                                    new: 0,
-                                                                                    updated: 0,
-                                                                                    deleted: 0,
-                                                                                });
-                                                                                start = util_1.getNow();
-                                                                                newestStartedAt =
-                                                                                    newestStartedAt === undefined
-                                                                                        ? startedAt
-                                                                                        : Math.max(newestStartedAt, startedAt);
-                                                                            }
-                                                                            /**
-                                                                             * If there are mutations in the outbox for a given id, those need to be
-                                                                             * merged individually. Otherwise, we can merge them in batches.
-                                                                             */
-                                                                            return [4 /*yield*/, this.storage.runExclusive(function (storage) { return __awaiter(_this, void 0, void 0, function () {
-                                                                                    var idsInOutbox, oneByOne, page, opTypeCount, oneByOne_1, oneByOne_1_1, item, opType, e_1_1, _a, _b, _c, counts;
-                                                                                    var e_1, _d;
-                                                                                    return __generator(this, function (_e) {
-                                                                                        switch (_e.label) {
-                                                                                            case 0: return [4 /*yield*/, this.outbox.getModelIds(storage)];
-                                                                                            case 1:
-                                                                                                idsInOutbox = _e.sent();
-                                                                                                oneByOne = [];
-                                                                                                page = items.filter(function (item) {
-                                                                                                    var itemId = utils_1.getIdentifierValue(modelDefinition, item);
-                                                                                                    if (!idsInOutbox.has(itemId)) {
-                                                                                                        return true;
+            _this.runningProcesses.isOpen &&
+                _this.runningProcesses.add(function (onTerminate) { return __awaiter(_this, void 0, void 0, function () {
+                    var terminated, _loop_1, this_1;
+                    var _this = this;
+                    return __generator(this, function (_a) {
+                        switch (_a.label) {
+                            case 0:
+                                terminated = false;
+                                _loop_1 = function () {
+                                    var count, modelLastSync, paginatingModels, newestFullSyncStartedAt, theInterval, start, duration, newestStartedAt, msNextFullSync;
+                                    return __generator(this, function (_a) {
+                                        switch (_a.label) {
+                                            case 0:
+                                                count = new WeakMap();
+                                                return [4 /*yield*/, this_1.getModelsMetadataWithNextFullSync(Date.now())];
+                                            case 1:
+                                                modelLastSync = _a.sent();
+                                                paginatingModels = new Set(modelLastSync.keys());
+                                                return [4 /*yield*/, new Promise(function (resolve, reject) {
+                                                        if (!_this.runningProcesses.isOpen)
+                                                            resolve();
+                                                        onTerminate.then(function () { return resolve(); });
+                                                        syncQueriesSubscription = _this.syncQueriesProcessor
+                                                            .start(modelLastSync)
+                                                            .subscribe({
+                                                            next: function (_a) {
+                                                                var namespace = _a.namespace, modelDefinition = _a.modelDefinition, items = _a.items, done = _a.done, startedAt = _a.startedAt, isFullSync = _a.isFullSync;
+                                                                return __awaiter(_this, void 0, void 0, function () {
+                                                                    var modelConstructor, modelName, modelMetadata_1, lastFullSync, fullSyncInterval, counts;
+                                                                    var _this = this;
+                                                                    return __generator(this, function (_b) {
+                                                                        switch (_b.label) {
+                                                                            case 0:
+                                                                                modelConstructor = this.userModelClasses[modelDefinition.name];
+                                                                                if (!count.has(modelConstructor)) {
+                                                                                    count.set(modelConstructor, {
+                                                                                        new: 0,
+                                                                                        updated: 0,
+                                                                                        deleted: 0,
+                                                                                    });
+                                                                                    start = util_1.getNow();
+                                                                                    newestStartedAt =
+                                                                                        newestStartedAt === undefined
+                                                                                            ? startedAt
+                                                                                            : Math.max(newestStartedAt, startedAt);
+                                                                                }
+                                                                                /**
+                                                                                 * If there are mutations in the outbox for a given id, those need to be
+                                                                                 * merged individually. Otherwise, we can merge them in batches.
+                                                                                 */
+                                                                                return [4 /*yield*/, this.storage.runExclusive(function (storage) { return __awaiter(_this, void 0, void 0, function () {
+                                                                                        var idsInOutbox, oneByOne, page, opTypeCount, oneByOne_1, oneByOne_1_1, item, opType, e_1_1, _a, _b, _c, counts;
+                                                                                        var e_1, _d;
+                                                                                        return __generator(this, function (_e) {
+                                                                                            switch (_e.label) {
+                                                                                                case 0: return [4 /*yield*/, this.outbox.getModelIds(storage)];
+                                                                                                case 1:
+                                                                                                    idsInOutbox = _e.sent();
+                                                                                                    oneByOne = [];
+                                                                                                    page = items.filter(function (item) {
+                                                                                                        var itemId = utils_1.getIdentifierValue(modelDefinition, item);
+                                                                                                        if (!idsInOutbox.has(itemId)) {
+                                                                                                            return true;
+                                                                                                        }
+                                                                                                        oneByOne.push(item);
+                                                                                                        return false;
+                                                                                                    });
+                                                                                                    opTypeCount = [];
+                                                                                                    _e.label = 2;
+                                                                                                case 2:
+                                                                                                    _e.trys.push([2, 7, 8, 9]);
+                                                                                                    oneByOne_1 = __values(oneByOne), oneByOne_1_1 = oneByOne_1.next();
+                                                                                                    _e.label = 3;
+                                                                                                case 3:
+                                                                                                    if (!!oneByOne_1_1.done) return [3 /*break*/, 6];
+                                                                                                    item = oneByOne_1_1.value;
+                                                                                                    return [4 /*yield*/, this.modelMerger.merge(storage, item, modelDefinition)];
+                                                                                                case 4:
+                                                                                                    opType = _e.sent();
+                                                                                                    if (opType !== undefined) {
+                                                                                                        opTypeCount.push([item, opType]);
                                                                                                     }
-                                                                                                    oneByOne.push(item);
-                                                                                                    return false;
-                                                                                                });
-                                                                                                opTypeCount = [];
-                                                                                                _e.label = 2;
-                                                                                            case 2:
-                                                                                                _e.trys.push([2, 7, 8, 9]);
-                                                                                                oneByOne_1 = __values(oneByOne), oneByOne_1_1 = oneByOne_1.next();
-                                                                                                _e.label = 3;
-                                                                                            case 3:
-                                                                                                if (!!oneByOne_1_1.done) return [3 /*break*/, 6];
-                                                                                                item = oneByOne_1_1.value;
-                                                                                                return [4 /*yield*/, this.modelMerger.merge(storage, item, modelDefinition)];
-                                                                                            case 4:
-                                                                                                opType = _e.sent();
-                                                                                                if (opType !== undefined) {
-                                                                                                    opTypeCount.push([item, opType]);
-                                                                                                }
-                                                                                                _e.label = 5;
-                                                                                            case 5:
-                                                                                                oneByOne_1_1 = oneByOne_1.next();
-                                                                                                return [3 /*break*/, 3];
-                                                                                            case 6: return [3 /*break*/, 9];
-                                                                                            case 7:
-                                                                                                e_1_1 = _e.sent();
-                                                                                                e_1 = { error: e_1_1 };
-                                                                                                return [3 /*break*/, 9];
-                                                                                            case 8:
-                                                                                                try {
-                                                                                                    if (oneByOne_1_1 && !oneByOne_1_1.done && (_d = oneByOne_1.return)) _d.call(oneByOne_1);
-                                                                                                }
-                                                                                                finally { if (e_1) throw e_1.error; }
-                                                                                                return [7 /*endfinally*/];
-                                                                                            case 9:
-                                                                                                _b = (_a = opTypeCount.push).apply;
-                                                                                                _c = [opTypeCount];
-                                                                                                return [4 /*yield*/, this.modelMerger.mergePage(storage, modelConstructor, page, modelDefinition)];
-                                                                                            case 10:
-                                                                                                _b.apply(_a, _c.concat([__spread.apply(void 0, [(_e.sent())])]));
-                                                                                                counts = count.get(modelConstructor);
-                                                                                                opTypeCount.forEach(function (_a) {
-                                                                                                    var _b = __read(_a, 2), opType = _b[1];
-                                                                                                    switch (opType) {
-                                                                                                        case types_1.OpType.INSERT:
-                                                                                                            counts.new++;
-                                                                                                            break;
-                                                                                                        case types_1.OpType.UPDATE:
-                                                                                                            counts.updated++;
-                                                                                                            break;
-                                                                                                        case types_1.OpType.DELETE:
-                                                                                                            counts.deleted++;
-                                                                                                            break;
-                                                                                                        default:
-                                                                                                            util_1.exhaustiveCheck(opType);
+                                                                                                    _e.label = 5;
+                                                                                                case 5:
+                                                                                                    oneByOne_1_1 = oneByOne_1.next();
+                                                                                                    return [3 /*break*/, 3];
+                                                                                                case 6: return [3 /*break*/, 9];
+                                                                                                case 7:
+                                                                                                    e_1_1 = _e.sent();
+                                                                                                    e_1 = { error: e_1_1 };
+                                                                                                    return [3 /*break*/, 9];
+                                                                                                case 8:
+                                                                                                    try {
+                                                                                                        if (oneByOne_1_1 && !oneByOne_1_1.done && (_d = oneByOne_1.return)) _d.call(oneByOne_1);
                                                                                                     }
-                                                                                                });
-                                                                                                return [2 /*return*/];
-                                                                                        }
-                                                                                    });
-                                                                                }); })];
-                                                                        case 1:
-                                                                            /**
-                                                                             * If there are mutations in the outbox for a given id, those need to be
-                                                                             * merged individually. Otherwise, we can merge them in batches.
-                                                                             */
-                                                                            _b.sent();
-                                                                            if (!done) return [3 /*break*/, 4];
-                                                                            modelName = modelDefinition.name;
-                                                                            return [4 /*yield*/, this.getModelMetadata(namespace, modelName)];
-                                                                        case 2:
-                                                                            modelMetadata_1 = _b.sent();
-                                                                            lastFullSync = modelMetadata_1.lastFullSync, fullSyncInterval = modelMetadata_1.fullSyncInterval;
-                                                                            theInterval = fullSyncInterval;
-                                                                            newestFullSyncStartedAt =
-                                                                                newestFullSyncStartedAt === undefined
-                                                                                    ? lastFullSync
-                                                                                    : Math.max(newestFullSyncStartedAt, isFullSync ? startedAt : lastFullSync);
-                                                                            modelMetadata_1 = this.modelClasses
-                                                                                .ModelMetadata.copyOf(modelMetadata_1, function (draft) {
-                                                                                draft.lastSync = startedAt;
-                                                                                draft.lastFullSync = isFullSync
-                                                                                    ? startedAt
-                                                                                    : modelMetadata_1.lastFullSync;
-                                                                            });
-                                                                            return [4 /*yield*/, this.storage.save(modelMetadata_1, undefined, ownSymbol)];
-                                                                        case 3:
-                                                                            _b.sent();
-                                                                            counts = count.get(modelConstructor);
-                                                                            this.modelSyncedStatus.set(modelConstructor, true);
-                                                                            observer.next({
-                                                                                type: ControlMessage.SYNC_ENGINE_MODEL_SYNCED,
-                                                                                data: {
-                                                                                    model: modelConstructor,
-                                                                                    isFullSync: isFullSync,
-                                                                                    isDeltaSync: !isFullSync,
-                                                                                    counts: counts,
-                                                                                },
-                                                                            });
-                                                                            paginatingModels.delete(modelDefinition);
-                                                                            if (paginatingModels.size === 0) {
-                                                                                duration = util_1.getNow() - start;
-                                                                                resolve();
+                                                                                                    finally { if (e_1) throw e_1.error; }
+                                                                                                    return [7 /*endfinally*/];
+                                                                                                case 9:
+                                                                                                    _b = (_a = opTypeCount.push).apply;
+                                                                                                    _c = [opTypeCount];
+                                                                                                    return [4 /*yield*/, this.modelMerger.mergePage(storage, modelConstructor, page, modelDefinition)];
+                                                                                                case 10:
+                                                                                                    _b.apply(_a, _c.concat([__spread.apply(void 0, [(_e.sent())])]));
+                                                                                                    counts = count.get(modelConstructor);
+                                                                                                    opTypeCount.forEach(function (_a) {
+                                                                                                        var _b = __read(_a, 2), opType = _b[1];
+                                                                                                        switch (opType) {
+                                                                                                            case types_1.OpType.INSERT:
+                                                                                                                counts.new++;
+                                                                                                                break;
+                                                                                                            case types_1.OpType.UPDATE:
+                                                                                                                counts.updated++;
+                                                                                                                break;
+                                                                                                            case types_1.OpType.DELETE:
+                                                                                                                counts.deleted++;
+                                                                                                                break;
+                                                                                                            default:
+                                                                                                                util_1.exhaustiveCheck(opType);
+                                                                                                        }
+                                                                                                    });
+                                                                                                    return [2 /*return*/];
+                                                                                            }
+                                                                                        });
+                                                                                    }); })];
+                                                                            case 1:
+                                                                                /**
+                                                                                 * If there are mutations in the outbox for a given id, those need to be
+                                                                                 * merged individually. Otherwise, we can merge them in batches.
+                                                                                 */
+                                                                                _b.sent();
+                                                                                if (!done) return [3 /*break*/, 4];
+                                                                                modelName = modelDefinition.name;
+                                                                                return [4 /*yield*/, this.getModelMetadata(namespace, modelName)];
+                                                                            case 2:
+                                                                                modelMetadata_1 = _b.sent();
+                                                                                lastFullSync = modelMetadata_1.lastFullSync, fullSyncInterval = modelMetadata_1.fullSyncInterval;
+                                                                                theInterval = fullSyncInterval;
+                                                                                newestFullSyncStartedAt =
+                                                                                    newestFullSyncStartedAt === undefined
+                                                                                        ? lastFullSync
+                                                                                        : Math.max(newestFullSyncStartedAt, isFullSync ? startedAt : lastFullSync);
+                                                                                modelMetadata_1 = this.modelClasses
+                                                                                    .ModelMetadata.copyOf(modelMetadata_1, function (draft) {
+                                                                                    draft.lastSync = startedAt;
+                                                                                    draft.lastFullSync = isFullSync
+                                                                                        ? startedAt
+                                                                                        : modelMetadata_1.lastFullSync;
+                                                                                });
+                                                                                return [4 /*yield*/, this.storage.save(modelMetadata_1, undefined, ownSymbol)];
+                                                                            case 3:
+                                                                                _b.sent();
+                                                                                counts = count.get(modelConstructor);
+                                                                                this.modelSyncedStatus.set(modelConstructor, true);
                                                                                 observer.next({
-                                                                                    type: ControlMessage.SYNC_ENGINE_SYNC_QUERIES_READY,
+                                                                                    type: ControlMessage.SYNC_ENGINE_MODEL_SYNCED,
+                                                                                    data: {
+                                                                                        model: modelConstructor,
+                                                                                        isFullSync: isFullSync,
+                                                                                        isDeltaSync: !isFullSync,
+                                                                                        counts: counts,
+                                                                                    },
                                                                                 });
-                                                                                syncQueriesSubscription.unsubscribe();
-                                                                            }
-                                                                            _b.label = 4;
-                                                                        case 4: return [2 /*return*/];
-                                                                    }
+                                                                                paginatingModels.delete(modelDefinition);
+                                                                                if (paginatingModels.size === 0) {
+                                                                                    duration = util_1.getNow() - start;
+                                                                                    resolve();
+                                                                                    observer.next({
+                                                                                        type: ControlMessage.SYNC_ENGINE_SYNC_QUERIES_READY,
+                                                                                    });
+                                                                                    syncQueriesSubscription.unsubscribe();
+                                                                                }
+                                                                                _b.label = 4;
+                                                                            case 4: return [2 /*return*/];
+                                                                        }
+                                                                    });
                                                                 });
+                                                            },
+                                                            error: function (error) {
+                                                                observer.error(error);
+                                                            },
+                                                        });
+                                                        observer.next({
+                                                            type: ControlMessage.SYNC_ENGINE_SYNC_QUERIES_STARTED,
+                                                            data: {
+                                                                models: Array.from(paginatingModels).map(function (_a) {
+                                                                    var name = _a.name;
+                                                                    return name;
+                                                                }),
+                                                            },
+                                                        });
+                                                    })];
+                                            case 2:
+                                                _a.sent();
+                                                msNextFullSync = newestFullSyncStartedAt +
+                                                    theInterval -
+                                                    (newestStartedAt + duration);
+                                                logger.debug("Next fullSync in " + msNextFullSync / 1000 + " seconds. (" + new Date(Date.now() + msNextFullSync) + ")");
+                                                // TODO: create `BackgroundProcessManager.sleep()` ... but, need to put
+                                                // a lot of thought into what that contract looks like to
+                                                //  support possible use-cases:
+                                                //
+                                                //  1. non-cancelable
+                                                //  2. cancelable, unsleep on exit()
+                                                //  3. cancelable, throw Error on exit()
+                                                //  4. cancelable, callback first on exit()?
+                                                //  5. ... etc. ? ...
+                                                //
+                                                // TLDR; this is a lot of complexity here for a sleep(),
+                                                // but, it's not clear to me yet how to support an
+                                                // extensible, centralized cancelable `sleep()` elegantly.
+                                                return [4 /*yield*/, this_1.runningProcesses.add(function (onTerminate) { return __awaiter(_this, void 0, void 0, function () {
+                                                        var sleepTimer, unsleep, sleep;
+                                                        return __generator(this, function (_a) {
+                                                            sleep = new Promise(function (_unsleep) {
+                                                                unsleep = _unsleep;
+                                                                sleepTimer = setTimeout(unsleep, msNextFullSync);
                                                             });
-                                                        },
-                                                        error: function (error) {
-                                                            observer.error(error);
-                                                        },
-                                                    });
-                                                    observer.next({
-                                                        type: ControlMessage.SYNC_ENGINE_SYNC_QUERIES_STARTED,
-                                                        data: {
-                                                            models: Array.from(paginatingModels).map(function (_a) {
-                                                                var name = _a.name;
-                                                                return name;
-                                                            }),
-                                                        },
-                                                    });
-                                                })];
-                                        case 2:
-                                            _a.sent();
-                                            msNextFullSync = newestFullSyncStartedAt +
-                                                theInterval -
-                                                (newestStartedAt + duration);
-                                            logger.debug("Next fullSync in " + msNextFullSync / 1000 + " seconds. (" + new Date(Date.now() + msNextFullSync) + ")");
-                                            return [4 /*yield*/, new Promise(function (res) {
-                                                    waitTimeoutId = setTimeout(res, msNextFullSync);
-                                                })];
-                                        case 3:
-                                            _a.sent();
-                                            return [2 /*return*/];
-                                    }
-                                });
-                            };
-                            this_1 = this;
-                            _a.label = 1;
-                        case 1:
-                            if (!!observer.closed) return [3 /*break*/, 3];
-                            return [5 /*yield**/, _loop_1()];
-                        case 2:
-                            _a.sent();
-                            return [3 /*break*/, 1];
-                        case 3: return [2 /*return*/];
-                    }
-                });
-            }); })();
-            return function () {
-                if (syncQueriesSubscription) {
-                    syncQueriesSubscription.unsubscribe();
-                }
-                if (waitTimeoutId) {
-                    clearTimeout(waitTimeoutId);
-                }
-            };
+                                                            onTerminate.then(function () {
+                                                                terminated = true;
+                                                                unsleep();
+                                                            });
+                                                            return [2 /*return*/, sleep];
+                                                        });
+                                                    }); }, 'syncQueriesObservable sleep')];
+                                            case 3:
+                                                // TODO: create `BackgroundProcessManager.sleep()` ... but, need to put
+                                                // a lot of thought into what that contract looks like to
+                                                //  support possible use-cases:
+                                                //
+                                                //  1. non-cancelable
+                                                //  2. cancelable, unsleep on exit()
+                                                //  3. cancelable, throw Error on exit()
+                                                //  4. cancelable, callback first on exit()?
+                                                //  5. ... etc. ? ...
+                                                //
+                                                // TLDR; this is a lot of complexity here for a sleep(),
+                                                // but, it's not clear to me yet how to support an
+                                                // extensible, centralized cancelable `sleep()` elegantly.
+                                                _a.sent();
+                                                return [2 /*return*/];
+                                        }
+                                    });
+                                };
+                                this_1 = this;
+                                _a.label = 1;
+                            case 1:
+                                if (!(!observer.closed && !terminated)) return [3 /*break*/, 3];
+                                return [5 /*yield**/, _loop_1()];
+                            case 2:
+                                _a.sent();
+                                return [3 /*break*/, 1];
+                            case 3: return [2 /*return*/];
+                        }
+                    });
+                }); }, 'syncQueriesObservable main');
         });
     };
     SyncEngine.prototype.disconnectionHandler = function () {
@@ -643,6 +715,57 @@ var SyncEngine = /** @class */ (function () {
     SyncEngine.prototype.unsubscribeConnectivity = function () {
         this.datastoreConnectivity.unsubscribe();
     };
+    /**
+     * Stops all subscription activities and resolves when all activies report
+     * that they're disconnected, done retrying, etc..
+     */
+    SyncEngine.prototype.stop = function () {
+        return __awaiter(this, void 0, void 0, function () {
+            return __generator(this, function (_a) {
+                switch (_a.label) {
+                    case 0:
+                        logger.debug('stopping sync engine');
+                        /**
+                         * Gracefully disconnecting subscribers first just prevents *more* work
+                         * from entering the pipelines.
+                         */
+                        this.unsubscribeConnectivity();
+                        /**
+                         * aggressively shut down any lingering background processes.
+                         * some of this might be semi-redundant with unsubscribing. however,
+                         * unsubscribing doesn't allow us to wait for settling.
+                         * (Whereas `stop()` does.)
+                         */
+                        return [4 /*yield*/, this.mutationsProcessor.stop()];
+                    case 1:
+                        /**
+                         * aggressively shut down any lingering background processes.
+                         * some of this might be semi-redundant with unsubscribing. however,
+                         * unsubscribing doesn't allow us to wait for settling.
+                         * (Whereas `stop()` does.)
+                         */
+                        _a.sent();
+                        return [4 /*yield*/, this.subscriptionsProcessor.stop()];
+                    case 2:
+                        _a.sent();
+                        return [4 /*yield*/, this.datastoreConnectivity.stop()];
+                    case 3:
+                        _a.sent();
+                        return [4 /*yield*/, this.syncQueriesProcessor.stop()];
+                    case 4:
+                        _a.sent();
+                        return [4 /*yield*/, this.runningProcesses.close()];
+                    case 5:
+                        _a.sent();
+                        return [4 /*yield*/, this.runningProcesses.open()];
+                    case 6:
+                        _a.sent();
+                        logger.debug('sync engine stopped and ready to restart');
+                        return [2 /*return*/];
+                }
+            });
+        });
+    };
     SyncEngine.prototype.setupModels = function (params) {
         return __awaiter(this, void 0, void 0, function () {
             var fullSyncInterval, ModelMetadataConstructor, models, savedModel, promises, result, _a, _b, modelMetadata, modelName, e_2_1;