dexie-cloud-addon 1.0.0-beta.10 → 1.0.0-beta.11

package/dist/modern/dexie-cloud-addon.js

@@ -8,7 +8,7 @@
  *
  * ==========================================================================
  *
- * Version 1.0.0-beta.10, Wed Sep 08 2021
+ * Version 1.0.0-beta.10, Wed Oct 06 2021
  *
  * https://dexie.org
  *
@@ -2951,6 +2951,105 @@ class BroadcastedAndLocalEvent extends Observable$1 {
     }
 }
 
+async function computeRealmSetHash({ realms, inviteRealms, }) {
+    const data = JSON.stringify([
+        ...realms.map((realmId) => ({ realmId, accepted: true })),
+        ...inviteRealms.map((realmId) => ({ realmId, accepted: false })),
+    ].sort((a, b) => a.realmId < b.realmId ? -1 : a.realmId > b.realmId ? 1 : 0));
+    const byteArray = new TextEncoder().encode(data);
+    const digestBytes = await crypto.subtle.digest('SHA-1', byteArray);
+    const base64 = b64encode(digestBytes);
+    return base64;
+}
+
+function getSyncableTables(db) {
+    return Object.entries(db.cloud.schema || {})
+        .filter(([, { markedForSync }]) => markedForSync)
+        .map(([tbl]) => db.table(tbl));
+}
+
+function getMutationTable(tableName) {
+    return `$${tableName}_mutations`;
+}
+
+function getTableFromMutationTable(mutationTable) {
+    var _a;
+    const tableName = (_a = /^\$(.*)_mutations$/.exec(mutationTable)) === null || _a === void 0 ? void 0 : _a[1];
+    if (!tableName)
+        throw new Error(`Given mutationTable ${mutationTable} is not correct`);
+    return tableName;
+}
+
+async function listClientChanges(mutationTables, db, { since = {}, limit = Infinity } = {}) {
+    const allMutsOnTables = await Promise.all(mutationTables.map(async (mutationTable) => {
+        const tableName = getTableFromMutationTable(mutationTable.name);
+        const lastRevision = since[tableName];
+        let query = lastRevision
+            ? mutationTable.where("rev").above(lastRevision)
+            : mutationTable;
+        if (limit < Infinity)
+            query = query.limit(limit);
+        const muts = await query.toArray();
+        //const objTable = db.table(tableName);
+        /*for (const mut of muts) {
+          if (mut.type === "insert" || mut.type === "upsert") {
+            mut.values = await objTable.bulkGet(mut.keys);
+          }
+        }*/
+        return {
+            table: tableName,
+            muts,
+        };
+    }));
+    // Filter out those tables that doesn't have any mutations:
+    return allMutsOnTables.filter(({ muts }) => muts.length > 0);
+}
+
+async function listSyncifiedChanges(tablesToSyncify, currentUser, schema, alreadySyncedRealms) {
+    if (currentUser.isLoggedIn) {
+        if (tablesToSyncify.length > 0) {
+            const ignoredRealms = new Set(alreadySyncedRealms || []);
+            const inserts = await Promise.all(tablesToSyncify.map(async (table) => {
+                const { extractKey } = table.core.schema.primaryKey;
+                if (!extractKey)
+                    return { table: table.name, muts: [] }; // Outbound tables are not synced.
+                const dexieCloudTableSchema = schema[table.name];
+                const query = (dexieCloudTableSchema === null || dexieCloudTableSchema === void 0 ? void 0 : dexieCloudTableSchema.generatedGlobalId)
+                    ? table.filter((item) => !ignoredRealms.has(item.realmId || "") && isValidSyncableID(extractKey(item)))
+                    : table.filter((item) => !ignoredRealms.has(item.realmId || "") && isValidAtID(extractKey(item), dexieCloudTableSchema === null || dexieCloudTableSchema === void 0 ? void 0 : dexieCloudTableSchema.idPrefix));
+                const unsyncedObjects = await query.toArray();
+                if (unsyncedObjects.length > 0) {
+                    const mut = {
+                        type: "insert",
+                        values: unsyncedObjects,
+                        keys: unsyncedObjects.map(extractKey),
+                        userId: currentUser.userId,
+                    };
+                    return {
+                        table: table.name,
+                        muts: [mut],
+                    };
+                }
+                else {
+                    return {
+                        table: table.name,
+                        muts: []
+                    };
+                }
+            }));
+            return inserts.filter(op => op.muts.length > 0);
+        }
+    }
+    return [];
+}
+
+function getTablesToSyncify(db, syncState) {
+    const syncedTables = (syncState === null || syncState === void 0 ? void 0 : syncState.syncedTables) || [];
+    const syncableTables = getSyncableTables(db);
+    const tablesToSyncify = syncableTables.filter((tbl) => !syncedTables.includes(tbl.name));
+    return tablesToSyncify;
+}
+
 const { toString: toStr } = {};
 function getToStringTag(val) {
     return toStr.call(val).slice(8, -1);
@@ -3439,22 +3538,6 @@ var undefinedDef = {
 //   else
 //     serverRev.rev = new FakeBigInt(server.rev)
 const hasBigIntSupport = typeof BigInt(0) === 'bigint';
-function getValueOfBigInt(x) {
-    if (typeof x === 'bigint') {
-        return x;
-    }
-    if (hasBigIntSupport) {
-        return typeof x === 'string' ? BigInt(x) : BigInt(x.v);
-    }
-    else {
-        return typeof x === 'string' ? Number(x) : Number(x.v);
-    }
-}
-function compareBigInts(a, b) {
-    const valA = getValueOfBigInt(a);
-    const valB = getValueOfBigInt(b);
-    return valA < valB ? -1 : valA > valB ? 1 : 0;
-}
 class FakeBigInt {
     constructor(value) {
         this.v = value;
@@ -3483,105 +3566,6 @@ const defs = {
 const TSON = TypesonSimplified(builtin, defs);
 const BISON = Bison(defs);
 
-async function computeRealmSetHash({ realms, inviteRealms, }) {
-    const data = JSON.stringify([
-        ...realms.map((realmId) => ({ realmId, accepted: true })),
-        ...inviteRealms.map((realmId) => ({ realmId, accepted: false })),
-    ].sort((a, b) => a.realmId < b.realmId ? -1 : a.realmId > b.realmId ? 1 : 0));
-    const byteArray = new TextEncoder().encode(data);
-    const digestBytes = await crypto.subtle.digest('SHA-1', byteArray);
-    const base64 = b64encode(digestBytes);
-    return base64;
-}
-
-function getSyncableTables(db) {
-    return Object.entries(db.cloud.schema || {})
-        .filter(([, { markedForSync }]) => markedForSync)
-        .map(([tbl]) => db.table(tbl));
-}
-
-function getMutationTable(tableName) {
-    return `$${tableName}_mutations`;
-}
-
-function getTableFromMutationTable(mutationTable) {
-    var _a;
-    const tableName = (_a = /^\$(.*)_mutations$/.exec(mutationTable)) === null || _a === void 0 ? void 0 : _a[1];
-    if (!tableName)
-        throw new Error(`Given mutationTable ${mutationTable} is not correct`);
-    return tableName;
-}
-
-async function listClientChanges(mutationTables, db, { since = {}, limit = Infinity } = {}) {
-    const allMutsOnTables = await Promise.all(mutationTables.map(async (mutationTable) => {
-        const tableName = getTableFromMutationTable(mutationTable.name);
-        const lastRevision = since[tableName];
-        let query = lastRevision
-            ? mutationTable.where("rev").above(lastRevision)
-            : mutationTable;
-        if (limit < Infinity)
-            query = query.limit(limit);
-        const muts = await query.toArray();
-        //const objTable = db.table(tableName);
-        /*for (const mut of muts) {
-          if (mut.type === "insert" || mut.type === "upsert") {
-            mut.values = await objTable.bulkGet(mut.keys);
-          }
-        }*/
-        return {
-            table: tableName,
-            muts,
-        };
-    }));
-    // Filter out those tables that doesn't have any mutations:
-    return allMutsOnTables.filter(({ muts }) => muts.length > 0);
-}
-
-async function listSyncifiedChanges(tablesToSyncify, currentUser, schema, alreadySyncedRealms) {
-    if (currentUser.isLoggedIn) {
-        if (tablesToSyncify.length > 0) {
-            const ignoredRealms = new Set(alreadySyncedRealms || []);
-            const inserts = await Promise.all(tablesToSyncify.map(async (table) => {
-                const { extractKey } = table.core.schema.primaryKey;
-                if (!extractKey)
-                    return { table: table.name, muts: [] }; // Outbound tables are not synced.
-                const dexieCloudTableSchema = schema[table.name];
-                const query = (dexieCloudTableSchema === null || dexieCloudTableSchema === void 0 ? void 0 : dexieCloudTableSchema.generatedGlobalId)
-                    ? table.filter((item) => !ignoredRealms.has(item.realmId || "") && isValidSyncableID(extractKey(item)))
-                    : table.filter((item) => !ignoredRealms.has(item.realmId || "") && isValidAtID(extractKey(item), dexieCloudTableSchema === null || dexieCloudTableSchema === void 0 ? void 0 : dexieCloudTableSchema.idPrefix));
-                const unsyncedObjects = await query.toArray();
-                if (unsyncedObjects.length > 0) {
-                    const mut = {
-                        type: "insert",
-                        values: unsyncedObjects,
-                        keys: unsyncedObjects.map(extractKey),
-                        userId: currentUser.userId,
-                    };
-                    return {
-                        table: table.name,
-                        muts: [mut],
-                    };
-                }
-                else {
-                    return {
-                        table: table.name,
-                        muts: []
-                    };
-                }
-            }));
-            return inserts.filter(op => op.muts.length > 0);
-        }
-    }
-    return [];
-}
-
-function getTablesToSyncify(db, syncState) {
-    const syncedTables = (syncState === null || syncState === void 0 ? void 0 : syncState.syncedTables) || [];
-    const syncableTables = getSyncableTables(db);
-    const tablesToSyncify = syncableTables.filter((tbl) => !syncedTables.includes(tbl.name));
-    return tablesToSyncify;
-}
-
 //import {BisonWebStreamReader} from "dreambase-library/dist/typeson-simplified/BisonWebStreamReader";
 async function syncWithServer(changes, syncState, baseRevs, db, databaseUrl, schema, clientIdentity) {
     //
@@ -3596,6 +3580,7 @@ async function syncWithServer(changes, syncState, baseRevs, db, databaseUrl, sch
         headers.Authorization = `Bearer ${accessToken}`;
     }
     const syncRequest = {
+        v: 2,
         dbID: syncState === null || syncState === void 0 ? void 0 : syncState.remoteDbId,
         clientIdentity,
         schema: schema || {},
@@ -3722,8 +3707,8 @@ async function updateBaseRevs(db, schema, latestRevisions, serverRev) {
 
 function getLatestRevisionsPerTable(clientChangeSet, lastRevisions = {}) {
     for (const { table, muts } of clientChangeSet) {
-        const lastRev = muts.length > 0 ? muts[muts.length - 1].rev || 0 : 0;
-        lastRevisions[table] = lastRev;
+        const lastRev = muts.length > 0 ? muts[muts.length - 1].rev : null;
+        lastRevisions[table] = lastRev || lastRevisions[table] || 0;
     }
     return lastRevisions;
 }
@@ -4044,16 +4029,35 @@ function MessagesFromServerConsumer(db) {
     const readyToServe = new BehaviorSubject(true);
     const event = new BehaviorSubject(null);
     let isWorking = false;
+    let loopWarning = 0;
+    let loopDetection = [0, 0, 0, 0, 0, 0, 0, 0, 0, Date.now()];
     event.subscribe(async () => {
         if (isWorking)
             return;
         if (queue.length > 0) {
             isWorking = true;
+            loopDetection.shift();
+            loopDetection.push(Date.now());
             readyToServe.next(false);
             try {
                 await consumeQueue();
             }
             finally {
+                if (loopDetection[loopDetection.length - 1] - loopDetection[0] < 10000) {
+                    // Ten loops within 10 seconds. Slow down!
+                    if (Date.now() - loopWarning < 5000) {
+                        // Last time we did this, we ended up here too. Wait for a minute.
+                        console.warn(`Slowing down websocket loop for one minute`);
+                        loopWarning = Date.now() + 60000;
+                        await new Promise(resolve => setTimeout(resolve, 60000));
+                    }
+                    else {
+                        // This is a one-time event. Just pause 10 seconds.
+                        console.warn(`Slowing down websocket loop for 10 seconds`);
+                        loopWarning = Date.now() + 10000;
+                        await new Promise(resolve => setTimeout(resolve, 10000));
+                    }
+                }
                 isWorking = false;
                 readyToServe.next(true);
             }
@@ -4069,6 +4073,9 @@ function MessagesFromServerConsumer(db) {
             const msg = queue.shift();
             try {
                 console.debug('processing msg', msg);
+                // If the sync worker or service worker is syncing, wait 'til thei're done.
+                // It's no need to have two channels at the same time - even though it wouldnt
+                // be a problem - this is an optimization.
                 await db.cloud.syncState
                     .pipe(filter(({ phase }) => phase === 'in-sync' || phase === 'error'), take(1))
                     .toPromise();
@@ -4092,26 +4099,23 @@ function MessagesFromServerConsumer(db) {
                         // in turn will lead to that connectWebSocket.ts will reconnect the socket with the
                         // new token. So we don't need to do anything more here.
                         break;
-                    case 'rev':
-                        if (!(persistedSyncState === null || persistedSyncState === void 0 ? void 0 : persistedSyncState.serverRevision) ||
-                            compareBigInts(persistedSyncState.serverRevision, msg.rev) < 0) {
-                            triggerSync(db, "pull");
-                        }
-                        break;
                     case 'realm-added':
                         if (!((_a = persistedSyncState === null || persistedSyncState === void 0 ? void 0 : persistedSyncState.realms) === null || _a === void 0 ? void 0 : _a.includes(msg.realm))) {
-                            triggerSync(db, "pull");
+                            triggerSync(db, 'pull');
                         }
                         break;
                     case 'realm-removed':
                         if ((_b = persistedSyncState === null || persistedSyncState === void 0 ? void 0 : persistedSyncState.realms) === null || _b === void 0 ? void 0 : _b.includes(msg.realm)) {
-                            triggerSync(db, "pull");
+                            triggerSync(db, 'pull');
                         }
                         break;
+                    case 'realms-changed':
+                        triggerSync(db, 'pull');
+                        break;
                     case 'changes':
                         console.debug('changes');
                         if (((_c = db.cloud.syncState.value) === null || _c === void 0 ? void 0 : _c.phase) === 'error') {
-                            triggerSync(db, "pull");
+                            triggerSync(db, 'pull');
                             break;
                         }
                         await db.transaction('rw', db.dx.tables, async (tx) => {
@@ -4134,16 +4138,34 @@ function MessagesFromServerConsumer(db) {
                                 return; // Initial sync must have taken place - otherwise, ignore this.
                             }
                             // Verify again in ACID tx that we're on same server revision.
-                            if (compareBigInts(msg.baseRev, syncState.serverRevision) !== 0) {
+                            if (msg.baseRev !== syncState.serverRevision) {
                                 console.debug(`baseRev (${msg.baseRev}) differs from our serverRevision in syncState (${syncState.serverRevision})`);
+                                // Should we trigger a sync now? No. This is a normal case
+                                // when another local peer (such as the SW or a websocket channel on other tab) has
+                                // updated syncState from new server information but we are not aware yet. It would
+                                // be unnescessary to do a sync in that case. Instead, the caller of this consumeQueue()
+                                // function will do readyToServe.next(true) right after this return, which will lead
+                                // to a "ready" message being sent to server with the new accurate serverRev we have,
+                                // so that the next message indeed will be correct.
+                                if (typeof msg.baseRev === 'string' && // v2 format
+                                    (typeof syncState.serverRevision === 'bigint' || // v1 format
+                                        typeof syncState.serverRevision === 'object') // v1 format old browser
+                                ) {
+                                    // The reason for the diff seems to be that server has migrated the revision format.
+                                    // Do a full sync to update revision format.
+                                    // If we don't do a sync request now, we could stuck in an endless loop.
+                                    triggerSync(db, 'pull');
+                                }
                                 return; // Ignore message
                             }
                             // Verify also that the message is based on the exact same set of realms
-                            const ourRealmSetHash = await Dexie.waitFor(computeRealmSetHash(syncState));
+                            const ourRealmSetHash = await Dexie.waitFor(
+                            // Keep TX in non-IDB work
+                            computeRealmSetHash(syncState));
                             console.debug('ourRealmSetHash', ourRealmSetHash);
                             if (ourRealmSetHash !== msg.realmSetHash) {
                                 console.debug('not same realmSetHash', msg.realmSetHash);
-                                triggerSync(db, "pull");
+                                triggerSync(db, 'pull');
                                 // The message isn't based on the same realms.
                                 // Trigger a sync instead to resolve all things up.
                                 return;
@@ -4155,12 +4177,14 @@ function MessagesFromServerConsumer(db) {
                                 clientChanges = await listClientChanges(mutationTables, db);
                                 console.debug('msg queue: client changes', clientChanges);
                             }
-                            const filteredChanges = filterServerChangesThroughAddedClientChanges(msg.changes, clientChanges);
-                            //
-                            // apply server changes
-                            //
-                            console.debug('applying filtered server changes', filteredChanges);
-                            await applyServerChanges(filteredChanges, db);
+                            if (msg.changes.length > 0) {
+                                const filteredChanges = filterServerChangesThroughAddedClientChanges(msg.changes, clientChanges);
+                                //
+                                // apply server changes
+                                //
+                                console.debug('applying filtered server changes', filteredChanges);
+                                await applyServerChanges(filteredChanges, db);
+                            }
                             // Update latest revisions per table in case there are unsynced changes
                             // This can be a real case in future when we allow non-eagery sync.
                             // And it can actually be realistic now also, but very rare.
@@ -5036,18 +5060,19 @@ const SERVER_PING_TIMEOUT = 20000;
 const CLIENT_PING_INTERVAL = 30000;
 const FAIL_RETRY_WAIT_TIME = 60000;
 class WSObservable extends Observable$1 {
-    constructor(databaseUrl, rev, clientIdentity, messageProducer, webSocketStatus, token, tokenExpiration) {
-        super((subscriber) => new WSConnection(databaseUrl, rev, clientIdentity, token, tokenExpiration, subscriber, messageProducer, webSocketStatus));
+    constructor(databaseUrl, rev, realmSetHash, clientIdentity, messageProducer, webSocketStatus, token, tokenExpiration) {
+        super((subscriber) => new WSConnection(databaseUrl, rev, realmSetHash, clientIdentity, token, tokenExpiration, subscriber, messageProducer, webSocketStatus));
     }
 }
 let counter = 0;
 class WSConnection extends Subscription$1 {
-    constructor(databaseUrl, rev, clientIdentity, token, tokenExpiration, subscriber, messageProducer, webSocketStatus) {
+    constructor(databaseUrl, rev, realmSetHash, clientIdentity, token, tokenExpiration, subscriber, messageProducer, webSocketStatus) {
         super(() => this.teardown());
         this.id = ++counter;
         console.debug('New WebSocket Connection', this.id, token ? 'authorized' : 'unauthorized');
         this.databaseUrl = databaseUrl;
         this.rev = rev;
+        this.realmSetHash = realmSetHash;
         this.clientIdentity = clientIdentity;
         this.token = token;
         this.tokenExpiration = tokenExpiration;
@@ -5152,7 +5177,9 @@ class WSConnection extends Subscription$1 {
         const searchParams = new URLSearchParams();
         if (this.subscriber.closed)
             return;
+        searchParams.set('v', "2");
         searchParams.set('rev', this.rev);
+        searchParams.set('realmsHash', this.realmSetHash);
         searchParams.set('clientId', this.clientIdentity);
         if (this.token) {
             searchParams.set('token', this.token);
@@ -5246,12 +5273,12 @@ function connectWebSocket(db) {
     function createObservable() {
         return db.cloud.persistedSyncState.pipe(filter(syncState => syncState === null || syncState === void 0 ? void 0 : syncState.serverRevision), // Don't connect before there's no initial sync performed.
         take(1), // Don't continue waking up whenever syncState change
-        switchMap(() => db.cloud.currentUser), switchMap((userLogin) => userIsReallyActive.pipe(map((isActive) => (isActive ? userLogin : null)))), switchMap((userLogin) => 
+        switchMap((syncState) => db.cloud.currentUser.pipe(map(userLogin => [userLogin, syncState]))), switchMap(([userLogin, syncState]) => userIsReallyActive.pipe(map((isActive) => [isActive ? userLogin : null, syncState]))), switchMap(async ([userLogin, syncState]) => [userLogin, await computeRealmSetHash(syncState)]), switchMap(([userLogin, realmSetHash]) => 
         // Let server end query changes from last entry of same client-ID and forward.
         // If no new entries, server won't bother the client. If new entries, server sends only those
         // and the baseRev of the last from same client-ID.
         userLogin
-            ? new WSObservable(db.cloud.options.databaseUrl, db.cloud.persistedSyncState.value.serverRevision, db.cloud.persistedSyncState.value.clientIdentity, messageProducer, db.cloud.webSocketStatus, userLogin.accessToken, userLogin.accessTokenExpiration)
+            ? new WSObservable(db.cloud.options.databaseUrl, db.cloud.persistedSyncState.value.serverRevision, realmSetHash, db.cloud.persistedSyncState.value.clientIdentity, messageProducer, db.cloud.webSocketStatus, userLogin.accessToken, userLogin.accessTokenExpiration)
             : from$1([])), catchError((error) => {
             if ((error === null || error === void 0 ? void 0 : error.name) === 'TokenExpiredError') {
                 console.debug('WebSocket observable: Token expired. Refreshing token...');
@@ -5371,22 +5398,26 @@ function LocalSyncWorker(db, cloudOptions, cloudSchema) {
     //let periodicSyncHandler: ((event: Event) => void) | null = null;
     let cancelToken = { cancelled: false };
     function syncAndRetry(purpose, retryNum = 1) {
-        syncIfPossible(db, cloudOptions, cloudSchema, {
-            cancelToken,
-            retryImmediatelyOnFetchError: true,
-            purpose
-        }).catch((e) => {
-            console.error('error in syncIfPossible()', e);
-            if (cancelToken.cancelled) {
-                stop();
-            }
-            else if (retryNum < 3) {
-                // Mimic service worker sync event: retry 3 times
-                // * first retry after 5 minutes
-                // * second retry 15 minutes later
-                setTimeout(() => syncAndRetry(purpose, retryNum + 1), [0, 5, 15][retryNum] * MINUTES);
-            }
-        });
+        // Use setTimeout() to get onto a clean stack and
+        // break free from possible active transaction:
+        setTimeout(() => {
+            syncIfPossible(db, cloudOptions, cloudSchema, {
+                cancelToken,
+                retryImmediatelyOnFetchError: true,
+                purpose,
+            }).catch((e) => {
+                console.error('error in syncIfPossible()', e);
+                if (cancelToken.cancelled) {
+                    stop();
+                }
+                else if (retryNum < 3) {
+                    // Mimic service worker sync event: retry 3 times
+                    // * first retry after 5 minutes
+                    // * second retry 15 minutes later
+                    setTimeout(() => syncAndRetry(purpose, retryNum + 1), [0, 5, 15][retryNum] * MINUTES);
+                }
+            });
+        }, 0);
     }
     const start = () => {
         // Sync eagerly whenever a change has happened (+ initially when there's no syncState yet)
@@ -5394,7 +5425,7 @@ function LocalSyncWorker(db, cloudOptions, cloudSchema) {
         console.debug('Starting LocalSyncWorker', db.localSyncEvent['id']);
         localSyncEventSubscription = db.localSyncEvent.subscribe(({ purpose }) => {
             try {
-                syncAndRetry(purpose || "pull");
+                syncAndRetry(purpose || 'pull');
             }
             catch (err) {
                 console.error('What-the....', err);