dexie-cloud-addon 1.0.0-beta.10 → 1.0.0-beta.11

package/dist/umd-modern/dexie-cloud-addon.js

@@ -2958,6 +2958,105 @@
         }
     }
 
+    async function computeRealmSetHash({ realms, inviteRealms, }) {
+        const data = JSON.stringify([
+            ...realms.map((realmId) => ({ realmId, accepted: true })),
+            ...inviteRealms.map((realmId) => ({ realmId, accepted: false })),
+        ].sort((a, b) => a.realmId < b.realmId ? -1 : a.realmId > b.realmId ? 1 : 0));
+        const byteArray = new TextEncoder().encode(data);
+        const digestBytes = await crypto.subtle.digest('SHA-1', byteArray);
+        const base64 = b64encode(digestBytes);
+        return base64;
+    }
+
+    function getSyncableTables(db) {
+        return Object.entries(db.cloud.schema || {})
+            .filter(([, { markedForSync }]) => markedForSync)
+            .map(([tbl]) => db.table(tbl));
+    }
+
+    function getMutationTable(tableName) {
+        return `$${tableName}_mutations`;
+    }
+
+    function getTableFromMutationTable(mutationTable) {
+        var _a;
+        const tableName = (_a = /^\$(.*)_mutations$/.exec(mutationTable)) === null || _a === void 0 ? void 0 : _a[1];
+        if (!tableName)
+            throw new Error(`Given mutationTable ${mutationTable} is not correct`);
+        return tableName;
+    }
+
+    async function listClientChanges(mutationTables, db, { since = {}, limit = Infinity } = {}) {
+        const allMutsOnTables = await Promise.all(mutationTables.map(async (mutationTable) => {
+            const tableName = getTableFromMutationTable(mutationTable.name);
+            const lastRevision = since[tableName];
+            let query = lastRevision
+                ? mutationTable.where("rev").above(lastRevision)
+                : mutationTable;
+            if (limit < Infinity)
+                query = query.limit(limit);
+            const muts = await query.toArray();
+            //const objTable = db.table(tableName);
+            /*for (const mut of muts) {
+              if (mut.type === "insert" || mut.type === "upsert") {
+                mut.values = await objTable.bulkGet(mut.keys);
+              }
+            }*/
+            return {
+                table: tableName,
+                muts,
+            };
+        }));
+        // Filter out those tables that doesn't have any mutations:
+        return allMutsOnTables.filter(({ muts }) => muts.length > 0);
+    }
+
+    async function listSyncifiedChanges(tablesToSyncify, currentUser, schema, alreadySyncedRealms) {
+        if (currentUser.isLoggedIn) {
+            if (tablesToSyncify.length > 0) {
+                const ignoredRealms = new Set(alreadySyncedRealms || []);
+                const inserts = await Promise.all(tablesToSyncify.map(async (table) => {
+                    const { extractKey } = table.core.schema.primaryKey;
+                    if (!extractKey)
+                        return { table: table.name, muts: [] }; // Outbound tables are not synced.
+                    const dexieCloudTableSchema = schema[table.name];
+                    const query = (dexieCloudTableSchema === null || dexieCloudTableSchema === void 0 ? void 0 : dexieCloudTableSchema.generatedGlobalId)
+                        ? table.filter((item) => !ignoredRealms.has(item.realmId || "") && isValidSyncableID(extractKey(item)))
+                        : table.filter((item) => !ignoredRealms.has(item.realmId || "") && isValidAtID(extractKey(item), dexieCloudTableSchema === null || dexieCloudTableSchema === void 0 ? void 0 : dexieCloudTableSchema.idPrefix));
+                    const unsyncedObjects = await query.toArray();
+                    if (unsyncedObjects.length > 0) {
+                        const mut = {
+                            type: "insert",
+                            values: unsyncedObjects,
+                            keys: unsyncedObjects.map(extractKey),
+                            userId: currentUser.userId,
+                        };
+                        return {
+                            table: table.name,
+                            muts: [mut],
+                        };
+                    }
+                    else {
+                        return {
+                            table: table.name,
+                            muts: []
+                        };
+                    }
+                }));
+                return inserts.filter(op => op.muts.length > 0);
+            }
+        }
+        return [];
+    }
+
+    function getTablesToSyncify(db, syncState) {
+        const syncedTables = (syncState === null || syncState === void 0 ? void 0 : syncState.syncedTables) || [];
+        const syncableTables = getSyncableTables(db);
+        const tablesToSyncify = syncableTables.filter((tbl) => !syncedTables.includes(tbl.name));
+        return tablesToSyncify;
+    }
+
     const { toString: toStr } = {};
     function getToStringTag(val) {
         return toStr.call(val).slice(8, -1);
@@ -3446,22 +3545,6 @@
     //   else
     //     serverRev.rev = new FakeBigInt(server.rev)
     const hasBigIntSupport = typeof BigInt(0) === 'bigint';
-    function getValueOfBigInt(x) {
-        if (typeof x === 'bigint') {
-            return x;
-        }
-        if (hasBigIntSupport) {
-            return typeof x === 'string' ? BigInt(x) : BigInt(x.v);
-        }
-        else {
-            return typeof x === 'string' ? Number(x) : Number(x.v);
-        }
-    }
-    function compareBigInts(a, b) {
-        const valA = getValueOfBigInt(a);
-        const valB = getValueOfBigInt(b);
-        return valA < valB ? -1 : valA > valB ? 1 : 0;
-    }
     class FakeBigInt {
         constructor(value) {
             this.v = value;
@@ -3490,105 +3573,6 @@
     const TSON = TypesonSimplified(builtin, defs);
     const BISON = Bison(defs);
 
-    async function computeRealmSetHash({ realms, inviteRealms, }) {
-        const data = JSON.stringify([
-            ...realms.map((realmId) => ({ realmId, accepted: true })),
-            ...inviteRealms.map((realmId) => ({ realmId, accepted: false })),
-        ].sort((a, b) => a.realmId < b.realmId ? -1 : a.realmId > b.realmId ? 1 : 0));
-        const byteArray = new TextEncoder().encode(data);
-        const digestBytes = await crypto.subtle.digest('SHA-1', byteArray);
-        const base64 = b64encode(digestBytes);
-        return base64;
-    }
-
-    function getSyncableTables(db) {
-        return Object.entries(db.cloud.schema || {})
-            .filter(([, { markedForSync }]) => markedForSync)
-            .map(([tbl]) => db.table(tbl));
-    }
-
-    function getMutationTable(tableName) {
-        return `$${tableName}_mutations`;
-    }
-
-    function getTableFromMutationTable(mutationTable) {
-        var _a;
-        const tableName = (_a = /^\$(.*)_mutations$/.exec(mutationTable)) === null || _a === void 0 ? void 0 : _a[1];
-        if (!tableName)
-            throw new Error(`Given mutationTable ${mutationTable} is not correct`);
-        return tableName;
-    }
-
-    async function listClientChanges(mutationTables, db, { since = {}, limit = Infinity } = {}) {
-        const allMutsOnTables = await Promise.all(mutationTables.map(async (mutationTable) => {
-            const tableName = getTableFromMutationTable(mutationTable.name);
-            const lastRevision = since[tableName];
-            let query = lastRevision
-                ? mutationTable.where("rev").above(lastRevision)
-                : mutationTable;
-            if (limit < Infinity)
-                query = query.limit(limit);
-            const muts = await query.toArray();
-            //const objTable = db.table(tableName);
-            /*for (const mut of muts) {
-              if (mut.type === "insert" || mut.type === "upsert") {
-                mut.values = await objTable.bulkGet(mut.keys);
-              }
-            }*/
-            return {
-                table: tableName,
-                muts,
-            };
-        }));
-        // Filter out those tables that doesn't have any mutations:
-        return allMutsOnTables.filter(({ muts }) => muts.length > 0);
-    }
-
-    async function listSyncifiedChanges(tablesToSyncify, currentUser, schema, alreadySyncedRealms) {
-        if (currentUser.isLoggedIn) {
-            if (tablesToSyncify.length > 0) {
-                const ignoredRealms = new Set(alreadySyncedRealms || []);
-                const inserts = await Promise.all(tablesToSyncify.map(async (table) => {
-                    const { extractKey } = table.core.schema.primaryKey;
-                    if (!extractKey)
-                        return { table: table.name, muts: [] }; // Outbound tables are not synced.
-                    const dexieCloudTableSchema = schema[table.name];
-                    const query = (dexieCloudTableSchema === null || dexieCloudTableSchema === void 0 ? void 0 : dexieCloudTableSchema.generatedGlobalId)
-                        ? table.filter((item) => !ignoredRealms.has(item.realmId || "") && isValidSyncableID(extractKey(item)))
-                        : table.filter((item) => !ignoredRealms.has(item.realmId || "") && isValidAtID(extractKey(item), dexieCloudTableSchema === null || dexieCloudTableSchema === void 0 ? void 0 : dexieCloudTableSchema.idPrefix));
-                    const unsyncedObjects = await query.toArray();
-                    if (unsyncedObjects.length > 0) {
-                        const mut = {
-                            type: "insert",
-                            values: unsyncedObjects,
-                            keys: unsyncedObjects.map(extractKey),
-                            userId: currentUser.userId,
-                        };
-                        return {
-                            table: table.name,
-                            muts: [mut],
-                        };
-                    }
-                    else {
-                        return {
-                            table: table.name,
-                            muts: []
-                        };
-                    }
-                }));
-                return inserts.filter(op => op.muts.length > 0);
-            }
-        }
-        return [];
-    }
-
-    function getTablesToSyncify(db, syncState) {
-        const syncedTables = (syncState === null || syncState === void 0 ? void 0 : syncState.syncedTables) || [];
-        const syncableTables = getSyncableTables(db);
-        const tablesToSyncify = syncableTables.filter((tbl) => !syncedTables.includes(tbl.name));
-        return tablesToSyncify;
-    }
-
     //import {BisonWebStreamReader} from "dreambase-library/dist/typeson-simplified/BisonWebStreamReader";
     async function syncWithServer(changes, syncState, baseRevs, db, databaseUrl, schema, clientIdentity) {
         //
@@ -3603,6 +3587,7 @@
             headers.Authorization = `Bearer ${accessToken}`;
         }
         const syncRequest = {
+            v: 2,
             dbID: syncState === null || syncState === void 0 ? void 0 : syncState.remoteDbId,
             clientIdentity,
             schema: schema || {},
@@ -3729,8 +3714,8 @@
 
     function getLatestRevisionsPerTable(clientChangeSet, lastRevisions = {}) {
         for (const { table, muts } of clientChangeSet) {
-            const lastRev = muts.length > 0 ? muts[muts.length - 1].rev || 0 : 0;
-            lastRevisions[table] = lastRev;
+            const lastRev = muts.length > 0 ? muts[muts.length - 1].rev : null;
+            lastRevisions[table] = lastRev || lastRevisions[table] || 0;
         }
         return lastRevisions;
     }
@@ -4051,16 +4036,35 @@
         const readyToServe = new rxjs.BehaviorSubject(true);
         const event = new rxjs.BehaviorSubject(null);
         let isWorking = false;
+        let loopWarning = 0;
+        let loopDetection = [0, 0, 0, 0, 0, 0, 0, 0, 0, Date.now()];
         event.subscribe(async () => {
             if (isWorking)
                 return;
             if (queue.length > 0) {
                 isWorking = true;
+                loopDetection.shift();
+                loopDetection.push(Date.now());
                 readyToServe.next(false);
                 try {
                     await consumeQueue();
                 }
                 finally {
+                    if (loopDetection[loopDetection.length - 1] - loopDetection[0] < 10000) {
+                        // Ten loops within 10 seconds. Slow down!
+                        if (Date.now() - loopWarning < 5000) {
+                            // Last time we did this, we ended up here too. Wait for a minute.
+                            console.warn(`Slowing down websocket loop for one minute`);
+                            loopWarning = Date.now() + 60000;
+                            await new Promise(resolve => setTimeout(resolve, 60000));
+                        }
+                        else {
+                            // This is a one-time event. Just pause 10 seconds.
+                            console.warn(`Slowing down websocket loop for 10 seconds`);
+                            loopWarning = Date.now() + 10000;
+                            await new Promise(resolve => setTimeout(resolve, 10000));
+                        }
+                    }
                     isWorking = false;
                     readyToServe.next(true);
                 }
@@ -4076,6 +4080,9 @@
                 const msg = queue.shift();
                 try {
                     console.debug('processing msg', msg);
+                    // If the sync worker or service worker is syncing, wait 'til thei're done.
+                    // It's no need to have two channels at the same time - even though it wouldnt
+                    // be a problem - this is an optimization.
                     await db.cloud.syncState
                         .pipe(filter(({ phase }) => phase === 'in-sync' || phase === 'error'), take(1))
                         .toPromise();
@@ -4099,26 +4106,23 @@
                             // in turn will lead to that connectWebSocket.ts will reconnect the socket with the
                             // new token. So we don't need to do anything more here.
                             break;
-                        case 'rev':
-                            if (!(persistedSyncState === null || persistedSyncState === void 0 ? void 0 : persistedSyncState.serverRevision) ||
-                                compareBigInts(persistedSyncState.serverRevision, msg.rev) < 0) {
-                                triggerSync(db, "pull");
-                            }
-                            break;
                         case 'realm-added':
                             if (!((_a = persistedSyncState === null || persistedSyncState === void 0 ? void 0 : persistedSyncState.realms) === null || _a === void 0 ? void 0 : _a.includes(msg.realm))) {
-                                triggerSync(db, "pull");
+                                triggerSync(db, 'pull');
                             }
                             break;
                         case 'realm-removed':
                             if ((_b = persistedSyncState === null || persistedSyncState === void 0 ? void 0 : persistedSyncState.realms) === null || _b === void 0 ? void 0 : _b.includes(msg.realm)) {
-                                triggerSync(db, "pull");
+                                triggerSync(db, 'pull');
                             }
                             break;
+                        case 'realms-changed':
+                            triggerSync(db, 'pull');
+                            break;
                         case 'changes':
                             console.debug('changes');
                             if (((_c = db.cloud.syncState.value) === null || _c === void 0 ? void 0 : _c.phase) === 'error') {
-                                triggerSync(db, "pull");
+                                triggerSync(db, 'pull');
                                 break;
                             }
                             await db.transaction('rw', db.dx.tables, async (tx) => {
@@ -4141,16 +4145,34 @@
                                     return; // Initial sync must have taken place - otherwise, ignore this.
                                 }
                                 // Verify again in ACID tx that we're on same server revision.
-                                if (compareBigInts(msg.baseRev, syncState.serverRevision) !== 0) {
+                                if (msg.baseRev !== syncState.serverRevision) {
                                     console.debug(`baseRev (${msg.baseRev}) differs from our serverRevision in syncState (${syncState.serverRevision})`);
+                                    // Should we trigger a sync now? No. This is a normal case
+                                    // when another local peer (such as the SW or a websocket channel on other tab) has
+                                    // updated syncState from new server information but we are not aware yet. It would
+                                    // be unnescessary to do a sync in that case. Instead, the caller of this consumeQueue()
+                                    // function will do readyToServe.next(true) right after this return, which will lead
+                                    // to a "ready" message being sent to server with the new accurate serverRev we have,
+                                    // so that the next message indeed will be correct.
+                                    if (typeof msg.baseRev === 'string' && // v2 format
+                                        (typeof syncState.serverRevision === 'bigint' || // v1 format
+                                            typeof syncState.serverRevision === 'object') // v1 format old browser
+                                    ) {
+                                        // The reason for the diff seems to be that server has migrated the revision format.
+                                        // Do a full sync to update revision format.
+                                        // If we don't do a sync request now, we could stuck in an endless loop.
+                                        triggerSync(db, 'pull');
+                                    }
                                     return; // Ignore message
                                 }
                                 // Verify also that the message is based on the exact same set of realms
-                                const ourRealmSetHash = await Dexie__default['default'].waitFor(computeRealmSetHash(syncState));
+                                const ourRealmSetHash = await Dexie__default['default'].waitFor(
+                                // Keep TX in non-IDB work
+                                computeRealmSetHash(syncState));
                                 console.debug('ourRealmSetHash', ourRealmSetHash);
                                 if (ourRealmSetHash !== msg.realmSetHash) {
                                     console.debug('not same realmSetHash', msg.realmSetHash);
-                                    triggerSync(db, "pull");
+                                    triggerSync(db, 'pull');
                                     // The message isn't based on the same realms.
                                     // Trigger a sync instead to resolve all things up.
                                     return;
@@ -4162,12 +4184,14 @@
                                     clientChanges = await listClientChanges(mutationTables, db);
                                     console.debug('msg queue: client changes', clientChanges);
                                 }
-                                const filteredChanges = filterServerChangesThroughAddedClientChanges(msg.changes, clientChanges);
-                                //
-                                // apply server changes
-                                //
-                                console.debug('applying filtered server changes', filteredChanges);
-                                await applyServerChanges(filteredChanges, db);
+                                if (msg.changes.length > 0) {
+                                    const filteredChanges = filterServerChangesThroughAddedClientChanges(msg.changes, clientChanges);
+                                    //
+                                    // apply server changes
+                                    //
+                                    console.debug('applying filtered server changes', filteredChanges);
+                                    await applyServerChanges(filteredChanges, db);
+                                }
                                 // Update latest revisions per table in case there are unsynced changes
                                 // This can be a real case in future when we allow non-eagery sync.
                                 // And it can actually be realistic now also, but very rare.
@@ -5043,18 +5067,19 @@
     const CLIENT_PING_INTERVAL = 30000;
     const FAIL_RETRY_WAIT_TIME = 60000;
     class WSObservable extends rxjs.Observable {
-        constructor(databaseUrl, rev, clientIdentity, messageProducer, webSocketStatus, token, tokenExpiration) {
-            super((subscriber) => new WSConnection(databaseUrl, rev, clientIdentity, token, tokenExpiration, subscriber, messageProducer, webSocketStatus));
+        constructor(databaseUrl, rev, realmSetHash, clientIdentity, messageProducer, webSocketStatus, token, tokenExpiration) {
+            super((subscriber) => new WSConnection(databaseUrl, rev, realmSetHash, clientIdentity, token, tokenExpiration, subscriber, messageProducer, webSocketStatus));
         }
     }
     let counter = 0;
     class WSConnection extends rxjs.Subscription {
-        constructor(databaseUrl, rev, clientIdentity, token, tokenExpiration, subscriber, messageProducer, webSocketStatus) {
+        constructor(databaseUrl, rev, realmSetHash, clientIdentity, token, tokenExpiration, subscriber, messageProducer, webSocketStatus) {
             super(() => this.teardown());
             this.id = ++counter;
             console.debug('New WebSocket Connection', this.id, token ? 'authorized' : 'unauthorized');
             this.databaseUrl = databaseUrl;
             this.rev = rev;
+            this.realmSetHash = realmSetHash;
             this.clientIdentity = clientIdentity;
             this.token = token;
             this.tokenExpiration = tokenExpiration;
@@ -5159,7 +5184,9 @@
             const searchParams = new URLSearchParams();
             if (this.subscriber.closed)
                 return;
+            searchParams.set('v', "2");
             searchParams.set('rev', this.rev);
+            searchParams.set('realmsHash', this.realmSetHash);
             searchParams.set('clientId', this.clientIdentity);
             if (this.token) {
                 searchParams.set('token', this.token);
@@ -5253,12 +5280,12 @@
         function createObservable() {
             return db.cloud.persistedSyncState.pipe(filter(syncState => syncState === null || syncState === void 0 ? void 0 : syncState.serverRevision), // Don't connect before there's no initial sync performed.
             take(1), // Don't continue waking up whenever syncState change
-            switchMap(() => db.cloud.currentUser), switchMap((userLogin) => userIsReallyActive.pipe(map((isActive) => (isActive ? userLogin : null)))), switchMap((userLogin) => 
+            switchMap((syncState) => db.cloud.currentUser.pipe(map(userLogin => [userLogin, syncState]))), switchMap(([userLogin, syncState]) => userIsReallyActive.pipe(map((isActive) => [isActive ? userLogin : null, syncState]))), switchMap(async ([userLogin, syncState]) => [userLogin, await computeRealmSetHash(syncState)]), switchMap(([userLogin, realmSetHash]) => 
             // Let server end query changes from last entry of same client-ID and forward.
             // If no new entries, server won't bother the client. If new entries, server sends only those
             // and the baseRev of the last from same client-ID.
             userLogin
-                ? new WSObservable(db.cloud.options.databaseUrl, db.cloud.persistedSyncState.value.serverRevision, db.cloud.persistedSyncState.value.clientIdentity, messageProducer, db.cloud.webSocketStatus, userLogin.accessToken, userLogin.accessTokenExpiration)
+                ? new WSObservable(db.cloud.options.databaseUrl, db.cloud.persistedSyncState.value.serverRevision, realmSetHash, db.cloud.persistedSyncState.value.clientIdentity, messageProducer, db.cloud.webSocketStatus, userLogin.accessToken, userLogin.accessTokenExpiration)
                 : rxjs.from([])), catchError((error) => {
                 if ((error === null || error === void 0 ? void 0 : error.name) === 'TokenExpiredError') {
                     console.debug('WebSocket observable: Token expired. Refreshing token...');
@@ -5378,22 +5405,26 @@
         //let periodicSyncHandler: ((event: Event) => void) | null = null;
         let cancelToken = { cancelled: false };
         function syncAndRetry(purpose, retryNum = 1) {
-            syncIfPossible(db, cloudOptions, cloudSchema, {
-                cancelToken,
-                retryImmediatelyOnFetchError: true,
-                purpose
-            }).catch((e) => {
-                console.error('error in syncIfPossible()', e);
-                if (cancelToken.cancelled) {
-                    stop();
-                }
-                else if (retryNum < 3) {
-                    // Mimic service worker sync event: retry 3 times
-                    // * first retry after 5 minutes
-                    // * second retry 15 minutes later
-                    setTimeout(() => syncAndRetry(purpose, retryNum + 1), [0, 5, 15][retryNum] * MINUTES);
-                }
-            });
+            // Use setTimeout() to get onto a clean stack and
+            // break free from possible active transaction:
+            setTimeout(() => {
+                syncIfPossible(db, cloudOptions, cloudSchema, {
+                    cancelToken,
+                    retryImmediatelyOnFetchError: true,
+                    purpose,
+                }).catch((e) => {
+                    console.error('error in syncIfPossible()', e);
+                    if (cancelToken.cancelled) {
+                        stop();
+                    }
+                    else if (retryNum < 3) {
+                        // Mimic service worker sync event: retry 3 times
+                        // * first retry after 5 minutes
+                        // * second retry 15 minutes later
+                        setTimeout(() => syncAndRetry(purpose, retryNum + 1), [0, 5, 15][retryNum] * MINUTES);
+                    }
+                });
+            }, 0);
         }
         const start = () => {
             // Sync eagerly whenever a change has happened (+ initially when there's no syncState yet)
@@ -5401,7 +5432,7 @@
             console.debug('Starting LocalSyncWorker', db.localSyncEvent['id']);
             localSyncEventSubscription = db.localSyncEvent.subscribe(({ purpose }) => {
                 try {
-                    syncAndRetry(purpose || "pull");
+                    syncAndRetry(purpose || 'pull');
                 }
                 catch (err) {
                     console.error('What-the....', err);