dexie-cloud-addon 4.1.0-beta.44 → 4.1.0-beta.46

package/dist/umd/service-worker.js

@@ -8,7 +8,7 @@
  *
  * ==========================================================================
  *
- * Version 4.1.0-beta.44, Fri Feb 14 2025
+ * Version 4.1.0-beta.46, Thu Apr 03 2025
  *
  * https://dexie.org
  *
@@ -2802,6 +2802,7 @@
                         break;
                     case 'u-s':
                         writeVarUint8Array(encoder, msg.u);
+                        writeVarString(encoder, msg.r || '');
                         break;
                 }
         }
@@ -3188,7 +3189,8 @@
                             table,
                             prop,
                             k,
-                            u: readVarUint8Array(decoder)
+                            u: readVarUint8Array(decoder),
+                            r: (decoder.pos < decoder.arr.length && readVarString(decoder)) || undefined,
                         };
                     default:
                         throw new TypeError(`Unknown message type: ${type}`);
@@ -4465,6 +4467,7 @@
                 baseRevs,
                 changes: encodeIdsForServer(db.dx.core.schema, currentUser, changes),
                 y,
+                dxcv: db.cloud.version
             };
             console.debug('Sync request', syncRequest);
             db.syncStateChangedEvent.next({
@@ -4607,9 +4610,11 @@
         return __awaiter(this, void 0, void 0, function* () {
             console.debug('Applying server changes', changes, Dexie.currentTransaction);
             for (const { table: tableName, muts } of changes) {
+                if (!db.dx._allTables[tableName]) {
+                    console.debug(`Server sent changes for table ${tableName} that we don't have. Ignoring.`);
+                    continue;
+                }
                 const table = db.table(tableName);
-                if (!table)
-                    continue; // If server sends changes on a table we don't have, ignore it.
                 const { primaryKey } = table.core.schema;
                 const keyDecoder = (key) => {
                     switch (key[0]) {
@@ -4795,9 +4800,15 @@
 
     function getUpdatesTable(db, table, ydocProp) {
         var _a, _b, _c;
+        if (!db.dx._allTables[table])
+            return undefined;
         const utbl = (_c = (_b = (_a = db.table(table)) === null || _a === void 0 ? void 0 : _a.schema.yProps) === null || _b === void 0 ? void 0 : _b.find(p => p.prop === ydocProp)) === null || _c === void 0 ? void 0 : _c.updatesTable;
-        if (!utbl)
-            throw new Error(`No updatesTable found for ${table}.${ydocProp}`);
+        if (!utbl) {
+            console.debug(`No updatesTable found for ${table}.${ydocProp}`);
+            return undefined;
+        }
+        if (!db.dx._allTables[utbl])
+            return undefined;
         return db.table(utbl);
     }
 
@@ -4808,74 +4819,91 @@
             let resyncNeeded = false;
             let yServerRevision;
             for (const m of yMessages) {
-                switch (m.type) {
-                    case 'u-s': {
-                        const utbl = getUpdatesTable(db, m.table, m.prop);
-                        receivedUntils[utbl.name] = yield utbl.add({
-                            k: m.k,
-                            u: m.u,
-                        });
-                        break;
-                    }
-                    case 'u-ack': {
-                        const utbl = getUpdatesTable(db, m.table, m.prop);
-                        yield db.transaction('rw', utbl, (tx) => __awaiter(this, void 0, void 0, function* () {
-                            let syncer = (yield tx
-                                .table(utbl.name)
-                                .get(DEXIE_CLOUD_SYNCER_ID));
-                            yield tx.table(utbl.name).put(Object.assign(Object.assign({}, (syncer || { i: DEXIE_CLOUD_SYNCER_ID })), { unsentFrom: Math.max((syncer === null || syncer === void 0 ? void 0 : syncer.unsentFrom) || 1, m.i + 1) }));
-                        }));
-                        break;
-                    }
-                    case 'u-reject': {
-                        // Acces control or constraint rejected the update.
-                        // We delete it. It's not going to be sent again.
-                        // What's missing is a way to notify consumers, such as Tiptap editor, that the update was rejected.
-                        // This is only an issue when the document is open. We could find the open document and
-                        // in a perfect world, we should send a reverse update to the open document to undo the change.
-                        // See my question in https://discuss.yjs.dev/t/generate-an-inverse-update/2765
-                        console.debug(`Y update rejected. Deleting it.`);
-                        const utbl = getUpdatesTable(db, m.table, m.prop);
-                        // Delete the rejected update and all local updates since (avoid holes in the CRDT)
-                        // and destroy it's open document if there is one.
-                        const primaryKey = (_a = (yield utbl.get(m.i))) === null || _a === void 0 ? void 0 : _a.k;
-                        if (primaryKey != null) {
-                            yield db.transaction('rw', utbl, (tx) => {
-                                // @ts-ignore
-                                tx.idbtrans._rejecting_y_ypdate = true; // Inform ydoc triggers that we delete because of a rejection and not GC
-                                return utbl
-                                    .where('i')
-                                    .aboveOrEqual(m.i)
-                                    .filter((u) => Dexie.cmp(u.k, primaryKey) === 0 && ((u.f || 0) & 1) === 1)
-                                    .delete();
-                            });
-                            // Destroy active doc
-                            const activeDoc = Dexie.DexieYProvider.getDocCache(db.dx).find(m.table, primaryKey, m.prop);
-                            if (activeDoc)
-                                activeDoc.destroy(); // Destroy the document so that editors don't continue to work on it
+                try {
+                    switch (m.type) {
+                        case 'u-s': {
+                            const utbl = getUpdatesTable(db, m.table, m.prop);
+                            if (utbl) {
+                                const updateRow = {
+                                    k: m.k,
+                                    u: m.u,
+                                };
+                                if (m.r) {
+                                    // @ts-ignore
+                                    updateRow.r = m.r;
+                                    yServerRevision = m.r;
+                                }
+                                receivedUntils[utbl.name] = yield utbl.add(updateRow);
+                            }
+                            break;
                         }
-                        break;
-                    }
-                    case 'in-sync': {
-                        const doc = Dexie.DexieYProvider.getDocCache(db.dx).find(m.table, m.k, m.prop);
-                        if (doc && !doc.isSynced) {
-                            doc.emit('sync', [true]);
+                        case 'u-ack': {
+                            const utbl = getUpdatesTable(db, m.table, m.prop);
+                            if (utbl) {
+                                yield db.transaction('rw', utbl, (tx) => __awaiter(this, void 0, void 0, function* () {
+                                    let syncer = (yield tx
+                                        .table(utbl.name)
+                                        .get(DEXIE_CLOUD_SYNCER_ID));
+                                    yield tx.table(utbl.name).put(Object.assign(Object.assign({}, (syncer || { i: DEXIE_CLOUD_SYNCER_ID })), { unsentFrom: Math.max((syncer === null || syncer === void 0 ? void 0 : syncer.unsentFrom) || 1, m.i + 1) }));
+                                }));
+                            }
+                            break;
                         }
-                        break;
-                    }
-                    case 'y-complete-sync-done': {
-                        yServerRevision = m.yServerRev;
-                        break;
+                        case 'u-reject': {
+                            // Acces control or constraint rejected the update.
+                            // We delete it. It's not going to be sent again.
+                            // What's missing is a way to notify consumers, such as Tiptap editor, that the update was rejected.
+                            // This is only an issue when the document is open. We could find the open document and
+                            // in a perfect world, we should send a reverse update to the open document to undo the change.
+                            // See my question in https://discuss.yjs.dev/t/generate-an-inverse-update/2765
+                            console.debug(`Y update rejected. Deleting it.`);
+                            const utbl = getUpdatesTable(db, m.table, m.prop);
+                            if (!utbl)
+                                break;
+                            // Delete the rejected update and all local updates since (avoid holes in the CRDT)
+                            // and destroy it's open document if there is one.
+                            const primaryKey = (_a = (yield utbl.get(m.i))) === null || _a === void 0 ? void 0 : _a.k;
+                            if (primaryKey != null) {
+                                yield db.transaction('rw', utbl, (tx) => {
+                                    // @ts-ignore
+                                    tx.idbtrans._rejecting_y_ypdate = true; // Inform ydoc triggers that we delete because of a rejection and not GC
+                                    return utbl
+                                        .where('i')
+                                        .aboveOrEqual(m.i)
+                                        .filter((u) => Dexie.cmp(u.k, primaryKey) === 0 && ((u.f || 0) & 1) === 1)
+                                        .delete();
+                                });
+                                // Destroy active doc
+                                const activeDoc = Dexie.DexieYProvider.getDocCache(db.dx).find(m.table, primaryKey, m.prop);
+                                if (activeDoc)
+                                    activeDoc.destroy(); // Destroy the document so that editors don't continue to work on it
+                            }
+                            break;
+                        }
+                        case 'in-sync': {
+                            const doc = Dexie.DexieYProvider.getDocCache(db.dx).find(m.table, m.k, m.prop);
+                            if (doc && !doc.isSynced) {
+                                doc.emit('sync', [true]);
+                            }
+                            break;
+                        }
+                        case 'y-complete-sync-done': {
+                            yServerRevision = m.yServerRev;
+                            break;
+                        }
+                        case 'outdated-server-rev':
+                            resyncNeeded = true;
+                            break;
                     }
-                    case 'outdated-server-rev':
-                        resyncNeeded = true;
-                        break;
+                }
+                catch (e) {
+                    console.error(`Failed to apply YMessage`, m, e);
                 }
             }
             return {
                 receivedUntils,
                 resyncNeeded,
-                yServerRevision
+                yServerRevision,
             };
         });
     }
@@ -4986,7 +5014,9 @@
                                     throw new Error(`Protocol error from ${databaseUrl}/y/download`);
                                 }
                                 const yTable = getUpdatesTable(db, currentTable, currentProp);
-                                yield yTable.bulkAdd(docsToInsert);
+                                if (yTable) {
+                                    yield yTable.bulkAdd(docsToInsert);
+                                }
                                 docsToInsert = [];
                             }
                             if (currentRealmId &&
@@ -5322,12 +5352,12 @@
             return false; // Not needed anymore
         });
     }
-    function deleteObjectsFromRemovedRealms(db, res, prevState) {
+    function deleteObjectsFromRemovedRealms(db, res, syncState) {
         return __awaiter(this, void 0, void 0, function* () {
             const deletedRealms = new Set();
             const rejectedRealms = new Set();
-            const previousRealmSet = prevState ? prevState.realms : [];
-            const previousInviteRealmSet = prevState ? prevState.inviteRealms : [];
+            const previousRealmSet = syncState ? syncState.realms : [];
+            const previousInviteRealmSet = syncState ? syncState.inviteRealms : [];
             const updatedRealmSet = new Set(res.realms);
             const updatedTotalRealmSet = new Set(res.realms.concat(res.inviteRealms));
             for (const realmId of previousRealmSet) {
@@ -5369,17 +5399,10 @@
                     }
                 }
             }
-            if (rejectedRealms.size > 0) {
-                // Remove rejected/deleted realms from yDownloadedRealms because of the following use case:
-                // 1. User becomes added to the realm
-                // 2. User syncs and all documents of the realm is downloaded (downloadYDocsFromServer.ts)
-                // 3. User leaves the realm and all docs are deleted locally (built-in-trigger of deleting their rows in this file)
-                // 4. User is yet again added to the realm. At this point, we must make sure the docs are not considered already downloaded.
-                const updateSpec = {};
+            if (rejectedRealms.size > 0 && (syncState === null || syncState === void 0 ? void 0 : syncState.yDownloadedRealms)) {
                 for (const realmId of rejectedRealms) {
-                    updateSpec[`yDownloadedRealms.${realmId}`] = undefined; // Setting to undefined will delete the property
+                    delete syncState.yDownloadedRealms[realmId];
                 }
-                yield db.$syncState.update('syncState', updateSpec);
             }
         });
     }
@@ -6902,13 +6925,13 @@
     const CLIENT_PING_INTERVAL = 30000;
     const FAIL_RETRY_WAIT_TIME = 60000;
     class WSObservable extends rxjs.Observable {
-        constructor(db, rev, realmSetHash, clientIdentity, messageProducer, webSocketStatus, user) {
-            super((subscriber) => new WSConnection(db, rev, realmSetHash, clientIdentity, user, subscriber, messageProducer, webSocketStatus));
+        constructor(db, rev, yrev, realmSetHash, clientIdentity, messageProducer, webSocketStatus, user) {
+            super((subscriber) => new WSConnection(db, rev, yrev, realmSetHash, clientIdentity, user, subscriber, messageProducer, webSocketStatus));
         }
     }
     let counter = 0;
     class WSConnection extends rxjs.Subscription {
-        constructor(db, rev, realmSetHash, clientIdentity, user, subscriber, messageProducer, webSocketStatus) {
+        constructor(db, rev, yrev, realmSetHash, clientIdentity, user, subscriber, messageProducer, webSocketStatus) {
             super(() => this.teardown());
             this.id = ++counter;
             this.subscriptions = new Set();
@@ -6917,6 +6940,7 @@
             this.db = db;
             this.databaseUrl = db.cloud.options.databaseUrl;
             this.rev = rev;
+            this.yrev = yrev;
             this.realmSetHash = realmSetHash;
             this.clientIdentity = clientIdentity;
             this.user = user;
@@ -6986,6 +7010,11 @@
                 }
                 this.webSocketStatus.next('connecting');
                 this.pinger = setInterval(() => __awaiter(this, void 0, void 0, function* () {
+                    // setInterval here causes unnecessary pings when server is proved active anyway.
+                    // TODO: Use setTimout() here instead. When triggered, check if we really need to ping.
+                    // In case we've had server activity, we don't need to ping. Then schedule then next ping
+                    // to the time when we should ping next time (based on lastServerActivity + CLIENT_PING_INTERVAL).
+                    // Else, ping now and schedule next ping to CLIENT_PING_INTERVAL from now.
                     if (this.closed) {
                         console.debug('pinger check', this.id, 'CLOSED.');
                         this.teardown();
@@ -7032,9 +7061,13 @@
                 if (this.subscriber.closed)
                     return;
                 searchParams.set('v', '2');
-                searchParams.set('rev', this.rev);
+                if (this.rev)
+                    searchParams.set('rev', this.rev);
+                if (this.yrev)
+                    searchParams.set('yrev', this.yrev);
                 searchParams.set('realmsHash', this.realmSetHash);
                 searchParams.set('clientId', this.clientIdentity);
+                searchParams.set('dxcv', this.db.cloud.version);
                 if (this.user.accessToken) {
                     searchParams.set('token', this.user.accessToken);
                 }
@@ -7071,8 +7104,8 @@
                                 }
                             }
                         }
-                        else if (msg.type === 'u-ack' || msg.type === 'u-reject' || msg.type === 'u-s' || msg.type === 'in-sync') {
-                            applyYServerMessages([msg], this.db);
+                        else if (msg.type === 'pong') {
+                            // Do nothing
                         }
                         else if (msg.type === 'doc-open') {
                             const docCache = Dexie.DexieYProvider.getDocCache(this.db.dx);
@@ -7081,11 +7114,26 @@
                                 getOpenDocSignal(doc).next(); // Make yHandler reopen the document on server.
                             }
                         }
-                        else if (msg.type === 'outdated-server-rev' || msg.type === 'y-complete-sync-done') {
-                            // Won't happen but need this for typing.
-                            throw new Error('Outdated server revision or y-complete-sync-done not expected over WebSocket - only in sync using fetch()');
+                        else if (msg.type === 'u-ack' || msg.type === 'u-reject' || msg.type === 'u-s' || msg.type === 'in-sync' || msg.type === 'outdated-server-rev' || msg.type === 'y-complete-sync-done') {
+                            applyYServerMessages([msg], this.db).then((_a) => __awaiter(this, [_a], void 0, function* ({ resyncNeeded, yServerRevision, receivedUntils }) {
+                                if (yServerRevision) {
+                                    yield this.db.$syncState.update('syncState', { yServerRevision: yServerRevision });
+                                }
+                                if (msg.type === 'u-s' && receivedUntils) {
+                                    const utbl = getUpdatesTable(this.db, msg.table, msg.prop);
+                                    if (utbl) {
+                                        const receivedUntil = receivedUntils[utbl.name];
+                                        if (receivedUntil) {
+                                            yield utbl.update(DEXIE_CLOUD_SYNCER_ID, { receivedUntil });
+                                        }
+                                    }
+                                }
+                                if (resyncNeeded) {
+                                    yield this.db.cloud.sync({ purpose: 'pull', wait: true });
+                                }
+                            }));
                         }
-                        else if (msg.type !== 'pong') {
+                        else {
                             // Forward the request to our subscriber, wich is in messageFromServerQueue.ts (via connectWebSocket's subscribe() at the end!)
                             this.subscriber.next(msg);
                         }
@@ -7218,7 +7266,7 @@
                 // If no new entries, server won't bother the client. If new entries, server sends only those
                 // and the baseRev of the last from same client-ID.
                 if (userLogin) {
-                    return new WSObservable(db, db.cloud.persistedSyncState.value.serverRevision, realmSetHash, db.cloud.persistedSyncState.value.clientIdentity, messageProducer, db.cloud.webSocketStatus, userLogin);
+                    return new WSObservable(db, db.cloud.persistedSyncState.value.serverRevision, db.cloud.persistedSyncState.value.yServerRevision, realmSetHash, db.cloud.persistedSyncState.value.clientIdentity, messageProducer, db.cloud.webSocketStatus, userLogin);
                 }
                 else {
                     return rxjs.from([]);
@@ -7348,61 +7396,95 @@
     }
 
     const SECONDS = 1000;
-    const MINUTES = 60 * SECONDS;
 
     function LocalSyncWorker(db, cloudOptions, cloudSchema) {
         let localSyncEventSubscription = null;
-        //let syncHandler: ((event: Event) => void) | null = null;
-        //let periodicSyncHandler: ((event: Event) => void) | null = null;
         let cancelToken = { cancelled: false };
-        let retryHandle = null;
-        let retryPurpose = null; // "pull" is superset of "push"
-        function syncAndRetry(purpose, retryNum = 1) {
+        let nextRetryTime = 0;
+        let syncStartTime = 0;
+        function syncAndRetry(retryNum = 1) {
             // Use setTimeout() to get onto a clean stack and
             // break free from possible active transaction:
             setTimeout(() => {
-                if (retryHandle)
-                    clearTimeout(retryHandle);
-                const combPurpose = retryPurpose === 'pull' ? 'pull' : purpose;
-                retryHandle = null;
-                retryPurpose = null;
+                const purpose = pullSignalled ? 'pull' : 'push';
+                syncStartTime = Date.now();
                 syncIfPossible(db, cloudOptions, cloudSchema, {
                     cancelToken,
                     retryImmediatelyOnFetchError: true, // workaround for "net::ERR_NETWORK_CHANGED" in chrome.
-                    purpose: combPurpose,
-                }).catch((e) => {
-                    console.error('error in syncIfPossible()', e);
+                    purpose,
+                }).then(() => {
+                    if (cancelToken.cancelled) {
+                        stop();
+                    }
+                    else {
+                        if (pullSignalled || pushSignalled) {
+                            // If we have signalled for more sync, do it now.
+                            pullSignalled = false;
+                            pushSignalled = false;
+                            return syncAndRetry();
+                        }
+                    }
+                    ongoingSync = false;
+                    nextRetryTime = 0;
+                    syncStartTime = 0;
+                }).catch((error) => {
+                    console.error('error in syncIfPossible()', error);
                     if (cancelToken.cancelled) {
                         stop();
+                        ongoingSync = false;
+                        nextRetryTime = 0;
+                        syncStartTime = 0;
                     }
-                    else if (retryNum < 3) {
-                        // Mimic service worker sync event: retry 3 times
-                        // * first retry after 5 minutes
-                        // * second retry 15 minutes later
-                        const combinedPurpose = retryPurpose && retryPurpose === 'pull' ? 'pull' : purpose;
-                        const handle = setTimeout(() => syncAndRetry(combinedPurpose, retryNum + 1), [0, 5, 15][retryNum] * MINUTES);
-                        // Cancel the previous retryHandle if it exists to avoid scheduling loads of retries.
-                        if (retryHandle)
-                            clearTimeout(retryHandle);
-                        retryHandle = handle;
-                        retryPurpose = combinedPurpose;
+                    else if (retryNum < 5) {
+                        // Mimic service worker sync event but a bit more eager: retry 4 times
+                        // * first retry after 20 seconds
+                        // * second retry 40 seconds later
+                        // * third retry 5 minutes later
+                        // * last retry 15 minutes later
+                        const retryIn = [0, 20, 40, 300, 900][retryNum] * SECONDS;
+                        nextRetryTime = Date.now() + retryIn;
+                        syncStartTime = 0;
+                        setTimeout(() => syncAndRetry(retryNum + 1), retryIn);
+                    }
+                    else {
+                        ongoingSync = false;
+                        nextRetryTime = 0;
+                        syncStartTime = 0;
                     }
                 });
             }, 0);
         }
+        let pullSignalled = false;
+        let pushSignalled = false;
+        let ongoingSync = false;
+        const consumer = (purpose) => {
+            if (cancelToken.cancelled)
+                return;
+            if (purpose === 'pull') {
+                pullSignalled = true;
+            }
+            if (purpose === 'push') {
+                pushSignalled = true;
+            }
+            if (ongoingSync) {
+                if (nextRetryTime) {
+                    console.debug(`Sync is paused until ${new Date(nextRetryTime).toISOString()} due to error in last sync attempt`);
+                }
+                else if (syncStartTime > 0 && Date.now() - syncStartTime > 20 * SECONDS) {
+                    console.debug(`An existing sync operation is taking more than 20 seconds. Will resync when done.`);
+                }
+                return;
+            }
+            ongoingSync = true;
+            syncAndRetry();
+        };
         const start = () => {
             // Sync eagerly whenever a change has happened (+ initially when there's no syncState yet)
             // This initial subscribe will also trigger an sync also now.
             console.debug('Starting LocalSyncWorker', db.localSyncEvent['id']);
             localSyncEventSubscription = db.localSyncEvent.subscribe(({ purpose }) => {
-                try {
-                    syncAndRetry(purpose || 'pull');
-                }
-                catch (err) {
-                    console.error('What-the....', err);
-                }
+                consumer(purpose || 'pull');
             });
-            //setTimeout(()=>db.localSyncEvent.next({}), 5000);
         };
         const stop = () => {
             console.debug('Stopping LocalSyncWorker');
@@ -8194,7 +8276,7 @@
         const syncComplete = new rxjs.Subject();
         dexie.cloud = {
             // @ts-ignore
-            version: "4.1.0-beta.44",
+            version: "4.1.0-beta.46",
             options: Object.assign({}, DEFAULT_OPTIONS),
             schema: null,
             get currentUserId() {
@@ -8512,7 +8594,7 @@
         }
     }
     // @ts-ignore
-    dexieCloud.version = "4.1.0-beta.44";
+    dexieCloud.version = "4.1.0-beta.46";
     Dexie.Cloud = dexieCloud;
 
     // In case the SW lives for a while, let it reuse already opened connections: