dexie-cloud-addon 4.1.0-beta.44 → 4.1.0-beta.45

package/dist/modern/service-worker.js

@@ -8,7 +8,7 @@
  *
  * ==========================================================================
  *
- * Version 4.1.0-beta.44, Fri Feb 14 2025
+ * Version 4.1.0-beta.45, Mon Mar 31 2025
  *
  * https://dexie.org
  *
@@ -2799,6 +2799,7 @@ function encodeYMessage(msg) {
                     break;
                 case 'u-s':
                     writeVarUint8Array(encoder, msg.u);
+                    writeVarString(encoder, msg.r || '');
                     break;
             }
     }
@@ -3185,7 +3186,8 @@ function decodeYMessage(a) {
                         table,
                         prop,
                         k,
-                        u: readVarUint8Array(decoder)
+                        u: readVarUint8Array(decoder),
+                        r: (decoder.pos < decoder.arr.length && readVarString(decoder)) || undefined,
                     };
                 default:
                     throw new TypeError(`Unknown message type: ${type}`);
@@ -4462,6 +4464,7 @@ function syncWithServer(changes, y, syncState, baseRevs, db, databaseUrl, schema
             baseRevs,
             changes: encodeIdsForServer(db.dx.core.schema, currentUser, changes),
             y,
+            dxcv: db.cloud.version
         };
         console.debug('Sync request', syncRequest);
         db.syncStateChangedEvent.next({
@@ -4604,9 +4607,11 @@ function applyServerChanges(changes, db) {
     return __awaiter(this, void 0, void 0, function* () {
         console.debug('Applying server changes', changes, Dexie.currentTransaction);
         for (const { table: tableName, muts } of changes) {
+            if (!db.dx._allTables[tableName]) {
+                console.debug(`Server sent changes for table ${tableName} that we don't have. Ignoring.`);
+                continue;
+            }
             const table = db.table(tableName);
-            if (!table)
-                continue; // If server sends changes on a table we don't have, ignore it.
             const { primaryKey } = table.core.schema;
             const keyDecoder = (key) => {
                 switch (key[0]) {
@@ -4792,9 +4797,15 @@ function listYClientMessagesAndStateVector(db, tablesToSync) {
 
 function getUpdatesTable(db, table, ydocProp) {
     var _a, _b, _c;
+    if (!db.dx._allTables[table])
+        return undefined;
     const utbl = (_c = (_b = (_a = db.table(table)) === null || _a === void 0 ? void 0 : _a.schema.yProps) === null || _b === void 0 ? void 0 : _b.find(p => p.prop === ydocProp)) === null || _c === void 0 ? void 0 : _c.updatesTable;
-    if (!utbl)
-        throw new Error(`No updatesTable found for ${table}.${ydocProp}`);
+    if (!utbl) {
+        console.debug(`No updatesTable found for ${table}.${ydocProp}`);
+        return undefined;
+    }
+    if (!db.dx._allTables[utbl])
+        return undefined;
     return db.table(utbl);
 }
 
@@ -4805,74 +4816,91 @@ function applyYServerMessages(yMessages, db) {
         let resyncNeeded = false;
         let yServerRevision;
         for (const m of yMessages) {
-            switch (m.type) {
-                case 'u-s': {
-                    const utbl = getUpdatesTable(db, m.table, m.prop);
-                    receivedUntils[utbl.name] = yield utbl.add({
-                        k: m.k,
-                        u: m.u,
-                    });
-                    break;
-                }
-                case 'u-ack': {
-                    const utbl = getUpdatesTable(db, m.table, m.prop);
-                    yield db.transaction('rw', utbl, (tx) => __awaiter(this, void 0, void 0, function* () {
-                        let syncer = (yield tx
-                            .table(utbl.name)
-                            .get(DEXIE_CLOUD_SYNCER_ID));
-                        yield tx.table(utbl.name).put(Object.assign(Object.assign({}, (syncer || { i: DEXIE_CLOUD_SYNCER_ID })), { unsentFrom: Math.max((syncer === null || syncer === void 0 ? void 0 : syncer.unsentFrom) || 1, m.i + 1) }));
-                    }));
-                    break;
-                }
-                case 'u-reject': {
-                    // Acces control or constraint rejected the update.
-                    // We delete it. It's not going to be sent again.
-                    // What's missing is a way to notify consumers, such as Tiptap editor, that the update was rejected.
-                    // This is only an issue when the document is open. We could find the open document and
-                    // in a perfect world, we should send a reverse update to the open document to undo the change.
-                    // See my question in https://discuss.yjs.dev/t/generate-an-inverse-update/2765
-                    console.debug(`Y update rejected. Deleting it.`);
-                    const utbl = getUpdatesTable(db, m.table, m.prop);
-                    // Delete the rejected update and all local updates since (avoid holes in the CRDT)
-                    // and destroy it's open document if there is one.
-                    const primaryKey = (_a = (yield utbl.get(m.i))) === null || _a === void 0 ? void 0 : _a.k;
-                    if (primaryKey != null) {
-                        yield db.transaction('rw', utbl, (tx) => {
-                            // @ts-ignore
-                            tx.idbtrans._rejecting_y_ypdate = true; // Inform ydoc triggers that we delete because of a rejection and not GC
-                            return utbl
-                                .where('i')
-                                .aboveOrEqual(m.i)
-                                .filter((u) => cmp(u.k, primaryKey) === 0 && ((u.f || 0) & 1) === 1)
-                                .delete();
-                        });
-                        // Destroy active doc
-                        const activeDoc = DexieYProvider.getDocCache(db.dx).find(m.table, primaryKey, m.prop);
-                        if (activeDoc)
-                            activeDoc.destroy(); // Destroy the document so that editors don't continue to work on it
+            try {
+                switch (m.type) {
+                    case 'u-s': {
+                        const utbl = getUpdatesTable(db, m.table, m.prop);
+                        if (utbl) {
+                            const updateRow = {
+                                k: m.k,
+                                u: m.u,
+                            };
+                            if (m.r) {
+                                // @ts-ignore
+                                updateRow.r = m.r;
+                                yServerRevision = m.r;
+                            }
+                            receivedUntils[utbl.name] = yield utbl.add(updateRow);
+                        }
+                        break;
                     }
-                    break;
-                }
-                case 'in-sync': {
-                    const doc = DexieYProvider.getDocCache(db.dx).find(m.table, m.k, m.prop);
-                    if (doc && !doc.isSynced) {
-                        doc.emit('sync', [true]);
+                    case 'u-ack': {
+                        const utbl = getUpdatesTable(db, m.table, m.prop);
+                        if (utbl) {
+                            yield db.transaction('rw', utbl, (tx) => __awaiter(this, void 0, void 0, function* () {
+                                let syncer = (yield tx
+                                    .table(utbl.name)
+                                    .get(DEXIE_CLOUD_SYNCER_ID));
+                                yield tx.table(utbl.name).put(Object.assign(Object.assign({}, (syncer || { i: DEXIE_CLOUD_SYNCER_ID })), { unsentFrom: Math.max((syncer === null || syncer === void 0 ? void 0 : syncer.unsentFrom) || 1, m.i + 1) }));
+                            }));
+                        }
+                        break;
                     }
-                    break;
-                }
-                case 'y-complete-sync-done': {
-                    yServerRevision = m.yServerRev;
-                    break;
+                    case 'u-reject': {
+                        // Acces control or constraint rejected the update.
+                        // We delete it. It's not going to be sent again.
+                        // What's missing is a way to notify consumers, such as Tiptap editor, that the update was rejected.
+                        // This is only an issue when the document is open. We could find the open document and
+                        // in a perfect world, we should send a reverse update to the open document to undo the change.
+                        // See my question in https://discuss.yjs.dev/t/generate-an-inverse-update/2765
+                        console.debug(`Y update rejected. Deleting it.`);
+                        const utbl = getUpdatesTable(db, m.table, m.prop);
+                        if (!utbl)
+                            break;
+                        // Delete the rejected update and all local updates since (avoid holes in the CRDT)
+                        // and destroy it's open document if there is one.
+                        const primaryKey = (_a = (yield utbl.get(m.i))) === null || _a === void 0 ? void 0 : _a.k;
+                        if (primaryKey != null) {
+                            yield db.transaction('rw', utbl, (tx) => {
+                                // @ts-ignore
+                                tx.idbtrans._rejecting_y_ypdate = true; // Inform ydoc triggers that we delete because of a rejection and not GC
+                                return utbl
+                                    .where('i')
+                                    .aboveOrEqual(m.i)
+                                    .filter((u) => cmp(u.k, primaryKey) === 0 && ((u.f || 0) & 1) === 1)
+                                    .delete();
+                            });
+                            // Destroy active doc
+                            const activeDoc = DexieYProvider.getDocCache(db.dx).find(m.table, primaryKey, m.prop);
+                            if (activeDoc)
+                                activeDoc.destroy(); // Destroy the document so that editors don't continue to work on it
+                        }
+                        break;
+                    }
+                    case 'in-sync': {
+                        const doc = DexieYProvider.getDocCache(db.dx).find(m.table, m.k, m.prop);
+                        if (doc && !doc.isSynced) {
+                            doc.emit('sync', [true]);
+                        }
+                        break;
+                    }
+                    case 'y-complete-sync-done': {
+                        yServerRevision = m.yServerRev;
+                        break;
+                    }
+                    case 'outdated-server-rev':
+                        resyncNeeded = true;
+                        break;
                 }
-                case 'outdated-server-rev':
-                    resyncNeeded = true;
-                    break;
+            }
+            catch (e) {
+                console.error(`Failed to apply YMessage`, m, e);
             }
         }
         return {
             receivedUntils,
             resyncNeeded,
-            yServerRevision
+            yServerRevision,
         };
     });
 }
@@ -4983,7 +5011,9 @@ function downloadYDocsFromServer(db_1, databaseUrl_1, _a) {
                                 throw new Error(`Protocol error from ${databaseUrl}/y/download`);
                             }
                             const yTable = getUpdatesTable(db, currentTable, currentProp);
-                            yield yTable.bulkAdd(docsToInsert);
+                            if (yTable) {
+                                yield yTable.bulkAdd(docsToInsert);
+                            }
                             docsToInsert = [];
                         }
                         if (currentRealmId &&
@@ -5319,12 +5349,12 @@ function _sync(db_1, options_1, schema_1) {
         return false; // Not needed anymore
     });
 }
-function deleteObjectsFromRemovedRealms(db, res, prevState) {
+function deleteObjectsFromRemovedRealms(db, res, syncState) {
     return __awaiter(this, void 0, void 0, function* () {
         const deletedRealms = new Set();
         const rejectedRealms = new Set();
-        const previousRealmSet = prevState ? prevState.realms : [];
-        const previousInviteRealmSet = prevState ? prevState.inviteRealms : [];
+        const previousRealmSet = syncState ? syncState.realms : [];
+        const previousInviteRealmSet = syncState ? syncState.inviteRealms : [];
         const updatedRealmSet = new Set(res.realms);
         const updatedTotalRealmSet = new Set(res.realms.concat(res.inviteRealms));
         for (const realmId of previousRealmSet) {
@@ -5366,17 +5396,10 @@ function deleteObjectsFromRemovedRealms(db, res, prevState) {
                 }
             }
         }
-        if (rejectedRealms.size > 0) {
-            // Remove rejected/deleted realms from yDownloadedRealms because of the following use case:
-            // 1. User becomes added to the realm
-            // 2. User syncs and all documents of the realm is downloaded (downloadYDocsFromServer.ts)
-            // 3. User leaves the realm and all docs are deleted locally (built-in-trigger of deleting their rows in this file)
-            // 4. User is yet again added to the realm. At this point, we must make sure the docs are not considered already downloaded.
-            const updateSpec = {};
+        if (rejectedRealms.size > 0 && (syncState === null || syncState === void 0 ? void 0 : syncState.yDownloadedRealms)) {
             for (const realmId of rejectedRealms) {
-                updateSpec[`yDownloadedRealms.${realmId}`] = undefined; // Setting to undefined will delete the property
+                delete syncState.yDownloadedRealms[realmId];
             }
-            yield db.$syncState.update('syncState', updateSpec);
         }
     });
 }
@@ -6899,13 +6922,13 @@ const SERVER_PING_TIMEOUT = 20000;
 const CLIENT_PING_INTERVAL = 30000;
 const FAIL_RETRY_WAIT_TIME = 60000;
 class WSObservable extends Observable$1 {
-    constructor(db, rev, realmSetHash, clientIdentity, messageProducer, webSocketStatus, user) {
-        super((subscriber) => new WSConnection(db, rev, realmSetHash, clientIdentity, user, subscriber, messageProducer, webSocketStatus));
+    constructor(db, rev, yrev, realmSetHash, clientIdentity, messageProducer, webSocketStatus, user) {
+        super((subscriber) => new WSConnection(db, rev, yrev, realmSetHash, clientIdentity, user, subscriber, messageProducer, webSocketStatus));
     }
 }
 let counter = 0;
 class WSConnection extends Subscription$1 {
-    constructor(db, rev, realmSetHash, clientIdentity, user, subscriber, messageProducer, webSocketStatus) {
+    constructor(db, rev, yrev, realmSetHash, clientIdentity, user, subscriber, messageProducer, webSocketStatus) {
         super(() => this.teardown());
         this.id = ++counter;
         this.subscriptions = new Set();
@@ -6914,6 +6937,7 @@ class WSConnection extends Subscription$1 {
         this.db = db;
         this.databaseUrl = db.cloud.options.databaseUrl;
         this.rev = rev;
+        this.yrev = yrev;
         this.realmSetHash = realmSetHash;
         this.clientIdentity = clientIdentity;
         this.user = user;
@@ -6983,6 +7007,11 @@ class WSConnection extends Subscription$1 {
             }
             this.webSocketStatus.next('connecting');
             this.pinger = setInterval(() => __awaiter(this, void 0, void 0, function* () {
+                // setInterval here causes unnecessary pings when server is proved active anyway.
+                // TODO: Use setTimout() here instead. When triggered, check if we really need to ping.
+                // In case we've had server activity, we don't need to ping. Then schedule then next ping
+                // to the time when we should ping next time (based on lastServerActivity + CLIENT_PING_INTERVAL).
+                // Else, ping now and schedule next ping to CLIENT_PING_INTERVAL from now.
                 if (this.closed) {
                     console.debug('pinger check', this.id, 'CLOSED.');
                     this.teardown();
@@ -7029,9 +7058,13 @@ class WSConnection extends Subscription$1 {
             if (this.subscriber.closed)
                 return;
             searchParams.set('v', '2');
-            searchParams.set('rev', this.rev);
+            if (this.rev)
+                searchParams.set('rev', this.rev);
+            if (this.yrev)
+                searchParams.set('yrev', this.yrev);
             searchParams.set('realmsHash', this.realmSetHash);
             searchParams.set('clientId', this.clientIdentity);
+            searchParams.set('dxcv', this.db.cloud.version);
             if (this.user.accessToken) {
                 searchParams.set('token', this.user.accessToken);
             }
@@ -7068,8 +7101,8 @@ class WSConnection extends Subscription$1 {
                             }
                         }
                     }
-                    else if (msg.type === 'u-ack' || msg.type === 'u-reject' || msg.type === 'u-s' || msg.type === 'in-sync') {
-                        applyYServerMessages([msg], this.db);
+                    else if (msg.type === 'pong') {
+                        // Do nothing
                     }
                     else if (msg.type === 'doc-open') {
                         const docCache = DexieYProvider.getDocCache(this.db.dx);
@@ -7078,11 +7111,26 @@ class WSConnection extends Subscription$1 {
                             getOpenDocSignal(doc).next(); // Make yHandler reopen the document on server.
                         }
                     }
-                    else if (msg.type === 'outdated-server-rev' || msg.type === 'y-complete-sync-done') {
-                        // Won't happen but need this for typing.
-                        throw new Error('Outdated server revision or y-complete-sync-done not expected over WebSocket - only in sync using fetch()');
+                    else if (msg.type === 'u-ack' || msg.type === 'u-reject' || msg.type === 'u-s' || msg.type === 'in-sync' || msg.type === 'outdated-server-rev' || msg.type === 'y-complete-sync-done') {
+                        applyYServerMessages([msg], this.db).then((_a) => __awaiter(this, [_a], void 0, function* ({ resyncNeeded, yServerRevision, receivedUntils }) {
+                            if (yServerRevision) {
+                                yield this.db.$syncState.update('syncState', { yServerRevision: yServerRevision });
+                            }
+                            if (msg.type === 'u-s' && receivedUntils) {
+                                const utbl = getUpdatesTable(this.db, msg.table, msg.prop);
+                                if (utbl) {
+                                    const receivedUntil = receivedUntils[utbl.name];
+                                    if (receivedUntil) {
+                                        yield utbl.update(DEXIE_CLOUD_SYNCER_ID, { receivedUntil });
+                                    }
+                                }
+                            }
+                            if (resyncNeeded) {
+                                yield this.db.cloud.sync({ purpose: 'pull', wait: true });
+                            }
+                        }));
                     }
-                    else if (msg.type !== 'pong') {
+                    else {
                         // Forward the request to our subscriber, wich is in messageFromServerQueue.ts (via connectWebSocket's subscribe() at the end!)
                         this.subscriber.next(msg);
                     }
@@ -7215,7 +7263,7 @@ function connectWebSocket(db) {
             // If no new entries, server won't bother the client. If new entries, server sends only those
             // and the baseRev of the last from same client-ID.
             if (userLogin) {
-                return new WSObservable(db, db.cloud.persistedSyncState.value.serverRevision, realmSetHash, db.cloud.persistedSyncState.value.clientIdentity, messageProducer, db.cloud.webSocketStatus, userLogin);
+                return new WSObservable(db, db.cloud.persistedSyncState.value.serverRevision, db.cloud.persistedSyncState.value.yServerRevision, realmSetHash, db.cloud.persistedSyncState.value.clientIdentity, messageProducer, db.cloud.webSocketStatus, userLogin);
             }
             else {
                 return from$1([]);
@@ -7345,61 +7393,95 @@ function syncIfPossible(db, cloudOptions, cloudSchema, options) {
 }
 
 const SECONDS = 1000;
-const MINUTES = 60 * SECONDS;
 
 function LocalSyncWorker(db, cloudOptions, cloudSchema) {
     let localSyncEventSubscription = null;
-    //let syncHandler: ((event: Event) => void) | null = null;
-    //let periodicSyncHandler: ((event: Event) => void) | null = null;
     let cancelToken = { cancelled: false };
-    let retryHandle = null;
-    let retryPurpose = null; // "pull" is superset of "push"
-    function syncAndRetry(purpose, retryNum = 1) {
+    let nextRetryTime = 0;
+    let syncStartTime = 0;
+    function syncAndRetry(retryNum = 1) {
         // Use setTimeout() to get onto a clean stack and
         // break free from possible active transaction:
         setTimeout(() => {
-            if (retryHandle)
-                clearTimeout(retryHandle);
-            const combPurpose = retryPurpose === 'pull' ? 'pull' : purpose;
-            retryHandle = null;
-            retryPurpose = null;
+            const purpose = pullSignalled ? 'pull' : 'push';
+            syncStartTime = Date.now();
             syncIfPossible(db, cloudOptions, cloudSchema, {
                 cancelToken,
                 retryImmediatelyOnFetchError: true, // workaround for "net::ERR_NETWORK_CHANGED" in chrome.
-                purpose: combPurpose,
-            }).catch((e) => {
-                console.error('error in syncIfPossible()', e);
+                purpose,
+            }).then(() => {
+                if (cancelToken.cancelled) {
+                    stop();
+                }
+                else {
+                    if (pullSignalled || pushSignalled) {
+                        // If we have signalled for more sync, do it now.
+                        pullSignalled = false;
+                        pushSignalled = false;
+                        return syncAndRetry();
+                    }
+                }
+                ongoingSync = false;
+                nextRetryTime = 0;
+                syncStartTime = 0;
+            }).catch((error) => {
+                console.error('error in syncIfPossible()', error);
                 if (cancelToken.cancelled) {
                     stop();
+                    ongoingSync = false;
+                    nextRetryTime = 0;
+                    syncStartTime = 0;
                 }
-                else if (retryNum < 3) {
-                    // Mimic service worker sync event: retry 3 times
-                    // * first retry after 5 minutes
-                    // * second retry 15 minutes later
-                    const combinedPurpose = retryPurpose && retryPurpose === 'pull' ? 'pull' : purpose;
-                    const handle = setTimeout(() => syncAndRetry(combinedPurpose, retryNum + 1), [0, 5, 15][retryNum] * MINUTES);
-                    // Cancel the previous retryHandle if it exists to avoid scheduling loads of retries.
-                    if (retryHandle)
-                        clearTimeout(retryHandle);
-                    retryHandle = handle;
-                    retryPurpose = combinedPurpose;
+                else if (retryNum < 5) {
+                    // Mimic service worker sync event but a bit more eager: retry 4 times
+                    // * first retry after 20 seconds
+                    // * second retry 40 seconds later
+                    // * third retry 5 minutes later
+                    // * last retry 15 minutes later
+                    const retryIn = [0, 20, 40, 300, 900][retryNum] * SECONDS;
+                    nextRetryTime = Date.now() + retryIn;
+                    syncStartTime = 0;
+                    setTimeout(() => syncAndRetry(retryNum + 1), retryIn);
+                }
+                else {
+                    ongoingSync = false;
+                    nextRetryTime = 0;
+                    syncStartTime = 0;
                 }
             });
         }, 0);
     }
+    let pullSignalled = false;
+    let pushSignalled = false;
+    let ongoingSync = false;
+    const consumer = (purpose) => {
+        if (cancelToken.cancelled)
+            return;
+        if (purpose === 'pull') {
+            pullSignalled = true;
+        }
+        if (purpose === 'push') {
+            pushSignalled = true;
+        }
+        if (ongoingSync) {
+            if (nextRetryTime) {
+                console.debug(`Sync is paused until ${new Date(nextRetryTime).toISOString()} due to error in last sync attempt`);
+            }
+            else if (syncStartTime > 0 && Date.now() - syncStartTime > 20 * SECONDS) {
+                console.debug(`An existing sync operation is taking more than 20 seconds. Will resync when done.`);
+            }
+            return;
+        }
+        ongoingSync = true;
+        syncAndRetry();
+    };
     const start = () => {
         // Sync eagerly whenever a change has happened (+ initially when there's no syncState yet)
         // This initial subscribe will also trigger an sync also now.
         console.debug('Starting LocalSyncWorker', db.localSyncEvent['id']);
         localSyncEventSubscription = db.localSyncEvent.subscribe(({ purpose }) => {
-            try {
-                syncAndRetry(purpose || 'pull');
-            }
-            catch (err) {
-                console.error('What-the....', err);
-            }
+            consumer(purpose || 'pull');
         });
-        //setTimeout(()=>db.localSyncEvent.next({}), 5000);
     };
     const stop = () => {
         console.debug('Stopping LocalSyncWorker');
@@ -8191,7 +8273,7 @@ function dexieCloud(dexie) {
     const syncComplete = new Subject();
     dexie.cloud = {
         // @ts-ignore
-        version: "4.1.0-beta.44",
+        version: "4.1.0-beta.45",
         options: Object.assign({}, DEFAULT_OPTIONS),
         schema: null,
         get currentUserId() {
@@ -8509,7 +8591,7 @@ function dexieCloud(dexie) {
     }
 }
 // @ts-ignore
-dexieCloud.version = "4.1.0-beta.44";
+dexieCloud.version = "4.1.0-beta.45";
 Dexie.Cloud = dexieCloud;
 
 // In case the SW lives for a while, let it reuse already opened connections: