@powersync/web 1.30.0 → 1.32.0

package/lib/src/worker/sync/SharedSyncImplementation.js

@@ -1,11 +1,11 @@
-import { AbortOperation, BaseObserver, ConnectionManager, createLogger, SqliteBucketStorage, SyncStatus } from '@powersync/common';
+import { AbortOperation, BaseObserver, ConnectionManager, SqliteBucketStorage, SyncStatus, createLogger } from '@powersync/common';
 import { Mutex } from 'async-mutex';
 import * as Comlink from 'comlink';
-import { WebRemote } from '../../db/sync/WebRemote';
-import { WebStreamingSyncImplementation } from '../../db/sync/WebStreamingSyncImplementation';
-import { LockedAsyncDatabaseAdapter } from '../../db/adapters/LockedAsyncDatabaseAdapter';
-import { WorkerWrappedAsyncDatabaseConnection } from '../../db/adapters/WorkerWrappedAsyncDatabaseConnection';
-import { BroadcastLogger } from './BroadcastLogger';
+import { WebRemote } from '../../db/sync/WebRemote.js';
+import { WebStreamingSyncImplementation } from '../../db/sync/WebStreamingSyncImplementation.js';
+import { LockedAsyncDatabaseAdapter } from '../../db/adapters/LockedAsyncDatabaseAdapter.js';
+import { WorkerWrappedAsyncDatabaseConnection } from '../../db/adapters/WorkerWrappedAsyncDatabaseConnection.js';
+import { BroadcastLogger } from './BroadcastLogger.js';
 /**
  * @internal
  * Manual message events for shared sync clients
@@ -35,7 +35,6 @@ export class SharedSyncImplementation extends BaseObserver {
     statusListener;
     fetchCredentialsController;
     uploadDataController;
-    dbAdapter;
     syncParams;
     logger;
     lastConnectOptions;
@@ -44,10 +43,10 @@ export class SharedSyncImplementation extends BaseObserver {
     connectionManager;
     syncStatus;
     broadCastLogger;
+    distributedDB;
     constructor() {
         super();
         this.ports = [];
-        this.dbAdapter = null;
         this.syncParams = null;
         this.logger = createLogger('shared-sync');
         this.lastConnectOptions = undefined;
@@ -60,26 +59,23 @@ export class SharedSyncImplementation extends BaseObserver {
                 }
             });
         });
+        // Should be configured once we get params
+        this.distributedDB = null;
         this.syncStatus = new SyncStatus({});
         this.broadCastLogger = new BroadcastLogger(this.ports);
         this.connectionManager = new ConnectionManager({
             createSyncImplementation: async () => {
-                return this.portMutex.runExclusive(async () => {
-                    await this.waitForReady();
-                    if (!this.dbAdapter) {
-                        await this.openInternalDB();
+                await this.waitForReady();
+                const sync = this.generateStreamingImplementation();
+                const onDispose = sync.registerListener({
+                    statusChanged: (status) => {
+                        this.updateAllStatuses(status.toJSON());
                     }
-                    const sync = this.generateStreamingImplementation();
-                    const onDispose = sync.registerListener({
-                        statusChanged: (status) => {
-                            this.updateAllStatuses(status.toJSON());
-                        }
-                    });
-                    return {
-                        sync,
-                        onDispose
-                    };
                 });
+                return {
+                    sync,
+                    onDispose
+                };
             },
             logger: this.logger
         });
@@ -90,6 +86,30 @@ export class SharedSyncImplementation extends BaseObserver {
     get isConnected() {
         return this.connectionManager.syncStreamImplementation?.isConnected ?? false;
     }
+    /**
+     * Gets the last client port which we know is safe from unexpected closes.
+     */
+    async getLastWrappedPort() {
+        // Find the last port which is not closing
+        return await this.portMutex.runExclusive(() => {
+            for (let i = this.ports.length - 1; i >= 0; i--) {
+                if (!this.ports[i].isClosing) {
+                    return this.ports[i];
+                }
+            }
+            return;
+        });
+    }
+    /**
+     * In some very rare cases a specific tab might not respond to requests.
+     * This returns a random port which is not closing.
+     */
+    async getRandomWrappedPort() {
+        return await this.portMutex.runExclusive(() => {
+            const nonClosingPorts = this.ports.filter((p) => !p.isClosing);
+            return nonClosingPorts[Math.floor(Math.random() * nonClosingPorts.length)];
+        });
+    }
     async waitForStatus(status) {
         return this.withSyncImplementation(async (sync) => {
             return sync.waitForStatus(status);
@@ -130,28 +150,53 @@ export class SharedSyncImplementation extends BaseObserver {
     async setParams(params) {
         await this.portMutex.runExclusive(async () => {
             this.collectActiveSubscriptions();
-            if (this.syncParams) {
-                // Cannot modify already existing sync implementation params
-                // But we can ask for a DB adapter, if required, at this point.
-                if (!this.dbAdapter) {
-                    await this.openInternalDB();
-                }
-                return;
-            }
-            // First time setting params
-            this.syncParams = params;
-            if (params.streamOptions?.flags?.broadcastLogs) {
-                this.logger = this.broadCastLogger;
-            }
-            self.onerror = (event) => {
-                // Share any uncaught events on the broadcast logger
-                this.logger.error('Uncaught exception in PowerSync shared sync worker', event);
-            };
-            if (!this.dbAdapter) {
-                await this.openInternalDB();
+        });
+        if (this.syncParams) {
+            // Cannot modify already existing sync implementation params
+            return;
+        }
+        // First time setting params
+        this.syncParams = params;
+        if (params.streamOptions?.flags?.broadcastLogs) {
+            this.logger = this.broadCastLogger;
+        }
+        const lockedAdapter = new LockedAsyncDatabaseAdapter({
+            name: params.dbParams.dbFilename,
+            openConnection: async () => {
+                // Gets a connection from the clients when a new connection is requested.
+                const db = await this.openInternalDB();
+                db.registerListener({
+                    closing: () => {
+                        lockedAdapter.reOpenInternalDB();
+                    }
+                });
+                return db;
+            },
+            logger: this.logger,
+            reOpenOnConnectionClosed: true
+        });
+        this.distributedDB = lockedAdapter;
+        await lockedAdapter.init();
+        lockedAdapter.registerListener({
+            databaseReOpened: () => {
+                // We may have missed some table updates while the database was closed.
+                // We can poke the crud in case we missed any updates.
+                this.connectionManager.syncStreamImplementation?.triggerCrudUpload();
+                /**
+                 * FIXME or IMPROVE ME
+                 * The Rust client implementation stores sync state on the connection level.
+                 * Reopening the database causes a state machine error which should cause the
+                 * StreamingSyncImplementation to reconnect. It would be nicer if we could trigger
+                 * this reconnect earlier.
+                 * This reconnect is not required for IndexedDB.
+                 */
             }
-            this.iterateListeners((l) => l.initialized?.());
         });
+        self.onerror = (event) => {
+            // Share any uncaught events on the broadcast logger
+            this.logger.error('Uncaught exception in PowerSync shared sync worker', event);
+        };
+        this.iterateListeners((l) => l.initialized?.());
     }
     async dispose() {
         await this.waitForReady();
@@ -180,7 +225,8 @@ export class SharedSyncImplementation extends BaseObserver {
                 port,
                 clientProvider: Comlink.wrap(port),
                 currentSubscriptions: [],
-                closeListeners: []
+                closeListeners: [],
+                isClosing: false
             };
             this.ports.push(portProvider);
             // Give the newly connected client the latest status
@@ -196,14 +242,16 @@ export class SharedSyncImplementation extends BaseObserver {
      * clients.
      */
     async removePort(port) {
+        // Ports might be removed faster than we can process them.
+        port.isClosing = true;
         // Remove the port within a mutex context.
         // Warns if the port is not found. This should not happen in practice.
         // We return early if the port is not found.
-        const { trackedPort, shouldReconnect } = await this.portMutex.runExclusive(async () => {
+        return await this.portMutex.runExclusive(async () => {
             const index = this.ports.findIndex((p) => p == port);
             if (index < 0) {
                 this.logger.warn(`Could not remove port ${port} since it is not present in active ports.`);
-                return {};
+                return () => { };
             }
             const trackedPort = this.ports[index];
             // Remove from the list of active ports
@@ -217,35 +265,13 @@ export class SharedSyncImplementation extends BaseObserver {
                     abortController.controller.abort(new AbortOperation('Closing pending requests after client port is removed'));
                 }
             });
-            const shouldReconnect = !!this.connectionManager.syncStreamImplementation && this.ports.length > 0;
-            return {
-                shouldReconnect,
-                trackedPort
-            };
-        });
-        if (!trackedPort) {
-            // We could not find the port to remove
-            return () => { };
-        }
-        for (const closeListener of trackedPort.closeListeners) {
-            await closeListener();
-        }
-        if (this.dbAdapter && this.dbAdapter == trackedPort.db) {
-            // Unconditionally close the connection because the database it's writing to has just been closed.
-            // The connection has been closed previously, this might throw. We should be able to ignore it.
-            await this.connectionManager
-                .disconnect()
-                .catch((ex) => this.logger.warn('Error while disconnecting. Will attempt to reconnect.', ex));
-            // Clearing the adapter will result in a new one being opened in connect
-            this.dbAdapter = null;
-            if (shouldReconnect) {
-                await this.connectionManager.connect(CONNECTOR_PLACEHOLDER, this.lastConnectOptions ?? {});
+            // Close the worker wrapped database connection, we can't accurately rely on this connection
+            for (const closeListener of trackedPort.closeListeners) {
+                await closeListener();
             }
-        }
-        // Re-index subscriptions, the subscriptions of the removed port would no longer be considered.
-        this.collectActiveSubscriptions();
-        // Release proxy
-        return () => trackedPort.clientProvider[Comlink.releaseProxy]();
+            this.collectActiveSubscriptions();
+            return () => trackedPort.clientProvider[Comlink.releaseProxy]();
+        });
     }
     triggerCrudUpload() {
         this.withSyncImplementation(async (sync) => {
@@ -282,10 +308,13 @@ export class SharedSyncImplementation extends BaseObserver {
         const syncParams = this.syncParams;
         // Create a new StreamingSyncImplementation for each connect call. This is usually done is all SDKs.
         return new WebStreamingSyncImplementation({
-            adapter: new SqliteBucketStorage(this.dbAdapter, this.logger),
+            adapter: new SqliteBucketStorage(this.distributedDB, this.logger),
             remote: new WebRemote({
                 invalidateCredentials: async () => {
-                    const lastPort = this.ports[this.ports.length - 1];
+                    const lastPort = await this.getLastWrappedPort();
+                    if (!lastPort) {
+                        throw new Error('No client port found to invalidate credentials');
+                    }
                     try {
                         this.logger.log('calling the last port client provider to invalidate credentials');
                         lastPort.clientProvider.invalidateCredentials();
@@ -295,7 +324,10 @@ export class SharedSyncImplementation extends BaseObserver {
                     }
                 },
                 fetchCredentials: async () => {
-                    const lastPort = this.ports[this.ports.length - 1];
+                    const lastPort = await this.getLastWrappedPort();
+                    if (!lastPort) {
+                        throw new Error('No client port found to fetch credentials');
+                    }
                     return new Promise(async (resolve, reject) => {
                         const abortController = new AbortController();
                         this.fetchCredentialsController = {
@@ -317,7 +349,10 @@ export class SharedSyncImplementation extends BaseObserver {
                 }
             }, this.logger),
             uploadCrud: async () => {
-                const lastPort = this.ports[this.ports.length - 1];
+                const lastPort = await this.getLastWrappedPort();
+                if (!lastPort) {
+                    throw new Error('No client port found to upload crud');
+                }
                 return new Promise(async (resolve, reject) => {
                     const abortController = new AbortController();
                     this.uploadDataController = {
@@ -343,38 +378,81 @@ export class SharedSyncImplementation extends BaseObserver {
             logger: this.logger
         });
     }
+    /**
+     * Opens a worker wrapped database connection. Using the last connected client port.
+     */
     async openInternalDB() {
-        const lastClient = this.ports[this.ports.length - 1];
-        if (!lastClient) {
+        const client = await this.getRandomWrappedPort();
+        if (!client) {
             // Should not really happen in practice
             throw new Error(`Could not open DB connection since no client is connected.`);
         }
-        const workerPort = await lastClient.clientProvider.getDBWorkerPort();
+        // Fail-safe timeout for opening a database connection.
+        const timeout = setTimeout(() => {
+            abortController.abort();
+        }, 10_000);
+        /**
+         * Handle cases where the client might close while opening a connection.
+         */
+        const abortController = new AbortController();
+        const closeListener = () => {
+            abortController.abort();
+        };
+        const removeCloseListener = () => {
+            const index = client.closeListeners.indexOf(closeListener);
+            if (index >= 0) {
+                client.closeListeners.splice(index, 1);
+            }
+        };
+        client.closeListeners.push(closeListener);
+        const workerPort = await withAbort({
+            action: () => client.clientProvider.getDBWorkerPort(),
+            signal: abortController.signal,
+            cleanupOnAbort: (port) => {
+                port.close();
+            }
+        }).catch((ex) => {
+            removeCloseListener();
+            throw ex;
+        });
         const remote = Comlink.wrap(workerPort);
         const identifier = this.syncParams.dbParams.dbFilename;
-        const db = await remote(this.syncParams.dbParams);
-        const locked = new LockedAsyncDatabaseAdapter({
-            name: identifier,
-            openConnection: async () => {
-                const wrapped = new WorkerWrappedAsyncDatabaseConnection({
-                    remote,
-                    baseConnection: db,
-                    identifier,
-                    // It's possible for this worker to outlive the client hosting the database for us. We need to be prepared for
-                    // that and ensure pending requests are aborted when the tab is closed.
-                    remoteCanCloseUnexpectedly: true
-                });
-                lastClient.closeListeners.push(async () => {
-                    this.logger.info('Aborting open connection because associated tab closed.');
-                    await wrapped.close().catch((ex) => this.logger.warn('error closing database connection', ex));
-                    wrapped.markRemoteClosed();
-                });
-                return wrapped;
-            },
-            logger: this.logger
+        /**
+         * The open could fail if the tab is closed while we're busy opening the database.
+         * This operation is typically executed inside an exclusive portMutex lock.
+         * We typically execute the closeListeners using the portMutex in a different context.
+         * We can't rely on the closeListeners to abort the operation if the tab is closed.
+         */
+        const db = await withAbort({
+            action: () => remote(this.syncParams.dbParams),
+            signal: abortController.signal,
+            cleanupOnAbort: (db) => {
+                db.close();
+            }
+        }).finally(() => {
+            // We can remove the close listener here since we no longer need it past this point.
+            removeCloseListener();
+        });
+        clearTimeout(timeout);
+        const wrapped = new WorkerWrappedAsyncDatabaseConnection({
+            remote,
+            baseConnection: db,
+            identifier,
+            // It's possible for this worker to outlive the client hosting the database for us. We need to be prepared for
+            // that and ensure pending requests are aborted when the tab is closed.
+            remoteCanCloseUnexpectedly: true
+        });
+        client.closeListeners.push(async () => {
+            this.logger.info('Aborting open connection because associated tab closed.');
+            /**
+             * Don't await this close operation. It might never resolve if the tab is closed.
+             * We mark the remote as closed first, this will reject any pending requests.
+             * We then call close. The close operation is configured to fire-and-forget, the main promise will reject immediately.
+             */
+            wrapped.markRemoteClosed();
+            wrapped.close().catch((ex) => this.logger.warn('error closing database connection', ex));
         });
-        await locked.init();
-        this.dbAdapter = lastClient.db = locked;
+        return wrapped;
     }
     /**
      * A method to update the all shared statuses for each
@@ -384,16 +462,34 @@ export class SharedSyncImplementation extends BaseObserver {
         this.syncStatus = new SyncStatus(status);
         this.ports.forEach((p) => p.clientProvider.statusChanged(status));
     }
-    /**
-     * A function only used for unit tests which updates the internal
-     * sync stream client and all tab client's sync status
-     */
-    async _testUpdateAllStatuses(status) {
-        if (!this.connectionManager.syncStreamImplementation) {
-            throw new Error('Cannot update status without a sync stream implementation');
+}
+/**
+ * Runs the action with an abort controller.
+ */
+function withAbort(options) {
+    const { action, signal, cleanupOnAbort } = options;
+    return new Promise((resolve, reject) => {
+        if (signal.aborted) {
+            reject(new AbortOperation('Operation aborted by abort controller'));
+            return;
         }
-        // Only assigning, don't call listeners for this test
-        this.connectionManager.syncStreamImplementation.syncStatus = new SyncStatus(status);
-        this.updateAllStatuses(status);
-    }
+        function handleAbort() {
+            signal.removeEventListener('abort', handleAbort);
+            reject(new AbortOperation('Operation aborted by abort controller'));
+        }
+        signal.addEventListener('abort', handleAbort, { once: true });
+        function completePromise(action) {
+            signal.removeEventListener('abort', handleAbort);
+            action();
+        }
+        action()
+            .then((data) => {
+            // We already rejected due to the abort, allow for cleanup
+            if (signal.aborted) {
+                return completePromise(() => cleanupOnAbort?.(data));
+            }
+            completePromise(() => resolve(data));
+        })
+            .catch((e) => completePromise(() => reject(e)));
+    });
 }

package/lib/src/worker/sync/SharedSyncImplementation.worker.js

@@ -1,11 +1,11 @@
 import { createBaseLogger } from '@powersync/common';
-import { SharedSyncImplementation } from './SharedSyncImplementation';
-import { WorkerClient } from './WorkerClient';
+import { SharedSyncImplementation } from './SharedSyncImplementation.js';
+import { WorkerClient } from './WorkerClient.js';
 const _self = self;
 const logger = createBaseLogger();
 logger.useDefaults();
 const sharedSyncImplementation = new SharedSyncImplementation();
 _self.onconnect = async function (event) {
     const port = event.ports[0];
-    await new WorkerClient(sharedSyncImplementation, port).initialize();
+    new WorkerClient(sharedSyncImplementation, port);
 };

package/lib/src/worker/sync/WorkerClient.d.ts

@@ -1,5 +1,5 @@
-import { SharedSyncImplementation, SharedSyncInitOptions } from './SharedSyncImplementation';
-import { ILogLevel, PowerSyncConnectionOptions, SubscribedStream, SyncStatusOptions } from '@powersync/common';
+import { ILogLevel, PowerSyncConnectionOptions, SubscribedStream } from '@powersync/common';
+import { SharedSyncImplementation, SharedSyncInitOptions, WrappedSyncPort } from './SharedSyncImplementation.js';
 /**
  * A client to the shared sync worker.
  *
@@ -10,8 +10,8 @@ export declare class WorkerClient {
     private readonly sync;
     private readonly port;
     private resolvedPort;
+    protected resolvedPortPromise: Promise<WrappedSyncPort> | null;
     constructor(sync: SharedSyncImplementation, port: MessagePort);
-    initialize(): Promise<void>;
     private removePort;
     /**
      * Called by a client after obtaining a lock with a random name.
@@ -19,7 +19,7 @@ export declare class WorkerClient {
      * When the client tab is closed, its lock will be returned. So when the shared worker attempts to acquire the lock,
      * it can consider the connection to be closed.
      */
-    addLockBasedCloseSignal(name: string): void;
+    addLockBasedCloseSignal(name: string): Promise<void>;
     setLogLevel(level: ILogLevel): void;
     triggerCrudUpload(): void;
     setParams(params: SharedSyncInitOptions, subscriptions: SubscribedStream[]): Promise<void>;
@@ -28,5 +28,4 @@ export declare class WorkerClient {
     connect(options?: PowerSyncConnectionOptions): Promise<void>;
     updateSubscriptions(subscriptions: SubscribedStream[]): void;
     disconnect(): Promise<void>;
-    _testUpdateAllStatuses(status: SyncStatusOptions): Promise<void>;
 }

package/lib/src/worker/sync/WorkerClient.js

@@ -1,6 +1,6 @@
 import * as Comlink from 'comlink';
-import { SharedSyncClientEvent } from './SharedSyncImplementation';
-import { getNavigatorLocks } from '../../shared/navigator';
+import { getNavigatorLocks } from '../../shared/navigator.js';
+import { SharedSyncClientEvent } from './SharedSyncImplementation.js';
 /**
  * A client to the shared sync worker.
  *
@@ -11,11 +11,11 @@ export class WorkerClient {
     sync;
     port;
     resolvedPort = null;
+    resolvedPortPromise = null;
     constructor(sync, port) {
         this.sync = sync;
         this.port = port;
-    }
-    async initialize() {
+        Comlink.expose(this, this.port);
         /**
          * Adds an extra listener which can remove this port
          * from the list of monitored ports.
@@ -26,8 +26,6 @@ export class WorkerClient {
                 await this.removePort();
             }
         });
-        this.resolvedPort = await this.sync.addPort(this.port);
-        Comlink.expose(this, this.port);
     }
     async removePort() {
         if (this.resolvedPort) {
@@ -48,7 +46,10 @@ export class WorkerClient {
      * When the client tab is closed, its lock will be returned. So when the shared worker attempts to acquire the lock,
      * it can consider the connection to be closed.
      */
-    addLockBasedCloseSignal(name) {
+    async addLockBasedCloseSignal(name) {
+        // Only add the port once the lock has been obtained on the client.
+        this.resolvedPort = await this.sync.addPort(this.port);
+        // Don't await this lock request
         getNavigatorLocks().request(name, async () => {
             await this.removePort();
         });
@@ -80,7 +81,4 @@ export class WorkerClient {
     disconnect() {
         return this.sync.disconnect();
     }
-    async _testUpdateAllStatuses(status) {
-        return this.sync._testUpdateAllStatuses(status);
-    }
 }