@powersync/web 1.36.0 → 1.37.0

package/lib/src/db/adapters/wa-sqlite/vfs.js

@@ -0,0 +1,91 @@
+/**
+ * List of currently tested virtual filesystems
+ */
+export var WASQLiteVFS;
+(function (WASQLiteVFS) {
+    WASQLiteVFS["IDBBatchAtomicVFS"] = "IDBBatchAtomicVFS";
+    WASQLiteVFS["OPFSCoopSyncVFS"] = "OPFSCoopSyncVFS";
+    WASQLiteVFS["AccessHandlePoolVFS"] = "AccessHandlePoolVFS";
+})(WASQLiteVFS || (WASQLiteVFS = {}));
+export function vfsRequiresDedicatedWorkers(vfs) {
+    return vfs != WASQLiteVFS.IDBBatchAtomicVFS;
+}
+/**
+ * @internal
+ */
+export const AsyncWASQLiteModuleFactory = async () => {
+    const { default: factory } = await import('@journeyapps/wa-sqlite/dist/wa-sqlite-async.mjs');
+    return factory();
+};
+/**
+ * @internal
+ */
+export const MultiCipherAsyncWASQLiteModuleFactory = async () => {
+    const { default: factory } = await import('@journeyapps/wa-sqlite/dist/mc-wa-sqlite-async.mjs');
+    return factory();
+};
+/**
+ * @internal
+ */
+export const SyncWASQLiteModuleFactory = async () => {
+    const { default: factory } = await import('@journeyapps/wa-sqlite/dist/wa-sqlite.mjs');
+    return factory();
+};
+/**
+ * @internal
+ */
+export const MultiCipherSyncWASQLiteModuleFactory = async () => {
+    const { default: factory } = await import('@journeyapps/wa-sqlite/dist/mc-wa-sqlite.mjs');
+    return factory();
+};
+/**
+ * @internal
+ */
+export const DEFAULT_MODULE_FACTORIES = {
+    [WASQLiteVFS.IDBBatchAtomicVFS]: async (options) => {
+        let module;
+        if (options.encryptionKey) {
+            module = await MultiCipherAsyncWASQLiteModuleFactory();
+        }
+        else {
+            module = await AsyncWASQLiteModuleFactory();
+        }
+        const { IDBBatchAtomicVFS } = await import('@journeyapps/wa-sqlite/src/examples/IDBBatchAtomicVFS.js');
+        return {
+            module,
+            // @ts-expect-error The types for this static method are missing upstream
+            vfs: await IDBBatchAtomicVFS.create(options.dbFileName, module, { lockPolicy: 'exclusive' })
+        };
+    },
+    [WASQLiteVFS.AccessHandlePoolVFS]: async (options) => {
+        let module;
+        if (options.encryptionKey) {
+            module = await MultiCipherSyncWASQLiteModuleFactory();
+        }
+        else {
+            module = await SyncWASQLiteModuleFactory();
+        }
+        // @ts-expect-error The types for this static method are missing upstream
+        const { AccessHandlePoolVFS } = await import('@journeyapps/wa-sqlite/src/examples/AccessHandlePoolVFS.js');
+        return {
+            module,
+            vfs: await AccessHandlePoolVFS.create(options.dbFileName, module)
+        };
+    },
+    [WASQLiteVFS.OPFSCoopSyncVFS]: async (options) => {
+        let module;
+        if (options.encryptionKey) {
+            module = await MultiCipherSyncWASQLiteModuleFactory();
+        }
+        else {
+            module = await SyncWASQLiteModuleFactory();
+        }
+        // @ts-expect-error The types for this static method are missing upstream
+        const { OPFSCoopSyncVFS } = await import('@journeyapps/wa-sqlite/src/examples/OPFSCoopSyncVFS.js');
+        const vfs = await OPFSCoopSyncVFS.create(options.dbFileName, module);
+        return {
+            module,
+            vfs
+        };
+    }
+};

package/lib/src/db/adapters/web-sql-flags.d.ts

@@ -62,6 +62,11 @@ export interface WebSQLOpenFactoryOptions extends SQLOpenOptions {
      * or a factory method that returns a worker.
      */
     worker?: string | URL | ((options: ResolvedWebSQLOpenOptions) => Worker | SharedWorker);
+    /**
+     * Use an existing port to an initialized worker.
+     * A worker will be initialized if none is provided
+     */
+    workerPort?: MessagePort;
     logger?: ILogger;
     /**
      * Where to store SQLite temporary files. Defaults to 'MEMORY'.

package/lib/src/db/sync/SSRWebStreamingSyncImplementation.d.ts

@@ -1,5 +1,4 @@
-import { AbstractStreamingSyncImplementationOptions, BaseObserver, LockOptions, PowerSyncConnectionOptions, StreamingSyncImplementation, SyncStatus, SyncStatusOptions } from '@powersync/common';
-import { Mutex } from 'async-mutex';
+import { AbstractStreamingSyncImplementationOptions, BaseObserver, LockOptions, Mutex, PowerSyncConnectionOptions, StreamingSyncImplementation, SyncStatus, SyncStatusOptions } from '@powersync/common';
 export declare class SSRStreamingSyncImplementation extends BaseObserver implements StreamingSyncImplementation {
     syncMutex: Mutex;
     crudMutex: Mutex;

package/lib/src/db/sync/SSRWebStreamingSyncImplementation.js

@@ -1,5 +1,4 @@
-import { BaseObserver, LockType, SyncStatus } from '@powersync/common';
-import { Mutex } from 'async-mutex';
+import { BaseObserver, LockType, Mutex, SyncStatus } from '@powersync/common';
 export class SSRStreamingSyncImplementation extends BaseObserver {
     syncMutex;
     crudMutex;
@@ -15,7 +14,7 @@ export class SSRStreamingSyncImplementation extends BaseObserver {
     }
     obtainLock(lockOptions) {
         const mutex = lockOptions.type == LockType.CRUD ? this.crudMutex : this.syncMutex;
-        return mutex.runExclusive(lockOptions.callback);
+        return mutex.runExclusive(lockOptions.callback, lockOptions.signal);
     }
     /**
      * This is a no-op in SSR mode

package/lib/src/db/sync/SharedWebStreamingSyncImplementation.js

@@ -1,9 +1,9 @@
 import * as Comlink from 'comlink';
-import { getNavigatorLocks } from '../../shared/navigator.js';
 import { AbstractSharedSyncClientProvider } from '../../worker/sync/AbstractSharedSyncClientProvider.js';
 import { SharedSyncClientEvent } from '../../worker/sync/SharedSyncImplementation.js';
 import { DEFAULT_CACHE_SIZE_KB, TemporaryStorageOption, resolveWebSQLFlags } from '../adapters/web-sql-flags.js';
 import { WebStreamingSyncImplementation } from './WebStreamingSyncImplementation.js';
+import { generateTabCloseSignal } from '../../shared/tab_close_signal.js';
 /**
  * The shared worker will trigger methods on this side of the message port
  * via this client provider.
@@ -156,24 +156,9 @@ export class SharedWebStreamingSyncImplementation extends WebStreamingSyncImplem
          *    - We resolve the top-level promise after the lock has been registered with the shared worker.
          * - The client sends the params to the shared worker after locks have been registered.
          */
-        await new Promise((resolve) => {
-            // Request a random lock until this client is disposed. The name of the lock is sent to the shared worker, which
-            // will also attempt to acquire it. Since the lock is returned when the tab is closed, this allows the share worker
-            // to free resources associated with this tab.
-            // We take hold of this lock as soon-as-possible in order to cater for potentially closed tabs.
-            getNavigatorLocks().request(`tab-close-signal-${crypto.randomUUID()}`, async (lock) => {
-                if (this.abortOnClose.signal.aborted) {
-                    return;
-                }
-                // Awaiting here ensures the worker is waiting for the lock
-                await this.syncManager.addLockBasedCloseSignal(lock.name);
-                // The lock has been registered, we can continue with the initialization
-                resolve();
-                await new Promise((r) => {
-                    this.abortOnClose.signal.onabort = () => r();
-                });
-            });
-        });
+        const closeSignal = await generateTabCloseSignal(this.abortOnClose.signal);
+        // Awaiting here ensures the worker is waiting for the lock
+        await this.syncManager.addLockBasedCloseSignal(closeSignal);
         const { crudUploadThrottleMs, identifier, retryDelayMs } = this.options;
         const flags = { ...this.webOptions.flags, workers: undefined };
         await this.syncManager.setParams({

package/lib/src/index.d.ts

@@ -1,10 +1,7 @@
 export * from '@powersync/common';
 export * from './attachments/IndexDBFileSystemAdapter.js';
 export * from './db/adapters/AbstractWebPowerSyncDatabaseOpenFactory.js';
-export * from './db/adapters/AbstractWebSQLOpenFactory.js';
-export * from './db/adapters/AsyncDatabaseConnection.js';
-export * from './db/adapters/wa-sqlite/WASQLiteConnection.js';
-export * from './db/adapters/wa-sqlite/WASQLiteDBAdapter.js';
+export { WASQLiteVFS } from './db/adapters/wa-sqlite/vfs.js';
 export * from './db/adapters/wa-sqlite/WASQLiteOpenFactory.js';
 export * from './db/adapters/wa-sqlite/WASQLitePowerSyncDatabaseOpenFactory.js';
 export * from './db/adapters/web-sql-flags.js';

package/lib/src/index.js

@@ -1,10 +1,7 @@
 export * from '@powersync/common';
 export * from './attachments/IndexDBFileSystemAdapter.js';
 export * from './db/adapters/AbstractWebPowerSyncDatabaseOpenFactory.js';
-export * from './db/adapters/AbstractWebSQLOpenFactory.js';
-export * from './db/adapters/AsyncDatabaseConnection.js';
-export * from './db/adapters/wa-sqlite/WASQLiteConnection.js';
-export * from './db/adapters/wa-sqlite/WASQLiteDBAdapter.js';
+export { WASQLiteVFS } from './db/adapters/wa-sqlite/vfs.js';
 export * from './db/adapters/wa-sqlite/WASQLiteOpenFactory.js';
 export * from './db/adapters/wa-sqlite/WASQLitePowerSyncDatabaseOpenFactory.js';
 export * from './db/adapters/web-sql-flags.js';

package/lib/src/shared/tab_close_signal.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Requests a random lock that will be released once the optional signal is aborted (or, if no signal is given, when the
+ * tab is closed).
+ *
+ * This allows sending the name of the lock to another context (e.g. a shared worker), which will also attempt to
+ * acquire it. Since the lock is returned when the tab is closed, this allows the shared worker to free resources
+ * assocatiated with this tab.
+ *
+ *  We take hold of this lock as soon-as-possible in order to cater for potentially closed tabs.
+ */
+export declare function generateTabCloseSignal(abort?: AbortSignal): Promise<string>;

package/lib/src/shared/tab_close_signal.js

@@ -0,0 +1,26 @@
+import { getNavigatorLocks } from './navigator.js';
+/**
+ * Requests a random lock that will be released once the optional signal is aborted (or, if no signal is given, when the
+ * tab is closed).
+ *
+ * This allows sending the name of the lock to another context (e.g. a shared worker), which will also attempt to
+ * acquire it. Since the lock is returned when the tab is closed, this allows the shared worker to free resources
+ * assocatiated with this tab.
+ *
+ *  We take hold of this lock as soon-as-possible in order to cater for potentially closed tabs.
+ */
+export function generateTabCloseSignal(abort) {
+    return new Promise((resolve, reject) => {
+        const options = { signal: abort };
+        getNavigatorLocks()
+            .request(`tab-close-signal-${crypto.randomUUID()}`, options, (lock) => {
+            resolve(lock.name);
+            return new Promise((resolve) => {
+                if (abort) {
+                    abort.addEventListener('abort', () => resolve());
+                }
+            });
+        })
+            .catch(reject);
+    });
+}

package/lib/src/worker/db/MultiDatabaseServer.d.ts

@@ -0,0 +1,17 @@
+import { ILogger } from '@powersync/common';
+import { ClientConnectionView } from '../../db/adapters/wa-sqlite/DatabaseServer.js';
+import { ResolvedWASQLiteOpenFactoryOptions, WorkerDBOpenerOptions } from '../../db/adapters/wa-sqlite/WASQLiteOpenFactory.js';
+/**
+ * Shared state to manage multiple database connections hosted by a worker.
+ */
+export declare class MultiDatabaseServer {
+    readonly logger: ILogger;
+    private activeDatabases;
+    constructor(logger: ILogger);
+    handleConnection(options: WorkerDBOpenerOptions): Promise<ClientConnectionView>;
+    connectToExisting(name: string, lockName: string): Promise<ClientConnectionView>;
+    openConnectionLocally(options: ResolvedWASQLiteOpenFactoryOptions, lockName?: string): Promise<ClientConnectionView>;
+    private databaseOpenAttempt;
+    closeAll(): Promise<void[]>;
+}
+export declare const isSharedWorker: boolean;

package/lib/src/worker/db/MultiDatabaseServer.js

@@ -0,0 +1,86 @@
+import * as Comlink from 'comlink';
+import { DatabaseServer } from '../../db/adapters/wa-sqlite/DatabaseServer.js';
+import { getNavigatorLocks } from '../../shared/navigator.js';
+import { RawSqliteConnection } from '../../db/adapters/wa-sqlite/RawSqliteConnection.js';
+import { ConcurrentSqliteConnection } from '../../db/adapters/wa-sqlite/ConcurrentConnection.js';
+const OPEN_DB_LOCK = 'open-wasqlite-db';
+/**
+ * Shared state to manage multiple database connections hosted by a worker.
+ */
+export class MultiDatabaseServer {
+    logger;
+    activeDatabases = new Map();
+    constructor(logger) {
+        this.logger = logger;
+    }
+    async handleConnection(options) {
+        this.logger.setLevel(options.logLevel);
+        return Comlink.proxy(await this.openConnectionLocally(options, options.lockName));
+    }
+    async connectToExisting(name, lockName) {
+        return getNavigatorLocks().request(OPEN_DB_LOCK, async () => {
+            const server = this.activeDatabases.get(name);
+            if (server == null) {
+                throw new Error(`connectToExisting(${name}) failed because the worker doesn't own a database with that name.`);
+            }
+            return Comlink.proxy(await server.connect(lockName));
+        });
+    }
+    async openConnectionLocally(options, lockName) {
+        // Especially on Firefox, we're sometimes seeing "NoModificationAllowedError"s when opening OPFS databases we can
+        // work around by retrying.
+        const maxAttempts = 3;
+        let server;
+        for (let count = 0; count < maxAttempts - 1; count++) {
+            try {
+                server = await this.databaseOpenAttempt(options);
+            }
+            catch (ex) {
+                this.logger.warn(`Attempt ${count + 1} of ${maxAttempts} to open database failed, retrying in 1 second...`, ex);
+                await new Promise((resolve) => setTimeout(resolve, 1000));
+            }
+        }
+        // Final attempt if we haven't been able to open the server - rethrow errors if we still can't open.
+        server ??= await this.databaseOpenAttempt(options);
+        return server.connect(lockName);
+    }
+    async databaseOpenAttempt(options) {
+        return getNavigatorLocks().request(OPEN_DB_LOCK, async () => {
+            const { dbFilename } = options;
+            let server = this.activeDatabases.get(dbFilename);
+            if (server == null) {
+                const needsNavigatorLocks = !isSharedWorker;
+                const connection = new RawSqliteConnection(options);
+                const withSafeConcurrency = new ConcurrentSqliteConnection(connection, needsNavigatorLocks);
+                // Initializing the RawSqliteConnection will run some pragmas that might write to the database file, so we want
+                // to do that in an exclusive lock. Note that OPEN_DB_LOCK is not enough for that, as another tab might have
+                // already created a connection (and is thus outside of OPEN_DB_LOCK) while currently writing to it.
+                const returnLease = await withSafeConcurrency.acquireMutex();
+                try {
+                    await connection.init();
+                }
+                catch (e) {
+                    returnLease();
+                    await connection.close();
+                    throw e;
+                }
+                returnLease();
+                const onClose = () => this.activeDatabases.delete(dbFilename);
+                server = new DatabaseServer({
+                    inner: withSafeConcurrency,
+                    logger: this.logger,
+                    onClose
+                });
+                this.activeDatabases.set(dbFilename, server);
+            }
+            return server;
+        });
+    }
+    closeAll() {
+        const existingDatabases = [...this.activeDatabases.values()];
+        return Promise.all(existingDatabases.map((db) => {
+            db.forceClose();
+        }));
+    }
+}
+export const isSharedWorker = 'SharedWorkerGlobalScope' in globalThis;

package/lib/src/worker/db/WASQLiteDB.worker.js

@@ -4,66 +4,27 @@
 import '@journeyapps/wa-sqlite';
 import { createBaseLogger, createLogger } from '@powersync/common';
 import * as Comlink from 'comlink';
-import { getNavigatorLocks } from '../../shared/navigator.js';
-import { SharedWASQLiteConnection } from './SharedWASQLiteConnection.js';
-import { WorkerWASQLiteConnection } from './WorkerWASQLiteConnection.js';
+import { isSharedWorker, MultiDatabaseServer } from './MultiDatabaseServer.js';
 const baseLogger = createBaseLogger();
 baseLogger.useDefaults();
 const logger = createLogger('db-worker');
-const DBMap = new Map();
-const OPEN_DB_LOCK = 'open-wasqlite-db';
-let nextClientId = 1;
-const openDBShared = async (options) => {
-    // Prevent multiple simultaneous opens from causing race conditions
-    return getNavigatorLocks().request(OPEN_DB_LOCK, async () => {
-        const clientId = nextClientId++;
-        const { dbFilename, logLevel } = options;
-        logger.setLevel(logLevel);
-        if (!DBMap.has(dbFilename)) {
-            const clientIds = new Set();
-            // This format returns proxy objects for function callbacks
-            const connection = new WorkerWASQLiteConnection(options);
-            await connection.init();
-            connection.registerListener({
-                holdOverwritten: async () => {
-                    /**
-                     * The previous hold has been overwritten, without being released.
-                     * we need to cleanup any resources associated with it.
-                     * We can perform a rollback to release any potential transactions that were started.
-                     */
-                    await connection.execute('ROLLBACK').catch(() => { });
-                }
-            });
-            DBMap.set(dbFilename, {
-                clientIds,
-                db: connection
-            });
-        }
-        // Associates this clientId with the shared connection entry
-        const sharedConnection = new SharedWASQLiteConnection({
-            dbMap: DBMap,
-            dbFilename,
-            clientId,
-            logger
-        });
-        return Comlink.proxy(sharedConnection);
-    });
+const server = new MultiDatabaseServer(logger);
+const exposedFunctions = {
+    connect: (config) => server.handleConnection(config),
+    connectToExisting: ({ identifier, lockName }) => server.connectToExisting(identifier, lockName)
 };
 // Check if we're in a SharedWorker context
-if (typeof SharedWorkerGlobalScope !== 'undefined') {
+if (isSharedWorker) {
     const _self = self;
     _self.onconnect = function (event) {
         const port = event.ports[0];
-        Comlink.expose(openDBShared, port);
+        Comlink.expose(exposedFunctions, port);
     };
 }
 else {
     // A dedicated worker can be shared externally
-    Comlink.expose(openDBShared);
+    Comlink.expose(exposedFunctions);
 }
 addEventListener('unload', () => {
-    Array.from(DBMap.values()).forEach(async (dbConnection) => {
-        const { db } = dbConnection;
-        db.close?.();
-    });
+    server.closeAll();
 });

package/lib/src/worker/db/open-worker-database.d.ts

@@ -1,6 +1,6 @@
 import * as Comlink from 'comlink';
-import { OpenAsyncDatabaseConnection } from '../../db/adapters/AsyncDatabaseConnection.js';
-import { WASQLiteVFS } from '../../db/adapters/wa-sqlite/WASQLiteConnection.js';
+import { WASQLiteVFS } from '../../db/adapters/wa-sqlite/vfs.js';
+import { OpenWorkerConnection } from '../../db/adapters/wa-sqlite/DatabaseClient.js';
 /**
  * Opens a shared or dedicated worker which exposes opening of database connections
  */
@@ -9,6 +9,6 @@ export declare function openWorkerDatabasePort(workerIdentifier: string, multipl
  * @returns A function which allows for opening database connections inside
  * a worker.
  */
-export declare function getWorkerDatabaseOpener(workerIdentifier: string, multipleTabs?: boolean, worker?: string | URL): Comlink.Remote<OpenAsyncDatabaseConnection>;
+export declare function getWorkerDatabaseOpener(workerIdentifier: string, multipleTabs?: boolean, worker?: string | URL): Comlink.Remote<OpenWorkerConnection>;
 export declare function resolveWorkerDatabasePortFactory(worker: () => Worker | SharedWorker): Worker | MessagePort;
 export declare function isSharedWorker(worker: Worker | SharedWorker): worker is SharedWorker;

package/lib/src/worker/db/open-worker-database.js

@@ -1,5 +1,5 @@
 import * as Comlink from 'comlink';
-import { WASQLiteVFS } from '../../db/adapters/wa-sqlite/WASQLiteConnection.js';
+import { WASQLiteVFS } from '../../db/adapters/wa-sqlite/vfs.js';
 /**
  * Opens a shared or dedicated worker which exposes opening of database connections
  */

package/lib/src/worker/sync/SharedSyncImplementation.d.ts

@@ -1,8 +1,6 @@
-import { BaseObserver, ConnectionManager, DBAdapter, SubscribedStream, SyncStatus, type ILogLevel, type ILogger, type PowerSyncConnectionOptions, type StreamingSyncImplementation, type StreamingSyncImplementationListener, type SyncStatusOptions } from '@powersync/common';
-import { Mutex } from 'async-mutex';
+import { BaseObserver, ConnectionManager, DBAdapter, SubscribedStream, SyncStatus, Mutex, type ILogLevel, type ILogger, type PowerSyncConnectionOptions, type StreamingSyncImplementation, type StreamingSyncImplementationListener, type SyncStatusOptions } from '@powersync/common';
 import * as Comlink from 'comlink';
 import { WebStreamingSyncImplementation, WebStreamingSyncImplementationOptions } from '../../db/sync/WebStreamingSyncImplementation.js';
-import { WorkerWrappedAsyncDatabaseConnection } from '../../db/adapters/WorkerWrappedAsyncDatabaseConnection.js';
 import { ResolvedWebSQLOpenOptions } from '../../db/adapters/web-sql-flags.js';
 import { AbstractSharedSyncClientProvider } from './AbstractSharedSyncClientProvider.js';
 /**
@@ -73,7 +71,7 @@ export declare class SharedSyncImplementation extends BaseObserver<SharedSyncImp
     protected connectionManager: ConnectionManager;
     syncStatus: SyncStatus;
     broadCastLogger: ILogger;
-    protected distributedDB: DBAdapter | null;
+    protected readonly database: DBAdapter;
     constructor();
     get lastSyncedAt(): Date | undefined;
     get isConnected(): boolean;
@@ -126,9 +124,10 @@ export declare class SharedSyncImplementation extends BaseObserver<SharedSyncImp
     protected withSyncImplementation<T>(callback: (sync: StreamingSyncImplementation) => Promise<T>): Promise<T>;
     protected generateStreamingImplementation(): WebStreamingSyncImplementation;
     /**
-     * Opens a worker wrapped database connection. Using the last connected client port.
+     * Requests a random client to share its database connection with us.
      */
-    protected openInternalDB(): Promise<WorkerWrappedAsyncDatabaseConnection<ResolvedWebSQLOpenOptions>>;
+    private openInternalDB;
+    private generateReconnectableDatabase;
     /**
      * A method to update the all shared statuses for each
      * client.