@powersync/web 0.0.0-dev-20260311103504 → 0.0.0-dev-20260503073249

package/lib/src/db/adapters/wa-sqlite/DatabaseClient.js

@@ -0,0 +1,227 @@
+import { DBGetUtilsDefaultMixin, BaseObserver, ConnectionClosedError } from '@powersync/common';
+import * as Comlink from 'comlink';
+/**
+ * A single-connection {@link ConnectionPool} implementation based on a worker connection.
+ */
+export class DatabaseClient extends BaseObserver {
+    options;
+    config;
+    #connection;
+    #shareConnectionAbortController = new AbortController();
+    #receiveTableUpdates;
+    constructor(options, config) {
+        super();
+        this.options = options;
+        this.config = config;
+        this.#connection = {
+            connection: options.connection,
+            notifyRemoteClosed: options.remoteCanCloseUnexpectedly ? new AbortController() : undefined,
+            traceQueries: config.debugMode === true
+        };
+        const { port1, port2 } = new MessageChannel();
+        options.connection.setUpdateListener(Comlink.transfer(port1, [port1]));
+        this.#receiveTableUpdates = port2;
+        port2.onmessage = (event) => {
+            const tables = event.data;
+            const notification = {
+                tables,
+                groupedUpdates: {},
+                rawUpdates: []
+            };
+            this.iterateListeners((l) => {
+                l.tablesUpdated && l.tablesUpdated(notification);
+            });
+        };
+    }
+    get name() {
+        return this.config.dbFilename;
+    }
+    /**
+     * Marks the remote as closed.
+     *
+     * This can sometimes happen outside of our control, e.g. when a shared worker requests a connection from a tab. When
+     * it happens, all outstanding requests on this pool would never resolve. To avoid livelocks in this scenario, we
+     * throw on all outstanding promises and forbid new calls.
+     */
+    markRemoteClosed() {
+        // Can non-null assert here because this function is only supposed to be called when remoteCanCloseUnexpectedly was
+        // set.
+        this.#connection.notifyRemoteClosed.abort();
+    }
+    async close() {
+        // This connection is no longer shared, so we can close locks held for shareConnection calls.
+        this.#shareConnectionAbortController.abort();
+        this.#receiveTableUpdates.close();
+        await useConnectionState(this.#connection, (c) => c.close(), true);
+        this.options.onClose?.();
+        this.options.source?.[Comlink.releaseProxy]();
+    }
+    readLock(fn, options) {
+        return this.#lock(false, fn, options);
+    }
+    writeLock(fn, options) {
+        return this.#lock(true, fn, options);
+    }
+    async #lock(write, fn, options) {
+        const token = await useConnectionState(this.#connection, (c) => c.requestAccess(write, options?.timeoutMs));
+        try {
+            return await fn(new ClientLockContext(this.#connection, token));
+        }
+        finally {
+            await useConnectionState(this.#connection, (c) => c.completeAccess(token));
+        }
+    }
+    async refreshSchema() {
+        // Currently a no-op on the web.
+    }
+    async shareConnection() {
+        /**
+         * Hold a navigator lock in order to avoid features such as Chrome's frozen tabs,
+         * or Edge's sleeping tabs from pausing the thread for this connection.
+         * This promise resolves once a lock is obtained.
+         * This lock will be held as long as this connection is open.
+         * The `shareConnection` method should not be called on multiple tabs concurrently.
+         */
+        const abort = this.#shareConnectionAbortController;
+        const source = this.options.source;
+        if (source == null) {
+            throw new Error(`shareConnection() is only available for connections based by workers.`);
+        }
+        await new Promise((resolve, reject) => navigator.locks
+            .request(`shared-connection-${this.name}-${Date.now()}-${Math.round(Math.random() * 10000)}`, {
+            signal: abort.signal
+        }, async () => {
+            resolve();
+            // Free the lock when the connection is already closed.
+            if (abort.signal.aborted) {
+                return;
+            }
+            // Hold the lock while the shared connection is in use.
+            await new Promise((releaseLock) => {
+                abort.signal.addEventListener('abort', () => {
+                    releaseLock();
+                });
+            });
+        })
+            // We aren't concerned with abort errors here
+            .catch((ex) => {
+            if (ex.name == 'AbortError') {
+                resolve();
+            }
+            else {
+                reject(ex);
+            }
+        }));
+        const newPort = await source[Comlink.createEndpoint]();
+        return { port: newPort, identifier: this.name };
+    }
+    getConfiguration() {
+        return this.config;
+    }
+}
+/**
+ * A {@link SqlExecutor} implemented by sending commands to a worker.
+ *
+ * While an instance is active, it has exclusive access to the underlying database connection (as represented by its
+ * token).
+ */
+class ClientSqlExecutor {
+    #connection;
+    #token;
+    constructor(connection, token) {
+        this.#connection = connection;
+        this.#token = token;
+    }
+    /**
+     * Requests an operation from the worker, potentially tracing it if that option has been enabled.
+     */
+    async maybeTrace(fn, describeForTrace) {
+        if (this.#connection.traceQueries) {
+            const start = performance.now();
+            const description = describeForTrace();
+            try {
+                const r = await useConnectionState(this.#connection, fn);
+                performance.measure(`[SQL] ${description}`, { start });
+                return r;
+            }
+            catch (e) {
+                performance.measure(`[SQL] [ERROR: ${e.message}] ${description}`, { start });
+                throw e;
+            }
+        }
+        else {
+            return useConnectionState(this.#connection, fn);
+        }
+    }
+    async execute(query, params) {
+        const rs = await this.#executeOnWorker(query, params);
+        let rows;
+        if (rs.resultSet) {
+            const resultSet = rs.resultSet;
+            function rowToJavaScriptObject(row) {
+                const obj = {};
+                resultSet.columns.forEach((key, idx) => (obj[key] = row[idx]));
+                return obj;
+            }
+            const mapped = resultSet.rows.map(rowToJavaScriptObject);
+            rows = {
+                _array: mapped,
+                length: mapped.length,
+                item: (idx) => mapped[idx]
+            };
+        }
+        return {
+            rowsAffected: rs.changes,
+            insertId: rs.lastInsertRowId,
+            rows
+        };
+    }
+    async executeRaw(query, params) {
+        const rs = await this.#executeOnWorker(query, params);
+        return rs.resultSet?.rows ?? [];
+    }
+    async #executeOnWorker(query, params) {
+        return this.maybeTrace((c) => c.execute(this.#token, query, params), () => query);
+    }
+    async executeBatch(query, params = []) {
+        const results = await this.maybeTrace((c) => c.executeBatch(this.#token, query, params), () => `${query} (batch of ${params.length})`);
+        const result = { insertId: undefined, rowsAffected: 0 };
+        for (const source of results) {
+            result.insertId = source.lastInsertRowId;
+            result.rowsAffected += source.changes;
+        }
+        return result;
+    }
+}
+class ClientLockContext extends DBGetUtilsDefaultMixin(ClientSqlExecutor) {
+}
+async function useConnectionState(state, workerPromise, fireActionOnAbort = false) {
+    const controller = state.notifyRemoteClosed;
+    if (controller) {
+        return new Promise((resolve, reject) => {
+            if (controller.signal.aborted) {
+                reject(new ConnectionClosedError('Called operation on closed remote'));
+                if (!fireActionOnAbort) {
+                    // Don't run the operation if we're going to reject
+                    // We might want to fire-and-forget the operation in some cases (like a close operation)
+                    return;
+                }
+            }
+            function handleAbort() {
+                reject(new ConnectionClosedError('Remote peer closed with request in flight'));
+            }
+            function completePromise(action) {
+                controller.signal.removeEventListener('abort', handleAbort);
+                action();
+            }
+            controller.signal.addEventListener('abort', handleAbort);
+            workerPromise(state.connection)
+                .then((data) => completePromise(() => resolve(data)))
+                .catch((e) => completePromise(() => reject(e)));
+        });
+    }
+    else {
+        // Can't close, so just return the inner worker promise unguarded.
+        return workerPromise(state.connection);
+    }
+}

package/lib/src/db/adapters/wa-sqlite/DatabaseServer.d.ts

@@ -0,0 +1,47 @@
+import { ILogger } from '@powersync/common';
+import { ConcurrentSqliteConnection } from './ConcurrentConnection.js';
+import { RawQueryResult } from './RawSqliteConnection.js';
+export interface DatabaseServerOptions {
+    inner: ConcurrentSqliteConnection;
+    onClose: () => void;
+    logger: ILogger;
+}
+/**
+ * Access to a WA-sqlite connection that can be shared with multiple clients sending queries over an RPC protocol built
+ * with the Comlink package.
+ */
+export declare class DatabaseServer {
+    #private;
+    constructor(options: DatabaseServerOptions);
+    /**
+     * Called by clients when they wish to connect to this database.
+     *
+     * @param lockName A lock that is currently held by the client. When the lock is returned, we know the client is gone
+     * and that we need to clean up resources.
+     */
+    connect(lockName?: string): Promise<ClientConnectionView>;
+    forceClose(): Promise<void>;
+}
+export interface ClientConnectionView {
+    close(): Promise<void>;
+    /**
+     * Only used for testing purposes.
+     */
+    debugIsAutoCommit(): Promise<boolean>;
+    /**
+     * Requests exclusive access to this database connection.
+     *
+     * Returns a token that can be used with the query methods. It must be returned with {@link completeAccess} to
+     * give other clients access to the database afterwards.
+     */
+    requestAccess(write: boolean, timeoutMs?: number): Promise<string>;
+    execute(token: string, sql: string, params: any[] | undefined): Promise<RawQueryResult>;
+    executeBatch(token: string, sql: string, params: any[][]): Promise<RawQueryResult[]>;
+    completeAccess(token: string): Promise<void>;
+    /**
+     * Sends update notifications to the given message port.
+     *
+     * Update notifications are posted as a `string[]` message.
+     */
+    setUpdateListener(listener: MessagePort): Promise<void>;
+}

package/lib/src/db/adapters/wa-sqlite/DatabaseServer.js

@@ -0,0 +1,145 @@
+/**
+ * Access to a WA-sqlite connection that can be shared with multiple clients sending queries over an RPC protocol built
+ * with the Comlink package.
+ */
+export class DatabaseServer {
+    #options;
+    #nextClientId = 0;
+    #activeClients = new Set();
+    // TODO: Don't use a broadcast channel for connections managed by a shared worker.
+    #updateBroadcastChannel;
+    #clientTableListeners = new Set();
+    constructor(options) {
+        this.#options = options;
+        const inner = options.inner;
+        this.#updateBroadcastChannel = new BroadcastChannel(`${inner.options.dbFilename}-table-updates`);
+        this.#updateBroadcastChannel.onmessage = ({ data }) => {
+            this.#pushTableUpdateToClients(data);
+        };
+    }
+    #pushTableUpdateToClients(changedTables) {
+        for (const listener of this.#clientTableListeners) {
+            listener.postMessage(changedTables);
+        }
+    }
+    get #inner() {
+        return this.#options.inner;
+    }
+    get #logger() {
+        return this.#options.logger;
+    }
+    /**
+     * Called by clients when they wish to connect to this database.
+     *
+     * @param lockName A lock that is currently held by the client. When the lock is returned, we know the client is gone
+     * and that we need to clean up resources.
+     */
+    async connect(lockName) {
+        let isOpen = true;
+        const clientId = this.#nextClientId++;
+        this.#activeClients.add(clientId);
+        let connectionLeases = new Map();
+        let currentTableListener;
+        function requireOpen() {
+            if (!isOpen) {
+                throw new Error('Client has already been closed');
+            }
+        }
+        function requireOpenAndLease(lease) {
+            requireOpen();
+            const token = connectionLeases.get(lease);
+            if (!token) {
+                throw new Error('Attempted to use a connection lease that has already been returned.');
+            }
+            return token;
+        }
+        const close = async () => {
+            if (isOpen) {
+                isOpen = false;
+                if (currentTableListener) {
+                    this.#clientTableListeners.delete(currentTableListener);
+                }
+                // If the client holds a connection lease it hasn't returned, return that now.
+                for (const { lease } of connectionLeases.values()) {
+                    this.#logger.debug(`Closing connection lease that hasn't been returned.`);
+                    await lease.returnLease();
+                }
+                this.#activeClients.delete(clientId);
+                if (this.#activeClients.size == 0) {
+                    await this.forceClose();
+                }
+                else {
+                    this.#logger.debug('Keeping underlying connection active since its used by other clients.');
+                }
+            }
+        };
+        if (lockName) {
+            navigator.locks.request(lockName, {}, () => {
+                close();
+            });
+        }
+        return {
+            close,
+            debugIsAutoCommit: async () => {
+                return this.#inner.unsafeUseInner().isAutoCommit();
+            },
+            requestAccess: async (write, timeoutMs) => {
+                requireOpen();
+                const lease = await this.#inner.acquireConnection(timeoutMs != null ? AbortSignal.timeout(timeoutMs) : undefined);
+                if (!isOpen) {
+                    // Race between requestAccess and close(), the connection was closed while we tried to acquire a lease.
+                    await lease.returnLease();
+                    return requireOpen();
+                }
+                const token = crypto.randomUUID();
+                connectionLeases.set(token, { lease, write });
+                return token;
+            },
+            completeAccess: async (token) => {
+                const lease = requireOpenAndLease(token);
+                connectionLeases.delete(token);
+                try {
+                    if (lease.write) {
+                        // Collect update hooks invoked while the client had the write connection.
+                        const { resultSet } = await lease.lease.use((conn) => conn.execute(`SELECT powersync_update_hooks('get')`));
+                        if (resultSet) {
+                            const updatedTables = JSON.parse(resultSet.rows[0][0]);
+                            if (updatedTables.length) {
+                                this.#updateBroadcastChannel.postMessage(updatedTables);
+                                this.#pushTableUpdateToClients(updatedTables);
+                            }
+                        }
+                    }
+                }
+                finally {
+                    await lease.lease.returnLease();
+                }
+            },
+            execute: async (token, sql, params) => {
+                const { lease } = requireOpenAndLease(token);
+                return await lease.use((db) => db.execute(sql, params));
+            },
+            executeBatch: async (token, sql, params) => {
+                const { lease } = requireOpenAndLease(token);
+                return await lease.use((db) => db.executeBatch(sql, params));
+            },
+            setUpdateListener: async (listener) => {
+                requireOpen();
+                if (currentTableListener) {
+                    this.#clientTableListeners.delete(currentTableListener);
+                }
+                currentTableListener = listener;
+                if (listener) {
+                    this.#clientTableListeners.add(listener);
+                }
+            }
+        };
+    }
+    async forceClose() {
+        this.#logger.debug(`Closing connection to ${this.#inner.options}.`);
+        const connection = this.#inner;
+        this.#options.onClose();
+        this.#updateBroadcastChannel.close();
+        await connection.close();
+    }
+}

package/lib/src/db/adapters/wa-sqlite/RawSqliteConnection.d.ts

@@ -0,0 +1,46 @@
+import { ResolvedWASQLiteOpenFactoryOptions } from './WASQLiteOpenFactory.js';
+export interface RawResultSet {
+    columns: string[];
+    rows: SQLiteCompatibleType[][];
+}
+export interface RawQueryResult {
+    changes: number;
+    lastInsertRowId: number;
+    autocommit: boolean;
+    resultSet: RawResultSet | undefined;
+}
+/**
+ * A small wrapper around WA-sqlite to help with opening databases and running statements by preparing them internally.
+ *
+ * This is an internal class, and it must never be used directly. Wrappers are required to ensure raw connections aren't
+ * used concurrently across tabs.
+ */
+export declare class RawSqliteConnection {
+    readonly options: ResolvedWASQLiteOpenFactoryOptions;
+    private _sqliteAPI;
+    /**
+     * The `sqlite3*` connection pointer.
+     */
+    private db;
+    private _moduleFactory;
+    constructor(options: ResolvedWASQLiteOpenFactoryOptions);
+    get isOpen(): boolean;
+    init(): Promise<void>;
+    private openSQLiteAPI;
+    requireSqlite(): SQLiteAPI;
+    /**
+     * Checks if the database connection is in autocommit mode.
+     * @returns true if in autocommit mode, false if in a transaction
+     */
+    isAutoCommit(): boolean;
+    execute(sql: string, bindings?: any[]): Promise<RawQueryResult>;
+    executeBatch(sql: string, bindings: any[][]): Promise<RawQueryResult[]>;
+    private wrapQueryResults;
+    /**
+     * This executes a single statement using SQLite3 and returns the results as a {@link RawResultSet}.
+     */
+    private executeSingleStatementRaw;
+    executeRaw(sql: string, bindings?: any[]): Promise<RawResultSet[]>;
+    private stepThroughStatement;
+    close(): Promise<void>;
+}

package/lib/src/db/adapters/wa-sqlite/RawSqliteConnection.js

@@ -0,0 +1,147 @@
+import { Factory as WaSqliteFactory, SQLITE_ROW } from '@journeyapps/wa-sqlite';
+import { DEFAULT_MODULE_FACTORIES } from './vfs.js';
+/**
+ * A small wrapper around WA-sqlite to help with opening databases and running statements by preparing them internally.
+ *
+ * This is an internal class, and it must never be used directly. Wrappers are required to ensure raw connections aren't
+ * used concurrently across tabs.
+ */
+export class RawSqliteConnection {
+    options;
+    _sqliteAPI = null;
+    /**
+     * The `sqlite3*` connection pointer.
+     */
+    db = 0;
+    _moduleFactory;
+    constructor(options) {
+        this.options = options;
+        this._moduleFactory = DEFAULT_MODULE_FACTORIES[this.options.vfs];
+    }
+    get isOpen() {
+        return this.db != 0;
+    }
+    async init() {
+        const api = (this._sqliteAPI = await this.openSQLiteAPI());
+        this.db = await api.open_v2(this.options.dbFilename, this.options.isReadOnly ? 1 /* SQLITE_OPEN_READONLY */ : 6 /* SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE */);
+        await this.executeRaw(`PRAGMA temp_store = ${this.options.temporaryStorage};`);
+        if (this.options.encryptionKey) {
+            const escapedKey = this.options.encryptionKey.replace("'", "''");
+            await this.executeRaw(`PRAGMA key = '${escapedKey}'`);
+        }
+        await this.executeRaw(`PRAGMA cache_size = -${this.options.cacheSizeKb};`);
+        await this.executeRaw(`SELECT powersync_update_hooks('install');`);
+    }
+    async openSQLiteAPI() {
+        const { module, vfs } = await this._moduleFactory({
+            dbFileName: this.options.dbFilename,
+            encryptionKey: this.options.encryptionKey
+        });
+        const sqlite3 = WaSqliteFactory(module);
+        sqlite3.vfs_register(vfs, true);
+        /**
+         * Register the PowerSync core SQLite extension
+         */
+        module.ccall('powersync_init_static', 'int', []);
+        /**
+         * Create the multiple cipher vfs if an encryption key is provided
+         */
+        if (this.options.encryptionKey) {
+            const createResult = module.ccall('sqlite3mc_vfs_create', 'int', ['string', 'int'], [this.options.dbFilename, 1]);
+            if (createResult !== 0) {
+                throw new Error('Failed to create multiple cipher vfs, Database encryption will not work');
+            }
+        }
+        return sqlite3;
+    }
+    requireSqlite() {
+        if (!this._sqliteAPI) {
+            throw new Error(`Initialization has not completed`);
+        }
+        return this._sqliteAPI;
+    }
+    /**
+     * Checks if the database connection is in autocommit mode.
+     * @returns true if in autocommit mode, false if in a transaction
+     */
+    isAutoCommit() {
+        return this.requireSqlite().get_autocommit(this.db) != 0;
+    }
+    async execute(sql, bindings) {
+        const resultSet = await this.executeSingleStatementRaw(sql, bindings);
+        return this.wrapQueryResults(this.requireSqlite(), resultSet);
+    }
+    async executeBatch(sql, bindings) {
+        const results = [];
+        const api = this.requireSqlite();
+        for await (const stmt of api.statements(this.db, sql)) {
+            let columns;
+            for (const parameterSet of bindings) {
+                const rs = await this.stepThroughStatement(api, stmt, parameterSet, columns, false);
+                results.push(this.wrapQueryResults(api, rs));
+            }
+            // executeBatch can only use a single statement
+            break;
+        }
+        return results;
+    }
+    wrapQueryResults(api, rs) {
+        return {
+            changes: api.changes(this.db),
+            lastInsertRowId: api.last_insert_id(this.db),
+            autocommit: api.get_autocommit(this.db) != 0,
+            resultSet: rs
+        };
+    }
+    /**
+     * This executes a single statement using SQLite3 and returns the results as a {@link RawResultSet}.
+     */
+    async executeSingleStatementRaw(sql, bindings) {
+        const results = await this.executeRaw(sql, bindings);
+        return results.length ? results[0] : undefined;
+    }
+    async executeRaw(sql, bindings) {
+        const results = [];
+        const api = this.requireSqlite();
+        for await (const stmt of api.statements(this.db, sql)) {
+            let columns;
+            const rs = await this.stepThroughStatement(api, stmt, bindings ?? [], columns);
+            columns = rs.columns;
+            if (columns.length) {
+                results.push(rs);
+            }
+            // When binding parameters, only a single statement is executed.
+            if (bindings) {
+                break;
+            }
+        }
+        return results;
+    }
+    async stepThroughStatement(api, stmt, bindings, knownColumns, includeResults = true) {
+        // TODO not sure why this is needed currently, but booleans break
+        bindings.forEach((b, index, arr) => {
+            if (typeof b == 'boolean') {
+                arr[index] = b ? 1 : 0;
+            }
+        });
+        api.reset(stmt);
+        if (bindings) {
+            api.bind_collection(stmt, bindings);
+        }
+        const rows = [];
+        while ((await api.step(stmt)) === SQLITE_ROW) {
+            if (includeResults) {
+                const row = api.row(stmt);
+                rows.push(row);
+            }
+        }
+        knownColumns ??= api.column_names(stmt);
+        return { columns: knownColumns, rows };
+    }
+    async close() {
+        if (this.isOpen) {
+            await this.requireSqlite().close(this.db);
+            this.db = 0;
+        }
+    }
+}

package/lib/src/db/adapters/wa-sqlite/WASQLiteOpenFactory.d.ts

@@ -1,23 +1,45 @@
-import { DBAdapter, type ILogLevel } from '@powersync/common';
-import { AbstractWebSQLOpenFactory } from '../AbstractWebSQLOpenFactory.js';
-import { AsyncDatabaseConnection } from '../AsyncDatabaseConnection.js';
+import { DBAdapter, SQLOpenFactory, type ILogLevel } from '@powersync/common';
 import { ResolvedWebSQLOpenOptions, WebSQLOpenFactoryOptions } from '../web-sql-flags.js';
-import { WASQLiteVFS } from './WASQLiteConnection.js';
+import { WASQLiteVFS } from './vfs.js';
+import { PoolConnection } from '../AsyncWebAdapter.js';
 export interface WASQLiteOpenFactoryOptions extends WebSQLOpenFactoryOptions {
     vfs?: WASQLiteVFS;
+    /**
+     * If the {@link vfs} supports it, an additional amount of read-only connections to open. Using additional read
+     * connections can speed up queries by dispatching them to multiple workers running them concurrently.
+     *
+     * {@link WASQLiteVFS.OPFSWriteAheadVFS} is the only VFS with support for multiple connections, so this option is
+     * ignored for other VFS implementations.
+     *
+     * Defaults to 1.
+     */
+    additionalReaders?: number;
 }
 export interface ResolvedWASQLiteOpenFactoryOptions extends ResolvedWebSQLOpenOptions {
     vfs: WASQLiteVFS;
+    /**
+     * Whether this is a read-only connection opened for the `OPFSWriteAheadVFS` file system.
+     */
+    isReadOnly: boolean;
 }
 export interface WorkerDBOpenerOptions extends ResolvedWASQLiteOpenFactoryOptions {
     logLevel: ILogLevel;
+    /**
+     * A lock that is currently held by the client. When the lock is returned, we know the client is gone and that we need
+     * to clean up resources.
+     */
+    lockName: string;
 }
 /**
  * Opens a SQLite connection using WA-SQLite.
  */
-export declare class WASQLiteOpenFactory extends AbstractWebSQLOpenFactory {
+export declare class WASQLiteOpenFactory implements SQLOpenFactory {
+    private options;
+    private resolvedFlags;
+    private logger;
     constructor(options: WASQLiteOpenFactoryOptions);
     get waOptions(): WASQLiteOpenFactoryOptions;
     protected openAdapter(): DBAdapter;
-    openConnection(): Promise<AsyncDatabaseConnection>;
+    openDB(): DBAdapter;
+    openConnection(): Promise<PoolConnection>;
 }