@powersync/common 1.7.1 → 1.8.1

package/lib/client/AbstractPowerSyncDatabase.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { Mutex } from 'async-mutex';
 import { ILogger } from 'js-logger';
 import { DBAdapter, QueryResult, Transaction } from '../db/DBAdapter';
@@ -10,7 +9,7 @@ import { PowerSyncBackendConnector } from './connection/PowerSyncBackendConnecto
 import { BucketStorageAdapter } from './sync/bucket/BucketStorageAdapter';
 import { CrudBatch } from './sync/bucket/CrudBatch';
 import { CrudTransaction } from './sync/bucket/CrudTransaction';
-import { AbstractStreamingSyncImplementation, StreamingSyncImplementationListener, StreamingSyncImplementation } from './sync/stream/AbstractStreamingSyncImplementation';
+import { AbstractStreamingSyncImplementation, StreamingSyncImplementationListener, StreamingSyncImplementation, PowerSyncConnectionOptions } from './sync/stream/AbstractStreamingSyncImplementation';
 export interface DisconnectAndClearOptions {
     /** When set to false, data in local-only tables is preserved. */
     clearLocal?: boolean;
@@ -153,7 +152,7 @@ export declare abstract class AbstractPowerSyncDatabase extends BaseObserver<Pow
     /**
      * Connects to stream of events from the PowerSync instance.
      */
-    connect(connector: PowerSyncBackendConnector): Promise<void>;
+    connect(connector: PowerSyncBackendConnector, options?: PowerSyncConnectionOptions): Promise<void>;
     /**
      * Close the sync connection.
      *

package/lib/client/AbstractPowerSyncDatabase.js

@@ -192,7 +192,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
     /**
      * Connects to stream of events from the PowerSync instance.
      */
-    async connect(connector) {
+    async connect(connector, options) {
         await this.waitForReady();
         // close connection if one is open
         await this.disconnect();
@@ -211,7 +211,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         });
         await this.syncStreamImplementation.waitForReady();
         this.syncStreamImplementation.triggerCrudUpload();
-        await this.syncStreamImplementation.connect();
+        await this.syncStreamImplementation.connect(options);
     }
     /**
      * Close the sync connection.
@@ -646,4 +646,3 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         return this.database.readLock((tx) => tx.execute(sql, params));
     }
 }
-//# sourceMappingURL=AbstractPowerSyncDatabase.js.map

package/lib/client/AbstractPowerSyncOpenFactory.js

@@ -22,4 +22,3 @@ export class AbstractPowerSyncDatabaseOpenFactory {
         return this.generateInstance(options);
     }
 }
-//# sourceMappingURL=AbstractPowerSyncOpenFactory.js.map

package/lib/client/connection/PowerSyncBackendConnector.js

@@ -1,2 +1 @@
 export {};
-//# sourceMappingURL=PowerSyncBackendConnector.js.map

package/lib/client/connection/PowerSyncCredentials.js

@@ -1,2 +1 @@
 export {};
-//# sourceMappingURL=PowerSyncCredentials.js.map

package/lib/client/sync/bucket/BucketStorageAdapter.js

@@ -6,4 +6,3 @@ export var PSInternalTable;
     PSInternalTable["OPLOG"] = "ps_oplog";
     PSInternalTable["UNTYPED"] = "ps_untyped";
 })(PSInternalTable || (PSInternalTable = {}));
-//# sourceMappingURL=BucketStorageAdapter.js.map

package/lib/client/sync/bucket/CrudBatch.js

@@ -23,4 +23,3 @@ export class CrudBatch {
         this.complete = complete;
     }
 }
-//# sourceMappingURL=CrudBatch.js.map

package/lib/client/sync/bucket/CrudEntry.js

@@ -82,4 +82,3 @@ export class CrudEntry {
         return [this.transactionId, this.clientId, this.op, this.table, this.id, this.opData];
     }
 }
-//# sourceMappingURL=CrudEntry.js.map

package/lib/client/sync/bucket/CrudTransaction.js

@@ -22,4 +22,3 @@ export class CrudTransaction extends CrudBatch {
         this.transactionId = transactionId;
     }
 }
-//# sourceMappingURL=CrudTransaction.js.map

package/lib/client/sync/bucket/OpType.js

@@ -20,4 +20,3 @@ export class OpType {
         return Object.entries(OpTypeEnum).find(([, value]) => value === this.value)[0];
     }
 }
-//# sourceMappingURL=OpType.js.map

package/lib/client/sync/bucket/OplogEntry.js

@@ -31,4 +31,3 @@ export class OplogEntry {
         };
     }
 }
-//# sourceMappingURL=OplogEntry.js.map

package/lib/client/sync/bucket/SqliteBucketStorage.js

@@ -1,4 +1,3 @@
-import { v4 as uuid } from 'uuid';
 import { extractTableUpdates } from '../../../db/DBAdapter';
 import { PSInternalTable } from './BucketStorageAdapter';
 import { OpTypeEnum } from './OpType';
@@ -84,10 +83,10 @@ export class SqliteBucketStorage extends BaseObserver {
         // To achieve this, we rename the bucket to a new temp name, and change all ops to remove.
         // By itself, this new bucket would ensure that the previous objects are deleted if they contain no more references.
         // If the old bucket is re-added, this new bucket would have no effect.
-        const newName = `$delete_${bucket}_${uuid()}`;
-        this.logger.debug('Deleting bucket', bucket);
-        // This
         await this.writeTransaction(async (tx) => {
+            const { uuid } = await tx.get('select uuid() as uuid');
+            const newName = `$delete_${bucket}_${uuid}`;
+            this.logger.debug('Deleting bucket', bucket);
             await tx.execute(`UPDATE ps_oplog SET op=${OpTypeEnum.REMOVE}, data=NULL WHERE op=${OpTypeEnum.PUT} AND superseded=0 AND bucket=?`, [bucket]);
             // Rename bucket
             await tx.execute('UPDATE ps_oplog SET bucket=? WHERE bucket=?', [newName, bucket]);
@@ -296,4 +295,3 @@ export class SqliteBucketStorage extends BaseObserver {
         // No-op for now
     }
 }
-//# sourceMappingURL=SqliteBucketStorage.js.map

package/lib/client/sync/bucket/SyncDataBatch.js

@@ -9,4 +9,3 @@ export class SyncDataBatch {
         this.buckets = buckets;
     }
 }
-//# sourceMappingURL=SyncDataBatch.js.map

package/lib/client/sync/bucket/SyncDataBucket.js

@@ -38,4 +38,3 @@ export class SyncDataBucket {
         };
     }
 }
-//# sourceMappingURL=SyncDataBucket.js.map

package/lib/client/sync/stream/AbstractRemote.d.ts

@@ -1,15 +1,41 @@
-/// <reference types="node" />
 import { ILogger } from 'js-logger';
+import { fetch } from 'cross-fetch';
 import { PowerSyncCredentials } from '../../connection/PowerSyncCredentials';
+import { StreamingSyncLine, StreamingSyncRequest } from './streaming-sync-types';
+import { DataStream } from '../../../utils/DataStream';
 export type RemoteConnector = {
     fetchCredentials: () => Promise<PowerSyncCredentials | null>;
 };
 export declare const DEFAULT_REMOTE_LOGGER: ILogger;
+export type SyncStreamOptions = {
+    path: string;
+    data: StreamingSyncRequest;
+    headers?: Record<string, string>;
+    abortSignal?: AbortSignal;
+    fetchOptions?: Request;
+};
+export type AbstractRemoteOptions = {
+    /**
+     * Transforms the PowerSync base URL which might contain
+     * `http(s)://` to the corresponding WebSocket variant
+     * e.g. `ws(s)://`
+     */
+    socketUrlTransformer: (url: string) => string;
+    /**
+     * Optionally provide the fetch implementation to use.
+     * Note that this usually needs to be bound to the global scope.
+     * Binding should be done before passing here.
+     */
+    fetchImplementation: typeof fetch;
+};
+export declare const DEFAULT_REMOTE_OPTIONS: AbstractRemoteOptions;
 export declare abstract class AbstractRemote {
     protected connector: RemoteConnector;
     protected logger: ILogger;
     protected credentials: PowerSyncCredentials | null;
-    constructor(connector: RemoteConnector, logger?: ILogger);
+    protected options: AbstractRemoteOptions;
+    constructor(connector: RemoteConnector, logger?: ILogger, options?: Partial<AbstractRemoteOptions>);
+    get fetch(): typeof globalThis.fetch;
     getCredentials(): Promise<PowerSyncCredentials | null>;
     protected buildRequest(path: string): Promise<{
         url: string;
@@ -18,8 +44,15 @@ export declare abstract class AbstractRemote {
             Authorization: string;
         };
     }>;
-    abstract post(path: string, data: any, headers?: Record<string, string>): Promise<any>;
-    abstract get(path: string, headers?: Record<string, string>): Promise<any>;
-    abstract postStreaming(path: string, data: any, headers?: Record<string, string>, signal?: AbortSignal): Promise<any>;
-    isAvailable(): boolean;
+    post(path: string, data: any, headers?: Record<string, string>): Promise<any>;
+    get(path: string, headers?: Record<string, string>): Promise<any>;
+    postStreaming(path: string, data: any, headers?: Record<string, string>, signal?: AbortSignal): Promise<any>;
+    /**
+     * Connects to the sync/stream websocket endpoint
+     */
+    socketStream(options: SyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
+    /**
+     * Connects to the sync/stream http endpoint
+     */
+    postStream(options: SyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
 }

package/lib/client/sync/stream/AbstractRemote.js

@@ -1,14 +1,42 @@
 import Logger from 'js-logger';
+import { fetch } from 'cross-fetch';
+import { DataStream } from '../../../utils/DataStream';
+import ndjsonStream from 'can-ndjson-stream';
+import { RSocketConnector } from 'rsocket-core';
+import { WebsocketClientTransport } from 'rsocket-websocket-client';
+import { serialize, deserialize } from 'bson';
+import { AbortOperation } from '../../../utils/AbortOperation';
+import { Buffer } from 'buffer';
 // Refresh at least 30 sec before it expires
 const REFRESH_CREDENTIALS_SAFETY_PERIOD_MS = 30_000;
+const SYNC_QUEUE_REQUEST_N = 10;
+const SYNC_QUEUE_REQUEST_LOW_WATER = 5;
+// Keep alive message is sent every period
+const KEEP_ALIVE_MS = 20_000;
+// The ACK must be received in this period
+const KEEP_ALIVE_LIFETIME_MS = 30_000;
 export const DEFAULT_REMOTE_LOGGER = Logger.get('PowerSyncRemote');
+export const DEFAULT_REMOTE_OPTIONS = {
+    socketUrlTransformer: (url) => url.replace(/^https?:\/\//, function (match) {
+        return match === 'https://' ? 'wss://' : 'ws://';
+    }),
+    fetchImplementation: fetch.bind(globalThis)
+};
 export class AbstractRemote {
     connector;
     logger;
     credentials = null;
-    constructor(connector, logger = DEFAULT_REMOTE_LOGGER) {
+    options;
+    constructor(connector, logger = DEFAULT_REMOTE_LOGGER, options) {
         this.connector = connector;
         this.logger = logger;
+        this.options = {
+            ...DEFAULT_REMOTE_OPTIONS,
+            ...(options ?? {})
+        };
+    }
+    get fetch() {
+        return this.options.fetchImplementation;
     }
     async getCredentials() {
         const { expiresAt } = this.credentials ?? {};
@@ -36,8 +64,299 @@ export class AbstractRemote {
             }
         };
     }
-    isAvailable() {
-        return true;
+    async post(path, data, headers = {}) {
+        const request = await this.buildRequest(path);
+        const res = await fetch(request.url, {
+            method: 'POST',
+            headers: {
+                ...headers,
+                ...request.headers
+            },
+            body: JSON.stringify(data)
+        });
+        if (!res.ok) {
+            throw new Error(`Received ${res.status} - ${res.statusText} when posting to ${path}: ${await res.text()}}`);
+        }
+        return res.json();
+    }
+    async get(path, headers) {
+        const request = await this.buildRequest(path);
+        const res = await this.fetch(request.url, {
+            method: 'GET',
+            headers: {
+                ...headers,
+                ...request.headers
+            }
+        });
+        if (!res.ok) {
+            throw new Error(`Received ${res.status} - ${res.statusText} when getting from ${path}: ${await res.text()}}`);
+        }
+        return res.json();
+    }
+    async postStreaming(path, data, headers = {}, signal) {
+        const request = await this.buildRequest(path);
+        const res = await this.fetch(request.url, {
+            method: 'POST',
+            headers: { ...headers, ...request.headers },
+            body: JSON.stringify(data),
+            signal,
+            cache: 'no-store'
+        }).catch((ex) => {
+            this.logger.error(`Caught ex when POST streaming to ${path}`, ex);
+            throw ex;
+        });
+        if (!res.ok) {
+            const text = await res.text();
+            this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
+            const error = new Error(`HTTP ${res.statusText}: ${text}`);
+            error.status = res.status;
+            throw error;
+        }
+        return res;
+    }
+    /**
+     * Connects to the sync/stream websocket endpoint
+     */
+    async socketStream(options) {
+        const { path } = options;
+        const request = await this.buildRequest(path);
+        const connector = new RSocketConnector({
+            transport: new WebsocketClientTransport({
+                url: this.options.socketUrlTransformer(request.url)
+            }),
+            setup: {
+                keepAlive: KEEP_ALIVE_MS,
+                lifetime: KEEP_ALIVE_LIFETIME_MS,
+                dataMimeType: 'application/bson',
+                metadataMimeType: 'application/bson',
+                payload: {
+                    data: null,
+                    metadata: Buffer.from(serialize({
+                        token: request.headers.Authorization
+                    }))
+                }
+            }
+        });
+        let rsocket;
+        try {
+            rsocket = await connector.connect();
+        }
+        catch (ex) {
+            /**
+             * On React native the connection exception can be `undefined` this causes issues
+             * with detecting the exception inside async-mutex
+             */
+            throw new Error(`Could not connect to PowerSync instance: ${JSON.stringify(ex)}`);
+        }
+        const stream = new DataStream({
+            logger: this.logger,
+            pressure: {
+                lowWaterMark: SYNC_QUEUE_REQUEST_LOW_WATER
+            }
+        });
+        let socketIsClosed = false;
+        const closeSocket = () => {
+            if (socketIsClosed) {
+                return;
+            }
+            socketIsClosed = true;
+            rsocket.close();
+        };
+        // We initially request this amount and expect these to arrive eventually
+        let pendingEventsCount = SYNC_QUEUE_REQUEST_N;
+        const socket = await new Promise((resolve, reject) => {
+            let connectionEstablished = false;
+            const res = rsocket.requestStream({
+                data: Buffer.from(serialize(options.data)),
+                metadata: Buffer.from(serialize({
+                    path
+                }))
+            }, SYNC_QUEUE_REQUEST_N, // The initial N amount
+            {
+                onError: (e) => {
+                    // Don't log closed as an error
+                    if (e.message !== 'Closed. ') {
+                        this.logger.error(e);
+                    }
+                    // RSocket will close this automatically
+                    // Attempting to close multiple times causes a console warning
+                    socketIsClosed = true;
+                    stream.close();
+                    // Handles cases where the connection failed e.g. auth error or connection error
+                    if (!connectionEstablished) {
+                        reject(e);
+                    }
+                },
+                onNext: (payload) => {
+                    // The connection is active
+                    if (!connectionEstablished) {
+                        connectionEstablished = true;
+                        resolve(res);
+                    }
+                    const { data } = payload;
+                    // Less events are now pending
+                    pendingEventsCount--;
+                    if (!data) {
+                        return;
+                    }
+                    const deserializedData = deserialize(data);
+                    stream.enqueueData(deserializedData);
+                },
+                onComplete: () => {
+                    stream.close();
+                },
+                onExtension: () => { }
+            });
+        });
+        const l = stream.registerListener({
+            lowWater: async () => {
+                // Request to fill up the queue
+                const required = SYNC_QUEUE_REQUEST_N - pendingEventsCount;
+                if (required > 0) {
+                    socket.request(SYNC_QUEUE_REQUEST_N - pendingEventsCount);
+                    pendingEventsCount = SYNC_QUEUE_REQUEST_N;
+                }
+            },
+            closed: () => {
+                closeSocket();
+                l?.();
+            }
+        });
+        /**
+         * Handle abort operations here.
+         * Unfortunately cannot insert them into the connection.
+         */
+        if (options.abortSignal?.aborted) {
+            stream.close();
+        }
+        else {
+            options.abortSignal?.addEventListener('abort', () => {
+                stream.close();
+            });
+        }
+        return stream;
+    }
+    /**
+     * Connects to the sync/stream http endpoint
+     */
+    async postStream(options) {
+        const { data, path, headers, abortSignal } = options;
+        const request = await this.buildRequest(path);
+        /**
+         * This abort controller will abort pending fetch requests.
+         * If the request has resolved, it will be used to close the readable stream.
+         * Which will cancel the network request.
+         *
+         * This nested controller is required since:
+         *  Aborting the active fetch request while it is being consumed seems to throw
+         *  an unhandled exception on the window level.
+         */
+        const controller = new AbortController();
+        let requestResolved = false;
+        abortSignal?.addEventListener('abort', () => {
+            if (!requestResolved) {
+                // Only abort via the abort controller if the request has not resolved yet
+                controller.abort(abortSignal.reason ??
+                    new AbortOperation('Cancelling network request before it resolves. Abort signal has been received.'));
+            }
+        });
+        const res = await this.fetch(request.url, {
+            method: 'POST',
+            headers: { ...headers, ...request.headers },
+            body: JSON.stringify(data),
+            signal: controller.signal,
+            cache: 'no-store',
+            ...options.fetchOptions
+        }).catch((ex) => {
+            if (ex.name == 'AbortError') {
+                throw new AbortOperation(`Pending fetch request to ${request.url} has been aborted.`);
+            }
+            throw ex;
+        });
+        if (!res) {
+            throw new Error('Fetch request was aborted');
+        }
+        requestResolved = true;
+        if (!res.ok || !res.body) {
+            const text = await res.text();
+            this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
+            const error = new Error(`HTTP ${res.statusText}: ${text}`);
+            error.status = res.status;
+            throw error;
+        }
+        /**
+         * The can-ndjson-stream does not handle aborted streams well.
+         * This will intercept the readable stream and close the stream if
+         * aborted.
+         */
+        const reader = res.body.getReader();
+        // This will close the network request and read stream
+        const closeReader = async () => {
+            try {
+                await reader.cancel();
+            }
+            catch (ex) {
+                // an error will throw if the reader hasn't been used yet
+            }
+            reader.releaseLock();
+        };
+        abortSignal?.addEventListener('abort', () => {
+            closeReader();
+        });
+        const outputStream = new ReadableStream({
+            start: (controller) => {
+                const processStream = async () => {
+                    while (!abortSignal?.aborted) {
+                        try {
+                            const { done, value } = await reader.read();
+                            // When no more data needs to be consumed, close the stream
+                            if (done) {
+                                break;
+                            }
+                            // Enqueue the next data chunk into our target stream
+                            controller.enqueue(value);
+                        }
+                        catch (ex) {
+                            this.logger.error('Caught exception when reading sync stream', ex);
+                            break;
+                        }
+                    }
+                    if (!abortSignal?.aborted) {
+                        // Close the downstream readable stream
+                        await closeReader();
+                    }
+                    controller.close();
+                };
+                processStream();
+            }
+        });
+        const jsonS = ndjsonStream(new Response(outputStream).body);
+        const stream = new DataStream({
+            logger: this.logger
+        });
+        const r = jsonS.getReader();
+        const l = stream.registerListener({
+            lowWater: async () => {
+                try {
+                    const { done, value } = await r.read();
+                    // Exit if we're done
+                    if (done) {
+                        stream.close();
+                        l?.();
+                        return;
+                    }
+                    stream.enqueueData(value);
+                }
+                catch (ex) {
+                    stream.close();
+                    throw ex;
+                }
+            },
+            closed: () => {
+                closeReader();
+                l?.();
+            }
+        });
+        return stream;
     }
 }
-//# sourceMappingURL=AbstractRemote.js.map

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts

@@ -1,6 +1,4 @@
-/// <reference types="node" />
 import { ILogger } from 'js-logger';
-import { StreamingSyncLine, StreamingSyncRequest } from './streaming-sync-types';
 import { AbstractRemote } from './AbstractRemote';
 import { BucketStorageAdapter } from '../bucket/BucketStorageAdapter';
 import { SyncStatus, SyncStatusOptions } from '../../../db/crud/SyncStatus';
@@ -9,6 +7,10 @@ export declare enum LockType {
     CRUD = "crud",
     SYNC = "sync"
 }
+export declare enum SyncStreamConnectionMethod {
+    HTTP = "http",
+    WEB_SOCKET = "web-socket"
+}
 /**
  * Abstract Lock to be implemented by various JS environments
  */
@@ -41,11 +43,23 @@ export interface StreamingSyncImplementationListener extends BaseListener {
      */
     statusChanged?: ((status: SyncStatus) => void) | undefined;
 }
+/**
+ * Configurable options to be used when connecting to the PowerSync
+ * backend instance.
+ */
+export interface PowerSyncConnectionOptions {
+    /**
+     * The connection method to use when streaming updates from
+     * the PowerSync backend instance.
+     * Defaults to a HTTP streaming connection.
+     */
+    connectionMethod?: SyncStreamConnectionMethod;
+}
 export interface StreamingSyncImplementation extends BaseObserver<StreamingSyncImplementationListener>, Disposable {
     /**
      * Connects to the sync service
      */
-    connect(): Promise<void>;
+    connect(options?: PowerSyncConnectionOptions): Promise<void>;
     /**
      * Disconnects from the sync services.
      * @throws if not connected or if abort is not controlled internally
@@ -66,6 +80,7 @@ export declare const DEFAULT_STREAMING_SYNC_OPTIONS: {
     logger: ILogger;
     crudUploadThrottleMs: number;
 };
+export declare const DEFAULT_STREAM_CONNECTION_OPTIONS: Required<PowerSyncConnectionOptions>;
 export declare abstract class AbstractStreamingSyncImplementation extends BaseObserver<StreamingSyncImplementationListener> implements StreamingSyncImplementation {
     protected _lastSyncedAt: Date | null;
     protected options: AbstractStreamingSyncImplementationOptions;
@@ -86,16 +101,15 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
     getWriteCheckpoint(): Promise<string>;
     protected _uploadAllCrud(): Promise<void>;
     protected uploadCrudBatch(): Promise<boolean>;
-    connect(): Promise<void>;
+    connect(options?: PowerSyncConnectionOptions): Promise<void>;
     disconnect(): Promise<void>;
     /**
      * @deprecated use [connect instead]
      */
-    streamingSync(signal?: AbortSignal): Promise<void>;
-    protected streamingSyncIteration(signal: AbortSignal, progress?: () => void): Promise<{
+    streamingSync(signal?: AbortSignal, options?: PowerSyncConnectionOptions): Promise<void>;
+    protected streamingSyncIteration(signal: AbortSignal, options?: PowerSyncConnectionOptions): Promise<{
         retry?: boolean;
     }>;
-    protected streamingSyncRequest(req: StreamingSyncRequest, signal?: AbortSignal): AsyncGenerator<StreamingSyncLine>;
     protected updateSyncStatus(options: SyncStatusOptions): void;
     private delayRetry;
 }