@powersync/common 0.0.0-dev-20240509081154 → 0.0.0-dev-20240516143814

package/lib/client/AbstractPowerSyncDatabase.d.ts

@@ -1,4 +1,3 @@
-/// <reference types="node" />
 import { Mutex } from 'async-mutex';
 import { ILogger } from 'js-logger';
 import { DBAdapter, QueryResult, Transaction } from '../db/DBAdapter';
@@ -10,7 +9,7 @@ import { PowerSyncBackendConnector } from './connection/PowerSyncBackendConnecto
 import { BucketStorageAdapter } from './sync/bucket/BucketStorageAdapter';
 import { CrudBatch } from './sync/bucket/CrudBatch';
 import { CrudTransaction } from './sync/bucket/CrudTransaction';
-import { AbstractStreamingSyncImplementation, StreamingSyncImplementationListener, StreamingSyncImplementation } from './sync/stream/AbstractStreamingSyncImplementation';
+import { AbstractStreamingSyncImplementation, StreamingSyncImplementationListener, StreamingSyncImplementation, PowerSyncConnectionOptions } from './sync/stream/AbstractStreamingSyncImplementation';
 export interface DisconnectAndClearOptions {
     /** When set to false, data in local-only tables is preserved. */
     clearLocal?: boolean;
@@ -153,7 +152,7 @@ export declare abstract class AbstractPowerSyncDatabase extends BaseObserver<Pow
     /**
      * Connects to stream of events from the PowerSync instance.
      */
-    connect(connector: PowerSyncBackendConnector): Promise<void>;
+    connect(connector: PowerSyncBackendConnector, options?: PowerSyncConnectionOptions): Promise<void>;
     /**
      * Close the sync connection.
      *

package/lib/client/AbstractPowerSyncDatabase.js

@@ -192,7 +192,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
     /**
      * Connects to stream of events from the PowerSync instance.
      */
-    async connect(connector) {
+    async connect(connector, options) {
         await this.waitForReady();
         // close connection if one is open
         await this.disconnect();
@@ -202,13 +202,16 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         this.syncStreamImplementation = this.generateSyncStreamImplementation(connector);
         this.syncStatusListenerDisposer = this.syncStreamImplementation.registerListener({
             statusChanged: (status) => {
-                this.currentStatus = new SyncStatus({ ...status.toJSON(), hasSynced: this.currentStatus?.hasSynced });
+                this.currentStatus = new SyncStatus({
+                    ...status.toJSON(),
+                    hasSynced: this.currentStatus?.hasSynced || !!status.lastSyncedAt
+                });
                 this.iterateListeners((cb) => cb.statusChanged?.(this.currentStatus));
             }
         });
         await this.syncStreamImplementation.waitForReady();
         this.syncStreamImplementation.triggerCrudUpload();
-        await this.syncStreamImplementation.connect();
+        await this.syncStreamImplementation.connect(options);
     }
     /**
      * Close the sync connection.
@@ -528,7 +531,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
                 .filter((row) => row.opcode == 'OpenRead' && row.p3 == 0 && typeof row.p2 == 'number')
                 .map((row) => row.p2);
             const tables = await this.getAll(`SELECT DISTINCT tbl_name FROM sqlite_master WHERE rootpage IN (SELECT json_each.value FROM json_each(?))`, [JSON.stringify(rootPages)]);
-            for (let table of tables) {
+            for (const table of tables) {
                 resolvedTables.push(table.tbl_name.replace(POWERSYNC_TABLE_MATCH, ''));
             }
         }
@@ -631,7 +634,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         const mappedTableNames = rawTableNames
             ? filteredTables
             : filteredTables.map((t) => t.replace(POWERSYNC_TABLE_MATCH, ''));
-        for (let table of mappedTableNames) {
+        for (const table of mappedTableNames) {
             changedTables.add(table);
         }
     }

package/lib/client/sync/bucket/CrudEntry.d.ts

@@ -82,5 +82,5 @@ export declare class CrudEntry {
     /**
      * Generates an array for use in deep comparison operations
      */
-    toComparisonArray(): (string | number | Record<string, any>)[];
+    toComparisonArray(): (string | number | Record<string, any> | undefined)[];
 }

package/lib/client/sync/bucket/CrudTransaction.d.ts

@@ -12,7 +12,7 @@ export declare class CrudTransaction extends CrudBatch {
     /**
      * If null, this contains a list of changes recorded without an explicit transaction associated.
      */
-    transactionId?: number;
+    transactionId?: number | undefined;
     constructor(
     /**
      * List of client-side changes.
@@ -25,5 +25,5 @@ export declare class CrudTransaction extends CrudBatch {
     /**
      * If null, this contains a list of changes recorded without an explicit transaction associated.
      */
-    transactionId?: number);
+    transactionId?: number | undefined);
 }

package/lib/client/sync/bucket/OplogEntry.d.ts

@@ -14,10 +14,10 @@ export declare class OplogEntry {
     op: OpType;
     checksum: number;
     subkey: string;
-    object_type?: string;
-    object_id?: string;
-    data?: string;
+    object_type?: string | undefined;
+    object_id?: string | undefined;
+    data?: string | undefined;
     static fromRow(row: OplogEntryJSON): OplogEntry;
-    constructor(op_id: OpId, op: OpType, checksum: number, subkey: string, object_type?: string, object_id?: string, data?: string);
+    constructor(op_id: OpId, op: OpType, checksum: number, subkey: string, object_type?: string | undefined, object_id?: string | undefined, data?: string | undefined);
     toJSON(): OplogEntryJSON;
 }

package/lib/client/sync/bucket/SqliteBucketStorage.js

@@ -1,4 +1,3 @@
-import { v4 as uuid } from 'uuid';
 import { extractTableUpdates } from '../../../db/DBAdapter';
 import { PSInternalTable } from './BucketStorageAdapter';
 import { OpTypeEnum } from './OpType';
@@ -84,10 +83,10 @@ export class SqliteBucketStorage extends BaseObserver {
         // To achieve this, we rename the bucket to a new temp name, and change all ops to remove.
         // By itself, this new bucket would ensure that the previous objects are deleted if they contain no more references.
         // If the old bucket is re-added, this new bucket would have no effect.
-        const newName = `$delete_${bucket}_${uuid()}`;
-        this.logger.debug('Deleting bucket', bucket);
-        // This
         await this.writeTransaction(async (tx) => {
+            const { uuid } = await tx.get('select uuid() as uuid');
+            const newName = `$delete_${bucket}_${uuid}`;
+            this.logger.debug('Deleting bucket', bucket);
             await tx.execute(`UPDATE ps_oplog SET op=${OpTypeEnum.REMOVE}, data=NULL WHERE op=${OpTypeEnum.PUT} AND superseded=0 AND bucket=?`, [bucket]);
             // Rename bucket
             await tx.execute('UPDATE ps_oplog SET bucket=? WHERE bucket=?', [newName, bucket]);

package/lib/client/sync/bucket/SyncDataBucket.d.ts

@@ -1,4 +1,3 @@
-import { OpId } from './CrudEntry';
 import { OplogEntry, OplogEntryJSON } from './OplogEntry';
 export type SyncDataBucketJSON = {
     bucket: string;
@@ -18,11 +17,11 @@ export declare class SyncDataBucket {
     /**
      * The `after` specified in the request.
      */
-    after?: OpId;
+    after?: string | undefined;
     /**
      * Use this for the next request.
      */
-    next_after?: OpId;
+    next_after?: string | undefined;
     static fromRow(row: SyncDataBucketJSON): SyncDataBucket;
     constructor(bucket: string, data: OplogEntry[], 
     /**
@@ -32,10 +31,10 @@ export declare class SyncDataBucket {
     /**
      * The `after` specified in the request.
      */
-    after?: OpId, 
+    after?: string | undefined, 
     /**
      * Use this for the next request.
      */
-    next_after?: OpId);
+    next_after?: string | undefined);
     toJSON(): SyncDataBucketJSON;
 }

package/lib/client/sync/stream/AbstractRemote.d.ts

@@ -1,15 +1,43 @@
-/// <reference types="node" />
 import { ILogger } from 'js-logger';
+import { fetch } from 'cross-fetch';
 import { PowerSyncCredentials } from '../../connection/PowerSyncCredentials';
+import { StreamingSyncLine, StreamingSyncRequest } from './streaming-sync-types';
+import { DataStream } from '../../../utils/DataStream';
+import type { BSON } from 'bson';
+export type BSONImplementation = typeof BSON;
 export type RemoteConnector = {
     fetchCredentials: () => Promise<PowerSyncCredentials | null>;
 };
 export declare const DEFAULT_REMOTE_LOGGER: ILogger;
+export type SyncStreamOptions = {
+    path: string;
+    data: StreamingSyncRequest;
+    headers?: Record<string, string>;
+    abortSignal?: AbortSignal;
+    fetchOptions?: Request;
+};
+export type AbstractRemoteOptions = {
+    /**
+     * Transforms the PowerSync base URL which might contain
+     * `http(s)://` to the corresponding WebSocket variant
+     * e.g. `ws(s)://`
+     */
+    socketUrlTransformer: (url: string) => string;
+    /**
+     * Optionally provide the fetch implementation to use.
+     * Note that this usually needs to be bound to the global scope.
+     * Binding should be done before passing here.
+     */
+    fetchImplementation: typeof fetch;
+};
+export declare const DEFAULT_REMOTE_OPTIONS: AbstractRemoteOptions;
 export declare abstract class AbstractRemote {
     protected connector: RemoteConnector;
     protected logger: ILogger;
     protected credentials: PowerSyncCredentials | null;
-    constructor(connector: RemoteConnector, logger?: ILogger);
+    protected options: AbstractRemoteOptions;
+    constructor(connector: RemoteConnector, logger?: ILogger, options?: Partial<AbstractRemoteOptions>);
+    get fetch(): typeof globalThis.fetch;
     getCredentials(): Promise<PowerSyncCredentials | null>;
     protected buildRequest(path: string): Promise<{
         url: string;
@@ -18,8 +46,19 @@ export declare abstract class AbstractRemote {
             Authorization: string;
         };
     }>;
-    abstract post(path: string, data: any, headers?: Record<string, string>): Promise<any>;
-    abstract get(path: string, headers?: Record<string, string>): Promise<any>;
-    abstract postStreaming(path: string, data: any, headers?: Record<string, string>, signal?: AbortSignal): Promise<any>;
-    isAvailable(): boolean;
+    post(path: string, data: any, headers?: Record<string, string>): Promise<any>;
+    get(path: string, headers?: Record<string, string>): Promise<any>;
+    postStreaming(path: string, data: any, headers?: Record<string, string>, signal?: AbortSignal): Promise<any>;
+    /**
+     * Provides a BSON implementation. The import nature of this varies depending on the platform
+     */
+    abstract getBSON(): Promise<BSONImplementation>;
+    /**
+     * Connects to the sync/stream websocket endpoint
+     */
+    socketStream(options: SyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
+    /**
+     * Connects to the sync/stream http endpoint
+     */
+    postStream(options: SyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
 }

package/lib/client/sync/stream/AbstractRemote.js

@@ -1,14 +1,41 @@
 import Logger from 'js-logger';
+import { fetch } from 'cross-fetch';
+import { DataStream } from '../../../utils/DataStream';
+import ndjsonStream from 'can-ndjson-stream';
+import { RSocketConnector } from 'rsocket-core';
+import { WebsocketClientTransport } from 'rsocket-websocket-client';
+import { AbortOperation } from '../../../utils/AbortOperation';
+import { Buffer } from 'buffer';
 // Refresh at least 30 sec before it expires
 const REFRESH_CREDENTIALS_SAFETY_PERIOD_MS = 30_000;
+const SYNC_QUEUE_REQUEST_N = 10;
+const SYNC_QUEUE_REQUEST_LOW_WATER = 5;
+// Keep alive message is sent every period
+const KEEP_ALIVE_MS = 20_000;
+// The ACK must be received in this period
+const KEEP_ALIVE_LIFETIME_MS = 30_000;
 export const DEFAULT_REMOTE_LOGGER = Logger.get('PowerSyncRemote');
+export const DEFAULT_REMOTE_OPTIONS = {
+    socketUrlTransformer: (url) => url.replace(/^https?:\/\//, function (match) {
+        return match === 'https://' ? 'wss://' : 'ws://';
+    }),
+    fetchImplementation: fetch.bind(globalThis)
+};
 export class AbstractRemote {
     connector;
     logger;
     credentials = null;
-    constructor(connector, logger = DEFAULT_REMOTE_LOGGER) {
+    options;
+    constructor(connector, logger = DEFAULT_REMOTE_LOGGER, options) {
         this.connector = connector;
         this.logger = logger;
+        this.options = {
+            ...DEFAULT_REMOTE_OPTIONS,
+            ...(options ?? {})
+        };
+    }
+    get fetch() {
+        return this.options.fetchImplementation;
     }
     async getCredentials() {
         const { expiresAt } = this.credentials ?? {};
@@ -36,7 +63,300 @@ export class AbstractRemote {
             }
         };
     }
-    isAvailable() {
-        return true;
+    async post(path, data, headers = {}) {
+        const request = await this.buildRequest(path);
+        const res = await fetch(request.url, {
+            method: 'POST',
+            headers: {
+                ...headers,
+                ...request.headers
+            },
+            body: JSON.stringify(data)
+        });
+        if (!res.ok) {
+            throw new Error(`Received ${res.status} - ${res.statusText} when posting to ${path}: ${await res.text()}}`);
+        }
+        return res.json();
+    }
+    async get(path, headers) {
+        const request = await this.buildRequest(path);
+        const res = await this.fetch(request.url, {
+            method: 'GET',
+            headers: {
+                ...headers,
+                ...request.headers
+            }
+        });
+        if (!res.ok) {
+            throw new Error(`Received ${res.status} - ${res.statusText} when getting from ${path}: ${await res.text()}}`);
+        }
+        return res.json();
+    }
+    async postStreaming(path, data, headers = {}, signal) {
+        const request = await this.buildRequest(path);
+        const res = await this.fetch(request.url, {
+            method: 'POST',
+            headers: { ...headers, ...request.headers },
+            body: JSON.stringify(data),
+            signal,
+            cache: 'no-store'
+        }).catch((ex) => {
+            this.logger.error(`Caught ex when POST streaming to ${path}`, ex);
+            throw ex;
+        });
+        if (!res.ok) {
+            const text = await res.text();
+            this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
+            const error = new Error(`HTTP ${res.statusText}: ${text}`);
+            error.status = res.status;
+            throw error;
+        }
+        return res;
+    }
+    /**
+     * Connects to the sync/stream websocket endpoint
+     */
+    async socketStream(options) {
+        const { path } = options;
+        const request = await this.buildRequest(path);
+        const BSON = await this.getBSON();
+        const connector = new RSocketConnector({
+            transport: new WebsocketClientTransport({
+                url: this.options.socketUrlTransformer(request.url)
+            }),
+            setup: {
+                keepAlive: KEEP_ALIVE_MS,
+                lifetime: KEEP_ALIVE_LIFETIME_MS,
+                dataMimeType: 'application/bson',
+                metadataMimeType: 'application/bson',
+                payload: {
+                    data: null,
+                    metadata: Buffer.from(BSON.serialize({
+                        token: request.headers.Authorization
+                    }))
+                }
+            }
+        });
+        let rsocket;
+        try {
+            rsocket = await connector.connect();
+        }
+        catch (ex) {
+            /**
+             * On React native the connection exception can be `undefined` this causes issues
+             * with detecting the exception inside async-mutex
+             */
+            throw new Error(`Could not connect to PowerSync instance: ${JSON.stringify(ex)}`);
+        }
+        const stream = new DataStream({
+            logger: this.logger,
+            pressure: {
+                lowWaterMark: SYNC_QUEUE_REQUEST_LOW_WATER
+            }
+        });
+        let socketIsClosed = false;
+        const closeSocket = () => {
+            if (socketIsClosed) {
+                return;
+            }
+            socketIsClosed = true;
+            rsocket.close();
+        };
+        // We initially request this amount and expect these to arrive eventually
+        let pendingEventsCount = SYNC_QUEUE_REQUEST_N;
+        const socket = await new Promise((resolve, reject) => {
+            let connectionEstablished = false;
+            const res = rsocket.requestStream({
+                data: Buffer.from(BSON.serialize(options.data)),
+                metadata: Buffer.from(BSON.serialize({
+                    path
+                }))
+            }, SYNC_QUEUE_REQUEST_N, // The initial N amount
+            {
+                onError: (e) => {
+                    // Don't log closed as an error
+                    if (e.message !== 'Closed. ') {
+                        this.logger.error(e);
+                    }
+                    // RSocket will close this automatically
+                    // Attempting to close multiple times causes a console warning
+                    socketIsClosed = true;
+                    stream.close();
+                    // Handles cases where the connection failed e.g. auth error or connection error
+                    if (!connectionEstablished) {
+                        reject(e);
+                    }
+                },
+                onNext: (payload) => {
+                    // The connection is active
+                    if (!connectionEstablished) {
+                        connectionEstablished = true;
+                        resolve(res);
+                    }
+                    const { data } = payload;
+                    // Less events are now pending
+                    pendingEventsCount--;
+                    if (!data) {
+                        return;
+                    }
+                    const deserializedData = BSON.deserialize(data);
+                    stream.enqueueData(deserializedData);
+                },
+                onComplete: () => {
+                    stream.close();
+                },
+                onExtension: () => { }
+            });
+        });
+        const l = stream.registerListener({
+            lowWater: async () => {
+                // Request to fill up the queue
+                const required = SYNC_QUEUE_REQUEST_N - pendingEventsCount;
+                if (required > 0) {
+                    socket.request(SYNC_QUEUE_REQUEST_N - pendingEventsCount);
+                    pendingEventsCount = SYNC_QUEUE_REQUEST_N;
+                }
+            },
+            closed: () => {
+                closeSocket();
+                l?.();
+            }
+        });
+        /**
+         * Handle abort operations here.
+         * Unfortunately cannot insert them into the connection.
+         */
+        if (options.abortSignal?.aborted) {
+            stream.close();
+        }
+        else {
+            options.abortSignal?.addEventListener('abort', () => {
+                stream.close();
+            });
+        }
+        return stream;
+    }
+    /**
+     * Connects to the sync/stream http endpoint
+     */
+    async postStream(options) {
+        const { data, path, headers, abortSignal } = options;
+        const request = await this.buildRequest(path);
+        /**
+         * This abort controller will abort pending fetch requests.
+         * If the request has resolved, it will be used to close the readable stream.
+         * Which will cancel the network request.
+         *
+         * This nested controller is required since:
+         *  Aborting the active fetch request while it is being consumed seems to throw
+         *  an unhandled exception on the window level.
+         */
+        const controller = new AbortController();
+        let requestResolved = false;
+        abortSignal?.addEventListener('abort', () => {
+            if (!requestResolved) {
+                // Only abort via the abort controller if the request has not resolved yet
+                controller.abort(abortSignal.reason ??
+                    new AbortOperation('Cancelling network request before it resolves. Abort signal has been received.'));
+            }
+        });
+        const res = await this.fetch(request.url, {
+            method: 'POST',
+            headers: { ...headers, ...request.headers },
+            body: JSON.stringify(data),
+            signal: controller.signal,
+            cache: 'no-store',
+            ...options.fetchOptions
+        }).catch((ex) => {
+            if (ex.name == 'AbortError') {
+                throw new AbortOperation(`Pending fetch request to ${request.url} has been aborted.`);
+            }
+            throw ex;
+        });
+        if (!res) {
+            throw new Error('Fetch request was aborted');
+        }
+        requestResolved = true;
+        if (!res.ok || !res.body) {
+            const text = await res.text();
+            this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
+            const error = new Error(`HTTP ${res.statusText}: ${text}`);
+            error.status = res.status;
+            throw error;
+        }
+        /**
+         * The can-ndjson-stream does not handle aborted streams well.
+         * This will intercept the readable stream and close the stream if
+         * aborted.
+         */
+        const reader = res.body.getReader();
+        // This will close the network request and read stream
+        const closeReader = async () => {
+            try {
+                await reader.cancel();
+            }
+            catch (ex) {
+                // an error will throw if the reader hasn't been used yet
+            }
+            reader.releaseLock();
+        };
+        abortSignal?.addEventListener('abort', () => {
+            closeReader();
+        });
+        const outputStream = new ReadableStream({
+            start: (controller) => {
+                const processStream = async () => {
+                    while (!abortSignal?.aborted) {
+                        try {
+                            const { done, value } = await reader.read();
+                            // When no more data needs to be consumed, close the stream
+                            if (done) {
+                                break;
+                            }
+                            // Enqueue the next data chunk into our target stream
+                            controller.enqueue(value);
+                        }
+                        catch (ex) {
+                            this.logger.error('Caught exception when reading sync stream', ex);
+                            break;
+                        }
+                    }
+                    if (!abortSignal?.aborted) {
+                        // Close the downstream readable stream
+                        await closeReader();
+                    }
+                    controller.close();
+                };
+                processStream();
+            }
+        });
+        const jsonS = ndjsonStream(new Response(outputStream).body);
+        const stream = new DataStream({
+            logger: this.logger
+        });
+        const r = jsonS.getReader();
+        const l = stream.registerListener({
+            lowWater: async () => {
+                try {
+                    const { done, value } = await r.read();
+                    // Exit if we're done
+                    if (done) {
+                        stream.close();
+                        l?.();
+                        return;
+                    }
+                    stream.enqueueData(value);
+                }
+                catch (ex) {
+                    stream.close();
+                    throw ex;
+                }
+            },
+            closed: () => {
+                closeReader();
+                l?.();
+            }
+        });
+        return stream;
     }
 }

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts

@@ -1,6 +1,4 @@
-/// <reference types="node" />
 import { ILogger } from 'js-logger';
-import { StreamingSyncLine, StreamingSyncRequest } from './streaming-sync-types';
 import { AbstractRemote } from './AbstractRemote';
 import { BucketStorageAdapter } from '../bucket/BucketStorageAdapter';
 import { SyncStatus, SyncStatusOptions } from '../../../db/crud/SyncStatus';
@@ -9,6 +7,10 @@ export declare enum LockType {
     CRUD = "crud",
     SYNC = "sync"
 }
+export declare enum SyncStreamConnectionMethod {
+    HTTP = "http",
+    WEB_SOCKET = "web-socket"
+}
 /**
  * Abstract Lock to be implemented by various JS environments
  */
@@ -41,11 +43,23 @@ export interface StreamingSyncImplementationListener extends BaseListener {
      */
     statusChanged?: ((status: SyncStatus) => void) | undefined;
 }
+/**
+ * Configurable options to be used when connecting to the PowerSync
+ * backend instance.
+ */
+export interface PowerSyncConnectionOptions {
+    /**
+     * The connection method to use when streaming updates from
+     * the PowerSync backend instance.
+     * Defaults to a HTTP streaming connection.
+     */
+    connectionMethod?: SyncStreamConnectionMethod;
+}
 export interface StreamingSyncImplementation extends BaseObserver<StreamingSyncImplementationListener>, Disposable {
     /**
      * Connects to the sync service
      */
-    connect(): Promise<void>;
+    connect(options?: PowerSyncConnectionOptions): Promise<void>;
     /**
      * Disconnects from the sync services.
      * @throws if not connected or if abort is not controlled internally
@@ -66,6 +80,7 @@ export declare const DEFAULT_STREAMING_SYNC_OPTIONS: {
     logger: ILogger;
     crudUploadThrottleMs: number;
 };
+export declare const DEFAULT_STREAM_CONNECTION_OPTIONS: Required<PowerSyncConnectionOptions>;
 export declare abstract class AbstractStreamingSyncImplementation extends BaseObserver<StreamingSyncImplementationListener> implements StreamingSyncImplementation {
     protected _lastSyncedAt: Date | null;
     protected options: AbstractStreamingSyncImplementationOptions;
@@ -77,7 +92,7 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
     constructor(options: AbstractStreamingSyncImplementationOptions);
     waitForReady(): Promise<void>;
     waitForStatus(status: SyncStatusOptions): Promise<void>;
-    get lastSyncedAt(): Date;
+    get lastSyncedAt(): Date | undefined;
     get isConnected(): boolean;
     protected get logger(): ILogger;
     dispose(): Promise<void>;
@@ -86,16 +101,15 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
     getWriteCheckpoint(): Promise<string>;
     protected _uploadAllCrud(): Promise<void>;
     protected uploadCrudBatch(): Promise<boolean>;
-    connect(): Promise<void>;
+    connect(options?: PowerSyncConnectionOptions): Promise<void>;
     disconnect(): Promise<void>;
     /**
      * @deprecated use [connect instead]
      */
-    streamingSync(signal?: AbortSignal): Promise<void>;
-    protected streamingSyncIteration(signal: AbortSignal, progress?: () => void): Promise<{
+    streamingSync(signal?: AbortSignal, options?: PowerSyncConnectionOptions): Promise<void>;
+    protected streamingSyncIteration(signal: AbortSignal, options?: PowerSyncConnectionOptions): Promise<{
         retry?: boolean;
     }>;
-    protected streamingSyncRequest(req: StreamingSyncRequest, signal?: AbortSignal): AsyncGenerator<StreamingSyncLine>;
     protected updateSyncStatus(options: SyncStatusOptions): void;
     private delayRetry;
 }

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js

@@ -1,7 +1,6 @@
 import throttle from 'lodash/throttle';
 import Logger from 'js-logger';
 import { isStreamingKeepalive, isStreamingSyncCheckpoint, isStreamingSyncCheckpointComplete, isStreamingSyncCheckpointDiff, isStreamingSyncData } from './streaming-sync-types';
-import ndjsonStream from 'can-ndjson-stream';
 import { SyncStatus } from '../../../db/crud/SyncStatus';
 import { SyncDataBucket } from '../bucket/SyncDataBucket';
 import { BaseObserver } from '../../../utils/BaseObserver';
@@ -11,12 +10,20 @@ export var LockType;
     LockType["CRUD"] = "crud";
     LockType["SYNC"] = "sync";
 })(LockType || (LockType = {}));
+export var SyncStreamConnectionMethod;
+(function (SyncStreamConnectionMethod) {
+    SyncStreamConnectionMethod["HTTP"] = "http";
+    SyncStreamConnectionMethod["WEB_SOCKET"] = "web-socket";
+})(SyncStreamConnectionMethod || (SyncStreamConnectionMethod = {}));
 export const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
 export const DEFAULT_STREAMING_SYNC_OPTIONS = {
     retryDelayMs: 5000,
     logger: Logger.get('PowerSyncStream'),
     crudUploadThrottleMs: DEFAULT_CRUD_UPLOAD_THROTTLE_MS
 };
+export const DEFAULT_STREAM_CONNECTION_OPTIONS = {
+    connectionMethod: SyncStreamConnectionMethod.HTTP
+};
 export class AbstractStreamingSyncImplementation extends BaseObserver {
     _lastSyncedAt;
     options;
@@ -139,12 +146,12 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
             return true;
         }
     }
-    async connect() {
+    async connect(options) {
         if (this.abortController) {
             await this.disconnect();
         }
         this.abortController = new AbortController();
-        this.streamingSyncPromise = this.streamingSync(this.abortController.signal);
+        this.streamingSyncPromise = this.streamingSync(this.abortController.signal, options);
         // Return a promise that resolves when the connection status is updated
         return new Promise((resolve) => {
             const l = this.registerListener({
@@ -189,7 +196,7 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
     /**
      * @deprecated use [connect instead]
      */
-    async streamingSync(signal) {
+    async streamingSync(signal, options) {
         if (!signal) {
             this.abortController = new AbortController();
             signal = this.abortController.signal;
@@ -231,7 +238,7 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
                 if (signal?.aborted) {
                     break;
                 }
-                const { retry } = await this.streamingSyncIteration(nestedAbortController.signal);
+                const { retry } = await this.streamingSyncIteration(nestedAbortController.signal, options);
                 if (!retry) {
                     /**
                      * A sync error ocurred that we cannot recover from here.
@@ -273,11 +280,15 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
         // Mark as disconnected if here
         this.updateSyncStatus({ connected: false });
     }
-    async streamingSyncIteration(signal, progress) {
+    async streamingSyncIteration(signal, options) {
         return await this.obtainLock({
             type: LockType.SYNC,
             signal,
             callback: async () => {
+                const resolvedOptions = {
+                    ...DEFAULT_STREAM_CONNECTION_OPTIONS,
+                    ...(options ?? {})
+                };
                 this.logger.debug('Streaming sync iteration started');
                 this.options.adapter.startSession();
                 const bucketEntries = await this.options.adapter.getBucketStates();
@@ -294,11 +305,34 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
                 let validatedCheckpoint = null;
                 let appliedCheckpoint = null;
                 let bucketSet = new Set(initialBuckets.keys());
-                for await (const line of this.streamingSyncRequest({
-                    buckets: req,
-                    include_checksum: true,
-                    raw_data: true
-                }, signal)) {
+                this.logger.debug('Requesting stream from server');
+                const syncOptions = {
+                    path: '/sync/stream',
+                    abortSignal: signal,
+                    data: {
+                        buckets: req,
+                        include_checksum: true,
+                        raw_data: true
+                    }
+                };
+                const stream = resolvedOptions?.connectionMethod == SyncStreamConnectionMethod.HTTP
+                    ? await this.options.remote.postStream(syncOptions)
+                    : await this.options.remote.socketStream(syncOptions);
+                this.logger.debug('Stream established. Processing events');
+                while (!stream.closed) {
+                    const line = await stream.read();
+                    if (!line) {
+                        // The stream has closed while waiting
+                        return { retry: true };
+                    }
+                    // A connection is active and messages are being received
+                    if (!this.syncStatus.connected) {
+                        // There is a connection now
+                        Promise.resolve().then(() => this.triggerCrudUpload());
+                        this.updateSyncStatus({
+                            connected: true
+                        });
+                    }
                     if (isStreamingSyncCheckpoint(line)) {
                         targetCheckpoint = line.checkpoint;
                         const bucketsToDelete = new Set(bucketSet);
@@ -385,6 +419,11 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
                         if (remaining_seconds == 0) {
                             // Connection would be closed automatically right after this
                             this.logger.debug('Token expiring; reconnect');
+                            /**
+                             * For a rare case where the backend connector does not update the token
+                             * (uses the same one), this should have some delay.
+                             */
+                            await this.delayRetry();
                             return { retry: true };
                         }
                         this.triggerCrudUpload();
@@ -421,7 +460,6 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
                             }
                         }
                     }
-                    progress?.();
                 }
                 this.logger.debug('Stream input empty');
                 // Connection closed. Likely due to auth issue.
@@ -429,31 +467,6 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
             }
         });
     }
-    async *streamingSyncRequest(req, signal) {
-        const body = await this.options.remote.postStreaming('/sync/stream', req, {}, signal);
-        // A connection is active
-        // There is a connection now
-        Promise.resolve().then(() => this.triggerCrudUpload());
-        this.updateSyncStatus({
-            connected: true
-        });
-        const stream = ndjsonStream(body);
-        const reader = stream.getReader();
-        try {
-            while (true) {
-                // Read from the stream
-                const { done, value } = await reader.read();
-                // Exit if we're done
-                if (done)
-                    return;
-                // Else yield the chunk
-                yield value;
-            }
-        }
-        finally {
-            reader.releaseLock();
-        }
-    }
     updateSyncStatus(options) {
         const updatedStatus = new SyncStatus({
             connected: options.connected ?? this.syncStatus.connected,

package/lib/db/Column.d.ts

@@ -11,9 +11,9 @@ export declare class Column {
     protected options: ColumnOptions;
     constructor(options: ColumnOptions);
     get name(): string;
-    get type(): ColumnType;
+    get type(): ColumnType | undefined;
     toJSON(): {
         name: string;
-        type: ColumnType;
+        type: ColumnType | undefined;
     };
 }

package/lib/db/crud/SyncStatus.d.ts

@@ -19,12 +19,12 @@ export declare class SyncStatus {
      *  Time that a last sync has fully completed, if any.
      *  Currently this is reset to null after a restart.
      */
-    get lastSyncedAt(): Date;
+    get lastSyncedAt(): Date | undefined;
     /**
      * Indicates whether there has been at least one full sync, if any.
      * Is undefined when unknown, for example when state is still being loaded from the database.
      */
-    get hasSynced(): boolean;
+    get hasSynced(): boolean | undefined;
     /**
      *  Upload/download status
      */

package/lib/db/schema/Index.d.ts

@@ -15,7 +15,7 @@ export declare class Index {
         name: string;
         columns: {
             name: string;
-            ascending: boolean;
+            ascending: boolean | undefined;
             type: import("../Column").ColumnType;
         }[];
     };

package/lib/db/schema/IndexedColumn.d.ts

@@ -10,10 +10,10 @@ export declare class IndexedColumn {
     static createAscending(column: string): IndexedColumn;
     constructor(options: IndexColumnOptions);
     get name(): string;
-    get ascending(): boolean;
+    get ascending(): boolean | undefined;
     toJSON(table: Table): {
         name: string;
-        ascending: boolean;
+        ascending: boolean | undefined;
         type: ColumnType;
     };
 }

package/lib/db/schema/Schema.d.ts

@@ -21,13 +21,13 @@ export declare class Schema<S extends SchemaType = SchemaType> {
             insert_only: boolean;
             columns: {
                 name: string;
-                type: import("../Column").ColumnType;
+                type: import("../Column").ColumnType | undefined;
             }[];
             indexes: {
                 name: string;
                 columns: {
                     name: string;
-                    ascending: boolean;
+                    ascending: boolean | undefined;
                     type: import("../Column").ColumnType;
                 }[];
             }[];

package/lib/db/schema/Table.d.ts

@@ -21,7 +21,7 @@ export declare class Table {
     static createTable(name: string, table: TableV2): Table;
     constructor(options: TableOptions);
     get name(): string;
-    get viewNameOverride(): string;
+    get viewNameOverride(): string | undefined;
     get viewName(): string;
     get columns(): Column[];
     get indexes(): Index[];
@@ -37,13 +37,13 @@ export declare class Table {
         insert_only: boolean;
         columns: {
             name: string;
-            type: import("../Column").ColumnType;
+            type: import("../Column").ColumnType | undefined;
         }[];
         indexes: {
             name: string;
             columns: {
                 name: string;
-                ascending: boolean;
+                ascending: boolean | undefined;
                 type: import("../Column").ColumnType;
             }[];
         }[];

package/lib/db/schema/TableV2.d.ts

@@ -4,9 +4,9 @@ export type BaseColumnType<T extends number | string | null> = {
     type: ColumnType;
 };
 export declare const column: {
-    text: BaseColumnType<string>;
-    integer: BaseColumnType<number>;
-    real: BaseColumnType<number>;
+    text: BaseColumnType<string | null>;
+    integer: BaseColumnType<number | null>;
+    real: BaseColumnType<number | null>;
 };
 export type ColumnsType = Record<string, BaseColumnType<any>>;
 export type ExtractColumnValueType<T extends BaseColumnType<any>> = T extends BaseColumnType<infer R> ? R : unknown;

package/lib/index.d.ts

@@ -28,5 +28,6 @@ export * from './db/schema/TableV2';
 export * from './utils/AbortOperation';
 export * from './utils/BaseObserver';
 export * from './utils/strings';
+export * from './utils/DataStream';
 export * from './utils/parseQuery';
 export * from './types/types';

package/lib/index.js

@@ -28,5 +28,6 @@ export * from './db/schema/TableV2';
 export * from './utils/AbortOperation';
 export * from './utils/BaseObserver';
 export * from './utils/strings';
+export * from './utils/DataStream';
 export * from './utils/parseQuery';
 export * from './types/types';

package/lib/utils/BaseObserver.d.ts

@@ -8,13 +8,12 @@ export type BaseListener = {
     [key: string]: ((...event: any) => any) | undefined;
 };
 export declare class BaseObserver<T extends BaseListener = BaseListener> implements BaseObserverInterface<T> {
-    protected listeners: {
-        [id: string]: Partial<T>;
-    };
+    protected listeners: Set<Partial<T>>;
     constructor();
     /**
      * Register a listener for updates to the PowerSync client.
      */
     registerListener(listener: Partial<T>): () => void;
     iterateListeners(cb: (listener: Partial<T>) => any): void;
+    iterateAsyncListeners(cb: (listener: Partial<T>) => Promise<any>): Promise<void>;
 }

package/lib/utils/BaseObserver.js

@@ -1,22 +1,23 @@
-import { v4 } from 'uuid';
 export class BaseObserver {
-    listeners;
-    constructor() {
-        this.listeners = {};
-    }
+    listeners = new Set();
+    constructor() { }
     /**
      * Register a listener for updates to the PowerSync client.
      */
     registerListener(listener) {
-        const id = v4();
-        this.listeners[id] = listener;
+        this.listeners.add(listener);
         return () => {
-            delete this.listeners[id];
+            this.listeners.delete(listener);
         };
     }
     iterateListeners(cb) {
-        for (const i in this.listeners) {
-            cb(this.listeners[i]);
+        for (const listener of this.listeners) {
+            cb(listener);
+        }
+    }
+    async iterateAsyncListeners(cb) {
+        for (let i of Array.from(this.listeners.values())) {
+            await cb(i);
         }
     }
 }

package/lib/utils/DataStream.d.ts

@@ -0,0 +1,63 @@
+import { ILogger } from 'js-logger';
+import { BaseListener, BaseObserver } from './BaseObserver';
+export type DataStreamOptions = {
+    /**
+     * Close the stream if any consumer throws an error
+     */
+    closeOnError?: boolean;
+    pressure?: {
+        highWaterMark?: number;
+        lowWaterMark?: number;
+    };
+    logger?: ILogger;
+};
+export type DataStreamCallback<Data extends any = any> = (data: Data) => Promise<void>;
+export interface DataStreamListener<Data extends any = any> extends BaseListener {
+    data: (data: Data) => Promise<void>;
+    closed: () => void;
+    error: (error: Error) => void;
+    highWater: () => Promise<void>;
+    lowWater: () => Promise<void>;
+}
+export declare const DEFAULT_PRESSURE_LIMITS: {
+    highWater: number;
+    lowWater: number;
+};
+/**
+ * A very basic implementation of a data stream with backpressure support which does not use
+ * native JS streams or async iterators.
+ * This is handy for environments such as React Native which need polyfills for the above.
+ */
+export declare class DataStream<Data extends any = any> extends BaseObserver<DataStreamListener<Data>> {
+    protected options?: DataStreamOptions | undefined;
+    dataQueue: Data[];
+    protected isClosed: boolean;
+    protected processingPromise: Promise<void> | null;
+    protected logger: ILogger;
+    constructor(options?: DataStreamOptions | undefined);
+    get highWatermark(): number;
+    get lowWatermark(): number;
+    get closed(): boolean;
+    close(): Promise<void>;
+    /**
+     * Enqueues data for the consumers to read
+     */
+    enqueueData(data: Data): void;
+    /**
+     * Reads data once from the data stream
+     * @returns a Data payload or Null if the stream closed.
+     */
+    read(): Promise<Data | null>;
+    /**
+     * Executes a callback for each data item in the stream
+     */
+    forEach(callback: DataStreamCallback<Data>): () => void;
+    protected processQueue(): Promise<void>;
+    /**
+     * Creates a new data stream which is a map of the original
+     */
+    map<ReturnData>(callback: (data: Data) => ReturnData): DataStream<ReturnData>;
+    protected hasDataReader(): boolean;
+    protected _processQueue(): Promise<void>;
+    protected iterateAsyncErrored(cb: (l: BaseListener) => Promise<void>): Promise<void>;
+}

package/lib/utils/DataStream.js

@@ -0,0 +1,162 @@
+import Logger from 'js-logger';
+import { BaseObserver } from './BaseObserver';
+export const DEFAULT_PRESSURE_LIMITS = {
+    highWater: 10,
+    lowWater: 0
+};
+/**
+ * A very basic implementation of a data stream with backpressure support which does not use
+ * native JS streams or async iterators.
+ * This is handy for environments such as React Native which need polyfills for the above.
+ */
+export class DataStream extends BaseObserver {
+    options;
+    dataQueue;
+    isClosed;
+    processingPromise;
+    logger;
+    constructor(options) {
+        super();
+        this.options = options;
+        this.processingPromise = null;
+        this.isClosed = false;
+        this.dataQueue = [];
+        this.logger = options?.logger ?? Logger.get('DataStream');
+        if (options?.closeOnError) {
+            const l = this.registerListener({
+                error: (ex) => {
+                    l?.();
+                    this.close();
+                }
+            });
+        }
+    }
+    get highWatermark() {
+        return this.options?.pressure?.highWaterMark ?? DEFAULT_PRESSURE_LIMITS.highWater;
+    }
+    get lowWatermark() {
+        return this.options?.pressure?.lowWaterMark ?? DEFAULT_PRESSURE_LIMITS.lowWater;
+    }
+    get closed() {
+        return this.isClosed;
+    }
+    async close() {
+        this.isClosed = true;
+        await this.processingPromise;
+        this.iterateListeners((l) => l.closed?.());
+        // Discard any data in the queue
+        this.dataQueue = [];
+        this.listeners.clear();
+    }
+    /**
+     * Enqueues data for the consumers to read
+     */
+    enqueueData(data) {
+        if (this.isClosed) {
+            throw new Error('Cannot enqueue data into closed stream.');
+        }
+        this.dataQueue.push(data);
+        this.processQueue();
+    }
+    /**
+     * Reads data once from the data stream
+     * @returns a Data payload or Null if the stream closed.
+     */
+    async read() {
+        if (this.dataQueue.length <= this.lowWatermark) {
+            await this.iterateAsyncErrored(async (l) => l.lowWater?.());
+        }
+        if (this.closed) {
+            return null;
+        }
+        return new Promise((resolve, reject) => {
+            const l = this.registerListener({
+                data: async (data) => {
+                    resolve(data);
+                    // Remove the listener
+                    l?.();
+                },
+                closed: () => {
+                    resolve(null);
+                    l?.();
+                },
+                error: (ex) => {
+                    reject(ex);
+                    l?.();
+                }
+            });
+            this.processQueue();
+        });
+    }
+    /**
+     * Executes a callback for each data item in the stream
+     */
+    forEach(callback) {
+        if (this.dataQueue.length <= this.lowWatermark) {
+            this.iterateAsyncErrored(async (l) => l.lowWater?.());
+        }
+        return this.registerListener({
+            data: callback
+        });
+    }
+    async processQueue() {
+        if (this.processingPromise) {
+            return;
+        }
+        /**
+         * Allow listeners to mutate the queue before processing.
+         * This allows for operations such as dropping or compressing data
+         * on high water or requesting more data on low water.
+         */
+        if (this.dataQueue.length >= this.highWatermark) {
+            await this.iterateAsyncErrored(async (l) => l.highWater?.());
+        }
+        return (this.processingPromise = this._processQueue());
+    }
+    /**
+     * Creates a new data stream which is a map of the original
+     */
+    map(callback) {
+        const stream = new DataStream(this.options);
+        const l = this.registerListener({
+            data: async (data) => {
+                stream.enqueueData(callback(data));
+            },
+            closed: () => {
+                stream.close();
+                l?.();
+            }
+        });
+        return stream;
+    }
+    hasDataReader() {
+        return Array.from(this.listeners.values()).some((l) => !!l.data);
+    }
+    async _processQueue() {
+        if (!this.dataQueue.length || this.isClosed || !this.hasDataReader()) {
+            Promise.resolve().then(() => (this.processingPromise = null));
+            return;
+        }
+        const data = this.dataQueue.shift();
+        await this.iterateAsyncErrored(async (l) => l.data?.(data));
+        if (this.dataQueue.length <= this.lowWatermark) {
+            await this.iterateAsyncErrored(async (l) => l.lowWater?.());
+        }
+        this.processingPromise = null;
+        if (this.dataQueue.length) {
+            // Next tick
+            setTimeout(() => this.processQueue());
+        }
+    }
+    async iterateAsyncErrored(cb) {
+        for (let i of Array.from(this.listeners.values())) {
+            try {
+                await cb(i);
+            }
+            catch (ex) {
+                this.logger.error(ex);
+                this.iterateListeners((l) => l.error?.(ex));
+            }
+        }
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@powersync/common",
-  "version": "0.0.0-dev-20240509081154",
+  "version": "0.0.0-dev-20240516143814",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -23,18 +23,22 @@
   "homepage": "https://docs.powersync.com/resources/api-reference",
   "dependencies": {
     "async-mutex": "^0.4.0",
+    "buffer": "^6.0.3",
     "can-ndjson-stream": "^1.0.2",
+    "cross-fetch": "^4.0.0",
     "event-iterator": "^2.0.0",
     "js-logger": "^1.6.1",
     "lodash": "^4.17.21",
-    "uuid": "^9.0.1"
+    "rsocket-core": "1.0.0-alpha.3",
+    "rsocket-websocket-client": "1.0.0-alpha.3"
   },
   "devDependencies": {
     "@types/lodash": "^4.14.197",
     "@types/node": "^20.5.9",
     "@types/uuid": "^9.0.1",
     "typescript": "^5.1.3",
-    "vitest": "^1.5.2"
+    "vitest": "^1.5.2",
+    "bson": "^6.6.0"
   },
   "scripts": {
     "build": "tsc -b",