@powersync/common 1.32.0 → 1.33.1

package/lib/client/sync/stream/AbstractRemote.js

@@ -1,5 +1,4 @@
 import { Buffer } from 'buffer';
-import ndjsonStream from 'can-ndjson-stream';
 import Logger from 'js-logger';
 import { RSocketConnector } from 'rsocket-core';
 import PACKAGE from '../../../../package.json' with { type: 'json' };
@@ -11,8 +10,12 @@ const POWERSYNC_JS_VERSION = PACKAGE.version;
 const SYNC_QUEUE_REQUEST_LOW_WATER = 5;
 // Keep alive message is sent every period
 const KEEP_ALIVE_MS = 20_000;
-// The ACK must be received in this period
-const KEEP_ALIVE_LIFETIME_MS = 30_000;
+// One message of any type must be received in this period.
+const SOCKET_TIMEOUT_MS = 30_000;
+// One keepalive message must be received in this period.
+// If there is a backlog of messages (for example on slow connections), keepalive messages could be delayed
+// significantly. Therefore this is longer than the socket timeout.
+const KEEP_ALIVE_LIFETIME_MS = 90_000;
 export const DEFAULT_REMOTE_LOGGER = Logger.get('PowerSyncRemote');
 export var FetchStrategy;
 (function (FetchStrategy) {
@@ -172,64 +175,75 @@ export class AbstractRemote {
         }
         return res.json();
     }
-    async postStreaming(path, data, headers = {}, signal) {
-        const request = await this.buildRequest(path);
-        const res = await this.fetch(request.url, {
-            method: 'POST',
-            headers: { ...headers, ...request.headers },
-            body: JSON.stringify(data),
-            signal,
-            cache: 'no-store'
-        }).catch((ex) => {
-            this.logger.error(`Caught ex when POST streaming to ${path}`, ex);
-            throw ex;
-        });
-        if (res.status === 401) {
-            this.invalidateCredentials();
-        }
-        if (!res.ok) {
-            const text = await res.text();
-            this.logger.error(`Could not POST streaming to ${path} - ${res.status} - ${res.statusText}: ${text}`);
-            const error = new Error(`HTTP ${res.statusText}: ${text}`);
-            error.status = res.status;
-            throw error;
-        }
-        return res;
-    }
     createSocket(url) {
         return new WebSocket(url);
     }
     /**
-     * Connects to the sync/stream websocket endpoint
+     * Connects to the sync/stream websocket endpoint and delivers sync lines by decoding the BSON events
+     * sent by the server.
      */
     async socketStream(options) {
+        const bson = await this.getBSON();
+        return await this.socketStreamRaw(options, (data) => bson.deserialize(data), bson);
+    }
+    /**
+     * Returns a data stream of sync line data.
+     *
+     * @param map Maps received payload frames to the typed event value.
+     * @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
+     * (required for compatibility with older sync services).
+     */
+    async socketStreamRaw(options, map, bson) {
         const { path, fetchStrategy = FetchStrategy.Buffered } = options;
+        const mimeType = bson == null ? 'application/json' : 'application/bson';
+        function toBuffer(js) {
+            let contents;
+            if (bson != null) {
+                contents = bson.serialize(js);
+            }
+            else {
+                contents = JSON.stringify(js);
+            }
+            return Buffer.from(contents);
+        }
         const syncQueueRequestSize = fetchStrategy == FetchStrategy.Buffered ? 10 : 1;
         const request = await this.buildRequest(path);
-        const bson = await this.getBSON();
         // Add the user agent in the setup payload - we can't set custom
         // headers with websockets on web. The browser userAgent is however added
         // automatically as a header.
         const userAgent = this.getUserAgent();
+        let keepAliveTimeout;
+        const resetTimeout = () => {
+            clearTimeout(keepAliveTimeout);
+            keepAliveTimeout = setTimeout(() => {
+                this.logger.error(`No data received on WebSocket in ${SOCKET_TIMEOUT_MS}ms, closing connection.`);
+                stream.close();
+            }, SOCKET_TIMEOUT_MS);
+        };
+        resetTimeout();
         const url = this.options.socketUrlTransformer(request.url);
         const connector = new RSocketConnector({
             transport: new WebsocketClientTransport({
                 url,
                 wsCreator: (url) => {
-                    return this.createSocket(url);
+                    const socket = this.createSocket(url);
+                    socket.addEventListener('message', (event) => {
+                        resetTimeout();
+                    });
+                    return socket;
                 }
             }),
             setup: {
                 keepAlive: KEEP_ALIVE_MS,
                 lifetime: KEEP_ALIVE_LIFETIME_MS,
-                dataMimeType: 'application/bson',
-                metadataMimeType: 'application/bson',
+                dataMimeType: mimeType,
+                metadataMimeType: mimeType,
                 payload: {
                     data: null,
-                    metadata: Buffer.from(bson.serialize({
+                    metadata: toBuffer({
                         token: request.headers.Authorization,
                         user_agent: userAgent
-                    }))
+                    })
                 }
             }
         });
@@ -239,16 +253,20 @@ export class AbstractRemote {
         }
         catch (ex) {
             this.logger.error(`Failed to connect WebSocket`, ex);
+            clearTimeout(keepAliveTimeout);
             throw ex;
         }
+        resetTimeout();
         const stream = new DataStream({
             logger: this.logger,
             pressure: {
                 lowWaterMark: SYNC_QUEUE_REQUEST_LOW_WATER
-            }
+            },
+            mapLine: map
         });
         let socketIsClosed = false;
         const closeSocket = () => {
+            clearTimeout(keepAliveTimeout);
             if (socketIsClosed) {
                 return;
             }
@@ -268,10 +286,10 @@ export class AbstractRemote {
         const socket = await new Promise((resolve, reject) => {
             let connectionEstablished = false;
             const res = rsocket.requestStream({
-                data: Buffer.from(bson.serialize(options.data)),
-                metadata: Buffer.from(bson.serialize({
+                data: toBuffer(options.data),
+                metadata: toBuffer({
                     path
-                }))
+                })
             }, syncQueueRequestSize, // The initial N amount
             {
                 onError: (e) => {
@@ -310,8 +328,7 @@ export class AbstractRemote {
                     if (!data) {
                         return;
                     }
-                    const deserializedData = bson.deserialize(data);
-                    stream.enqueueData(deserializedData);
+                    stream.enqueueData(data);
                 },
                 onComplete: () => {
                     stream.close();
@@ -347,9 +364,17 @@ export class AbstractRemote {
         return stream;
     }
     /**
-     * Connects to the sync/stream http endpoint
+     * Connects to the sync/stream http endpoint, parsing lines as JSON.
      */
     async postStream(options) {
+        return await this.postStreamRaw(options, (line) => {
+            return JSON.parse(line);
+        });
+    }
+    /**
+     * Connects to the sync/stream http endpoint, mapping and emitting each received string line.
+     */
+    async postStreamRaw(options, mapLine) {
         const { data, path, headers, abortSignal } = options;
         const request = await this.buildRequest(path);
         /**
@@ -395,11 +420,8 @@ export class AbstractRemote {
             error.status = res.status;
             throw error;
         }
-        /**
-         * The can-ndjson-stream does not handle aborted streams well.
-         * This will intercept the readable stream and close the stream if
-         * aborted.
-         */
+        // Create a new stream splitting the response at line endings while also handling cancellations
+        // by closing the reader.
         const reader = res.body.getReader();
         // This will close the network request and read stream
         const closeReader = async () => {
@@ -414,49 +436,39 @@ export class AbstractRemote {
         abortSignal?.addEventListener('abort', () => {
             closeReader();
         });
-        const outputStream = new ReadableStream({
-            start: (controller) => {
-                const processStream = async () => {
-                    while (!abortSignal?.aborted) {
-                        try {
-                            const { done, value } = await reader.read();
-                            // When no more data needs to be consumed, close the stream
-                            if (done) {
-                                break;
-                            }
-                            // Enqueue the next data chunk into our target stream
-                            controller.enqueue(value);
-                        }
-                        catch (ex) {
-                            this.logger.error('Caught exception when reading sync stream', ex);
-                            break;
-                        }
-                    }
-                    if (!abortSignal?.aborted) {
-                        // Close the downstream readable stream
-                        await closeReader();
-                    }
-                    controller.close();
-                };
-                processStream();
-            }
-        });
-        const jsonS = ndjsonStream(outputStream);
+        const decoder = new TextDecoder();
+        let buffer = '';
         const stream = new DataStream({
-            logger: this.logger
+            logger: this.logger,
+            mapLine: mapLine
         });
-        const r = jsonS.getReader();
         const l = stream.registerListener({
             lowWater: async () => {
                 try {
-                    const { done, value } = await r.read();
-                    // Exit if we're done
-                    if (done) {
-                        stream.close();
-                        l?.();
-                        return;
+                    let didCompleteLine = false;
+                    while (!didCompleteLine) {
+                        const { done, value } = await reader.read();
+                        if (done) {
+                            const remaining = buffer.trim();
+                            if (remaining.length != 0) {
+                                stream.enqueueData(remaining);
+                            }
+                            stream.close();
+                            await closeReader();
+                            return;
+                        }
+                        const data = decoder.decode(value, { stream: true });
+                        buffer += data;
+                        const lines = buffer.split('\n');
+                        for (var i = 0; i < lines.length - 1; i++) {
+                            var l = lines[i].trim();
+                            if (l.length > 0) {
+                                stream.enqueueData(l);
+                                didCompleteLine = true;
+                            }
+                        }
+                        buffer = lines[lines.length - 1];
                     }
-                    stream.enqueueData(value);
                 }
                 catch (ex) {
                     stream.close();

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts

@@ -12,6 +12,48 @@ export declare enum SyncStreamConnectionMethod {
     HTTP = "http",
     WEB_SOCKET = "web-socket"
 }
+export declare enum SyncClientImplementation {
+    /**
+     * Decodes and handles sync lines received from the sync service in JavaScript.
+     *
+     * This is the default option.
+     *
+     * @deprecated Don't use {@link SyncClientImplementation.JAVASCRIPT} directly. Instead, use
+     * {@link DEFAULT_SYNC_CLIENT_IMPLEMENTATION} or omit the option. The explicit choice to use
+     * the JavaScript-based sync implementation will be removed from a future version of the SDK.
+     */
+    JAVASCRIPT = "js",
+    /**
+     * This implementation offloads the sync line decoding and handling into the PowerSync
+     * core extension.
+     *
+     * @experimental
+     * While this implementation is more performant than {@link SyncClientImplementation.JAVASCRIPT},
+     * it has seen less real-world testing and is marked as __experimental__ at the moment.
+     *
+     * ## Compatibility warning
+     *
+     * The Rust sync client stores sync data in a format that is slightly different than the one used
+     * by the old {@link JAVASCRIPT} implementation. When adopting the {@link RUST} client on existing
+     * databases, the PowerSync SDK will migrate the format automatically.
+     * Further, the {@link JAVASCRIPT} client in recent versions of the PowerSync JS SDK (starting from
+     * the version introducing {@link RUST} as an option) also supports the new format, so you can switch
+     * back to {@link JAVASCRIPT} later.
+     *
+     * __However__: Upgrading the SDK version, then adopting {@link RUST} as a sync client and later
+     * downgrading the SDK to an older version (necessarily using the JavaScript-based implementation then)
+     * can lead to sync issues.
+     */
+    RUST = "rust"
+}
+/**
+ * The default {@link SyncClientImplementation} to use.
+ *
+ * Please use this field instead of {@link SyncClientImplementation.JAVASCRIPT} directly. A future version
+ * of the PowerSync SDK will enable {@link SyncClientImplementation.RUST} by default and remove the JavaScript
+ * option.
+ */
+export declare const DEFAULT_SYNC_CLIENT_IMPLEMENTATION = SyncClientImplementation.JAVASCRIPT;
 /**
  * Abstract Lock to be implemented by various JS environments
  */
@@ -50,6 +92,14 @@ export interface PowerSyncConnectionOptions extends BaseConnectionOptions, Addit
 }
 /** @internal */
 export interface BaseConnectionOptions {
+    /**
+     * Whether to use a JavaScript implementation to handle received sync lines from the sync
+     * service, or whether this work should be offloaded to the PowerSync core extension.
+     *
+     * This defaults to the JavaScript implementation ({@link SyncClientImplementation.JAVASCRIPT})
+     * since the ({@link SyncClientImplementation.RUST}) implementation is experimental at the moment.
+     */
+    clientImplementation?: SyncClientImplementation;
     /**
      * The connection method to use when streaming updates from
      * the PowerSync backend instance.
@@ -117,6 +167,7 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
     protected crudUpdateListener?: () => void;
     protected streamingSyncPromise?: Promise<void>;
     private pendingCrudUpload?;
+    private notifyCompletedUploads?;
     syncStatus: SyncStatus;
     triggerCrudUpload: () => void;
     constructor(options: AbstractStreamingSyncImplementationOptions);
@@ -138,7 +189,25 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
      */
     streamingSync(signal?: AbortSignal, options?: PowerSyncConnectionOptions): Promise<void>;
     private collectLocalBucketState;
+    /**
+     * Older versions of the JS SDK used to encode subkeys as JSON in {@link OplogEntry.toJSON}.
+     * Because subkeys are always strings, this leads to quotes being added around them in `ps_oplog`.
+     * While this is not a problem as long as it's done consistently, it causes issues when a database
+     * created by the JS SDK is used with other SDKs, or (more likely) when the new Rust sync client
+     * is enabled.
+     *
+     * So, we add a migration from the old key format (with quotes) to the new one (no quotes). The
+     * migration is only triggered when necessary (for now). The function returns whether the new format
+     * should be used, so that the JS SDK is able to write to updated databases.
+     *
+     * @param requireFixedKeyFormat Whether we require the new format or also support the old one.
+     *        The Rust client requires the new subkey format.
+     * @returns Whether the database is now using the new, fixed subkey format.
+     */
+    private requireKeyFormat;
     protected streamingSyncIteration(signal: AbortSignal, options?: PowerSyncConnectionOptions): Promise<void>;
+    private legacyStreamingSyncIteration;
+    private rustSyncIteration;
     private updateSyncStatusForStartingCheckpoint;
     private applyCheckpoint;
     protected updateSyncStatus(options: SyncStatusOptions): void;