@powersync/common 0.0.0-dev-20250710153817 → 0.0.0-dev-20250711131120

package/lib/client/sync/stream/AbstractRemote.d.ts

@@ -3,7 +3,7 @@ import { type fetch } from 'cross-fetch';
 import Logger, { ILogger } from 'js-logger';
 import { DataStream } from '../../../utils/DataStream.js';
 import { PowerSyncCredentials } from '../../connection/PowerSyncCredentials.js';
-import { StreamingSyncLine, StreamingSyncRequest } from './streaming-sync-types.js';
+import { StreamingSyncRequest } from './streaming-sync-types.js';
 export type BSONImplementation = typeof BSON;
 export type RemoteConnector = {
     fetchCredentials: () => Promise<PowerSyncCredentials | null>;
@@ -120,11 +120,6 @@ export declare abstract class AbstractRemote {
      */
     abstract getBSON(): Promise<BSONImplementation>;
     protected createSocket(url: string): WebSocket;
-    /**
-     * Connects to the sync/stream websocket endpoint and delivers sync lines by decoding the BSON events
-     * sent by the server.
-     */
-    socketStream(options: SocketSyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
     /**
      * Returns a data stream of sync line data.
      *
@@ -133,10 +128,6 @@ export declare abstract class AbstractRemote {
      * (required for compatibility with older sync services).
      */
     socketStreamRaw<T>(options: SocketSyncStreamOptions, map: (buffer: Uint8Array) => T, bson?: typeof BSON): Promise<DataStream<T>>;
-    /**
-     * Connects to the sync/stream http endpoint, parsing lines as JSON.
-     */
-    postStream(options: SyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
     /**
      * Connects to the sync/stream http endpoint, mapping and emitting each received string line.
      */

package/lib/client/sync/stream/AbstractRemote.js

@@ -178,14 +178,6 @@ export class AbstractRemote {
     createSocket(url) {
         return new WebSocket(url);
     }
-    /**
-     * Connects to the sync/stream websocket endpoint and delivers sync lines by decoding the BSON events
-     * sent by the server.
-     */
-    async socketStream(options) {
-        const bson = await this.getBSON();
-        return await this.socketStreamRaw(options, (data) => bson.deserialize(data), bson);
-    }
     /**
      * Returns a data stream of sync line data.
      *
@@ -363,14 +355,6 @@ export class AbstractRemote {
         }
         return stream;
     }
-    /**
-     * Connects to the sync/stream http endpoint, parsing lines as JSON.
-     */
-    async postStream(options) {
-        return await this.postStreamRaw(options, (line) => {
-            return JSON.parse(line);
-        });
-    }
     /**
      * Connects to the sync/stream http endpoint, mapping and emitting each received string line.
      */

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts

@@ -166,7 +166,7 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
     protected abortController: AbortController | null;
     protected crudUpdateListener?: () => void;
     protected streamingSyncPromise?: Promise<void>;
-    private pendingCrudUpload?;
+    private isUploadingCrud;
     private notifyCompletedUploads?;
     syncStatus: SyncStatus;
     triggerCrudUpload: () => void;

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js

@@ -3,7 +3,7 @@ import { SyncStatus } from '../../../db/crud/SyncStatus.js';
 import { FULL_SYNC_PRIORITY } from '../../../db/crud/SyncProgress.js';
 import { AbortOperation } from '../../../utils/AbortOperation.js';
 import { BaseObserver } from '../../../utils/BaseObserver.js';
-import { onAbortPromise, throttleLeadingTrailing } from '../../../utils/async.js';
+import { throttleLeadingTrailing } from '../../../utils/async.js';
 import { PowerSyncControlCommand } from '../bucket/BucketStorageAdapter.js';
 import { SyncDataBucket } from '../bucket/SyncDataBucket.js';
 import { FetchStrategy } from './AbstractRemote.js';
@@ -85,7 +85,7 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
     abortController;
     crudUpdateListener;
     streamingSyncPromise;
-    pendingCrudUpload;
+    isUploadingCrud = false;
     notifyCompletedUploads;
     syncStatus;
     triggerCrudUpload;
@@ -103,15 +103,13 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
         });
         this.abortController = null;
         this.triggerCrudUpload = throttleLeadingTrailing(() => {
-            if (!this.syncStatus.connected || this.pendingCrudUpload != null) {
+            if (!this.syncStatus.connected || this.isUploadingCrud) {
                 return;
             }
-            this.pendingCrudUpload = new Promise((resolve) => {
-                this._uploadAllCrud().finally(() => {
-                    this.notifyCompletedUploads?.();
-                    this.pendingCrudUpload = undefined;
-                    resolve();
-                });
+            this.isUploadingCrud = true;
+            this._uploadAllCrud().finally(() => {
+                this.notifyCompletedUploads?.();
+                this.isUploadingCrud = false;
             });
         }, this.options.crudUploadThrottleMs);
     }
@@ -368,6 +366,7 @@ The next upload iteration will be delayed.`);
                 });
             }
             finally {
+                this.notifyCompletedUploads = undefined;
                 if (!signal.aborted) {
                     nestedAbortController.abort(new AbortOperation('Closing sync stream network requests before retry.'));
                     nestedAbortController = new AbortController();
@@ -445,10 +444,9 @@ The next upload iteration will be delayed.`);
         this.logger.debug('Streaming sync iteration started');
         this.options.adapter.startSession();
         let [req, bucketMap] = await this.collectLocalBucketState();
-        // These are compared by reference
         let targetCheckpoint = null;
-        let validatedCheckpoint = null;
-        let appliedCheckpoint = null;
+        // A checkpoint that has been validated but not applied (e.g. due to pending local writes)
+        let pendingValidatedCheckpoint = null;
         const clientId = await this.options.adapter.getClientId();
         const usingFixedKeyFormat = await this.requireKeyFormat(false);
         this.logger.debug('Requesting stream from server');
@@ -465,21 +463,55 @@ The next upload iteration will be delayed.`);
         };
         let stream;
         if (resolvedOptions?.connectionMethod == SyncStreamConnectionMethod.HTTP) {
-            stream = await this.options.remote.postStream(syncOptions);
+            stream = await this.options.remote.postStreamRaw(syncOptions, (line) => {
+                if (typeof line == 'string') {
+                    return JSON.parse(line);
+                }
+                else {
+                    // Directly enqueued by us
+                    return line;
+                }
+            });
         }
         else {
-            stream = await this.options.remote.socketStream({
+            const bson = await this.options.remote.getBSON();
+            stream = await this.options.remote.socketStreamRaw({
                 ...syncOptions,
                 ...{ fetchStrategy: resolvedOptions.fetchStrategy }
-            });
+            }, (payload) => {
+                if (payload instanceof Uint8Array) {
+                    return bson.deserialize(payload);
+                }
+                else {
+                    // Directly enqueued by us
+                    return payload;
+                }
+            }, bson);
         }
         this.logger.debug('Stream established. Processing events');
+        this.notifyCompletedUploads = () => {
+            if (!stream.closed) {
+                stream.enqueueData({ crud_upload_completed: null });
+            }
+        };
         while (!stream.closed) {
             const line = await stream.read();
             if (!line) {
                 // The stream has closed while waiting
                 return;
             }
+            if ('crud_upload_completed' in line) {
+                if (pendingValidatedCheckpoint != null) {
+                    const { applied, endIteration } = await this.applyCheckpoint(pendingValidatedCheckpoint);
+                    if (applied) {
+                        pendingValidatedCheckpoint = null;
+                    }
+                    else if (endIteration) {
+                        break;
+                    }
+                }
+                continue;
+            }
             // A connection is active and messages are being received
             if (!this.syncStatus.connected) {
                 // There is a connection now
@@ -508,14 +540,13 @@ The next upload iteration will be delayed.`);
                 await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
             }
             else if (isStreamingSyncCheckpointComplete(line)) {
-                const result = await this.applyCheckpoint(targetCheckpoint, signal);
+                const result = await this.applyCheckpoint(targetCheckpoint);
                 if (result.endIteration) {
                     return;
                 }
-                else if (result.applied) {
-                    appliedCheckpoint = targetCheckpoint;
+                else if (!result.applied) {
+                    pendingValidatedCheckpoint = targetCheckpoint;
                 }
-                validatedCheckpoint = targetCheckpoint;
             }
             else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
                 const priority = line.partial_checkpoint_complete.priority;
@@ -625,26 +656,7 @@ The next upload iteration will be delayed.`);
                 this.triggerCrudUpload();
             }
             else {
-                this.logger.debug('Sync complete');
-                if (targetCheckpoint === appliedCheckpoint) {
-                    this.updateSyncStatus({
-                        connected: true,
-                        lastSyncedAt: new Date(),
-                        priorityStatusEntries: [],
-                        dataFlow: {
-                            downloadError: undefined
-                        }
-                    });
-                }
-                else if (validatedCheckpoint === targetCheckpoint) {
-                    const result = await this.applyCheckpoint(targetCheckpoint, signal);
-                    if (result.endIteration) {
-                        return;
-                    }
-                    else if (result.applied) {
-                        appliedCheckpoint = targetCheckpoint;
-                    }
-                }
+                this.logger.debug('Received unknown sync line', line);
             }
         }
         this.logger.debug('Stream input empty');
@@ -860,9 +872,8 @@ The next upload iteration will be delayed.`);
             }
         });
     }
-    async applyCheckpoint(checkpoint, abort) {
+    async applyCheckpoint(checkpoint) {
         let result = await this.options.adapter.syncLocalDatabase(checkpoint);
-        const pending = this.pendingCrudUpload;
         if (!result.checkpointValid) {
             this.logger.debug('Checksum mismatch in checkpoint, will reconnect');
             // This means checksums failed. Start again with a new checkpoint.
@@ -870,36 +881,21 @@ The next upload iteration will be delayed.`);
             await new Promise((resolve) => setTimeout(resolve, 50));
             return { applied: false, endIteration: true };
         }
-        else if (!result.ready && pending != null) {
-            // We have pending entries in the local upload queue or are waiting to confirm a write
-            // checkpoint, which prevented this checkpoint from applying. Wait for that to complete and
-            // try again.
-            this.logger.debug(`Could not apply checkpoint ${checkpoint.last_op_id} due to local data. Waiting for in-progress upload before retrying.`);
-            await Promise.race([pending, onAbortPromise(abort)]);
-            this.logger.debug(`Pending uploads complete, retrying local checkpoint at ${checkpoint.last_op_id}`);
-            if (abort.aborted) {
-                return { applied: false, endIteration: true };
-            }
-            // Try again now that uploads have completed.
-            result = await this.options.adapter.syncLocalDatabase(checkpoint);
-        }
-        if (result.checkpointValid && result.ready) {
-            this.logger.debug('validated checkpoint', checkpoint);
-            this.updateSyncStatus({
-                connected: true,
-                lastSyncedAt: new Date(),
-                dataFlow: {
-                    downloading: false,
-                    downloadProgress: null,
-                    downloadError: undefined
-                }
-            });
-            return { applied: true, endIteration: false };
-        }
-        else {
-            this.logger.debug('Could not apply checkpoint. Waiting for next sync complete line.');
+        else if (!result.ready) {
+            this.logger.debug('Could not apply checkpoint due to local data. We will retry applying the checkpoint after that upload is completed.');
             return { applied: false, endIteration: false };
         }
+        this.logger.debug('validated checkpoint', checkpoint);
+        this.updateSyncStatus({
+            connected: true,
+            lastSyncedAt: new Date(),
+            dataFlow: {
+                downloading: false,
+                downloadProgress: null,
+                downloadError: undefined
+            }
+        });
+        return { applied: true, endIteration: false };
     }
     updateSyncStatus(options) {
         const updatedStatus = new SyncStatus({

package/lib/client/sync/stream/streaming-sync-types.d.ts

@@ -101,6 +101,10 @@ export interface StreamingSyncKeepalive {
     token_expires_in: number;
 }
 export type StreamingSyncLine = StreamingSyncDataJSON | StreamingSyncCheckpoint | StreamingSyncCheckpointDiff | StreamingSyncCheckpointComplete | StreamingSyncCheckpointPartiallyComplete | StreamingSyncKeepalive;
+export type CrudUploadNotification = {
+    crud_upload_completed: null;
+};
+export type StreamingSyncLineOrCrudUploadComplete = StreamingSyncLine | CrudUploadNotification;
 export interface BucketRequest {
     name: string;
     /**

package/lib/utils/async.d.ts

@@ -12,4 +12,3 @@ export declare function throttleTrailing(func: () => void, wait: number): () =>
  * Roughly equivalent to lodash/throttle with {leading: true, trailing: true}
  */
 export declare function throttleLeadingTrailing(func: () => void, wait: number): () => void;
-export declare function onAbortPromise(signal: AbortSignal): Promise<void>;

package/lib/utils/async.js

@@ -43,13 +43,3 @@ export function throttleLeadingTrailing(func, wait) {
         }
     };
 }
-export function onAbortPromise(signal) {
-    return new Promise((resolve) => {
-        if (signal.aborted) {
-            resolve();
-        }
-        else {
-            signal.onabort = () => resolve();
-        }
-    });
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@powersync/common",
-  "version": "0.0.0-dev-20250710153817",
+  "version": "0.0.0-dev-20250711131120",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"