@powersync/common 1.33.2 → 1.35.0

package/lib/client/sync/bucket/SqliteBucketStorage.d.ts

@@ -1,4 +1,3 @@
-import { Mutex } from 'async-mutex';
 import { ILogger } from 'js-logger';
 import { DBAdapter, Transaction } from '../../../db/DBAdapter.js';
 import { BaseObserver } from '../../../utils/BaseObserver.js';
@@ -8,13 +7,12 @@ import { CrudEntry } from './CrudEntry.js';
 import { SyncDataBatch } from './SyncDataBatch.js';
 export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListener> implements BucketStorageAdapter {
     private db;
-    private mutex;
     private logger;
     tableNames: Set<string>;
     private _hasCompletedSync;
     private updateListener;
     private _clientId?;
-    constructor(db: DBAdapter, mutex: Mutex, logger?: ILogger);
+    constructor(db: DBAdapter, logger?: ILogger);
     init(): Promise<void>;
     dispose(): Promise<void>;
     _getClientId(): Promise<string>;

package/lib/client/sync/bucket/SqliteBucketStorage.js

@@ -6,16 +6,14 @@ import { PSInternalTable } from './BucketStorageAdapter.js';
 import { CrudEntry } from './CrudEntry.js';
 export class SqliteBucketStorage extends BaseObserver {
     db;
-    mutex;
     logger;
     tableNames;
     _hasCompletedSync;
     updateListener;
     _clientId;
-    constructor(db, mutex, logger = Logger.get('SqliteBucketStorage')) {
+    constructor(db, logger = Logger.get('SqliteBucketStorage')) {
         super();
         this.db = db;
-        this.mutex = mutex;
         this.logger = logger;
         this._hasCompletedSync = false;
         this.tableNames = new Set();
@@ -66,11 +64,11 @@ export class SqliteBucketStorage extends BaseObserver {
     async saveSyncData(batch, fixedKeyFormat = false) {
         await this.writeTransaction(async (tx) => {
             for (const b of batch.buckets) {
-                const result = await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
+                await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
                     'save',
                     JSON.stringify({ buckets: [b.toJSON(fixedKeyFormat)] })
                 ]);
-                this.logger.debug('saveSyncData', JSON.stringify(result));
+                this.logger.debug(`Saved batch of data for  bucket: ${b.bucket}, operations: ${b.data.length}`);
             }
         });
     }
@@ -86,7 +84,7 @@ export class SqliteBucketStorage extends BaseObserver {
         await this.writeTransaction(async (tx) => {
             await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', ['delete_bucket', bucket]);
         });
-        this.logger.debug('done deleting bucket');
+        this.logger.debug(`Done deleting bucket ${bucket}`);
     }
     async hasCompletedSync() {
         if (this._hasCompletedSync) {
@@ -108,6 +106,12 @@ export class SqliteBucketStorage extends BaseObserver {
             }
             return { ready: false, checkpointValid: false, checkpointFailures: r.checkpointFailures };
         }
+        if (priority == null) {
+            this.logger.debug(`Validated checksums checkpoint ${checkpoint.last_op_id}`);
+        }
+        else {
+            this.logger.debug(`Validated checksums for partial checkpoint ${checkpoint.last_op_id}, priority ${priority}`);
+        }
         let buckets = checkpoint.buckets;
         if (priority !== undefined) {
             buckets = buckets.filter((b) => hasMatchingPriority(priority, b));
@@ -124,7 +128,6 @@ export class SqliteBucketStorage extends BaseObserver {
         });
         const valid = await this.updateObjectsFromBuckets(checkpoint, priority);
         if (!valid) {
-            this.logger.debug('Not at a consistent checkpoint - cannot update local db');
             return { ready: false, checkpointValid: true };
         }
         return {
@@ -177,7 +180,6 @@ export class SqliteBucketStorage extends BaseObserver {
             JSON.stringify({ ...checkpoint })
         ]);
         const resultItem = rs.rows?.item(0);
-        this.logger.debug('validateChecksums priority, checkpoint, result item', priority, checkpoint, resultItem);
         if (!resultItem) {
             return {
                 checkpointValid: false,
@@ -210,30 +212,26 @@ export class SqliteBucketStorage extends BaseObserver {
         }
         const seqBefore = rs[0]['seq'];
         const opId = await cb();
-        this.logger.debug(`[updateLocalTarget] Updating target to checkpoint ${opId}`);
         return this.writeTransaction(async (tx) => {
             const anyData = await tx.execute('SELECT 1 FROM ps_crud LIMIT 1');
             if (anyData.rows?.length) {
                 // if isNotEmpty
-                this.logger.debug('updateLocalTarget', 'ps crud is not empty');
+                this.logger.debug(`New data uploaded since write checkpoint ${opId} - need new write checkpoint`);
                 return false;
             }
             const rs = await tx.execute("SELECT seq FROM sqlite_sequence WHERE name = 'ps_crud'");
             if (!rs.rows?.length) {
                 // assert isNotEmpty
-                throw new Error('SQlite Sequence should not be empty');
+                throw new Error('SQLite Sequence should not be empty');
             }
             const seqAfter = rs.rows?.item(0)['seq'];
-            this.logger.debug('seqAfter', JSON.stringify(rs.rows?.item(0)));
             if (seqAfter != seqBefore) {
-                this.logger.debug('seqAfter != seqBefore', seqAfter, seqBefore);
+                this.logger.debug(`New data uploaded since write checpoint ${opId} - need new write checkpoint (sequence updated)`);
                 // New crud data may have been uploaded since we got the checkpoint. Abort.
                 return false;
             }
-            const response = await tx.execute("UPDATE ps_buckets SET target_op = CAST(? as INTEGER) WHERE name='$local'", [
-                opId
-            ]);
-            this.logger.debug(['[updateLocalTarget] Response from updating target_op ', JSON.stringify(response)]);
+            this.logger.debug(`Updating target write checkpoint to ${opId}`);
+            await tx.execute("UPDATE ps_buckets SET target_op = CAST(? as INTEGER) WHERE name='$local'", [opId]);
             return true;
         });
     }

package/lib/client/sync/stream/AbstractRemote.d.ts

@@ -3,7 +3,7 @@ import { type fetch } from 'cross-fetch';
 import Logger, { ILogger } from 'js-logger';
 import { DataStream } from '../../../utils/DataStream.js';
 import { PowerSyncCredentials } from '../../connection/PowerSyncCredentials.js';
-import { StreamingSyncLine, StreamingSyncRequest } from './streaming-sync-types.js';
+import { StreamingSyncRequest } from './streaming-sync-types.js';
 export type BSONImplementation = typeof BSON;
 export type RemoteConnector = {
     fetchCredentials: () => Promise<PowerSyncCredentials | null>;
@@ -120,11 +120,6 @@ export declare abstract class AbstractRemote {
      */
     abstract getBSON(): Promise<BSONImplementation>;
     protected createSocket(url: string): WebSocket;
-    /**
-     * Connects to the sync/stream websocket endpoint and delivers sync lines by decoding the BSON events
-     * sent by the server.
-     */
-    socketStream(options: SocketSyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
     /**
      * Returns a data stream of sync line data.
      *
@@ -133,10 +128,6 @@ export declare abstract class AbstractRemote {
      * (required for compatibility with older sync services).
      */
     socketStreamRaw<T>(options: SocketSyncStreamOptions, map: (buffer: Uint8Array) => T, bson?: typeof BSON): Promise<DataStream<T>>;
-    /**
-     * Connects to the sync/stream http endpoint, parsing lines as JSON.
-     */
-    postStream(options: SyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
     /**
      * Connects to the sync/stream http endpoint, mapping and emitting each received string line.
      */

package/lib/client/sync/stream/AbstractRemote.js

@@ -178,14 +178,6 @@ export class AbstractRemote {
     createSocket(url) {
         return new WebSocket(url);
     }
-    /**
-     * Connects to the sync/stream websocket endpoint and delivers sync lines by decoding the BSON events
-     * sent by the server.
-     */
-    async socketStream(options) {
-        const bson = await this.getBSON();
-        return await this.socketStreamRaw(options, (data) => bson.deserialize(data), bson);
-    }
     /**
      * Returns a data stream of sync line data.
      *
@@ -212,6 +204,22 @@ export class AbstractRemote {
         // headers with websockets on web. The browser userAgent is however added
         // automatically as a header.
         const userAgent = this.getUserAgent();
+        const stream = new DataStream({
+            logger: this.logger,
+            pressure: {
+                lowWaterMark: SYNC_QUEUE_REQUEST_LOW_WATER
+            },
+            mapLine: map
+        });
+        // Handle upstream abort
+        if (options.abortSignal?.aborted) {
+            throw new AbortOperation('Connection request aborted');
+        }
+        else {
+            options.abortSignal?.addEventListener('abort', () => {
+                stream.close();
+            }, { once: true });
+        }
         let keepAliveTimeout;
         const resetTimeout = () => {
             clearTimeout(keepAliveTimeout);
@@ -221,12 +229,22 @@ export class AbstractRemote {
             }, SOCKET_TIMEOUT_MS);
         };
         resetTimeout();
+        // Typescript complains about this being `never` if it's not assigned here.
+        // This is assigned in `wsCreator`.
+        let disposeSocketConnectionTimeout = () => { };
         const url = this.options.socketUrlTransformer(request.url);
         const connector = new RSocketConnector({
             transport: new WebsocketClientTransport({
                 url,
                 wsCreator: (url) => {
                     const socket = this.createSocket(url);
+                    disposeSocketConnectionTimeout = stream.registerListener({
+                        closed: () => {
+                            // Allow closing the underlying WebSocket if the stream was closed before the
+                            // RSocket connect completed. This should effectively abort the request.
+                            socket.close();
+                        }
+                    });
                     socket.addEventListener('message', (event) => {
                         resetTimeout();
                     });
@@ -250,20 +268,18 @@ export class AbstractRemote {
         let rsocket;
         try {
             rsocket = await connector.connect();
+            // The connection is established, we no longer need to monitor the initial timeout
+            disposeSocketConnectionTimeout();
         }
         catch (ex) {
             this.logger.error(`Failed to connect WebSocket`, ex);
             clearTimeout(keepAliveTimeout);
+            if (!stream.closed) {
+                await stream.close();
+            }
             throw ex;
         }
         resetTimeout();
-        const stream = new DataStream({
-            logger: this.logger,
-            pressure: {
-                lowWaterMark: SYNC_QUEUE_REQUEST_LOW_WATER
-            },
-            mapLine: map
-        });
         let socketIsClosed = false;
         const closeSocket = () => {
             clearTimeout(keepAliveTimeout);
@@ -349,28 +365,8 @@ export class AbstractRemote {
                 l();
             }
         });
-        /**
-         * Handle abort operations here.
-         * Unfortunately cannot insert them into the connection.
-         */
-        if (options.abortSignal?.aborted) {
-            stream.close();
-        }
-        else {
-            options.abortSignal?.addEventListener('abort', () => {
-                stream.close();
-            });
-        }
         return stream;
     }
-    /**
-     * Connects to the sync/stream http endpoint, parsing lines as JSON.
-     */
-    async postStream(options) {
-        return await this.postStreamRaw(options, (line) => {
-            return JSON.parse(line);
-        });
-    }
     /**
      * Connects to the sync/stream http endpoint, mapping and emitting each received string line.
      */

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts

@@ -1,6 +1,6 @@
-import Logger, { ILogger } from 'js-logger';
+import { ILogger } from 'js-logger';
 import { SyncStatus, SyncStatusOptions } from '../../../db/crud/SyncStatus.js';
-import { BaseListener, BaseObserver, Disposable } from '../../../utils/BaseObserver.js';
+import { BaseListener, BaseObserver, BaseObserverInterface, Disposable } from '../../../utils/BaseObserver.js';
 import { BucketStorageAdapter } from '../bucket/BucketStorageAdapter.js';
 import { AbstractRemote, FetchStrategy } from './AbstractRemote.js';
 import { StreamingSyncRequestParameterType } from './streaming-sync-types.js';
@@ -88,7 +88,8 @@ export interface StreamingSyncImplementationListener extends BaseListener {
  * Configurable options to be used when connecting to the PowerSync
  * backend instance.
  */
-export interface PowerSyncConnectionOptions extends BaseConnectionOptions, AdditionalConnectionOptions {
+export type PowerSyncConnectionOptions = Omit<InternalConnectionOptions, 'serializedSchema'>;
+export interface InternalConnectionOptions extends BaseConnectionOptions, AdditionalConnectionOptions {
 }
 /** @internal */
 export interface BaseConnectionOptions {
@@ -114,6 +115,10 @@ export interface BaseConnectionOptions {
      * These parameters are passed to the sync rules, and will be available under the`user_parameters` object.
      */
     params?: Record<string, StreamingSyncRequestParameterType>;
+    /**
+     * The serialized schema - mainly used to forward information about raw tables to the sync client.
+     */
+    serializedSchema?: any;
 }
 /** @internal */
 export interface AdditionalConnectionOptions {
@@ -131,11 +136,11 @@ export interface AdditionalConnectionOptions {
 }
 /** @internal */
 export type RequiredAdditionalConnectionOptions = Required<AdditionalConnectionOptions>;
-export interface StreamingSyncImplementation extends BaseObserver<StreamingSyncImplementationListener>, Disposable {
+export interface StreamingSyncImplementation extends BaseObserverInterface<StreamingSyncImplementationListener>, Disposable {
     /**
      * Connects to the sync service
      */
-    connect(options?: PowerSyncConnectionOptions): Promise<void>;
+    connect(options?: InternalConnectionOptions): Promise<void>;
     /**
      * Disconnects from the sync services.
      * @throws if not connected or if abort is not controlled internally
@@ -155,7 +160,6 @@ export declare const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
 export declare const DEFAULT_RETRY_DELAY_MS = 5000;
 export declare const DEFAULT_STREAMING_SYNC_OPTIONS: {
     retryDelayMs: number;
-    logger: Logger.ILogger;
     crudUploadThrottleMs: number;
 };
 export type RequiredPowerSyncConnectionOptions = Required<BaseConnectionOptions>;
@@ -164,9 +168,11 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
     protected _lastSyncedAt: Date | null;
     protected options: AbstractStreamingSyncImplementationOptions;
     protected abortController: AbortController | null;
+    protected uploadAbortController: AbortController | null;
     protected crudUpdateListener?: () => void;
     protected streamingSyncPromise?: Promise<void>;
-    private pendingCrudUpload?;
+    protected logger: ILogger;
+    private isUploadingCrud;
     private notifyCompletedUploads?;
     syncStatus: SyncStatus;
     triggerCrudUpload: () => void;
@@ -176,7 +182,6 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
     waitUntilStatusMatches(predicate: (status: SyncStatus) => boolean): Promise<void>;
     get lastSyncedAt(): Date | undefined;
     get isConnected(): boolean;
-    protected get logger(): Logger.ILogger;
     dispose(): Promise<void>;
     abstract obtainLock<T>(lockOptions: LockOptions<T>): Promise<T>;
     hasCompletedSync(): Promise<boolean>;