@powersync/common 1.33.0 → 1.33.1

package/lib/client/ConnectionManager.js

@@ -136,7 +136,6 @@ export class ConnectionManager extends BaseObserver {
         await this.disconnectingPromise;
         this.logger.debug('Attempting to connect to PowerSync instance');
         await this.syncStreamImplementation?.connect(appliedOptions);
-        this.syncStreamImplementation?.triggerCrudUpload();
     }
     /**
      * Close the sync connection.

package/lib/client/sync/bucket/BucketStorageAdapter.d.ts

@@ -82,14 +82,6 @@ export interface BucketStorageAdapter extends BaseObserver<BucketStorageListener
     getCrudBatch(limit?: number): Promise<CrudBatch | null>;
     hasCompletedSync(): Promise<boolean>;
     updateLocalTarget(cb: () => Promise<string>): Promise<boolean>;
-    /**
-     * Exposed for tests only.
-     */
-    autoCompact(): Promise<void>;
-    /**
-     * Exposed for tests only.
-     */
-    forceCompact(): Promise<void>;
     getMaxOpId(): string;
     /**
      * Get an unique client id.
@@ -98,5 +90,5 @@ export interface BucketStorageAdapter extends BaseObserver<BucketStorageListener
     /**
      * Invokes the `powersync_control` function for the sync client.
      */
-    control(op: PowerSyncControlCommand, payload: string | ArrayBuffer | null): Promise<string>;
+    control(op: PowerSyncControlCommand, payload: string | Uint8Array | null): Promise<string>;
 }

package/lib/client/sync/bucket/SqliteBucketStorage.d.ts

@@ -11,14 +11,9 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
     private mutex;
     private logger;
     tableNames: Set<string>;
-    private pendingBucketDeletes;
     private _hasCompletedSync;
     private updateListener;
     private _clientId?;
-    /**
-     * Count up, and do a compact on startup.
-     */
-    private compactCounter;
     constructor(db: DBAdapter, mutex: Mutex, logger?: ILogger);
     init(): Promise<void>;
     dispose(): Promise<void>;
@@ -46,13 +41,6 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
      */
     private updateObjectsFromBuckets;
     validateChecksums(checkpoint: Checkpoint, priority: number | undefined): Promise<SyncLocalDatabaseResult>;
-    /**
-     * Force a compact, for tests.
-     */
-    forceCompact(): Promise<void>;
-    autoCompact(): Promise<void>;
-    private deletePendingBuckets;
-    private clearRemoveOps;
     updateLocalTarget(cb: () => Promise<string>): Promise<boolean>;
     nextCrudItem(): Promise<CrudEntry | undefined>;
     hasCrud(): Promise<boolean>;
@@ -68,7 +56,7 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
      * Set a target checkpoint.
      */
     setTargetCheckpoint(checkpoint: Checkpoint): Promise<void>;
-    control(op: PowerSyncControlCommand, payload: string | ArrayBuffer | null): Promise<string>;
+    control(op: PowerSyncControlCommand, payload: string | Uint8Array | ArrayBuffer | null): Promise<string>;
     hasMigratedSubkeys(): Promise<boolean>;
     migrateToFixedSubkeys(): Promise<void>;
     static _subkeyMigrationKey: string;

package/lib/client/sync/bucket/SqliteBucketStorage.js

@@ -4,27 +4,20 @@ import { BaseObserver } from '../../../utils/BaseObserver.js';
 import { MAX_OP_ID } from '../../constants.js';
 import { PSInternalTable } from './BucketStorageAdapter.js';
 import { CrudEntry } from './CrudEntry.js';
-const COMPACT_OPERATION_INTERVAL = 1_000;
 export class SqliteBucketStorage extends BaseObserver {
     db;
     mutex;
     logger;
     tableNames;
-    pendingBucketDeletes;
     _hasCompletedSync;
     updateListener;
     _clientId;
-    /**
-     * Count up, and do a compact on startup.
-     */
-    compactCounter = COMPACT_OPERATION_INTERVAL;
     constructor(db, mutex, logger = Logger.get('SqliteBucketStorage')) {
         super();
         this.db = db;
         this.mutex = mutex;
         this.logger = logger;
         this._hasCompletedSync = false;
-        this.pendingBucketDeletes = true;
         this.tableNames = new Set();
         this.updateListener = db.registerListener({
             tablesUpdated: (update) => {
@@ -72,16 +65,13 @@ export class SqliteBucketStorage extends BaseObserver {
     }
     async saveSyncData(batch, fixedKeyFormat = false) {
         await this.writeTransaction(async (tx) => {
-            let count = 0;
             for (const b of batch.buckets) {
                 const result = await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
                     'save',
                     JSON.stringify({ buckets: [b.toJSON(fixedKeyFormat)] })
                 ]);
                 this.logger.debug('saveSyncData', JSON.stringify(result));
-                count += b.data.length;
             }
-            this.compactCounter += count;
         });
     }
     async removeBuckets(buckets) {
@@ -97,7 +87,6 @@ export class SqliteBucketStorage extends BaseObserver {
             await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', ['delete_bucket', bucket]);
         });
         this.logger.debug('done deleting bucket');
-        this.pendingBucketDeletes = true;
     }
     async hasCompletedSync() {
         if (this._hasCompletedSync) {
@@ -138,7 +127,6 @@ export class SqliteBucketStorage extends BaseObserver {
             this.logger.debug('Not at a consistent checkpoint - cannot update local db');
             return { ready: false, checkpointValid: true };
         }
-        await this.forceCompact();
         return {
             ready: true,
             checkpointValid: true
@@ -209,36 +197,6 @@ export class SqliteBucketStorage extends BaseObserver {
             };
         }
     }
-    /**
-     * Force a compact, for tests.
-     */
-    async forceCompact() {
-        this.compactCounter = COMPACT_OPERATION_INTERVAL;
-        this.pendingBucketDeletes = true;
-        await this.autoCompact();
-    }
-    async autoCompact() {
-        await this.deletePendingBuckets();
-        await this.clearRemoveOps();
-    }
-    async deletePendingBuckets() {
-        if (this.pendingBucketDeletes !== false) {
-            await this.writeTransaction(async (tx) => {
-                await tx.execute('INSERT INTO powersync_operations(op, data) VALUES (?, ?)', ['delete_pending_buckets', '']);
-            });
-            // Executed once after start-up, and again when there are pending deletes.
-            this.pendingBucketDeletes = false;
-        }
-    }
-    async clearRemoveOps() {
-        if (this.compactCounter < COMPACT_OPERATION_INTERVAL) {
-            return;
-        }
-        await this.writeTransaction(async (tx) => {
-            await tx.execute('INSERT INTO powersync_operations(op, data) VALUES (?, ?)', ['clear_remove_ops', '']);
-        });
-        this.compactCounter = 0;
-    }
     async updateLocalTarget(cb) {
         const rs1 = await this.db.getAll("SELECT target_op FROM ps_buckets WHERE name = '$local' AND target_op = CAST(? as INTEGER)", [MAX_OP_ID]);
         if (!rs1.length) {

package/lib/client/sync/stream/AbstractRemote.d.ts

@@ -1,5 +1,4 @@
 import type { BSON } from 'bson';
-import { Buffer } from 'buffer';
 import { type fetch } from 'cross-fetch';
 import Logger, { ILogger } from 'js-logger';
 import { DataStream } from '../../../utils/DataStream.js';
@@ -133,7 +132,7 @@ export declare abstract class AbstractRemote {
      * @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
      * (required for compatibility with older sync services).
      */
-    socketStreamRaw<T>(options: SocketSyncStreamOptions, map: (buffer: Buffer) => T, bson?: typeof BSON): Promise<DataStream>;
+    socketStreamRaw<T>(options: SocketSyncStreamOptions, map: (buffer: Uint8Array) => T, bson?: typeof BSON): Promise<DataStream<T>>;
     /**
      * Connects to the sync/stream http endpoint, parsing lines as JSON.
      */

package/lib/client/sync/stream/AbstractRemote.js

@@ -10,8 +10,12 @@ const POWERSYNC_JS_VERSION = PACKAGE.version;
 const SYNC_QUEUE_REQUEST_LOW_WATER = 5;
 // Keep alive message is sent every period
 const KEEP_ALIVE_MS = 20_000;
-// The ACK must be received in this period
-const KEEP_ALIVE_LIFETIME_MS = 30_000;
+// One message of any type must be received in this period.
+const SOCKET_TIMEOUT_MS = 30_000;
+// One keepalive message must be received in this period.
+// If there is a backlog of messages (for example on slow connections), keepalive messages could be delayed
+// significantly. Therefore this is longer than the socket timeout.
+const KEEP_ALIVE_LIFETIME_MS = 90_000;
 export const DEFAULT_REMOTE_LOGGER = Logger.get('PowerSyncRemote');
 export var FetchStrategy;
 (function (FetchStrategy) {
@@ -208,12 +212,25 @@ export class AbstractRemote {
         // headers with websockets on web. The browser userAgent is however added
         // automatically as a header.
         const userAgent = this.getUserAgent();
+        let keepAliveTimeout;
+        const resetTimeout = () => {
+            clearTimeout(keepAliveTimeout);
+            keepAliveTimeout = setTimeout(() => {
+                this.logger.error(`No data received on WebSocket in ${SOCKET_TIMEOUT_MS}ms, closing connection.`);
+                stream.close();
+            }, SOCKET_TIMEOUT_MS);
+        };
+        resetTimeout();
         const url = this.options.socketUrlTransformer(request.url);
         const connector = new RSocketConnector({
             transport: new WebsocketClientTransport({
                 url,
                 wsCreator: (url) => {
-                    return this.createSocket(url);
+                    const socket = this.createSocket(url);
+                    socket.addEventListener('message', (event) => {
+                        resetTimeout();
+                    });
+                    return socket;
                 }
             }),
             setup: {
@@ -236,16 +253,20 @@ export class AbstractRemote {
         }
         catch (ex) {
             this.logger.error(`Failed to connect WebSocket`, ex);
+            clearTimeout(keepAliveTimeout);
             throw ex;
         }
+        resetTimeout();
         const stream = new DataStream({
             logger: this.logger,
             pressure: {
                 lowWaterMark: SYNC_QUEUE_REQUEST_LOW_WATER
-            }
+            },
+            mapLine: map
         });
         let socketIsClosed = false;
         const closeSocket = () => {
+            clearTimeout(keepAliveTimeout);
             if (socketIsClosed) {
                 return;
             }
@@ -307,7 +328,7 @@ export class AbstractRemote {
                     if (!data) {
                         return;
                     }
-                    stream.enqueueData(map(data));
+                    stream.enqueueData(data);
                 },
                 onComplete: () => {
                     stream.close();
@@ -418,7 +439,8 @@ export class AbstractRemote {
         const decoder = new TextDecoder();
         let buffer = '';
         const stream = new DataStream({
-            logger: this.logger
+            logger: this.logger,
+            mapLine: mapLine
         });
         const l = stream.registerListener({
             lowWater: async () => {
@@ -429,7 +451,7 @@ export class AbstractRemote {
                         if (done) {
                             const remaining = buffer.trim();
                             if (remaining.length != 0) {
-                                stream.enqueueData(mapLine(remaining));
+                                stream.enqueueData(remaining);
                             }
                             stream.close();
                             await closeReader();
@@ -441,7 +463,7 @@ export class AbstractRemote {
                         for (var i = 0; i < lines.length - 1; i++) {
                             var l = lines[i].trim();
                             if (l.length > 0) {
-                                stream.enqueueData(mapLine(l));
+                                stream.enqueueData(l);
                                 didCompleteLine = true;
                             }
                         }

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js

@@ -182,17 +182,17 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
                  */
                 let checkedCrudItem;
                 while (true) {
-                    this.updateSyncStatus({
-                        dataFlow: {
-                            uploading: true
-                        }
-                    });
                     try {
                         /**
                          * This is the first item in the FIFO CRUD queue.
                          */
                         const nextCrudItem = await this.options.adapter.nextCrudItem();
                         if (nextCrudItem) {
+                            this.updateSyncStatus({
+                                dataFlow: {
+                                    uploading: true
+                                }
+                            });
                             if (nextCrudItem.clientId == checkedCrudItem?.clientId) {
                                 // This will force a higher log level than exceptions which are caught here.
                                 this.logger.warn(`Potentially previously uploaded CRUD entries are still present in the upload queue.
@@ -247,23 +247,17 @@ The next upload iteration will be delayed.`);
         const controller = new AbortController();
         this.abortController = controller;
         this.streamingSyncPromise = this.streamingSync(this.abortController.signal, options);
-        // Return a promise that resolves when the connection status is updated
+        // Return a promise that resolves when the connection status is updated to indicate that we're connected.
         return new Promise((resolve) => {
             const disposer = this.registerListener({
-                statusUpdated: (update) => {
-                    // This is triggered as soon as a connection is read from
-                    if (typeof update.connected == 'undefined') {
-                        // only concern with connection updates
-                        return;
-                    }
-                    if (update.connected == false) {
-                        /**
-                         * This function does not reject if initial connect attempt failed.
-                         * Connected can be false if the connection attempt was aborted or if the initial connection
-                         * attempt failed.
-                         */
+                statusChanged: (status) => {
+                    if (status.dataFlowStatus.downloadError != null) {
                         this.logger.warn('Initial connect attempt did not successfully connect to server');
                     }
+                    else if (status.connecting) {
+                        // Still connecting.
+                        return;
+                    }
                     disposer();
                     resolve();
                 }
@@ -655,6 +649,7 @@ The next upload iteration will be delayed.`);
         const adapter = this.options.adapter;
         const remote = this.options.remote;
         let receivingLines = null;
+        let hadSyncLine = false;
         const abortController = new AbortController();
         signal.addEventListener('abort', () => abortController.abort());
         // Pending sync lines received from the service, as well as local events that trigger a powersync_control
@@ -668,20 +663,39 @@ The next upload iteration will be delayed.`);
                 data: instr.request
             };
             if (resolvedOptions.connectionMethod == SyncStreamConnectionMethod.HTTP) {
-                controlInvocations = await remote.postStreamRaw(syncOptions, (line) => ({
-                    command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
-                    payload: line
-                }));
+                controlInvocations = await remote.postStreamRaw(syncOptions, (line) => {
+                    if (typeof line == 'string') {
+                        return {
+                            command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
+                            payload: line
+                        };
+                    }
+                    else {
+                        // Directly enqueued by us
+                        return line;
+                    }
+                });
             }
             else {
                 controlInvocations = await remote.socketStreamRaw({
                     ...syncOptions,
                     fetchStrategy: resolvedOptions.fetchStrategy
-                }, (buffer) => ({
-                    command: PowerSyncControlCommand.PROCESS_BSON_LINE,
-                    payload: buffer
-                }));
+                }, (payload) => {
+                    if (payload instanceof Uint8Array) {
+                        return {
+                            command: PowerSyncControlCommand.PROCESS_BSON_LINE,
+                            payload: payload
+                        };
+                    }
+                    else {
+                        // Directly enqueued by us
+                        return payload;
+                    }
+                });
             }
+            // The rust client will set connected: true after the first sync line because that's when it gets invoked, but
+            // we're already connected here and can report that.
+            syncImplementation.updateSyncStatus({ connected: true });
             try {
                 while (!controlInvocations.closed) {
                     const line = await controlInvocations.read();
@@ -689,6 +703,10 @@ The next upload iteration will be delayed.`);
                         return;
                     }
                     await control(line.command, line.payload);
+                    if (!hadSyncLine) {
+                        syncImplementation.triggerCrudUpload();
+                        hadSyncLine = true;
+                    }
                 }
             }
             finally {
@@ -726,7 +744,7 @@ The next upload iteration will be delayed.`);
                     return {
                         priority: status.priority,
                         hasSynced: status.has_synced ?? undefined,
-                        lastSyncedAt: status?.last_synced_at != null ? new Date(status.last_synced_at) : undefined
+                        lastSyncedAt: status?.last_synced_at != null ? new Date(status.last_synced_at * 1000) : undefined
                     };
                 }
                 const info = instruction.UpdateSyncStatus.status;

package/lib/utils/DataStream.d.ts

@@ -1,6 +1,7 @@
 import { ILogger } from 'js-logger';
 import { BaseListener, BaseObserver } from './BaseObserver.js';
-export type DataStreamOptions = {
+export type DataStreamOptions<ParsedData, SourceData> = {
+    mapLine?: (line: SourceData) => ParsedData;
     /**
      * Close the stream if any consumer throws an error
      */
@@ -28,13 +29,15 @@ export declare const DEFAULT_PRESSURE_LIMITS: {
  * native JS streams or async iterators.
  * This is handy for environments such as React Native which need polyfills for the above.
  */
-export declare class DataStream<Data extends any = any> extends BaseObserver<DataStreamListener<Data>> {
-    protected options?: DataStreamOptions | undefined;
-    dataQueue: Data[];
+export declare class DataStream<ParsedData, SourceData = any> extends BaseObserver<DataStreamListener<ParsedData>> {
+    protected options?: DataStreamOptions<ParsedData, SourceData> | undefined;
+    dataQueue: SourceData[];
     protected isClosed: boolean;
     protected processingPromise: Promise<void> | null;
+    protected notifyDataAdded: (() => void) | null;
     protected logger: ILogger;
-    constructor(options?: DataStreamOptions | undefined);
+    protected mapLine: (line: SourceData) => ParsedData;
+    constructor(options?: DataStreamOptions<ParsedData, SourceData> | undefined);
     get highWatermark(): number;
     get lowWatermark(): number;
     get closed(): boolean;
@@ -42,22 +45,18 @@ export declare class DataStream<Data extends any = any> extends BaseObserver<Dat
     /**
      * Enqueues data for the consumers to read
      */
-    enqueueData(data: Data): void;
+    enqueueData(data: SourceData): void;
     /**
      * Reads data once from the data stream
      * @returns a Data payload or Null if the stream closed.
      */
-    read(): Promise<Data | null>;
+    read(): Promise<ParsedData | null>;
     /**
      * Executes a callback for each data item in the stream
      */
-    forEach(callback: DataStreamCallback<Data>): () => void;
-    protected processQueue(): Promise<void>;
-    /**
-     * Creates a new data stream which is a map of the original
-     */
-    map<ReturnData>(callback: (data: Data) => ReturnData): DataStream<ReturnData>;
+    forEach(callback: DataStreamCallback<ParsedData>): () => void;
+    protected processQueue(): Promise<void> | undefined;
     protected hasDataReader(): boolean;
     protected _processQueue(): Promise<void>;
-    protected iterateAsyncErrored(cb: (l: BaseListener) => Promise<void>): Promise<void>;
+    protected iterateAsyncErrored(cb: (l: Partial<DataStreamListener<ParsedData>>) => Promise<void>): Promise<void>;
 }

package/lib/utils/DataStream.js

@@ -14,13 +14,16 @@ export class DataStream extends BaseObserver {
     dataQueue;
     isClosed;
     processingPromise;
+    notifyDataAdded;
     logger;
+    mapLine;
     constructor(options) {
         super();
         this.options = options;
         this.processingPromise = null;
         this.isClosed = false;
         this.dataQueue = [];
+        this.mapLine = options?.mapLine ?? ((line) => line);
         this.logger = options?.logger ?? Logger.get('DataStream');
         if (options?.closeOnError) {
             const l = this.registerListener({
@@ -56,6 +59,7 @@ export class DataStream extends BaseObserver {
             throw new Error('Cannot enqueue data into closed stream.');
         }
         this.dataQueue.push(data);
+        this.notifyDataAdded?.();
         this.processQueue();
     }
     /**
@@ -96,10 +100,20 @@ export class DataStream extends BaseObserver {
             data: callback
         });
     }
-    async processQueue() {
+    processQueue() {
         if (this.processingPromise) {
             return;
         }
+        const promise = (this.processingPromise = this._processQueue());
+        promise.finally(() => {
+            return (this.processingPromise = null);
+        });
+        return promise;
+    }
+    hasDataReader() {
+        return Array.from(this.listeners.values()).some((l) => !!l.data);
+    }
+    async _processQueue() {
         /**
          * Allow listeners to mutate the queue before processing.
          * This allows for operations such as dropping or compressing data
@@ -108,47 +122,31 @@ export class DataStream extends BaseObserver {
         if (this.dataQueue.length >= this.highWatermark) {
             await this.iterateAsyncErrored(async (l) => l.highWater?.());
         }
-        return (this.processingPromise = this._processQueue());
-    }
-    /**
-     * Creates a new data stream which is a map of the original
-     */
-    map(callback) {
-        const stream = new DataStream(this.options);
-        const l = this.registerListener({
-            data: async (data) => {
-                stream.enqueueData(callback(data));
-            },
-            closed: () => {
-                stream.close();
-                l?.();
-            }
-        });
-        return stream;
-    }
-    hasDataReader() {
-        return Array.from(this.listeners.values()).some((l) => !!l.data);
-    }
-    async _processQueue() {
         if (this.isClosed || !this.hasDataReader()) {
-            Promise.resolve().then(() => (this.processingPromise = null));
             return;
         }
         if (this.dataQueue.length) {
             const data = this.dataQueue.shift();
-            await this.iterateAsyncErrored(async (l) => l.data?.(data));
+            const mapped = this.mapLine(data);
+            await this.iterateAsyncErrored(async (l) => l.data?.(mapped));
         }
         if (this.dataQueue.length <= this.lowWatermark) {
-            await this.iterateAsyncErrored(async (l) => l.lowWater?.());
+            const dataAdded = new Promise((resolve) => {
+                this.notifyDataAdded = resolve;
+            });
+            await Promise.race([this.iterateAsyncErrored(async (l) => l.lowWater?.()), dataAdded]);
+            this.notifyDataAdded = null;
         }
-        this.processingPromise = null;
-        if (this.dataQueue.length) {
+        if (this.dataQueue.length > 0) {
             // Next tick
             setTimeout(() => this.processQueue());
         }
     }
     async iterateAsyncErrored(cb) {
-        for (let i of Array.from(this.listeners.values())) {
+        // Important: We need to copy the listeners, as calling a listener could result in adding another
+        // listener, resulting in infinite loops.
+        const listeners = Array.from(this.listeners.values());
+        for (let i of listeners) {
             try {
                 await cb(i);
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@powersync/common",
-  "version": "1.33.0",
+  "version": "1.33.1",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -13,8 +13,9 @@
   "exports": {
     ".": {
       "import": "./dist/bundle.mjs",
-      "default": "./dist/bundle.mjs",
-      "types": "./lib/index.d.ts"
+      "require": "./dist/bundle.cjs",
+      "types": "./lib/index.d.ts",
+      "default": "./dist/bundle.mjs"
     }
   },
   "author": "JOURNEYAPPS",