@powersync/common 0.0.0-dev-20250625140957 → 0.0.0-dev-20250710151329

package/lib/client/AbstractPowerSyncDatabase.d.ts

@@ -1,5 +1,5 @@
 import { Mutex } from 'async-mutex';
-import Logger, { ILogger } from 'js-logger';
+import { ILogger } from 'js-logger';
 import { DBAdapter, QueryResult, Transaction } from '../db/DBAdapter.js';
 import { SyncStatus } from '../db/crud/SyncStatus.js';
 import { UploadQueueStats } from '../db/crud/UploadQueueStatus.js';
@@ -84,7 +84,6 @@ export declare const DEFAULT_POWERSYNC_CLOSE_OPTIONS: PowerSyncCloseOptions;
 export declare const DEFAULT_WATCH_THROTTLE_MS = 30;
 export declare const DEFAULT_POWERSYNC_DB_OPTIONS: {
     retryDelayMs: number;
-    logger: Logger.ILogger;
     crudUploadThrottleMs: number;
 };
 export declare const DEFAULT_CRUD_BATCH_LIMIT = 100;
@@ -123,6 +122,7 @@ export declare abstract class AbstractPowerSyncDatabase extends BaseObserver<Pow
     protected _schema: Schema;
     private _database;
     protected runExclusiveMutex: Mutex;
+    logger: ILogger;
     constructor(options: PowerSyncDatabaseOptionsWithDBAdapter);
     constructor(options: PowerSyncDatabaseOptionsWithOpenFactory);
     constructor(options: PowerSyncDatabaseOptionsWithSettings);
@@ -185,7 +185,6 @@ export declare abstract class AbstractPowerSyncDatabase extends BaseObserver<Pow
      * Cannot be used while connected - this should only be called before {@link AbstractPowerSyncDatabase.connect}.
      */
     updateSchema(schema: Schema): Promise<void>;
-    get logger(): Logger.ILogger;
     /**
      * Wait for initialization to complete.
      * While initializing is automatic, this helps to catch and report initialization errors.

package/lib/client/AbstractPowerSyncDatabase.js

@@ -27,7 +27,6 @@ export const DEFAULT_POWERSYNC_CLOSE_OPTIONS = {
 export const DEFAULT_WATCH_THROTTLE_MS = 30;
 export const DEFAULT_POWERSYNC_DB_OPTIONS = {
     retryDelayMs: 5000,
-    logger: Logger.get('PowerSyncDatabase'),
     crudUploadThrottleMs: DEFAULT_CRUD_UPLOAD_THROTTLE_MS
 };
 export const DEFAULT_CRUD_BATCH_LIMIT = 100;
@@ -70,6 +69,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
     _schema;
     _database;
     runExclusiveMutex;
+    logger;
     constructor(options) {
         super();
         this.options = options;
@@ -89,6 +89,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         else {
             throw new Error('The provided `database` option is invalid.');
         }
+        this.logger = options.logger ?? Logger.get(`PowerSyncDatabase[${this._database.name}]`);
         this.bucketStorageAdapter = this.generateBucketStorageAdapter();
         this.closed = false;
         this.currentStatus = new SyncStatus({});
@@ -268,16 +269,13 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
             schema.validate();
         }
         catch (ex) {
-            this.options.logger?.warn('Schema validation failed. Unexpected behaviour could occur', ex);
+            this.logger.warn('Schema validation failed. Unexpected behaviour could occur', ex);
         }
         this._schema = schema;
         await this.database.execute('SELECT powersync_replace_schema(?)', [JSON.stringify(this.schema.toJSON())]);
         await this.database.refreshSchema();
         this.iterateListeners(async (cb) => cb.schemaChanged?.(schema));
     }
-    get logger() {
-        return this.options.logger;
-    }
     /**
      * Wait for initialization to complete.
      * While initializing is automatic, this helps to catch and report initialization errors.
@@ -617,7 +615,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
      * @param options Options for configuring watch behavior
      */
     watchWithCallback(sql, parameters, handler, options) {
-        const { onResult, onError = (e) => this.options.logger?.error(e) } = handler ?? {};
+        const { onResult, onError = (e) => this.logger.error(e) } = handler ?? {};
         if (!onResult) {
             throw new Error('onResult is required');
         }
@@ -723,7 +721,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
      * @returns A dispose function to stop watching for changes
      */
     onChangeWithCallback(handler, options) {
-        const { onChange, onError = (e) => this.options.logger?.error(e) } = handler ?? {};
+        const { onChange, onError = (e) => this.logger.error(e) } = handler ?? {};
         if (!onChange) {
             throw new Error('onChange is required');
         }

package/lib/client/ConnectionManager.js

@@ -136,7 +136,6 @@ export class ConnectionManager extends BaseObserver {
         await this.disconnectingPromise;
         this.logger.debug('Attempting to connect to PowerSync instance');
         await this.syncStreamImplementation?.connect(appliedOptions);
-        this.syncStreamImplementation?.triggerCrudUpload();
     }
     /**
      * Close the sync connection.

package/lib/client/sync/bucket/BucketStorageAdapter.d.ts

@@ -90,5 +90,5 @@ export interface BucketStorageAdapter extends BaseObserver<BucketStorageListener
     /**
      * Invokes the `powersync_control` function for the sync client.
      */
-    control(op: PowerSyncControlCommand, payload: string | ArrayBuffer | null): Promise<string>;
+    control(op: PowerSyncControlCommand, payload: string | Uint8Array | null): Promise<string>;
 }

package/lib/client/sync/bucket/SqliteBucketStorage.d.ts

@@ -56,7 +56,7 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
      * Set a target checkpoint.
      */
     setTargetCheckpoint(checkpoint: Checkpoint): Promise<void>;
-    control(op: PowerSyncControlCommand, payload: string | ArrayBuffer | null): Promise<string>;
+    control(op: PowerSyncControlCommand, payload: string | Uint8Array | ArrayBuffer | null): Promise<string>;
     hasMigratedSubkeys(): Promise<boolean>;
     migrateToFixedSubkeys(): Promise<void>;
     static _subkeyMigrationKey: string;

package/lib/client/sync/stream/AbstractRemote.d.ts

@@ -1,5 +1,4 @@
 import type { BSON } from 'bson';
-import { Buffer } from 'buffer';
 import { type fetch } from 'cross-fetch';
 import Logger, { ILogger } from 'js-logger';
 import { DataStream } from '../../../utils/DataStream.js';
@@ -133,7 +132,7 @@ export declare abstract class AbstractRemote {
      * @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
      * (required for compatibility with older sync services).
      */
-    socketStreamRaw<T>(options: SocketSyncStreamOptions, map: (buffer: Buffer) => T, bson?: typeof BSON): Promise<DataStream>;
+    socketStreamRaw<T>(options: SocketSyncStreamOptions, map: (buffer: Uint8Array) => T, bson?: typeof BSON): Promise<DataStream<T>>;
     /**
      * Connects to the sync/stream http endpoint, parsing lines as JSON.
      */

package/lib/client/sync/stream/AbstractRemote.js

@@ -10,8 +10,12 @@ const POWERSYNC_JS_VERSION = PACKAGE.version;
 const SYNC_QUEUE_REQUEST_LOW_WATER = 5;
 // Keep alive message is sent every period
 const KEEP_ALIVE_MS = 20_000;
-// The ACK must be received in this period
-const KEEP_ALIVE_LIFETIME_MS = 30_000;
+// One message of any type must be received in this period.
+const SOCKET_TIMEOUT_MS = 30_000;
+// One keepalive message must be received in this period.
+// If there is a backlog of messages (for example on slow connections), keepalive messages could be delayed
+// significantly. Therefore this is longer than the socket timeout.
+const KEEP_ALIVE_LIFETIME_MS = 90_000;
 export const DEFAULT_REMOTE_LOGGER = Logger.get('PowerSyncRemote');
 export var FetchStrategy;
 (function (FetchStrategy) {
@@ -208,12 +212,25 @@ export class AbstractRemote {
         // headers with websockets on web. The browser userAgent is however added
         // automatically as a header.
         const userAgent = this.getUserAgent();
+        let keepAliveTimeout;
+        const resetTimeout = () => {
+            clearTimeout(keepAliveTimeout);
+            keepAliveTimeout = setTimeout(() => {
+                this.logger.error(`No data received on WebSocket in ${SOCKET_TIMEOUT_MS}ms, closing connection.`);
+                stream.close();
+            }, SOCKET_TIMEOUT_MS);
+        };
+        resetTimeout();
         const url = this.options.socketUrlTransformer(request.url);
         const connector = new RSocketConnector({
             transport: new WebsocketClientTransport({
                 url,
                 wsCreator: (url) => {
-                    return this.createSocket(url);
+                    const socket = this.createSocket(url);
+                    socket.addEventListener('message', (event) => {
+                        resetTimeout();
+                    });
+                    return socket;
                 }
             }),
             setup: {
@@ -236,16 +253,20 @@ export class AbstractRemote {
         }
         catch (ex) {
             this.logger.error(`Failed to connect WebSocket`, ex);
+            clearTimeout(keepAliveTimeout);
             throw ex;
         }
+        resetTimeout();
         const stream = new DataStream({
             logger: this.logger,
             pressure: {
                 lowWaterMark: SYNC_QUEUE_REQUEST_LOW_WATER
-            }
+            },
+            mapLine: map
         });
         let socketIsClosed = false;
         const closeSocket = () => {
+            clearTimeout(keepAliveTimeout);
             if (socketIsClosed) {
                 return;
             }
@@ -307,7 +328,7 @@ export class AbstractRemote {
                     if (!data) {
                         return;
                     }
-                    stream.enqueueData(map(data));
+                    stream.enqueueData(data);
                 },
                 onComplete: () => {
                     stream.close();
@@ -418,7 +439,8 @@ export class AbstractRemote {
         const decoder = new TextDecoder();
         let buffer = '';
         const stream = new DataStream({
-            logger: this.logger
+            logger: this.logger,
+            mapLine: mapLine
         });
         const l = stream.registerListener({
             lowWater: async () => {
@@ -429,7 +451,7 @@ export class AbstractRemote {
                         if (done) {
                             const remaining = buffer.trim();
                             if (remaining.length != 0) {
-                                stream.enqueueData(mapLine(remaining));
+                                stream.enqueueData(remaining);
                             }
                             stream.close();
                             await closeReader();
@@ -441,7 +463,7 @@ export class AbstractRemote {
                         for (var i = 0; i < lines.length - 1; i++) {
                             var l = lines[i].trim();
                             if (l.length > 0) {
-                                stream.enqueueData(mapLine(l));
+                                stream.enqueueData(l);
                                 didCompleteLine = true;
                             }
                         }

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js

@@ -169,9 +169,12 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
     }
     async getWriteCheckpoint() {
         const clientId = await this.options.adapter.getClientId();
+        this.logger.debug(`Creating write checkpoint for ${clientId}`);
         let path = `/write-checkpoint2.json?client_id=${clientId}`;
         const response = await this.options.remote.get(path);
-        return response['data']['write_checkpoint'];
+        const checkpoint = response['data']['write_checkpoint'];
+        this.logger.debug(`Got write checkpoint: ${checkpoint}`);
+        return checkpoint;
     }
     async _uploadAllCrud() {
         return this.obtainLock({
@@ -182,17 +185,17 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
                  */
                 let checkedCrudItem;
                 while (true) {
-                    this.updateSyncStatus({
-                        dataFlow: {
-                            uploading: true
-                        }
-                    });
                     try {
                         /**
                          * This is the first item in the FIFO CRUD queue.
                          */
                         const nextCrudItem = await this.options.adapter.nextCrudItem();
                         if (nextCrudItem) {
+                            this.updateSyncStatus({
+                                dataFlow: {
+                                    uploading: true
+                                }
+                            });
                             if (nextCrudItem.clientId == checkedCrudItem?.clientId) {
                                 // This will force a higher log level than exceptions which are caught here.
                                 this.logger.warn(`Potentially previously uploaded CRUD entries are still present in the upload queue.
@@ -210,6 +213,7 @@ The next upload iteration will be delayed.`);
                         }
                         else {
                             // Uploading is completed
+                            this.logger.debug('Upload complete, updating write checkpoint');
                             await this.options.adapter.updateLocalTarget(() => this.getWriteCheckpoint());
                             break;
                         }
@@ -247,23 +251,17 @@ The next upload iteration will be delayed.`);
         const controller = new AbortController();
         this.abortController = controller;
         this.streamingSyncPromise = this.streamingSync(this.abortController.signal, options);
-        // Return a promise that resolves when the connection status is updated
+        // Return a promise that resolves when the connection status is updated to indicate that we're connected.
         return new Promise((resolve) => {
             const disposer = this.registerListener({
-                statusUpdated: (update) => {
-                    // This is triggered as soon as a connection is read from
-                    if (typeof update.connected == 'undefined') {
-                        // only concern with connection updates
-                        return;
-                    }
-                    if (update.connected == false) {
-                        /**
-                         * This function does not reject if initial connect attempt failed.
-                         * Connected can be false if the connection attempt was aborted or if the initial connection
-                         * attempt failed.
-                         */
+                statusChanged: (status) => {
+                    if (status.dataFlowStatus.downloadError != null) {
                         this.logger.warn('Initial connect attempt did not successfully connect to server');
                     }
+                    else if (status.connecting) {
+                        // Still connecting.
+                        return;
+                    }
                     disposer();
                     resolve();
                 }
@@ -655,6 +653,7 @@ The next upload iteration will be delayed.`);
         const adapter = this.options.adapter;
         const remote = this.options.remote;
         let receivingLines = null;
+        let hadSyncLine = false;
         const abortController = new AbortController();
         signal.addEventListener('abort', () => abortController.abort());
         // Pending sync lines received from the service, as well as local events that trigger a powersync_control
@@ -668,20 +667,39 @@ The next upload iteration will be delayed.`);
                 data: instr.request
             };
             if (resolvedOptions.connectionMethod == SyncStreamConnectionMethod.HTTP) {
-                controlInvocations = await remote.postStreamRaw(syncOptions, (line) => ({
-                    command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
-                    payload: line
-                }));
+                controlInvocations = await remote.postStreamRaw(syncOptions, (line) => {
+                    if (typeof line == 'string') {
+                        return {
+                            command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
+                            payload: line
+                        };
+                    }
+                    else {
+                        // Directly enqueued by us
+                        return line;
+                    }
+                });
             }
             else {
                 controlInvocations = await remote.socketStreamRaw({
                     ...syncOptions,
                     fetchStrategy: resolvedOptions.fetchStrategy
-                }, (buffer) => ({
-                    command: PowerSyncControlCommand.PROCESS_BSON_LINE,
-                    payload: buffer
-                }));
+                }, (payload) => {
+                    if (payload instanceof Uint8Array) {
+                        return {
+                            command: PowerSyncControlCommand.PROCESS_BSON_LINE,
+                            payload: payload
+                        };
+                    }
+                    else {
+                        // Directly enqueued by us
+                        return payload;
+                    }
+                });
             }
+            // The rust client will set connected: true after the first sync line because that's when it gets invoked, but
+            // we're already connected here and can report that.
+            syncImplementation.updateSyncStatus({ connected: true });
             try {
                 while (!controlInvocations.closed) {
                     const line = await controlInvocations.read();
@@ -689,6 +707,10 @@ The next upload iteration will be delayed.`);
                         return;
                     }
                     await control(line.command, line.payload);
+                    if (!hadSyncLine) {
+                        syncImplementation.triggerCrudUpload();
+                        hadSyncLine = true;
+                    }
                 }
             }
             finally {
@@ -726,7 +748,7 @@ The next upload iteration will be delayed.`);
                     return {
                         priority: status.priority,
                         hasSynced: status.has_synced ?? undefined,
-                        lastSyncedAt: status?.last_synced_at != null ? new Date(status.last_synced_at) : undefined
+                        lastSyncedAt: status?.last_synced_at != null ? new Date(status.last_synced_at * 1000) : undefined
                     };
                 }
                 const info = instruction.UpdateSyncStatus.status;

package/lib/db/crud/SyncProgress.d.ts

@@ -1,4 +1,4 @@
-import { BucketProgress } from 'src/client/sync/stream/core-instruction.js';
+import type { BucketProgress } from '../../client/sync/stream/core-instruction.js';
 /** @internal */
 export type InternalProgressInformation = Record<string, BucketProgress>;
 /**

package/lib/utils/DataStream.d.ts

@@ -1,6 +1,7 @@
 import { ILogger } from 'js-logger';
 import { BaseListener, BaseObserver } from './BaseObserver.js';
-export type DataStreamOptions = {
+export type DataStreamOptions<ParsedData, SourceData> = {
+    mapLine?: (line: SourceData) => ParsedData;
     /**
      * Close the stream if any consumer throws an error
      */
@@ -28,13 +29,15 @@ export declare const DEFAULT_PRESSURE_LIMITS: {
  * native JS streams or async iterators.
  * This is handy for environments such as React Native which need polyfills for the above.
  */
-export declare class DataStream<Data extends any = any> extends BaseObserver<DataStreamListener<Data>> {
-    protected options?: DataStreamOptions | undefined;
-    dataQueue: Data[];
+export declare class DataStream<ParsedData, SourceData = any> extends BaseObserver<DataStreamListener<ParsedData>> {
+    protected options?: DataStreamOptions<ParsedData, SourceData> | undefined;
+    dataQueue: SourceData[];
     protected isClosed: boolean;
     protected processingPromise: Promise<void> | null;
+    protected notifyDataAdded: (() => void) | null;
     protected logger: ILogger;
-    constructor(options?: DataStreamOptions | undefined);
+    protected mapLine: (line: SourceData) => ParsedData;
+    constructor(options?: DataStreamOptions<ParsedData, SourceData> | undefined);
     get highWatermark(): number;
     get lowWatermark(): number;
     get closed(): boolean;
@@ -42,22 +45,18 @@ export declare class DataStream<Data extends any = any> extends BaseObserver<Dat
     /**
      * Enqueues data for the consumers to read
      */
-    enqueueData(data: Data): void;
+    enqueueData(data: SourceData): void;
     /**
      * Reads data once from the data stream
      * @returns a Data payload or Null if the stream closed.
      */
-    read(): Promise<Data | null>;
+    read(): Promise<ParsedData | null>;
     /**
      * Executes a callback for each data item in the stream
      */
-    forEach(callback: DataStreamCallback<Data>): () => void;
-    protected processQueue(): Promise<void>;
-    /**
-     * Creates a new data stream which is a map of the original
-     */
-    map<ReturnData>(callback: (data: Data) => ReturnData): DataStream<ReturnData>;
+    forEach(callback: DataStreamCallback<ParsedData>): () => void;
+    protected processQueue(): Promise<void> | undefined;
     protected hasDataReader(): boolean;
     protected _processQueue(): Promise<void>;
-    protected iterateAsyncErrored(cb: (l: BaseListener) => Promise<void>): Promise<void>;
+    protected iterateAsyncErrored(cb: (l: Partial<DataStreamListener<ParsedData>>) => Promise<void>): Promise<void>;
 }

package/lib/utils/DataStream.js

@@ -14,13 +14,16 @@ export class DataStream extends BaseObserver {
     dataQueue;
     isClosed;
     processingPromise;
+    notifyDataAdded;
     logger;
+    mapLine;
     constructor(options) {
         super();
         this.options = options;
         this.processingPromise = null;
         this.isClosed = false;
         this.dataQueue = [];
+        this.mapLine = options?.mapLine ?? ((line) => line);
         this.logger = options?.logger ?? Logger.get('DataStream');
         if (options?.closeOnError) {
             const l = this.registerListener({
@@ -56,6 +59,7 @@ export class DataStream extends BaseObserver {
             throw new Error('Cannot enqueue data into closed stream.');
         }
         this.dataQueue.push(data);
+        this.notifyDataAdded?.();
         this.processQueue();
     }
     /**
@@ -96,10 +100,20 @@ export class DataStream extends BaseObserver {
             data: callback
         });
     }
-    async processQueue() {
+    processQueue() {
         if (this.processingPromise) {
             return;
         }
+        const promise = (this.processingPromise = this._processQueue());
+        promise.finally(() => {
+            return (this.processingPromise = null);
+        });
+        return promise;
+    }
+    hasDataReader() {
+        return Array.from(this.listeners.values()).some((l) => !!l.data);
+    }
+    async _processQueue() {
         /**
          * Allow listeners to mutate the queue before processing.
          * This allows for operations such as dropping or compressing data
@@ -108,47 +122,31 @@ export class DataStream extends BaseObserver {
         if (this.dataQueue.length >= this.highWatermark) {
             await this.iterateAsyncErrored(async (l) => l.highWater?.());
         }
-        return (this.processingPromise = this._processQueue());
-    }
-    /**
-     * Creates a new data stream which is a map of the original
-     */
-    map(callback) {
-        const stream = new DataStream(this.options);
-        const l = this.registerListener({
-            data: async (data) => {
-                stream.enqueueData(callback(data));
-            },
-            closed: () => {
-                stream.close();
-                l?.();
-            }
-        });
-        return stream;
-    }
-    hasDataReader() {
-        return Array.from(this.listeners.values()).some((l) => !!l.data);
-    }
-    async _processQueue() {
         if (this.isClosed || !this.hasDataReader()) {
-            Promise.resolve().then(() => (this.processingPromise = null));
             return;
         }
         if (this.dataQueue.length) {
             const data = this.dataQueue.shift();
-            await this.iterateAsyncErrored(async (l) => l.data?.(data));
+            const mapped = this.mapLine(data);
+            await this.iterateAsyncErrored(async (l) => l.data?.(mapped));
         }
         if (this.dataQueue.length <= this.lowWatermark) {
-            await this.iterateAsyncErrored(async (l) => l.lowWater?.());
+            const dataAdded = new Promise((resolve) => {
+                this.notifyDataAdded = resolve;
+            });
+            await Promise.race([this.iterateAsyncErrored(async (l) => l.lowWater?.()), dataAdded]);
+            this.notifyDataAdded = null;
         }
-        this.processingPromise = null;
-        if (this.dataQueue.length) {
+        if (this.dataQueue.length > 0) {
             // Next tick
             setTimeout(() => this.processQueue());
         }
     }
     async iterateAsyncErrored(cb) {
-        for (let i of Array.from(this.listeners.values())) {
+        // Important: We need to copy the listeners, as calling a listener could result in adding another
+        // listener, resulting in infinite loops.
+        const listeners = Array.from(this.listeners.values());
+        for (let i of listeners) {
             try {
                 await cb(i);
             }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@powersync/common",
-  "version": "0.0.0-dev-20250625140957",
+  "version": "0.0.0-dev-20250710151329",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"