@powersync/common 0.0.0-dev-20250207081035 → 0.0.0-dev-20250319141441

package/lib/client/AbstractPowerSyncDatabase.d.ts

@@ -10,7 +10,7 @@ import { PowerSyncBackendConnector } from './connection/PowerSyncBackendConnecto
 import { BucketStorageAdapter } from './sync/bucket/BucketStorageAdapter.js';
 import { CrudBatch } from './sync/bucket/CrudBatch.js';
 import { CrudTransaction } from './sync/bucket/CrudTransaction.js';
-import { type AdditionalConnectionOptions, type PowerSyncConnectionOptions, StreamingSyncImplementation, StreamingSyncImplementationListener, type RequiredAdditionalConnectionOptions } from './sync/stream/AbstractStreamingSyncImplementation.js';
+import { StreamingSyncImplementation, StreamingSyncImplementationListener, type AdditionalConnectionOptions, type PowerSyncConnectionOptions, type RequiredAdditionalConnectionOptions } from './sync/stream/AbstractStreamingSyncImplementation.js';
 export interface DisconnectAndClearOptions {
     /** When set to false, data in local-only tables is preserved. */
     clearLocal?: boolean;
@@ -153,9 +153,18 @@ export declare abstract class AbstractPowerSyncDatabase extends BaseObserver<Pow
      */
     waitForReady(): Promise<void>;
     /**
+     * Wait for the first sync operation to complete.
+     *
+     * @argument request Either an abort signal (after which the promise will complete regardless of
+     * whether a full sync was completed) or an object providing an abort signal and a priority target.
+     * When a priority target is set, the promise may complete when all buckets with the given (or higher)
+     * priorities have been synchronized. This can be earlier than a complete sync.
      * @returns A promise which will resolve once the first full sync has completed.
      */
-    waitForFirstSync(signal?: AbortSignal): Promise<void>;
+    waitForFirstSync(request?: AbortSignal | {
+        signal?: AbortSignal;
+        priority?: number;
+    }): Promise<void>;
     /**
      * Allows for extended implementations to execute custom initialization
      * logic as part of the total init process
@@ -256,6 +265,7 @@ export declare abstract class AbstractPowerSyncDatabase extends BaseObserver<Pow
      * and optionally return results.
      */
     execute(sql: string, parameters?: any[]): Promise<QueryResult>;
+    executeRaw(sql: string, parameters?: any[]): Promise<any[][]>;
     /**
      * Execute a write query (INSERT/UPDATE/DELETE) multiple times with each parameter set
      * and optionally return results.

package/lib/client/AbstractPowerSyncDatabase.js

@@ -9,12 +9,12 @@ import { ControlledExecutor } from '../utils/ControlledExecutor.js';
 import { mutexRunExclusive } from '../utils/mutex.js';
 import { throttleTrailing } from '../utils/throttle.js';
 import { isDBAdapter, isSQLOpenFactory, isSQLOpenOptions } from './SQLOpenFactory.js';
+import { runOnSchemaChange } from './runOnSchemaChange.js';
 import { PSInternalTable } from './sync/bucket/BucketStorageAdapter.js';
 import { CrudBatch } from './sync/bucket/CrudBatch.js';
 import { CrudEntry } from './sync/bucket/CrudEntry.js';
 import { CrudTransaction } from './sync/bucket/CrudTransaction.js';
 import { DEFAULT_CRUD_UPLOAD_THROTTLE_MS, DEFAULT_RETRY_DELAY_MS } from './sync/stream/AbstractStreamingSyncImplementation.js';
-import { runOnSchemaChange } from './runOnSchemaChange.js';
 const POWERSYNC_TABLE_MATCH = /(^ps_data__|^ps_data_local__)/;
 const DEFAULT_DISCONNECT_CLEAR_OPTIONS = {
     clearLocal: true
@@ -42,6 +42,10 @@ export const DEFAULT_LOCK_TIMEOUT_MS = 120_000; // 2 mins
 export const isPowerSyncDatabaseOptionsWithSettings = (test) => {
     return typeof test == 'object' && isSQLOpenOptions(test.database);
 };
+/**
+ * The priority used by the core extension to indicate that a full sync was completed.
+ */
+const FULL_SYNC_PRIORITY = 2147483647;
 export class AbstractPowerSyncDatabase extends BaseObserver {
     options;
     /**
@@ -127,16 +131,27 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         await this._isReadyPromise;
     }
     /**
+     * Wait for the first sync operation to complete.
+     *
+     * @argument request Either an abort signal (after which the promise will complete regardless of
+     * whether a full sync was completed) or an object providing an abort signal and a priority target.
+     * When a priority target is set, the promise may complete when all buckets with the given (or higher)
+     * priorities have been synchronized. This can be earlier than a complete sync.
      * @returns A promise which will resolve once the first full sync has completed.
      */
-    async waitForFirstSync(signal) {
-        if (this.currentStatus.hasSynced) {
+    async waitForFirstSync(request) {
+        const signal = request instanceof AbortSignal ? request : request?.signal;
+        const priority = request && 'priority' in request ? request.priority : undefined;
+        const statusMatches = priority === undefined
+            ? (status) => status.hasSynced
+            : (status) => status.statusForPriority(priority).hasSynced;
+        if (statusMatches(this.currentStatus)) {
             return;
         }
         return new Promise((resolve) => {
             const dispose = this.registerListener({
                 statusChanged: (status) => {
-                    if (status.hasSynced) {
+                    if (statusMatches(status)) {
                         dispose();
                         resolve();
                     }
@@ -177,19 +192,36 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
                 .map((n) => parseInt(n));
         }
         catch (e) {
-            throw new Error(`Unsupported powersync extension version. Need >=0.2.0 <1.0.0, got: ${this.sdkVersion}. Details: ${e.message}`);
+            throw new Error(`Unsupported powersync extension version. Need >=0.3.11 <1.0.0, got: ${this.sdkVersion}. Details: ${e.message}`);
         }
-        // Validate >=0.2.0 <1.0.0
-        if (versionInts[0] != 0 || versionInts[1] < 2 || versionInts[2] < 0) {
-            throw new Error(`Unsupported powersync extension version. Need >=0.2.0 <1.0.0, got: ${this.sdkVersion}`);
+        // Validate >=0.3.11 <1.0.0
+        if (versionInts[0] != 0 || versionInts[1] < 3 || (versionInts[1] == 3 && versionInts[2] < 11)) {
+            throw new Error(`Unsupported powersync extension version. Need >=0.3.11 <1.0.0, got: ${this.sdkVersion}`);
         }
     }
     async updateHasSynced() {
-        const result = await this.database.get('SELECT powersync_last_synced_at() as synced_at');
-        const hasSynced = result.synced_at != null;
-        const syncedAt = result.synced_at != null ? new Date(result.synced_at + 'Z') : undefined;
-        if (hasSynced != this.currentStatus.hasSynced) {
-            this.currentStatus = new SyncStatus({ ...this.currentStatus.toJSON(), hasSynced, lastSyncedAt: syncedAt });
+        const result = await this.database.getAll('SELECT priority, last_synced_at FROM ps_sync_state ORDER BY priority DESC');
+        let lastCompleteSync;
+        const priorityStatusEntries = [];
+        for (const { priority, last_synced_at } of result) {
+            const parsedDate = new Date(last_synced_at + 'Z');
+            if (priority == FULL_SYNC_PRIORITY) {
+                // This lowest-possible priority represents a complete sync.
+                lastCompleteSync = parsedDate;
+            }
+            else {
+                priorityStatusEntries.push({ priority, hasSynced: true, lastSyncedAt: parsedDate });
+            }
+        }
+        const hasSynced = lastCompleteSync != null;
+        const updatedStatus = new SyncStatus({
+            ...this.currentStatus.toJSON(),
+            hasSynced,
+            priorityStatusEntries,
+            lastSyncedAt: lastCompleteSync
+        });
+        if (!updatedStatus.isEqual(this.currentStatus)) {
+            this.currentStatus = updatedStatus;
             this.iterateListeners((l) => l.statusChanged?.(this.currentStatus));
         }
     }
@@ -245,7 +277,7 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         const { retryDelayMs, crudUploadThrottleMs } = this.resolvedConnectionOptions(options);
         this.syncStreamImplementation = this.generateSyncStreamImplementation(connector, {
             retryDelayMs,
-            crudUploadThrottleMs,
+            crudUploadThrottleMs
         });
         this.syncStatusListenerDisposer = this.syncStreamImplementation.registerListener({
             statusChanged: (status) => {
@@ -302,12 +334,15 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
      */
     async close(options = DEFAULT_POWERSYNC_CLOSE_OPTIONS) {
         await this.waitForReady();
+        if (this.closed) {
+            return;
+        }
         const { disconnect } = options;
         if (disconnect) {
             await this.disconnect();
         }
         await this.syncStreamImplementation?.dispose();
-        this.database.close();
+        await this.database.close();
         this.closed = true;
     }
     /**
@@ -424,6 +459,10 @@ export class AbstractPowerSyncDatabase extends BaseObserver {
         await this.waitForReady();
         return this.database.execute(sql, parameters);
     }
+    async executeRaw(sql, parameters) {
+        await this.waitForReady();
+        return this.database.executeRaw(sql, parameters);
+    }
     /**
      * Execute a write query (INSERT/UPDATE/DELETE) multiple times with each parameter set
      * and optionally return results.

package/lib/client/sync/bucket/BucketStorageAdapter.d.ts

@@ -2,6 +2,10 @@ import { BaseListener, BaseObserver, Disposable } from '../../../utils/BaseObser
 import { CrudBatch } from './CrudBatch.js';
 import { CrudEntry, OpId } from './CrudEntry.js';
 import { SyncDataBatch } from './SyncDataBatch.js';
+export interface BucketDescription {
+    name: string;
+    priority: number;
+}
 export interface Checkpoint {
     last_op_id: OpId;
     buckets: BucketChecksum[];
@@ -25,6 +29,7 @@ export interface SyncLocalDatabaseResult {
 }
 export interface BucketChecksum {
     bucket: string;
+    priority?: number;
     /**
      * 32-bit unsigned hash.
      */
@@ -51,7 +56,7 @@ export interface BucketStorageAdapter extends BaseObserver<BucketStorageListener
     setTargetCheckpoint(checkpoint: Checkpoint): Promise<void>;
     startSession(): void;
     getBucketStates(): Promise<BucketState[]>;
-    syncLocalDatabase(checkpoint: Checkpoint): Promise<{
+    syncLocalDatabase(checkpoint: Checkpoint, priority?: number): Promise<{
         checkpointValid: boolean;
         ready: boolean;
         failures?: any[];

package/lib/client/sync/bucket/SqliteBucketStorage.d.ts

@@ -37,14 +37,14 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
      */
     private deleteBucket;
     hasCompletedSync(): Promise<boolean>;
-    syncLocalDatabase(checkpoint: Checkpoint): Promise<SyncLocalDatabaseResult>;
+    syncLocalDatabase(checkpoint: Checkpoint, priority?: number): Promise<SyncLocalDatabaseResult>;
     /**
      * Atomically update the local state to the current checkpoint.
      *
      * This includes creating new tables, dropping old tables, and copying data over from the oplog.
      */
     private updateObjectsFromBuckets;
-    validateChecksums(checkpoint: Checkpoint): Promise<SyncLocalDatabaseResult>;
+    validateChecksums(checkpoint: Checkpoint, priority: number | undefined): Promise<SyncLocalDatabaseResult>;
     /**
      * Force a compact, for tests.
      */

package/lib/client/sync/bucket/SqliteBucketStorage.js

@@ -106,8 +106,8 @@ export class SqliteBucketStorage extends BaseObserver {
         }
         return completed;
     }
-    async syncLocalDatabase(checkpoint) {
-        const r = await this.validateChecksums(checkpoint);
+    async syncLocalDatabase(checkpoint, priority) {
+        const r = await this.validateChecksums(checkpoint, priority);
         if (!r.checkpointValid) {
             this.logger.error('Checksums failed for', r.checkpointFailures);
             for (const b of r.checkpointFailures ?? []) {
@@ -115,17 +115,21 @@ export class SqliteBucketStorage extends BaseObserver {
             }
             return { ready: false, checkpointValid: false, checkpointFailures: r.checkpointFailures };
         }
-        const bucketNames = checkpoint.buckets.map((b) => b.bucket);
+        const buckets = checkpoint.buckets;
+        if (priority !== undefined) {
+            buckets.filter((b) => hasMatchingPriority(priority, b));
+        }
+        const bucketNames = buckets.map((b) => b.bucket);
         await this.writeTransaction(async (tx) => {
             await tx.execute(`UPDATE ps_buckets SET last_op = ? WHERE name IN (SELECT json_each.value FROM json_each(?))`, [
                 checkpoint.last_op_id,
                 JSON.stringify(bucketNames)
             ]);
-            if (checkpoint.write_checkpoint) {
+            if (priority == null && checkpoint.write_checkpoint) {
                 await tx.execute("UPDATE ps_buckets SET last_op = ? WHERE name = '$local'", [checkpoint.write_checkpoint]);
             }
         });
-        const valid = await this.updateObjectsFromBuckets(checkpoint);
+        const valid = await this.updateObjectsFromBuckets(checkpoint, priority);
         if (!valid) {
             this.logger.debug('Not at a consistent checkpoint - cannot update local db');
             return { ready: false, checkpointValid: true };
@@ -141,19 +145,36 @@ export class SqliteBucketStorage extends BaseObserver {
      *
      * This includes creating new tables, dropping old tables, and copying data over from the oplog.
      */
-    async updateObjectsFromBuckets(checkpoint) {
+    async updateObjectsFromBuckets(checkpoint, priority) {
+        let arg = '';
+        if (priority !== undefined) {
+            const affectedBuckets = [];
+            for (const desc of checkpoint.buckets) {
+                if (hasMatchingPriority(priority, desc)) {
+                    affectedBuckets.push(desc.bucket);
+                }
+            }
+            arg = JSON.stringify({ priority, buckets: affectedBuckets });
+        }
         return this.writeTransaction(async (tx) => {
             const { insertId: result } = await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
                 'sync_local',
-                ''
+                arg
             ]);
             return result == 1;
         });
     }
-    async validateChecksums(checkpoint) {
-        const rs = await this.db.execute('SELECT powersync_validate_checkpoint(?) as result', [JSON.stringify(checkpoint)]);
+    async validateChecksums(checkpoint, priority) {
+        if (priority !== undefined) {
+            // Only validate the buckets within the priority we care about
+            const newBuckets = checkpoint.buckets.filter((cs) => hasMatchingPriority(priority, cs));
+            checkpoint = { ...checkpoint, buckets: newBuckets };
+        }
+        const rs = await this.db.execute('SELECT powersync_validate_checkpoint(?) as result', [
+            JSON.stringify({ ...checkpoint })
+        ]);
         const resultItem = rs.rows?.item(0);
-        this.logger.debug('validateChecksums result item', resultItem);
+        this.logger.debug('validateChecksums priority, checkpoint, result item', priority, checkpoint, resultItem);
         if (!resultItem) {
             return {
                 checkpointValid: false,
@@ -304,3 +325,6 @@ export class SqliteBucketStorage extends BaseObserver {
         // No-op for now
     }
 }
+function hasMatchingPriority(priority, bucket) {
+    return bucket.priority != null && bucket.priority <= priority;
+}

package/lib/client/sync/stream/AbstractRemote.d.ts

@@ -16,6 +16,21 @@ export type SyncStreamOptions = {
     abortSignal?: AbortSignal;
     fetchOptions?: Request;
 };
+export declare enum FetchStrategy {
+    /**
+     * Queues multiple sync events before processing, reducing round-trips.
+     * This comes at the cost of more processing overhead, which may cause ACK timeouts on older/weaker devices for big enough datasets.
+     */
+    Buffered = "buffered",
+    /**
+     * Processes each sync event immediately before requesting the next.
+     * This reduces processing overhead and improves real-time responsiveness.
+     */
+    Sequential = "sequential"
+}
+export type SocketSyncStreamOptions = SyncStreamOptions & {
+    fetchStrategy: FetchStrategy;
+};
 export type FetchImplementation = typeof fetch;
 /**
  * Class wrapper for providing a fetch implementation.
@@ -72,7 +87,7 @@ export declare abstract class AbstractRemote {
     /**
      * Connects to the sync/stream websocket endpoint
      */
-    socketStream(options: SyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
+    socketStream(options: SocketSyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
     /**
      * Connects to the sync/stream http endpoint
      */

package/lib/client/sync/stream/AbstractRemote.js

@@ -3,19 +3,32 @@ import ndjsonStream from 'can-ndjson-stream';
 import Logger from 'js-logger';
 import { RSocketConnector } from 'rsocket-core';
 import { WebsocketClientTransport } from 'rsocket-websocket-client';
+import PACKAGE from '../../../../package.json' with { type: 'json' };
 import { AbortOperation } from '../../../utils/AbortOperation.js';
 import { DataStream } from '../../../utils/DataStream.js';
-import { version as POWERSYNC_JS_VERSION } from '../../../../package.json';
 const POWERSYNC_TRAILING_SLASH_MATCH = /\/+$/;
+const POWERSYNC_JS_VERSION = PACKAGE.version;
 // Refresh at least 30 sec before it expires
 const REFRESH_CREDENTIALS_SAFETY_PERIOD_MS = 30_000;
-const SYNC_QUEUE_REQUEST_N = 1;
 const SYNC_QUEUE_REQUEST_LOW_WATER = 5;
 // Keep alive message is sent every period
-const KEEP_ALIVE_MS = 60_000;
+const KEEP_ALIVE_MS = 20_000;
 // The ACK must be received in this period
-const KEEP_ALIVE_LIFETIME_MS = 90_000;
+const KEEP_ALIVE_LIFETIME_MS = 30_000;
 export const DEFAULT_REMOTE_LOGGER = Logger.get('PowerSyncRemote');
+export var FetchStrategy;
+(function (FetchStrategy) {
+    /**
+     * Queues multiple sync events before processing, reducing round-trips.
+     * This comes at the cost of more processing overhead, which may cause ACK timeouts on older/weaker devices for big enough datasets.
+     */
+    FetchStrategy["Buffered"] = "buffered";
+    /**
+     * Processes each sync event immediately before requesting the next.
+     * This reduces processing overhead and improves real-time responsiveness.
+     */
+    FetchStrategy["Sequential"] = "sequential";
+})(FetchStrategy || (FetchStrategy = {}));
 /**
  * Class wrapper for providing a fetch implementation.
  * The class wrapper is used to distinguish the fetchImplementation
@@ -144,7 +157,8 @@ export class AbstractRemote {
      * Connects to the sync/stream websocket endpoint
      */
     async socketStream(options) {
-        const { path } = options;
+        const { path, fetchStrategy = FetchStrategy.Buffered } = options;
+        const syncQueueRequestSize = fetchStrategy == FetchStrategy.Buffered ? 10 : 1;
         const request = await this.buildRequest(path);
         const bson = await this.getBSON();
         // Add the user agent in the setup payload - we can't set custom
@@ -197,7 +211,7 @@ export class AbstractRemote {
         // Helps to prevent double close scenarios
         rsocket.onClose(() => (socketIsClosed = true));
         // We initially request this amount and expect these to arrive eventually
-        let pendingEventsCount = SYNC_QUEUE_REQUEST_N;
+        let pendingEventsCount = syncQueueRequestSize;
         const disposeClosedListener = stream.registerListener({
             closed: () => {
                 closeSocket();
@@ -211,7 +225,7 @@ export class AbstractRemote {
                 metadata: Buffer.from(bson.serialize({
                     path
                 }))
-            }, SYNC_QUEUE_REQUEST_N, // The initial N amount
+            }, syncQueueRequestSize, // The initial N amount
             {
                 onError: (e) => {
                     // Don't log closed as an error
@@ -250,10 +264,10 @@ export class AbstractRemote {
         const l = stream.registerListener({
             lowWater: async () => {
                 // Request to fill up the queue
-                const required = SYNC_QUEUE_REQUEST_N - pendingEventsCount;
+                const required = syncQueueRequestSize - pendingEventsCount;
                 if (required > 0) {
-                    socket.request(SYNC_QUEUE_REQUEST_N - pendingEventsCount);
-                    pendingEventsCount = SYNC_QUEUE_REQUEST_N;
+                    socket.request(syncQueueRequestSize - pendingEventsCount);
+                    pendingEventsCount = syncQueueRequestSize;
                 }
             },
             closed: () => {

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts

@@ -2,7 +2,7 @@ import Logger, { ILogger } from 'js-logger';
 import { SyncStatus, SyncStatusOptions } from '../../../db/crud/SyncStatus.js';
 import { BaseListener, BaseObserver, Disposable } from '../../../utils/BaseObserver.js';
 import { BucketStorageAdapter } from '../bucket/BucketStorageAdapter.js';
-import { AbstractRemote } from './AbstractRemote.js';
+import { AbstractRemote, FetchStrategy } from './AbstractRemote.js';
 import { StreamingSyncRequestParameterType } from './streaming-sync-types.js';
 export declare enum LockType {
     CRUD = "crud",
@@ -56,6 +56,10 @@ export interface BaseConnectionOptions {
      * Defaults to a HTTP streaming connection.
      */
     connectionMethod?: SyncStreamConnectionMethod;
+    /**
+     * The fetch strategy to use when streaming updates from the PowerSync backend instance.
+     */
+    fetchStrategy?: FetchStrategy;
     /**
      * These parameters are passed to the sync rules, and will be available under the`user_parameters` object.
      */
@@ -95,6 +99,7 @@ export interface StreamingSyncImplementation extends BaseObserver<StreamingSyncI
     triggerCrudUpload: () => void;
     waitForReady(): Promise<void>;
     waitForStatus(status: SyncStatusOptions): Promise<void>;
+    waitUntilStatusMatches(predicate: (status: SyncStatus) => boolean): Promise<void>;
 }
 export declare const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
 export declare const DEFAULT_RETRY_DELAY_MS = 5000;
@@ -116,6 +121,7 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
     constructor(options: AbstractStreamingSyncImplementationOptions);
     waitForReady(): Promise<void>;
     waitForStatus(status: SyncStatusOptions): Promise<void>;
+    waitUntilStatusMatches(predicate: (status: SyncStatus) => boolean): Promise<void>;
     get lastSyncedAt(): Date | undefined;
     get isConnected(): boolean;
     protected get logger(): Logger.ILogger;
@@ -130,6 +136,7 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
      * @deprecated use [connect instead]
      */
     streamingSync(signal?: AbortSignal, options?: PowerSyncConnectionOptions): Promise<void>;
+    private collectLocalBucketState;
     protected streamingSyncIteration(signal: AbortSignal, options?: PowerSyncConnectionOptions): Promise<{
         retry?: boolean;
     }>;