@powersync/common 0.0.0-dev-20250207081035 → 0.0.0-dev-20250319141441

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js

@@ -4,7 +4,8 @@ import { AbortOperation } from '../../../utils/AbortOperation.js';
 import { BaseObserver } from '../../../utils/BaseObserver.js';
 import { throttleLeadingTrailing } from '../../../utils/throttle.js';
 import { SyncDataBucket } from '../bucket/SyncDataBucket.js';
-import { isStreamingKeepalive, isStreamingSyncCheckpoint, isStreamingSyncCheckpointComplete, isStreamingSyncCheckpointDiff, isStreamingSyncData } from './streaming-sync-types.js';
+import { FetchStrategy } from './AbstractRemote.js';
+import { isStreamingKeepalive, isStreamingSyncCheckpoint, isStreamingSyncCheckpointComplete, isStreamingSyncCheckpointDiff, isStreamingSyncCheckpointPartiallyComplete, isStreamingSyncData } from './streaming-sync-types.js';
 export var LockType;
 (function (LockType) {
     LockType["CRUD"] = "crud";
@@ -24,8 +25,14 @@ export const DEFAULT_STREAMING_SYNC_OPTIONS = {
 };
 export const DEFAULT_STREAM_CONNECTION_OPTIONS = {
     connectionMethod: SyncStreamConnectionMethod.WEB_SOCKET,
+    fetchStrategy: FetchStrategy.Buffered,
     params: {}
 };
+// The priority we assume when we receive checkpoint lines where no priority is set.
+// This is the default priority used by the sync service, but can be set to an arbitrary
+// value since sync services without priorities also won't send partial sync completion
+// messages.
+const FALLBACK_PRIORITY = 3;
 export class AbstractStreamingSyncImplementation extends BaseObserver {
     _lastSyncedAt;
     options;
@@ -56,23 +63,32 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
     }
     async waitForReady() { }
     waitForStatus(status) {
+        return this.waitUntilStatusMatches((currentStatus) => {
+            /**
+             * Match only the partial status options provided in the
+             * matching status
+             */
+            const matchPartialObject = (compA, compB) => {
+                return Object.entries(compA).every(([key, value]) => {
+                    const comparisonBValue = compB[key];
+                    if (typeof value == 'object' && typeof comparisonBValue == 'object') {
+                        return matchPartialObject(value, comparisonBValue);
+                    }
+                    return value == comparisonBValue;
+                });
+            };
+            return matchPartialObject(status, currentStatus);
+        });
+    }
+    waitUntilStatusMatches(predicate) {
         return new Promise((resolve) => {
+            if (predicate(this.syncStatus)) {
+                resolve();
+                return;
+            }
             const l = this.registerListener({
                 statusChanged: (updatedStatus) => {
-                    /**
-                     * Match only the partial status options provided in the
-                     * matching status
-                     */
-                    const matchPartialObject = (compA, compB) => {
-                        return Object.entries(compA).every(([key, value]) => {
-                            const comparisonBValue = compB[key];
-                            if (typeof value == 'object' && typeof comparisonBValue == 'object') {
-                                return matchPartialObject(value, comparisonBValue);
-                            }
-                            return value == comparisonBValue;
-                        });
-                    };
-                    if (matchPartialObject(status, updatedStatus.toJSON())) {
+                    if (predicate(updatedStatus)) {
                         resolve();
                         l?.();
                     }
@@ -301,6 +317,18 @@ The next upload iteration will be delayed.`);
         // Mark as disconnected if here
         this.updateSyncStatus({ connected: false, connecting: false });
     }
+    async collectLocalBucketState() {
+        const bucketEntries = await this.options.adapter.getBucketStates();
+        const req = bucketEntries.map((entry) => ({
+            name: entry.bucket,
+            after: entry.op_id
+        }));
+        const localDescriptions = new Map();
+        for (const entry of bucketEntries) {
+            localDescriptions.set(entry.bucket, null);
+        }
+        return [req, localDescriptions];
+    }
     async streamingSyncIteration(signal, options) {
         return await this.obtainLock({
             type: LockType.SYNC,
@@ -312,20 +340,11 @@ The next upload iteration will be delayed.`);
                 };
                 this.logger.debug('Streaming sync iteration started');
                 this.options.adapter.startSession();
-                const bucketEntries = await this.options.adapter.getBucketStates();
-                const initialBuckets = new Map();
-                bucketEntries.forEach((entry) => {
-                    initialBuckets.set(entry.bucket, entry.op_id);
-                });
-                const req = Array.from(initialBuckets.entries()).map(([bucket, after]) => ({
-                    name: bucket,
-                    after: after
-                }));
+                let [req, bucketMap] = await this.collectLocalBucketState();
                 // These are compared by reference
                 let targetCheckpoint = null;
                 let validatedCheckpoint = null;
                 let appliedCheckpoint = null;
-                let bucketSet = new Set(initialBuckets.keys());
                 const clientId = await this.options.adapter.getClientId();
                 this.logger.debug('Requesting stream from server');
                 const syncOptions = {
@@ -339,9 +358,16 @@ The next upload iteration will be delayed.`);
                         client_id: clientId
                     }
                 };
-                const stream = resolvedOptions?.connectionMethod == SyncStreamConnectionMethod.HTTP
-                    ? await this.options.remote.postStream(syncOptions)
-                    : await this.options.remote.socketStream(syncOptions);
+                let stream;
+                if (resolvedOptions?.connectionMethod == SyncStreamConnectionMethod.HTTP) {
+                    stream = await this.options.remote.postStream(syncOptions);
+                }
+                else {
+                    stream = await this.options.remote.socketStream({
+                        ...syncOptions,
+                        ...{ fetchStrategy: resolvedOptions.fetchStrategy }
+                    });
+                }
                 this.logger.debug('Stream established. Processing events');
                 while (!stream.closed) {
                     const line = await stream.read();
@@ -359,16 +385,19 @@ The next upload iteration will be delayed.`);
                     }
                     if (isStreamingSyncCheckpoint(line)) {
                         targetCheckpoint = line.checkpoint;
-                        const bucketsToDelete = new Set(bucketSet);
-                        const newBuckets = new Set();
+                        const bucketsToDelete = new Set(bucketMap.keys());
+                        const newBuckets = new Map();
                         for (const checksum of line.checkpoint.buckets) {
-                            newBuckets.add(checksum.bucket);
+                            newBuckets.set(checksum.bucket, {
+                                name: checksum.bucket,
+                                priority: checksum.priority ?? FALLBACK_PRIORITY
+                            });
                             bucketsToDelete.delete(checksum.bucket);
                         }
                         if (bucketsToDelete.size > 0) {
                             this.logger.debug('Removing buckets', [...bucketsToDelete]);
                         }
-                        bucketSet = newBuckets;
+                        bucketMap = newBuckets;
                         await this.options.adapter.removeBuckets([...bucketsToDelete]);
                         await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
                     }
@@ -399,6 +428,35 @@ The next upload iteration will be delayed.`);
                         }
                         validatedCheckpoint = targetCheckpoint;
                     }
+                    else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
+                        const priority = line.partial_checkpoint_complete.priority;
+                        this.logger.debug('Partial checkpoint complete', priority);
+                        const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint, priority);
+                        if (!result.checkpointValid) {
+                            // This means checksums failed. Start again with a new checkpoint.
+                            // TODO: better back-off
+                            await new Promise((resolve) => setTimeout(resolve, 50));
+                            return { retry: true };
+                        }
+                        else if (!result.ready) {
+                            // Need more data for a consistent partial sync within a priority - continue waiting.
+                        }
+                        else {
+                            // We'll keep on downloading, but can report that this priority is synced now.
+                            this.logger.debug('partial checkpoint validation succeeded');
+                            // All states with a higher priority can be deleted since this partial sync includes them.
+                            const priorityStates = this.syncStatus.priorityStatusEntries.filter((s) => s.priority <= priority);
+                            priorityStates.push({
+                                priority,
+                                lastSyncedAt: new Date(),
+                                hasSynced: true
+                            });
+                            this.updateSyncStatus({
+                                connected: true,
+                                priorityStatusEntries: priorityStates
+                            });
+                        }
+                    }
                     else if (isStreamingSyncCheckpointDiff(line)) {
                         // TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint
                         if (targetCheckpoint == null) {
@@ -421,7 +479,11 @@ The next upload iteration will be delayed.`);
                             write_checkpoint: diff.write_checkpoint
                         };
                         targetCheckpoint = newCheckpoint;
-                        bucketSet = new Set(newBuckets.keys());
+                        bucketMap = new Map();
+                        newBuckets.forEach((checksum, name) => bucketMap.set(name, {
+                            name: checksum.bucket,
+                            priority: checksum.priority ?? FALLBACK_PRIORITY
+                        }));
                         const bucketsToDelete = diff.removed_buckets;
                         if (bucketsToDelete.length > 0) {
                             this.logger.debug('Remove buckets', bucketsToDelete);
@@ -457,7 +519,8 @@ The next upload iteration will be delayed.`);
                         if (targetCheckpoint === appliedCheckpoint) {
                             this.updateSyncStatus({
                                 connected: true,
-                                lastSyncedAt: new Date()
+                                lastSyncedAt: new Date(),
+                                priorityStatusEntries: []
                             });
                         }
                         else if (validatedCheckpoint === targetCheckpoint) {
@@ -477,6 +540,7 @@ The next upload iteration will be delayed.`);
                                 this.updateSyncStatus({
                                     connected: true,
                                     lastSyncedAt: new Date(),
+                                    priorityStatusEntries: [],
                                     dataFlow: {
                                         downloading: false
                                     }
@@ -499,7 +563,8 @@ The next upload iteration will be delayed.`);
             dataFlow: {
                 ...this.syncStatus.dataFlowStatus,
                 ...options.dataFlow
-            }
+            },
+            priorityStatusEntries: options.priorityStatusEntries ?? this.syncStatus.priorityStatusEntries
         });
         if (!this.syncStatus.isEqual(updatedStatus)) {
             this.syncStatus = updatedStatus;

package/lib/client/sync/stream/streaming-sync-types.d.ts

@@ -90,11 +90,17 @@ export interface StreamingSyncCheckpointComplete {
         last_op_id: OpId;
     };
 }
+export interface StreamingSyncCheckpointPartiallyComplete {
+    partial_checkpoint_complete: {
+        priority: number;
+        last_op_id: OpId;
+    };
+}
 export interface StreamingSyncKeepalive {
     /** If specified, token expires in this many seconds. */
     token_expires_in: number;
 }
-export type StreamingSyncLine = StreamingSyncDataJSON | StreamingSyncCheckpoint | StreamingSyncCheckpointDiff | StreamingSyncCheckpointComplete | StreamingSyncKeepalive;
+export type StreamingSyncLine = StreamingSyncDataJSON | StreamingSyncCheckpoint | StreamingSyncCheckpointDiff | StreamingSyncCheckpointComplete | StreamingSyncCheckpointPartiallyComplete | StreamingSyncKeepalive;
 export interface BucketRequest {
     name: string;
     /**
@@ -106,6 +112,7 @@ export declare function isStreamingSyncData(line: StreamingSyncLine): line is St
 export declare function isStreamingKeepalive(line: StreamingSyncLine): line is StreamingSyncKeepalive;
 export declare function isStreamingSyncCheckpoint(line: StreamingSyncLine): line is StreamingSyncCheckpoint;
 export declare function isStreamingSyncCheckpointComplete(line: StreamingSyncLine): line is StreamingSyncCheckpointComplete;
+export declare function isStreamingSyncCheckpointPartiallyComplete(line: StreamingSyncLine): line is StreamingSyncCheckpointPartiallyComplete;
 export declare function isStreamingSyncCheckpointDiff(line: StreamingSyncLine): line is StreamingSyncCheckpointDiff;
 export declare function isContinueCheckpointRequest(request: SyncRequest): request is ContinueCheckpointRequest;
 export declare function isSyncNewCheckpointRequest(request: SyncRequest): request is SyncNewCheckpointRequest;

package/lib/client/sync/stream/streaming-sync-types.js

@@ -10,6 +10,9 @@ export function isStreamingSyncCheckpoint(line) {
 export function isStreamingSyncCheckpointComplete(line) {
     return line.checkpoint_complete != null;
 }
+export function isStreamingSyncCheckpointPartiallyComplete(line) {
+    return line.partial_checkpoint_complete != null;
+}
 export function isStreamingSyncCheckpointDiff(line) {
     return line.checkpoint_diff != null;
 }

package/lib/db/DBAdapter.d.ts

@@ -39,6 +39,8 @@ export interface DBGetUtils {
 export interface LockContext extends DBGetUtils {
     /** Execute a single write statement. */
     execute: (query: string, params?: any[] | undefined) => Promise<QueryResult>;
+    /** Execute a single write statement and return raw results. */
+    executeRaw: (query: string, params?: any[] | undefined) => Promise<any[][]>;
 }
 export interface Transaction extends LockContext {
     /** Commit multiple changes to the local DB using the Transaction context. */
@@ -82,8 +84,9 @@ export interface DBLockOptions {
     timeoutMs?: number;
 }
 export interface DBAdapter extends BaseObserverInterface<DBAdapterListener>, DBGetUtils {
-    close: () => void;
+    close: () => void | Promise<void>;
     execute: (query: string, params?: any[]) => Promise<QueryResult>;
+    executeRaw: (query: string, params?: any[]) => Promise<any[][]>;
     executeBatch: (query: string, params?: any[][]) => Promise<QueryResult>;
     name: string;
     readLock: <T>(fn: (tx: LockContext) => Promise<T>, options?: DBLockOptions) => Promise<T>;

package/lib/db/crud/SyncStatus.d.ts

@@ -2,12 +2,18 @@ export type SyncDataFlowStatus = Partial<{
     downloading: boolean;
     uploading: boolean;
 }>;
+export interface SyncPriorityStatus {
+    priority: number;
+    lastSyncedAt?: Date;
+    hasSynced?: boolean;
+}
 export type SyncStatusOptions = {
     connected?: boolean;
     connecting?: boolean;
     dataFlow?: SyncDataFlowStatus;
     lastSyncedAt?: Date;
     hasSynced?: boolean;
+    priorityStatusEntries?: SyncPriorityStatus[];
 };
 export declare class SyncStatus {
     protected options: SyncStatusOptions;
@@ -34,7 +40,28 @@ export declare class SyncStatus {
         downloading: boolean;
         uploading: boolean;
     }>;
+    /**
+     * Partial sync status for involved bucket priorities.
+     */
+    get priorityStatusEntries(): SyncPriorityStatus[];
+    /**
+     * Reports a pair of {@link SyncStatus#hasSynced} and {@link SyncStatus#lastSyncedAt} fields that apply
+     * to a specific bucket priority instead of the entire sync operation.
+     *
+     * When buckets with different priorities are declared, PowerSync may choose to synchronize higher-priority
+     * buckets first. When a consistent view over all buckets for all priorities up until the given priority is
+     * reached, PowerSync makes data from those buckets available before lower-priority buckets have finished
+     * synchronizing.
+     * When PowerSync makes data for a given priority available, all buckets in higher-priorities are guaranteed to
+     * be consistent with that checkpoint. For this reason, this method may also return the status for lower priorities.
+     * In a state where the PowerSync just finished synchronizing buckets in priority level 3, calling this method
+     * with a priority of 1 may return information for priority level 3.
+     *
+     * @param priority The bucket priority for which the status should be reported.
+     */
+    statusForPriority(priority: number): SyncPriorityStatus;
     isEqual(status: SyncStatus): boolean;
     getMessage(): string;
     toJSON(): SyncStatusOptions;
+    private static comparePriorities;
 }

package/lib/db/crud/SyncStatus.js

@@ -42,6 +42,42 @@ export class SyncStatus {
             uploading: false
         });
     }
+    /**
+     * Partial sync status for involved bucket priorities.
+     */
+    get priorityStatusEntries() {
+        return (this.options.priorityStatusEntries ?? []).slice().sort(SyncStatus.comparePriorities);
+    }
+    /**
+     * Reports a pair of {@link SyncStatus#hasSynced} and {@link SyncStatus#lastSyncedAt} fields that apply
+     * to a specific bucket priority instead of the entire sync operation.
+     *
+     * When buckets with different priorities are declared, PowerSync may choose to synchronize higher-priority
+     * buckets first. When a consistent view over all buckets for all priorities up until the given priority is
+     * reached, PowerSync makes data from those buckets available before lower-priority buckets have finished
+     * synchronizing.
+     * When PowerSync makes data for a given priority available, all buckets in higher-priorities are guaranteed to
+     * be consistent with that checkpoint. For this reason, this method may also return the status for lower priorities.
+     * In a state where the PowerSync just finished synchronizing buckets in priority level 3, calling this method
+     * with a priority of 1 may return information for priority level 3.
+     *
+     * @param priority The bucket priority for which the status should be reported.
+     */
+    statusForPriority(priority) {
+        // priorityStatusEntries are sorted by ascending priorities (so higher numbers to lower numbers).
+        for (const known of this.priorityStatusEntries) {
+            // We look for the first entry that doesn't have a higher priority.
+            if (known.priority >= priority) {
+                return known;
+            }
+        }
+        // If we have a complete sync, that necessarily includes all priorities.
+        return {
+            priority,
+            lastSyncedAt: this.lastSyncedAt,
+            hasSynced: this.hasSynced
+        };
+    }
     isEqual(status) {
         return JSON.stringify(this.options) == JSON.stringify(status.options);
     }
@@ -55,7 +91,11 @@ export class SyncStatus {
             connecting: this.connecting,
             dataFlow: this.dataFlowStatus,
             lastSyncedAt: this.lastSyncedAt,
-            hasSynced: this.hasSynced
+            hasSynced: this.hasSynced,
+            priorityStatusEntries: this.priorityStatusEntries
         };
     }
+    static comparePriorities(a, b) {
+        return b.priority - a.priority; // Reverse because higher priorities have lower numbers
+    }
 }

package/lib/utils/DataStream.js

@@ -63,9 +63,6 @@ export class DataStream extends BaseObserver {
      * @returns a Data payload or Null if the stream closed.
      */
     async read() {
-        if (this.dataQueue.length <= this.lowWatermark) {
-            await this.iterateAsyncErrored(async (l) => l.lowWater?.());
-        }
         if (this.closed) {
             return null;
         }
@@ -133,12 +130,14 @@ export class DataStream extends BaseObserver {
         return Array.from(this.listeners.values()).some((l) => !!l.data);
     }
     async _processQueue() {
-        if (!this.dataQueue.length || this.isClosed || !this.hasDataReader()) {
+        if (this.isClosed || !this.hasDataReader()) {
             Promise.resolve().then(() => (this.processingPromise = null));
             return;
         }
-        const data = this.dataQueue.shift();
-        await this.iterateAsyncErrored(async (l) => l.data?.(data));
+        if (this.dataQueue.length) {
+            const data = this.dataQueue.shift();
+            await this.iterateAsyncErrored(async (l) => l.data?.(data));
+        }
         if (this.dataQueue.length <= this.lowWatermark) {
             await this.iterateAsyncErrored(async (l) => l.lowWater?.());
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@powersync/common",
-  "version": "0.0.0-dev-20250207081035",
+  "version": "0.0.0-dev-20250319141441",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -52,8 +52,6 @@
     "rsocket-core": "1.0.0-alpha.3",
     "rsocket-websocket-client": "1.0.0-alpha.3",
     "text-encoding": "^0.7.0",
-    "typescript": "^5.5.3",
-    "vitest": "^2.0.5",
     "web-streams-polyfill": "3.2.1"
   },
   "scripts": {