@powersync/common 1.28.0 → 1.29.0

package/lib/client/AbstractPowerSyncDatabase.js

@@ -15,6 +15,7 @@ import { CrudBatch } from './sync/bucket/CrudBatch.js';
 import { CrudEntry } from './sync/bucket/CrudEntry.js';
 import { CrudTransaction } from './sync/bucket/CrudTransaction.js';
 import { DEFAULT_CRUD_UPLOAD_THROTTLE_MS, DEFAULT_RETRY_DELAY_MS } from './sync/stream/AbstractStreamingSyncImplementation.js';
+import { FULL_SYNC_PRIORITY } from '../db/crud/SyncProgress.js';
 const POWERSYNC_TABLE_MATCH = /(^ps_data__|^ps_data_local__)/;
 const DEFAULT_DISCONNECT_CLEAR_OPTIONS = {
     clearLocal: true
@@ -42,10 +43,6 @@ export const DEFAULT_LOCK_TIMEOUT_MS = 120_000; // 2 mins
 export const isPowerSyncDatabaseOptionsWithSettings = (test) => {
     return typeof test == 'object' && isSQLOpenOptions(test.database);
 };
-/**
- * The priority used by the core extension to indicate that a full sync was completed.
- */
-const FULL_SYNC_PRIORITY = 2147483647;
 export class AbstractPowerSyncDatabase extends BaseObserver {
     options;
     /**

package/lib/client/sync/bucket/BucketStorageAdapter.d.ts

@@ -27,6 +27,11 @@ export interface SyncLocalDatabaseResult {
     checkpointValid: boolean;
     checkpointFailures?: string[];
 }
+export type SavedProgress = {
+    atLast: number;
+    sinceLast: number;
+};
+export type BucketOperationProgress = Record<string, SavedProgress>;
 export interface BucketChecksum {
     bucket: string;
     priority?: number;
@@ -56,6 +61,7 @@ export interface BucketStorageAdapter extends BaseObserver<BucketStorageListener
     setTargetCheckpoint(checkpoint: Checkpoint): Promise<void>;
     startSession(): void;
     getBucketStates(): Promise<BucketState[]>;
+    getBucketOperationProgress(): Promise<BucketOperationProgress>;
     syncLocalDatabase(checkpoint: Checkpoint, priority?: number): Promise<{
         checkpointValid: boolean;
         ready: boolean;

package/lib/client/sync/bucket/SqliteBucketStorage.d.ts

@@ -2,7 +2,7 @@ import { Mutex } from 'async-mutex';
 import { ILogger } from 'js-logger';
 import { DBAdapter, Transaction } from '../../../db/DBAdapter.js';
 import { BaseObserver } from '../../../utils/BaseObserver.js';
-import { BucketState, BucketStorageAdapter, BucketStorageListener, Checkpoint, SyncLocalDatabaseResult } from './BucketStorageAdapter.js';
+import { BucketOperationProgress, BucketState, BucketStorageAdapter, BucketStorageListener, Checkpoint, SyncLocalDatabaseResult } from './BucketStorageAdapter.js';
 import { CrudBatch } from './CrudBatch.js';
 import { CrudEntry } from './CrudEntry.js';
 import { SyncDataBatch } from './SyncDataBatch.js';
@@ -30,6 +30,7 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
      */
     startSession(): void;
     getBucketStates(): Promise<BucketState[]>;
+    getBucketOperationProgress(): Promise<BucketOperationProgress>;
     saveSyncData(batch: SyncDataBatch): Promise<void>;
     removeBuckets(buckets: string[]): Promise<void>;
     /**

package/lib/client/sync/bucket/SqliteBucketStorage.js

@@ -66,6 +66,10 @@ export class SqliteBucketStorage extends BaseObserver {
         const result = await this.db.getAll("SELECT name as bucket, cast(last_op as TEXT) as op_id FROM ps_buckets WHERE pending_delete = 0 AND name != '$local'");
         return result;
     }
+    async getBucketOperationProgress() {
+        const rows = await this.db.getAll('SELECT name, count_at_last, count_since_last FROM ps_buckets');
+        return Object.fromEntries(rows.map((r) => [r.name, { atLast: r.count_at_last, sinceLast: r.count_since_last }]));
+    }
     async saveSyncData(batch) {
         await this.writeTransaction(async (tx) => {
             let count = 0;
@@ -161,7 +165,18 @@ export class SqliteBucketStorage extends BaseObserver {
                 'sync_local',
                 arg
             ]);
-            return result == 1;
+            if (result == 1) {
+                if (priority == null) {
+                    const bucketToCount = Object.fromEntries(checkpoint.buckets.map((b) => [b.bucket, b.count]));
+                    // The two parameters could be replaced with one, but: https://github.com/powersync-ja/better-sqlite3/pull/6
+                    const jsonBucketCount = JSON.stringify(bucketToCount);
+                    await tx.execute("UPDATE ps_buckets SET count_since_last = 0, count_at_last = ?->name WHERE name != '$local' AND ?->name IS NOT NULL", [jsonBucketCount, jsonBucketCount]);
+                }
+                return true;
+            }
+            else {
+                return false;
+            }
         });
     }
     async validateChecksums(checkpoint, priority) {

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.d.ts

@@ -139,6 +139,7 @@ export declare abstract class AbstractStreamingSyncImplementation extends BaseOb
     streamingSync(signal?: AbortSignal, options?: PowerSyncConnectionOptions): Promise<void>;
     private collectLocalBucketState;
     protected streamingSyncIteration(signal: AbortSignal, options?: PowerSyncConnectionOptions): Promise<void>;
+    private updateSyncStatusForStartingCheckpoint;
     private applyCheckpoint;
     protected updateSyncStatus(options: SyncStatusOptions): void;
     private delayRetry;

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js

@@ -270,7 +270,8 @@ The next upload iteration will be delayed.`);
                 connected: false,
                 connecting: false,
                 dataFlow: {
-                    downloading: false
+                    downloading: false,
+                    downloadProgress: null
                 }
             });
         });
@@ -409,6 +410,7 @@ The next upload iteration will be delayed.`);
                         bucketMap = newBuckets;
                         await this.options.adapter.removeBuckets([...bucketsToDelete]);
                         await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
+                        await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
                     }
                     else if (isStreamingSyncCheckpointComplete(line)) {
                         const result = await this.applyCheckpoint(targetCheckpoint, signal);
@@ -472,6 +474,7 @@ The next upload iteration will be delayed.`);
                             write_checkpoint: diff.write_checkpoint
                         };
                         targetCheckpoint = newCheckpoint;
+                        await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
                         bucketMap = new Map();
                         newBuckets.forEach((checksum, name) => bucketMap.set(name, {
                             name: checksum.bucket,
@@ -486,9 +489,22 @@ The next upload iteration will be delayed.`);
                     }
                     else if (isStreamingSyncData(line)) {
                         const { data } = line;
+                        const previousProgress = this.syncStatus.dataFlowStatus.downloadProgress;
+                        let updatedProgress = null;
+                        if (previousProgress) {
+                            updatedProgress = { ...previousProgress };
+                            const progressForBucket = updatedProgress[data.bucket];
+                            if (progressForBucket) {
+                                updatedProgress[data.bucket] = {
+                                    ...progressForBucket,
+                                    sinceLast: progressForBucket.sinceLast + data.data.length
+                                };
+                            }
+                        }
                         this.updateSyncStatus({
                             dataFlow: {
-                                downloading: true
+                                downloading: true,
+                                downloadProgress: updatedProgress
                             }
                         });
                         await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] });
@@ -536,6 +552,27 @@ The next upload iteration will be delayed.`);
             }
         });
     }
+    async updateSyncStatusForStartingCheckpoint(checkpoint) {
+        const localProgress = await this.options.adapter.getBucketOperationProgress();
+        const progress = {};
+        for (const bucket of checkpoint.buckets) {
+            const savedProgress = localProgress[bucket.bucket];
+            progress[bucket.bucket] = {
+                // The fallback priority doesn't matter here, but 3 is the one newer versions of the sync service
+                // will use by default.
+                priority: bucket.priority ?? 3,
+                atLast: savedProgress?.atLast ?? 0,
+                sinceLast: savedProgress?.sinceLast ?? 0,
+                targetCount: bucket.count ?? 0
+            };
+        }
+        this.updateSyncStatus({
+            dataFlow: {
+                downloading: true,
+                downloadProgress: progress
+            }
+        });
+    }
     async applyCheckpoint(checkpoint, abort) {
         let result = await this.options.adapter.syncLocalDatabase(checkpoint);
         const pending = this.pendingCrudUpload;
@@ -565,6 +602,7 @@ The next upload iteration will be delayed.`);
                 lastSyncedAt: new Date(),
                 dataFlow: {
                     downloading: false,
+                    downloadProgress: null,
                     downloadError: undefined
                 }
             });

package/lib/client/sync/stream/streaming-sync-types.d.ts

@@ -79,7 +79,7 @@ export interface StreamingSyncCheckpointDiff {
         last_op_id: OpId;
         updated_buckets: BucketChecksum[];
         removed_buckets: string[];
-        write_checkpoint: string;
+        write_checkpoint?: string;
     };
 }
 export interface StreamingSyncDataJSON {

package/lib/db/crud/SyncProgress.d.ts

@@ -0,0 +1,72 @@
+/** @internal */
+export type InternalProgressInformation = Record<string, {
+    priority: number;
+    atLast: number;
+    sinceLast: number;
+    targetCount: number;
+}>;
+/**
+ * @internal The priority used by the core extension to indicate that a full sync was completed.
+ */
+export declare const FULL_SYNC_PRIORITY = 2147483647;
+/**
+ * Information about a progressing download made by the PowerSync SDK.
+ *
+ * To obtain these values, use {@link SyncProgress}, available through
+ * {@link SyncStatus#downloadProgress}.
+ */
+export interface ProgressWithOperations {
+    /**
+     * The total amount of operations to download for the current sync iteration
+     * to complete.
+     */
+    totalOperations: number;
+    /**
+     * The amount of operations that have already been downloaded.
+     */
+    downloadedOperations: number;
+    /**
+     * Relative progress, as {@link downloadedOperations} of {@link totalOperations}.
+     *
+     * This will be a number between `0.0` and `1.0` (inclusive).
+     *
+     * When this number reaches `1.0`, all changes have been received from the sync service.
+     * Actually applying these changes happens before the `downloadProgress` field is cleared from
+     * {@link SyncStatus}, so progress can stay at `1.0` for a short while before completing.
+     */
+    downloadedFraction: number;
+}
+/**
+ * Provides realtime progress on how PowerSync is downloading rows.
+ *
+ * The progress until the next complete sync is available through the fields on {@link ProgressWithOperations},
+ * which this class implements.
+ * Additionally, the {@link SyncProgress.untilPriority} method can be used to otbain progress towards
+ * a specific priority (instead of the progress for the entire download).
+ *
+ * The reported progress always reflects the status towards th end of a sync iteration (after
+ * which a consistent snapshot of all buckets is available locally).
+ *
+ * In rare cases (in particular, when a [compacting](https://docs.powersync.com/usage/lifecycle-maintenance/compacting-buckets)
+ * operation takes place between syncs), it's possible for the returned numbers to be slightly
+ * inaccurate. For this reason, {@link SyncProgress} should be seen as an approximation of progress.
+ * The information returned is good enough to build progress bars, but not exact enough to track
+ * individual download counts.
+ *
+ * Also note that data is downloaded in bulk, which means that individual counters are unlikely
+ * to be updated one-by-one.
+ */
+export declare class SyncProgress implements ProgressWithOperations {
+    protected internal: InternalProgressInformation;
+    totalOperations: number;
+    downloadedOperations: number;
+    downloadedFraction: number;
+    constructor(internal: InternalProgressInformation);
+    /**
+     * Returns download progress towards all data up until the specified priority being received.
+     *
+     * The returned {@link ProgressWithOperations} tracks the target amount of operations that need
+     * to be downloaded in total and how many of them have already been received.
+     */
+    untilPriority(priority: number): ProgressWithOperations;
+}

package/lib/db/crud/SyncProgress.js

@@ -0,0 +1,60 @@
+/**
+ * @internal The priority used by the core extension to indicate that a full sync was completed.
+ */
+export const FULL_SYNC_PRIORITY = 2147483647;
+/**
+ * Provides realtime progress on how PowerSync is downloading rows.
+ *
+ * The progress until the next complete sync is available through the fields on {@link ProgressWithOperations},
+ * which this class implements.
+ * Additionally, the {@link SyncProgress.untilPriority} method can be used to otbain progress towards
+ * a specific priority (instead of the progress for the entire download).
+ *
+ * The reported progress always reflects the status towards th end of a sync iteration (after
+ * which a consistent snapshot of all buckets is available locally).
+ *
+ * In rare cases (in particular, when a [compacting](https://docs.powersync.com/usage/lifecycle-maintenance/compacting-buckets)
+ * operation takes place between syncs), it's possible for the returned numbers to be slightly
+ * inaccurate. For this reason, {@link SyncProgress} should be seen as an approximation of progress.
+ * The information returned is good enough to build progress bars, but not exact enough to track
+ * individual download counts.
+ *
+ * Also note that data is downloaded in bulk, which means that individual counters are unlikely
+ * to be updated one-by-one.
+ */
+export class SyncProgress {
+    internal;
+    totalOperations;
+    downloadedOperations;
+    downloadedFraction;
+    constructor(internal) {
+        this.internal = internal;
+        const untilCompletion = this.untilPriority(FULL_SYNC_PRIORITY);
+        this.totalOperations = untilCompletion.totalOperations;
+        this.downloadedOperations = untilCompletion.downloadedOperations;
+        this.downloadedFraction = untilCompletion.downloadedFraction;
+    }
+    /**
+     * Returns download progress towards all data up until the specified priority being received.
+     *
+     * The returned {@link ProgressWithOperations} tracks the target amount of operations that need
+     * to be downloaded in total and how many of them have already been received.
+     */
+    untilPriority(priority) {
+        let total = 0;
+        let downloaded = 0;
+        for (const progress of Object.values(this.internal)) {
+            // Include higher-priority buckets, which are represented by lower numbers.
+            if (progress.priority <= priority) {
+                downloaded += progress.sinceLast;
+                total += progress.targetCount - progress.atLast;
+            }
+        }
+        let progress = total == 0 ? 0.0 : downloaded / total;
+        return {
+            totalOperations: total,
+            downloadedOperations: downloaded,
+            downloadedFraction: progress
+        };
+    }
+}

package/lib/db/crud/SyncStatus.d.ts

@@ -1,3 +1,4 @@
+import { InternalProgressInformation, SyncProgress } from './SyncProgress.js';
 export type SyncDataFlowStatus = Partial<{
     downloading: boolean;
     uploading: boolean;
@@ -12,6 +13,12 @@ export type SyncDataFlowStatus = Partial<{
      * Cleared on the next successful upload.
      */
     uploadError?: Error;
+    /**
+     * Internal information about how far we are downloading operations in buckets.
+     *
+     * Please use the {@link SyncStatus#downloadProgress} property to track sync progress.
+     */
+    downloadProgress: InternalProgressInformation | null;
 }>;
 export interface SyncPriorityStatus {
     priority: number;
@@ -77,6 +84,12 @@ export declare class SyncStatus {
          * Cleared on the next successful upload.
          */
         uploadError?: Error;
+        /**
+         * Internal information about how far we are downloading operations in buckets.
+         *
+         * Please use the {@link SyncStatus#downloadProgress} property to track sync progress.
+         */
+        downloadProgress: InternalProgressInformation | null;
     }>;
     /**
      * Provides sync status information for all bucket priorities, sorted by priority (highest first).
@@ -85,6 +98,13 @@ export declare class SyncStatus {
      * sorted with highest priorities (lower numbers) first.
      */
     get priorityStatusEntries(): SyncPriorityStatus[];
+    /**
+     * A realtime progress report on how many operations have been downloaded and
+     * how many are necessary in total to complete the next sync iteration.
+     *
+     * This field is only set when {@link SyncDataFlowStatus#downloading} is also true.
+     */
+    get downloadProgress(): SyncProgress | null;
     /**
      * Reports the sync status (a pair of {@link SyncStatus#hasSynced} and {@link SyncStatus#lastSyncedAt} fields)
      * for a specific bucket priority level.

package/lib/db/crud/SyncStatus.js

@@ -1,3 +1,4 @@
+import { SyncProgress } from './SyncProgress.js';
 export class SyncStatus {
     options;
     constructor(options) {
@@ -67,6 +68,19 @@ export class SyncStatus {
     get priorityStatusEntries() {
         return (this.options.priorityStatusEntries ?? []).slice().sort(SyncStatus.comparePriorities);
     }
+    /**
+     * A realtime progress report on how many operations have been downloaded and
+     * how many are necessary in total to complete the next sync iteration.
+     *
+     * This field is only set when {@link SyncDataFlowStatus#downloading} is also true.
+     */
+    get downloadProgress() {
+        const internalProgress = this.options.dataFlow?.downloadProgress;
+        if (internalProgress == null) {
+            return null;
+        }
+        return new SyncProgress(internalProgress);
+    }
     /**
      * Reports the sync status (a pair of {@link SyncStatus#hasSynced} and {@link SyncStatus#lastSyncedAt} fields)
      * for a specific bucket priority level.

package/lib/index.d.ts

@@ -18,6 +18,7 @@ export * from './client/sync/stream/AbstractRemote.js';
 export * from './client/sync/stream/AbstractStreamingSyncImplementation.js';
 export * from './client/sync/stream/streaming-sync-types.js';
 export { MAX_OP_ID } from './client/constants.js';
+export { ProgressWithOperations, SyncProgress } from './db/crud/SyncProgress.js';
 export * from './db/crud/SyncStatus.js';
 export * from './db/crud/UploadQueueStatus.js';
 export * from './db/schema/Schema.js';

package/lib/index.js

@@ -18,6 +18,7 @@ export * from './client/sync/stream/AbstractRemote.js';
 export * from './client/sync/stream/AbstractStreamingSyncImplementation.js';
 export * from './client/sync/stream/streaming-sync-types.js';
 export { MAX_OP_ID } from './client/constants.js';
+export { SyncProgress } from './db/crud/SyncProgress.js';
 export * from './db/crud/SyncStatus.js';
 export * from './db/crud/UploadQueueStatus.js';
 export * from './db/schema/Schema.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@powersync/common",
-  "version": "1.28.0",
+  "version": "1.29.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"