@powersync/common 0.0.0-dev-20250423120133 → 0.0.0-dev-20250526133243

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js

@@ -196,11 +196,12 @@ The next upload iteration will be delayed.`);
         if (this.abortController) {
             await this.disconnect();
         }
-        this.abortController = new AbortController();
+        const controller = new AbortController();
+        this.abortController = controller;
         this.streamingSyncPromise = this.streamingSync(this.abortController.signal, options);
         // Return a promise that resolves when the connection status is updated
         return new Promise((resolve) => {
-            const l = this.registerListener({
+            const disposer = this.registerListener({
                 statusUpdated: (update) => {
                     // This is triggered as soon as a connection is read from
                     if (typeof update.connected == 'undefined') {
@@ -209,12 +210,14 @@ The next upload iteration will be delayed.`);
                     }
                     if (update.connected == false) {
                         /**
-                         * This function does not reject if initial connect attempt failed
+                         * This function does not reject if initial connect attempt failed.
+                         * Connected can be false if the connection attempt was aborted or if the initial connection
+                         * attempt failed.
                          */
                         this.logger.warn('Initial connect attempt did not successfully connect to server');
                     }
+                    disposer();
                     resolve();
-                    l();
                 }
             });
         });
@@ -270,7 +273,8 @@ The next upload iteration will be delayed.`);
                 connected: false,
                 connecting: false,
                 dataFlow: {
-                    downloading: false
+                    downloading: false,
+                    downloadProgress: null
                 }
             });
         });
@@ -355,6 +359,9 @@ The next upload iteration will be delayed.`);
                 let validatedCheckpoint = null;
                 let appliedCheckpoint = null;
                 const clientId = await this.options.adapter.getClientId();
+                if (signal.aborted) {
+                    return;
+                }
                 this.logger.debug('Requesting stream from server');
                 const syncOptions = {
                     path: '/sync/stream',
@@ -409,6 +416,7 @@ The next upload iteration will be delayed.`);
                         bucketMap = newBuckets;
                         await this.options.adapter.removeBuckets([...bucketsToDelete]);
                         await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
+                        await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
                     }
                     else if (isStreamingSyncCheckpointComplete(line)) {
                         const result = await this.applyCheckpoint(targetCheckpoint, signal);
@@ -472,6 +480,7 @@ The next upload iteration will be delayed.`);
                             write_checkpoint: diff.write_checkpoint
                         };
                         targetCheckpoint = newCheckpoint;
+                        await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
                         bucketMap = new Map();
                         newBuckets.forEach((checksum, name) => bucketMap.set(name, {
                             name: checksum.bucket,
@@ -486,9 +495,22 @@ The next upload iteration will be delayed.`);
                     }
                     else if (isStreamingSyncData(line)) {
                         const { data } = line;
+                        const previousProgress = this.syncStatus.dataFlowStatus.downloadProgress;
+                        let updatedProgress = null;
+                        if (previousProgress) {
+                            updatedProgress = { ...previousProgress };
+                            const progressForBucket = updatedProgress[data.bucket];
+                            if (progressForBucket) {
+                                updatedProgress[data.bucket] = {
+                                    ...progressForBucket,
+                                    sinceLast: progressForBucket.sinceLast + data.data.length
+                                };
+                            }
+                        }
                         this.updateSyncStatus({
                             dataFlow: {
-                                downloading: true
+                                downloading: true,
+                                downloadProgress: updatedProgress
                             }
                         });
                         await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] });
@@ -498,6 +520,7 @@ The next upload iteration will be delayed.`);
                         if (remaining_seconds == 0) {
                             // Connection would be closed automatically right after this
                             this.logger.debug('Token expiring; reconnect');
+                            this.options.remote.invalidateCredentials();
                             /**
                              * For a rare case where the backend connector does not update the token
                              * (uses the same one), this should have some delay.
@@ -505,6 +528,12 @@ The next upload iteration will be delayed.`);
                             await this.delayRetry();
                             return;
                         }
+                        else if (remaining_seconds < 30) {
+                            this.logger.debug('Token will expire soon; reconnect');
+                            // Pre-emptively refresh the token
+                            this.options.remote.invalidateCredentials();
+                            return;
+                        }
                         this.triggerCrudUpload();
                     }
                     else {
@@ -536,6 +565,27 @@ The next upload iteration will be delayed.`);
             }
         });
     }
+    async updateSyncStatusForStartingCheckpoint(checkpoint) {
+        const localProgress = await this.options.adapter.getBucketOperationProgress();
+        const progress = {};
+        for (const bucket of checkpoint.buckets) {
+            const savedProgress = localProgress[bucket.bucket];
+            progress[bucket.bucket] = {
+                // The fallback priority doesn't matter here, but 3 is the one newer versions of the sync service
+                // will use by default.
+                priority: bucket.priority ?? 3,
+                atLast: savedProgress?.atLast ?? 0,
+                sinceLast: savedProgress?.sinceLast ?? 0,
+                targetCount: bucket.count ?? 0
+            };
+        }
+        this.updateSyncStatus({
+            dataFlow: {
+                downloading: true,
+                downloadProgress: progress
+            }
+        });
+    }
     async applyCheckpoint(checkpoint, abort) {
         let result = await this.options.adapter.syncLocalDatabase(checkpoint);
         const pending = this.pendingCrudUpload;
@@ -565,6 +615,7 @@ The next upload iteration will be delayed.`);
                 lastSyncedAt: new Date(),
                 dataFlow: {
                     downloading: false,
+                    downloadProgress: null,
                     downloadError: undefined
                 }
             });

package/lib/client/sync/stream/streaming-sync-types.d.ts

@@ -79,7 +79,7 @@ export interface StreamingSyncCheckpointDiff {
         last_op_id: OpId;
         updated_buckets: BucketChecksum[];
         removed_buckets: string[];
-        write_checkpoint: string;
+        write_checkpoint?: string;
     };
 }
 export interface StreamingSyncDataJSON {

package/lib/db/crud/SyncProgress.d.ts

@@ -0,0 +1,72 @@
+/** @internal */
+export type InternalProgressInformation = Record<string, {
+    priority: number;
+    atLast: number;
+    sinceLast: number;
+    targetCount: number;
+}>;
+/**
+ * @internal The priority used by the core extension to indicate that a full sync was completed.
+ */
+export declare const FULL_SYNC_PRIORITY = 2147483647;
+/**
+ * Information about a progressing download made by the PowerSync SDK.
+ *
+ * To obtain these values, use {@link SyncProgress}, available through
+ * {@link SyncStatus#downloadProgress}.
+ */
+export interface ProgressWithOperations {
+    /**
+     * The total amount of operations to download for the current sync iteration
+     * to complete.
+     */
+    totalOperations: number;
+    /**
+     * The amount of operations that have already been downloaded.
+     */
+    downloadedOperations: number;
+    /**
+     * Relative progress, as {@link downloadedOperations} of {@link totalOperations}.
+     *
+     * This will be a number between `0.0` and `1.0` (inclusive).
+     *
+     * When this number reaches `1.0`, all changes have been received from the sync service.
+     * Actually applying these changes happens before the `downloadProgress` field is cleared from
+     * {@link SyncStatus}, so progress can stay at `1.0` for a short while before completing.
+     */
+    downloadedFraction: number;
+}
+/**
+ * Provides realtime progress on how PowerSync is downloading rows.
+ *
+ * The progress until the next complete sync is available through the fields on {@link ProgressWithOperations},
+ * which this class implements.
+ * Additionally, the {@link SyncProgress.untilPriority} method can be used to otbain progress towards
+ * a specific priority (instead of the progress for the entire download).
+ *
+ * The reported progress always reflects the status towards the end of a sync iteration (after
+ * which a consistent snapshot of all buckets is available locally).
+ *
+ * In rare cases (in particular, when a [compacting](https://docs.powersync.com/usage/lifecycle-maintenance/compacting-buckets)
+ * operation takes place between syncs), it's possible for the returned numbers to be slightly
+ * inaccurate. For this reason, {@link SyncProgress} should be seen as an approximation of progress.
+ * The information returned is good enough to build progress bars, but not exact enough to track
+ * individual download counts.
+ *
+ * Also note that data is downloaded in bulk, which means that individual counters are unlikely
+ * to be updated one-by-one.
+ */
+export declare class SyncProgress implements ProgressWithOperations {
+    protected internal: InternalProgressInformation;
+    totalOperations: number;
+    downloadedOperations: number;
+    downloadedFraction: number;
+    constructor(internal: InternalProgressInformation);
+    /**
+     * Returns download progress towards all data up until the specified priority being received.
+     *
+     * The returned {@link ProgressWithOperations} tracks the target amount of operations that need
+     * to be downloaded in total and how many of them have already been received.
+     */
+    untilPriority(priority: number): ProgressWithOperations;
+}

package/lib/db/crud/SyncProgress.js

@@ -0,0 +1,60 @@
+/**
+ * @internal The priority used by the core extension to indicate that a full sync was completed.
+ */
+export const FULL_SYNC_PRIORITY = 2147483647;
+/**
+ * Provides realtime progress on how PowerSync is downloading rows.
+ *
+ * The progress until the next complete sync is available through the fields on {@link ProgressWithOperations},
+ * which this class implements.
+ * Additionally, the {@link SyncProgress.untilPriority} method can be used to otbain progress towards
+ * a specific priority (instead of the progress for the entire download).
+ *
+ * The reported progress always reflects the status towards the end of a sync iteration (after
+ * which a consistent snapshot of all buckets is available locally).
+ *
+ * In rare cases (in particular, when a [compacting](https://docs.powersync.com/usage/lifecycle-maintenance/compacting-buckets)
+ * operation takes place between syncs), it's possible for the returned numbers to be slightly
+ * inaccurate. For this reason, {@link SyncProgress} should be seen as an approximation of progress.
+ * The information returned is good enough to build progress bars, but not exact enough to track
+ * individual download counts.
+ *
+ * Also note that data is downloaded in bulk, which means that individual counters are unlikely
+ * to be updated one-by-one.
+ */
+export class SyncProgress {
+    internal;
+    totalOperations;
+    downloadedOperations;
+    downloadedFraction;
+    constructor(internal) {
+        this.internal = internal;
+        const untilCompletion = this.untilPriority(FULL_SYNC_PRIORITY);
+        this.totalOperations = untilCompletion.totalOperations;
+        this.downloadedOperations = untilCompletion.downloadedOperations;
+        this.downloadedFraction = untilCompletion.downloadedFraction;
+    }
+    /**
+     * Returns download progress towards all data up until the specified priority being received.
+     *
+     * The returned {@link ProgressWithOperations} tracks the target amount of operations that need
+     * to be downloaded in total and how many of them have already been received.
+     */
+    untilPriority(priority) {
+        let total = 0;
+        let downloaded = 0;
+        for (const progress of Object.values(this.internal)) {
+            // Include higher-priority buckets, which are represented by lower numbers.
+            if (progress.priority <= priority) {
+                downloaded += progress.sinceLast;
+                total += progress.targetCount - progress.atLast;
+            }
+        }
+        let progress = total == 0 ? 0.0 : downloaded / total;
+        return {
+            totalOperations: total,
+            downloadedOperations: downloaded,
+            downloadedFraction: progress
+        };
+    }
+}

package/lib/db/crud/SyncStatus.d.ts

@@ -1,3 +1,4 @@
+import { InternalProgressInformation, SyncProgress } from './SyncProgress.js';
 export type SyncDataFlowStatus = Partial<{
     downloading: boolean;
     uploading: boolean;
@@ -12,6 +13,12 @@ export type SyncDataFlowStatus = Partial<{
      * Cleared on the next successful upload.
      */
     uploadError?: Error;
+    /**
+     * Internal information about how far we are downloading operations in buckets.
+     *
+     * Please use the {@link SyncStatus#downloadProgress} property to track sync progress.
+     */
+    downloadProgress: InternalProgressInformation | null;
 }>;
 export interface SyncPriorityStatus {
     priority: number;
@@ -77,6 +84,12 @@ export declare class SyncStatus {
          * Cleared on the next successful upload.
          */
         uploadError?: Error;
+        /**
+         * Internal information about how far we are downloading operations in buckets.
+         *
+         * Please use the {@link SyncStatus#downloadProgress} property to track sync progress.
+         */
+        downloadProgress: InternalProgressInformation | null;
     }>;
     /**
      * Provides sync status information for all bucket priorities, sorted by priority (highest first).
@@ -85,6 +98,13 @@ export declare class SyncStatus {
      * sorted with highest priorities (lower numbers) first.
      */
     get priorityStatusEntries(): SyncPriorityStatus[];
+    /**
+     * A realtime progress report on how many operations have been downloaded and
+     * how many are necessary in total to complete the next sync iteration.
+     *
+     * This field is only set when {@link SyncDataFlowStatus#downloading} is also true.
+     */
+    get downloadProgress(): SyncProgress | null;
     /**
      * Reports the sync status (a pair of {@link SyncStatus#hasSynced} and {@link SyncStatus#lastSyncedAt} fields)
      * for a specific bucket priority level.

package/lib/db/crud/SyncStatus.js

@@ -1,3 +1,4 @@
+import { SyncProgress } from './SyncProgress.js';
 export class SyncStatus {
     options;
     constructor(options) {
@@ -67,6 +68,19 @@ export class SyncStatus {
     get priorityStatusEntries() {
         return (this.options.priorityStatusEntries ?? []).slice().sort(SyncStatus.comparePriorities);
     }
+    /**
+     * A realtime progress report on how many operations have been downloaded and
+     * how many are necessary in total to complete the next sync iteration.
+     *
+     * This field is only set when {@link SyncDataFlowStatus#downloading} is also true.
+     */
+    get downloadProgress() {
+        const internalProgress = this.options.dataFlow?.downloadProgress;
+        if (internalProgress == null) {
+            return null;
+        }
+        return new SyncProgress(internalProgress);
+    }
     /**
      * Reports the sync status (a pair of {@link SyncStatus#hasSynced} and {@link SyncStatus#lastSyncedAt} fields)
      * for a specific bucket priority level.

package/lib/db/schema/Schema.d.ts

@@ -18,6 +18,10 @@ export declare class Schema<S extends SchemaType = SchemaType> {
             view_name: string;
             local_only: boolean;
             insert_only: boolean;
+            include_old: any;
+            include_old_only_when_changed: boolean;
+            include_metadata: boolean;
+            ignore_empty_update: boolean;
             columns: {
                 name: string;
                 type: import("./Column.js").ColumnType | undefined;

package/lib/db/schema/Schema.js

@@ -1,4 +1,3 @@
-import { Table } from './Table.js';
 /**
  * A schema is a collection of tables. It is used to define the structure of a database.
  */
@@ -41,15 +40,7 @@ export class Schema {
     }
     convertToClassicTables(props) {
         return Object.entries(props).map(([name, table]) => {
-            const convertedTable = new Table({
-                name,
-                columns: table.columns,
-                indexes: table.indexes,
-                localOnly: table.localOnly,
-                insertOnly: table.insertOnly,
-                viewName: table.viewNameOverride || name
-            });
-            return convertedTable;
+            return table.copyWithName(name);
         });
     }
 }

package/lib/db/schema/Table.d.ts

@@ -1,16 +1,32 @@
 import { Column, ColumnsType, ColumnType, ExtractColumnValueType } from './Column.js';
 import { Index } from './Index.js';
 import { TableV2 } from './TableV2.js';
-export interface TableOptions {
+interface SharedTableOptions {
+    localOnly?: boolean;
+    insertOnly?: boolean;
+    viewName?: string;
+    trackPrevious?: boolean | TrackPreviousOptions;
+    trackMetadata?: boolean;
+    ignoreEmptyUpdates?: boolean;
+}
+/** Whether to include previous column values when PowerSync tracks local changes.
+ *
+ * Including old values may be helpful for some backend connector implementations, which is
+ * why it can be enabled on per-table or per-columm basis.
+ */
+export interface TrackPreviousOptions {
+    /** When defined, a list of column names for which old values should be tracked. */
+    columns?: string[];
+    /** When enabled, only include values that have actually been changed by an update. */
+    onlyWhenChanged?: boolean;
+}
+export interface TableOptions extends SharedTableOptions {
     /**
      * The synced table name, matching sync rules
      */
     name: string;
     columns: Column[];
     indexes?: Index[];
-    localOnly?: boolean;
-    insertOnly?: boolean;
-    viewName?: string;
 }
 export type RowType<T extends TableV2<any>> = {
     [K in keyof T['columnMap']]: ExtractColumnValueType<T['columnMap'][K]>;
@@ -18,16 +34,16 @@ export type RowType<T extends TableV2<any>> = {
     id: string;
 };
 export type IndexShorthand = Record<string, string[]>;
-export interface TableV2Options {
+export interface TableV2Options extends SharedTableOptions {
     indexes?: IndexShorthand;
-    localOnly?: boolean;
-    insertOnly?: boolean;
-    viewName?: string;
 }
 export declare const DEFAULT_TABLE_OPTIONS: {
     indexes: never[];
     insertOnly: boolean;
     localOnly: boolean;
+    trackPrevious: boolean;
+    trackMetadata: boolean;
+    ignoreEmptyUpdates: boolean;
 };
 export declare const InvalidSQLCharacters: RegExp;
 export declare class Table<Columns extends ColumnsType = ColumnsType> {
@@ -96,9 +112,11 @@ export declare class Table<Columns extends ColumnsType = ColumnsType> {
      *```
      */
     constructor(options: TableOptions);
+    copyWithName(name: string): Table;
     private isTableV1;
     private initTableV1;
     private initTableV2;
+    private applyDefaultOptions;
     get name(): string;
     get viewNameOverride(): string | undefined;
     get viewName(): string;
@@ -107,6 +125,9 @@ export declare class Table<Columns extends ColumnsType = ColumnsType> {
     get indexes(): Index[];
     get localOnly(): boolean;
     get insertOnly(): boolean;
+    get trackPrevious(): boolean | TrackPreviousOptions;
+    get trackMetadata(): boolean;
+    get ignoreEmptyUpdates(): boolean;
     get internalName(): string;
     get validName(): boolean;
     validate(): void;
@@ -115,6 +136,10 @@ export declare class Table<Columns extends ColumnsType = ColumnsType> {
         view_name: string;
         local_only: boolean;
         insert_only: boolean;
+        include_old: any;
+        include_old_only_when_changed: boolean;
+        include_metadata: boolean;
+        ignore_empty_update: boolean;
         columns: {
             name: string;
             type: ColumnType | undefined;
@@ -129,3 +154,4 @@ export declare class Table<Columns extends ColumnsType = ColumnsType> {
         }[];
     };
 }
+export {};

package/lib/db/schema/Table.js

@@ -4,7 +4,10 @@ import { IndexedColumn } from './IndexedColumn.js';
 export const DEFAULT_TABLE_OPTIONS = {
     indexes: [],
     insertOnly: false,
-    localOnly: false
+    localOnly: false,
+    trackPrevious: false,
+    trackMetadata: false,
+    ignoreEmptyUpdates: false
 };
 export const InvalidSQLCharacters = /["'%,.#\s[\]]/;
 export class Table {
@@ -41,16 +44,21 @@ export class Table {
             this.initTableV2(optionsOrColumns, v2Options);
         }
     }
+    copyWithName(name) {
+        return new Table({
+            ...this.options,
+            name
+        });
+    }
     isTableV1(arg) {
         return 'columns' in arg && Array.isArray(arg.columns);
     }
     initTableV1(options) {
         this.options = {
             ...options,
-            indexes: options.indexes || [],
-            insertOnly: options.insertOnly ?? DEFAULT_TABLE_OPTIONS.insertOnly,
-            localOnly: options.localOnly ?? DEFAULT_TABLE_OPTIONS.localOnly
+            indexes: options.indexes || []
         };
+        this.applyDefaultOptions();
     }
     initTableV2(columns, options) {
         const convertedColumns = Object.entries(columns).map(([name, columnInfo]) => new Column({ name, type: columnInfo.type }));
@@ -65,12 +73,23 @@ export class Table {
             name: '',
             columns: convertedColumns,
             indexes: convertedIndexes,
-            insertOnly: options?.insertOnly ?? DEFAULT_TABLE_OPTIONS.insertOnly,
-            localOnly: options?.localOnly ?? DEFAULT_TABLE_OPTIONS.localOnly,
-            viewName: options?.viewName
+            viewName: options?.viewName,
+            insertOnly: options?.insertOnly,
+            localOnly: options?.localOnly,
+            trackPrevious: options?.trackPrevious,
+            trackMetadata: options?.trackMetadata,
+            ignoreEmptyUpdates: options?.ignoreEmptyUpdates
         };
+        this.applyDefaultOptions();
         this._mappedColumns = columns;
     }
+    applyDefaultOptions() {
+        this.options.insertOnly ??= DEFAULT_TABLE_OPTIONS.insertOnly;
+        this.options.localOnly ??= DEFAULT_TABLE_OPTIONS.localOnly;
+        this.options.trackPrevious ??= DEFAULT_TABLE_OPTIONS.trackPrevious;
+        this.options.trackMetadata ??= DEFAULT_TABLE_OPTIONS.trackMetadata;
+        this.options.ignoreEmptyUpdates ??= DEFAULT_TABLE_OPTIONS.ignoreEmptyUpdates;
+    }
     get name() {
         return this.options.name;
     }
@@ -94,10 +113,19 @@ export class Table {
         return this.options.indexes ?? [];
     }
     get localOnly() {
-        return this.options.localOnly ?? false;
+        return this.options.localOnly;
     }
     get insertOnly() {
-        return this.options.insertOnly ?? false;
+        return this.options.insertOnly;
+    }
+    get trackPrevious() {
+        return this.options.trackPrevious;
+    }
+    get trackMetadata() {
+        return this.options.trackMetadata;
+    }
+    get ignoreEmptyUpdates() {
+        return this.options.ignoreEmptyUpdates;
     }
     get internalName() {
         if (this.options.localOnly) {
@@ -124,6 +152,12 @@ export class Table {
         if (this.columns.length > MAX_AMOUNT_OF_COLUMNS) {
             throw new Error(`Table has too many columns. The maximum number of columns is ${MAX_AMOUNT_OF_COLUMNS}.`);
         }
+        if (this.trackMetadata && this.localOnly) {
+            throw new Error(`Can't include metadata for local-only tables.`);
+        }
+        if (this.trackPrevious != false && this.localOnly) {
+            throw new Error(`Can't include old values for local-only tables.`);
+        }
         const columnNames = new Set();
         columnNames.add('id');
         for (const column of this.columns) {
@@ -156,11 +190,16 @@ export class Table {
         }
     }
     toJSON() {
+        const trackPrevious = this.trackPrevious;
         return {
             name: this.name,
             view_name: this.viewName,
             local_only: this.localOnly,
             insert_only: this.insertOnly,
+            include_old: trackPrevious && (trackPrevious.columns ?? true),
+            include_old_only_when_changed: typeof trackPrevious == 'object' && trackPrevious.onlyWhenChanged == true,
+            include_metadata: this.trackMetadata,
+            ignore_empty_update: this.ignoreEmptyUpdates,
             columns: this.columns.map((c) => c.toJSON()),
             indexes: this.indexes.map((e) => e.toJSON(this))
         };

package/lib/index.d.ts

@@ -18,6 +18,7 @@ export * from './client/sync/stream/AbstractRemote.js';
 export * from './client/sync/stream/AbstractStreamingSyncImplementation.js';
 export * from './client/sync/stream/streaming-sync-types.js';
 export { MAX_OP_ID } from './client/constants.js';
+export { ProgressWithOperations, SyncProgress } from './db/crud/SyncProgress.js';
 export * from './db/crud/SyncStatus.js';
 export * from './db/crud/UploadQueueStatus.js';
 export * from './db/schema/Schema.js';

package/lib/index.js

@@ -18,6 +18,7 @@ export * from './client/sync/stream/AbstractRemote.js';
 export * from './client/sync/stream/AbstractStreamingSyncImplementation.js';
 export * from './client/sync/stream/streaming-sync-types.js';
 export { MAX_OP_ID } from './client/constants.js';
+export { SyncProgress } from './db/crud/SyncProgress.js';
 export * from './db/crud/SyncStatus.js';
 export * from './db/crud/UploadQueueStatus.js';
 export * from './db/schema/Schema.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@powersync/common",
-  "version": "0.0.0-dev-20250423120133",
+  "version": "0.0.0-dev-20250526133243",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"