@powersync/common 1.33.1 → 1.34.0

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js

@@ -3,7 +3,7 @@ import { SyncStatus } from '../../../db/crud/SyncStatus.js';
 import { FULL_SYNC_PRIORITY } from '../../../db/crud/SyncProgress.js';
 import { AbortOperation } from '../../../utils/AbortOperation.js';
 import { BaseObserver } from '../../../utils/BaseObserver.js';
-import { onAbortPromise, throttleLeadingTrailing } from '../../../utils/async.js';
+import { throttleLeadingTrailing } from '../../../utils/async.js';
 import { PowerSyncControlCommand } from '../bucket/BucketStorageAdapter.js';
 import { SyncDataBucket } from '../bucket/SyncDataBucket.js';
 import { FetchStrategy } from './AbstractRemote.js';
@@ -65,14 +65,14 @@ export const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
 export const DEFAULT_RETRY_DELAY_MS = 5000;
 export const DEFAULT_STREAMING_SYNC_OPTIONS = {
     retryDelayMs: DEFAULT_RETRY_DELAY_MS,
-    logger: Logger.get('PowerSyncStream'),
     crudUploadThrottleMs: DEFAULT_CRUD_UPLOAD_THROTTLE_MS
 };
 export const DEFAULT_STREAM_CONNECTION_OPTIONS = {
     connectionMethod: SyncStreamConnectionMethod.WEB_SOCKET,
     clientImplementation: DEFAULT_SYNC_CLIENT_IMPLEMENTATION,
     fetchStrategy: FetchStrategy.Buffered,
-    params: {}
+    params: {},
+    serializedSchema: undefined
 };
 // The priority we assume when we receive checkpoint lines where no priority is set.
 // This is the default priority used by the sync service, but can be set to an arbitrary
@@ -85,13 +85,15 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
     abortController;
     crudUpdateListener;
     streamingSyncPromise;
-    pendingCrudUpload;
+    logger;
+    isUploadingCrud = false;
     notifyCompletedUploads;
     syncStatus;
     triggerCrudUpload;
     constructor(options) {
         super();
         this.options = { ...DEFAULT_STREAMING_SYNC_OPTIONS, ...options };
+        this.logger = options.logger ?? Logger.get('PowerSyncStream');
         this.syncStatus = new SyncStatus({
             connected: false,
             connecting: false,
@@ -103,15 +105,13 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
         });
         this.abortController = null;
         this.triggerCrudUpload = throttleLeadingTrailing(() => {
-            if (!this.syncStatus.connected || this.pendingCrudUpload != null) {
+            if (!this.syncStatus.connected || this.isUploadingCrud) {
                 return;
             }
-            this.pendingCrudUpload = new Promise((resolve) => {
-                this._uploadAllCrud().finally(() => {
-                    this.notifyCompletedUploads?.();
-                    this.pendingCrudUpload = undefined;
-                    resolve();
-                });
+            this.isUploadingCrud = true;
+            this._uploadAllCrud().finally(() => {
+                this.notifyCompletedUploads?.();
+                this.isUploadingCrud = false;
             });
         }, this.options.crudUploadThrottleMs);
     }
@@ -157,9 +157,6 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
     get isConnected() {
         return this.syncStatus.connected;
     }
-    get logger() {
-        return this.options.logger;
-    }
     async dispose() {
         this.crudUpdateListener?.();
         this.crudUpdateListener = undefined;
@@ -171,7 +168,9 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
         const clientId = await this.options.adapter.getClientId();
         let path = `/write-checkpoint2.json?client_id=${clientId}`;
         const response = await this.options.remote.get(path);
-        return response['data']['write_checkpoint'];
+        const checkpoint = response['data']['write_checkpoint'];
+        this.logger.debug(`Created write checkpoint: ${checkpoint}`);
+        return checkpoint;
     }
     async _uploadAllCrud() {
         return this.obtainLock({
@@ -210,7 +209,11 @@ The next upload iteration will be delayed.`);
                         }
                         else {
                             // Uploading is completed
-                            await this.options.adapter.updateLocalTarget(() => this.getWriteCheckpoint());
+                            const neededUpdate = await this.options.adapter.updateLocalTarget(() => this.getWriteCheckpoint());
+                            if (neededUpdate == false && checkedCrudItem != null) {
+                                // Only log this if there was something to upload
+                                this.logger.debug('Upload complete, no write checkpoint needed.');
+                            }
                             break;
                         }
                     }
@@ -361,6 +364,7 @@ The next upload iteration will be delayed.`);
                 });
             }
             finally {
+                this.notifyCompletedUploads = undefined;
                 if (!signal.aborted) {
                     nestedAbortController.abort(new AbortOperation('Closing sync stream network requests before retry.'));
                     nestedAbortController = new AbortController();
@@ -435,13 +439,15 @@ The next upload iteration will be delayed.`);
         });
     }
     async legacyStreamingSyncIteration(signal, resolvedOptions) {
+        if (resolvedOptions.serializedSchema?.raw_tables != null) {
+            this.logger.warn('Raw tables require the Rust-based sync client. The JS client will ignore them.');
+        }
         this.logger.debug('Streaming sync iteration started');
         this.options.adapter.startSession();
         let [req, bucketMap] = await this.collectLocalBucketState();
-        // These are compared by reference
         let targetCheckpoint = null;
-        let validatedCheckpoint = null;
-        let appliedCheckpoint = null;
+        // A checkpoint that has been validated but not applied (e.g. due to pending local writes)
+        let pendingValidatedCheckpoint = null;
         const clientId = await this.options.adapter.getClientId();
         const usingFixedKeyFormat = await this.requireKeyFormat(false);
         this.logger.debug('Requesting stream from server');
@@ -458,21 +464,55 @@ The next upload iteration will be delayed.`);
         };
         let stream;
         if (resolvedOptions?.connectionMethod == SyncStreamConnectionMethod.HTTP) {
-            stream = await this.options.remote.postStream(syncOptions);
+            stream = await this.options.remote.postStreamRaw(syncOptions, (line) => {
+                if (typeof line == 'string') {
+                    return JSON.parse(line);
+                }
+                else {
+                    // Directly enqueued by us
+                    return line;
+                }
+            });
         }
         else {
-            stream = await this.options.remote.socketStream({
+            const bson = await this.options.remote.getBSON();
+            stream = await this.options.remote.socketStreamRaw({
                 ...syncOptions,
                 ...{ fetchStrategy: resolvedOptions.fetchStrategy }
-            });
+            }, (payload) => {
+                if (payload instanceof Uint8Array) {
+                    return bson.deserialize(payload);
+                }
+                else {
+                    // Directly enqueued by us
+                    return payload;
+                }
+            }, bson);
         }
         this.logger.debug('Stream established. Processing events');
+        this.notifyCompletedUploads = () => {
+            if (!stream.closed) {
+                stream.enqueueData({ crud_upload_completed: null });
+            }
+        };
         while (!stream.closed) {
             const line = await stream.read();
             if (!line) {
                 // The stream has closed while waiting
                 return;
             }
+            if ('crud_upload_completed' in line) {
+                if (pendingValidatedCheckpoint != null) {
+                    const { applied, endIteration } = await this.applyCheckpoint(pendingValidatedCheckpoint);
+                    if (applied) {
+                        pendingValidatedCheckpoint = null;
+                    }
+                    else if (endIteration) {
+                        break;
+                    }
+                }
+                continue;
+            }
             // A connection is active and messages are being received
             if (!this.syncStatus.connected) {
                 // There is a connection now
@@ -483,6 +523,8 @@ The next upload iteration will be delayed.`);
             }
             if (isStreamingSyncCheckpoint(line)) {
                 targetCheckpoint = line.checkpoint;
+                // New checkpoint - existing validated checkpoint is no longer valid
+                pendingValidatedCheckpoint = null;
                 const bucketsToDelete = new Set(bucketMap.keys());
                 const newBuckets = new Map();
                 for (const checksum of line.checkpoint.buckets) {
@@ -501,14 +543,20 @@ The next upload iteration will be delayed.`);
                 await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
             }
             else if (isStreamingSyncCheckpointComplete(line)) {
-                const result = await this.applyCheckpoint(targetCheckpoint, signal);
+                const result = await this.applyCheckpoint(targetCheckpoint);
                 if (result.endIteration) {
                     return;
                 }
-                else if (result.applied) {
-                    appliedCheckpoint = targetCheckpoint;
+                else if (!result.applied) {
+                    // "Could not apply checkpoint due to local data". We need to retry after
+                    // finishing uploads.
+                    pendingValidatedCheckpoint = targetCheckpoint;
+                }
+                else {
+                    // Nothing to retry later. This would likely already be null from the last
+                    // checksum or checksum_diff operation, but we make sure.
+                    pendingValidatedCheckpoint = null;
                 }
-                validatedCheckpoint = targetCheckpoint;
             }
             else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
                 const priority = line.partial_checkpoint_complete.priority;
@@ -545,6 +593,8 @@ The next upload iteration will be delayed.`);
                 if (targetCheckpoint == null) {
                     throw new Error('Checkpoint diff without previous checkpoint');
                 }
+                // New checkpoint - existing validated checkpoint is no longer valid
+                pendingValidatedCheckpoint = null;
                 const diff = line.checkpoint_diff;
                 const newBuckets = new Map();
                 for (const checksum of targetCheckpoint.buckets) {
@@ -618,26 +668,7 @@ The next upload iteration will be delayed.`);
                 this.triggerCrudUpload();
             }
             else {
-                this.logger.debug('Sync complete');
-                if (targetCheckpoint === appliedCheckpoint) {
-                    this.updateSyncStatus({
-                        connected: true,
-                        lastSyncedAt: new Date(),
-                        priorityStatusEntries: [],
-                        dataFlow: {
-                            downloadError: undefined
-                        }
-                    });
-                }
-                else if (validatedCheckpoint === targetCheckpoint) {
-                    const result = await this.applyCheckpoint(targetCheckpoint, signal);
-                    if (result.endIteration) {
-                        return;
-                    }
-                    else if (result.applied) {
-                        appliedCheckpoint = targetCheckpoint;
-                    }
-                }
+                this.logger.debug('Received unknown sync line', line);
             }
         }
         this.logger.debug('Stream input empty');
@@ -803,9 +834,11 @@ The next upload iteration will be delayed.`);
             }
         }
         try {
-            await control(PowerSyncControlCommand.START, JSON.stringify({
-                parameters: resolvedOptions.params
-            }));
+            const options = { parameters: resolvedOptions.params };
+            if (resolvedOptions.serializedSchema) {
+                options.schema = resolvedOptions.serializedSchema;
+            }
+            await control(PowerSyncControlCommand.START, JSON.stringify(options));
             this.notifyCompletedUploads = () => {
                 controlInvocations?.enqueueData({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
             };
@@ -853,45 +886,30 @@ The next upload iteration will be delayed.`);
             }
         });
     }
-    async applyCheckpoint(checkpoint, abort) {
+    async applyCheckpoint(checkpoint) {
         let result = await this.options.adapter.syncLocalDatabase(checkpoint);
-        const pending = this.pendingCrudUpload;
         if (!result.checkpointValid) {
-            this.logger.debug('Checksum mismatch in checkpoint, will reconnect');
+            this.logger.debug(`Checksum mismatch in checkpoint ${checkpoint.last_op_id}, will reconnect`);
             // This means checksums failed. Start again with a new checkpoint.
             // TODO: better back-off
             await new Promise((resolve) => setTimeout(resolve, 50));
             return { applied: false, endIteration: true };
         }
-        else if (!result.ready && pending != null) {
-            // We have pending entries in the local upload queue or are waiting to confirm a write
-            // checkpoint, which prevented this checkpoint from applying. Wait for that to complete and
-            // try again.
-            this.logger.debug('Could not apply checkpoint due to local data. Waiting for in-progress upload before retrying.');
-            await Promise.race([pending, onAbortPromise(abort)]);
-            if (abort.aborted) {
-                return { applied: false, endIteration: true };
-            }
-            // Try again now that uploads have completed.
-            result = await this.options.adapter.syncLocalDatabase(checkpoint);
-        }
-        if (result.checkpointValid && result.ready) {
-            this.logger.debug('validated checkpoint', checkpoint);
-            this.updateSyncStatus({
-                connected: true,
-                lastSyncedAt: new Date(),
-                dataFlow: {
-                    downloading: false,
-                    downloadProgress: null,
-                    downloadError: undefined
-                }
-            });
-            return { applied: true, endIteration: false };
-        }
-        else {
-            this.logger.debug('Could not apply checkpoint. Waiting for next sync complete line.');
+        else if (!result.ready) {
+            this.logger.debug(`Could not apply checkpoint ${checkpoint.last_op_id} due to local data. We will retry applying the checkpoint after that upload is completed.`);
             return { applied: false, endIteration: false };
         }
+        this.logger.debug(`Applied checkpoint ${checkpoint.last_op_id}`, checkpoint);
+        this.updateSyncStatus({
+            connected: true,
+            lastSyncedAt: new Date(),
+            dataFlow: {
+                downloading: false,
+                downloadProgress: null,
+                downloadError: undefined
+            }
+        });
+        return { applied: true, endIteration: false };
     }
     updateSyncStatus(options) {
         const updatedStatus = new SyncStatus({

package/lib/client/sync/stream/streaming-sync-types.d.ts

@@ -101,6 +101,10 @@ export interface StreamingSyncKeepalive {
     token_expires_in: number;
 }
 export type StreamingSyncLine = StreamingSyncDataJSON | StreamingSyncCheckpoint | StreamingSyncCheckpointDiff | StreamingSyncCheckpointComplete | StreamingSyncCheckpointPartiallyComplete | StreamingSyncKeepalive;
+export type CrudUploadNotification = {
+    crud_upload_completed: null;
+};
+export type StreamingSyncLineOrCrudUploadComplete = StreamingSyncLine | CrudUploadNotification;
 export interface BucketRequest {
     name: string;
     /**

package/lib/db/crud/SyncProgress.d.ts

@@ -1,4 +1,4 @@
-import { BucketProgress } from 'src/client/sync/stream/core-instruction.js';
+import type { BucketProgress } from '../../client/sync/stream/core-instruction.js';
 /** @internal */
 export type InternalProgressInformation = Record<string, BucketProgress>;
 /**

package/lib/db/schema/RawTable.d.ts

@@ -0,0 +1,61 @@
+/**
+ * A pending variant of a {@link RawTable} that doesn't have a name (because it would be inferred when creating the
+ * schema).
+ */
+export type RawTableType = {
+    /**
+     * The statement to run when PowerSync detects that a row needs to be inserted or updated.
+     */
+    put: PendingStatement;
+    /**
+     * The statement to run when PowerSync detects that a row needs to be deleted.
+     */
+    delete: PendingStatement;
+};
+/**
+ * A parameter to use as part of {@link PendingStatement}.
+ *
+ * For delete statements, only the `"Id"` value is supported - the sync client will replace it with the id of the row to
+ * be synced.
+ *
+ * For insert and replace operations, the values of columns in the table are available as parameters through
+ * `{Column: 'name'}`.
+ */
+export type PendingStatementParameter = 'Id' | {
+    Column: string;
+};
+/**
+ * A statement that the PowerSync client should use to insert or delete data into a table managed by the user.
+ */
+export type PendingStatement = {
+    sql: string;
+    params: PendingStatementParameter[];
+};
+/**
+ * Instructs PowerSync to sync data into a "raw" table.
+ *
+ * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
+ * using client-side table and column constraints.
+ *
+ * To collect local writes to raw tables with PowerSync, custom triggers are required. See
+ * {@link https://docs.powersync.com/usage/use-case-examples/raw-tables the documentation} for details and an example on
+ * using raw tables.
+ *
+ * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
+ *
+ * @experimental Please note that this feature is experimental at the moment, and not covered by PowerSync semver or
+ * stability guarantees.
+ */
+export declare class RawTable implements RawTableType {
+    /**
+     * The name of the table.
+     *
+     * This does not have to match the actual table name in the schema - {@link put} and {@link delete} are free to use
+     * another table. Instead, this name is used by the sync client to recognize that operations on this table (as it
+     * appears in the source / backend database) are to be handled specially.
+     */
+    name: string;
+    put: PendingStatement;
+    delete: PendingStatement;
+    constructor(name: string, type: RawTableType);
+}

package/lib/db/schema/RawTable.js

@@ -0,0 +1,32 @@
+/**
+ * Instructs PowerSync to sync data into a "raw" table.
+ *
+ * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
+ * using client-side table and column constraints.
+ *
+ * To collect local writes to raw tables with PowerSync, custom triggers are required. See
+ * {@link https://docs.powersync.com/usage/use-case-examples/raw-tables the documentation} for details and an example on
+ * using raw tables.
+ *
+ * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
+ *
+ * @experimental Please note that this feature is experimental at the moment, and not covered by PowerSync semver or
+ * stability guarantees.
+ */
+export class RawTable {
+    /**
+     * The name of the table.
+     *
+     * This does not have to match the actual table name in the schema - {@link put} and {@link delete} are free to use
+     * another table. Instead, this name is used by the sync client to recognize that operations on this table (as it
+     * appears in the source / backend database) are to be handled specially.
+     */
+    name;
+    put;
+    delete;
+    constructor(name, type) {
+        this.name = name;
+        this.put = type.put;
+        this.delete = type.delete;
+    }
+}

package/lib/db/schema/Schema.d.ts

@@ -1,3 +1,4 @@
+import { RawTable, RawTableType } from './RawTable.js';
 import { RowType, Table } from './Table.js';
 type SchemaType = Record<string, Table<any>>;
 export type SchemaTableType<S extends SchemaType> = {
@@ -10,7 +11,19 @@ export declare class Schema<S extends SchemaType = SchemaType> {
     readonly types: SchemaTableType<S>;
     readonly props: S;
     readonly tables: Table[];
+    readonly rawTables: RawTable[];
     constructor(tables: Table[] | S);
+    /**
+     * Adds raw tables to this schema. Raw tables are identified by their name, but entirely managed by the application
+     * developer instead of automatically by PowerSync.
+     * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
+     * using client-side table and column constraints.
+     * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
+     *
+     * @param tables An object of (table name, raw table definition) entries.
+     * @experimental Note that the raw tables API is still experimental and may change in the future.
+     */
+    withRawTables(tables: Record<string, RawTableType>): void;
     validate(): void;
     toJSON(): {
         tables: {
@@ -35,6 +48,7 @@ export declare class Schema<S extends SchemaType = SchemaType> {
                 }[];
             }[];
         }[];
+        raw_tables: RawTable[];
     };
     private convertToClassicTables;
 }

package/lib/db/schema/Schema.js

@@ -1,3 +1,4 @@
+import { RawTable } from './RawTable.js';
 /**
  * A schema is a collection of tables. It is used to define the structure of a database.
  */
@@ -8,6 +9,7 @@ export class Schema {
     types;
     props;
     tables;
+    rawTables;
     constructor(tables) {
         if (Array.isArray(tables)) {
             /*
@@ -26,6 +28,22 @@ export class Schema {
             this.props = tables;
             this.tables = this.convertToClassicTables(this.props);
         }
+        this.rawTables = [];
+    }
+    /**
+     * Adds raw tables to this schema. Raw tables are identified by their name, but entirely managed by the application
+     * developer instead of automatically by PowerSync.
+     * Since raw tables are not backed by JSON, running complex queries on them may be more efficient. Further, they allow
+     * using client-side table and column constraints.
+     * Note that raw tables are only supported when using the new `SyncClientImplementation.rust` sync client.
+     *
+     * @param tables An object of (table name, raw table definition) entries.
+     * @experimental Note that the raw tables API is still experimental and may change in the future.
+     */
+    withRawTables(tables) {
+        for (const [name, rawTableDefinition] of Object.entries(tables)) {
+            this.rawTables.push(new RawTable(name, rawTableDefinition));
+        }
     }
     validate() {
         for (const table of this.tables) {
@@ -35,7 +53,8 @@ export class Schema {
     toJSON() {
         return {
             // This is required because "name" field is not present in TableV2
-            tables: this.tables.map((t) => t.toJSON())
+            tables: this.tables.map((t) => t.toJSON()),
+            raw_tables: this.rawTables
         };
     }
     convertToClassicTables(props) {

package/lib/utils/async.d.ts

@@ -12,4 +12,3 @@ export declare function throttleTrailing(func: () => void, wait: number): () =>
  * Roughly equivalent to lodash/throttle with {leading: true, trailing: true}
  */
 export declare function throttleLeadingTrailing(func: () => void, wait: number): () => void;
-export declare function onAbortPromise(signal: AbortSignal): Promise<void>;

package/lib/utils/async.js

@@ -43,13 +43,3 @@ export function throttleLeadingTrailing(func, wait) {
         }
     };
 }
-export function onAbortPromise(signal) {
-    return new Promise((resolve) => {
-        if (signal.aborted) {
-            resolve();
-        }
-        else {
-            signal.onabort = () => resolve();
-        }
-    });
-}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@powersync/common",
-  "version": "1.33.1",
+  "version": "1.34.0",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"