@powersync/common 1.32.0 → 1.33.1

package/lib/client/ConnectionManager.js

@@ -136,7 +136,6 @@ export class ConnectionManager extends BaseObserver {
         await this.disconnectingPromise;
         this.logger.debug('Attempting to connect to PowerSync instance');
         await this.syncStreamImplementation?.connect(appliedOptions);
-        this.syncStreamImplementation?.triggerCrudUpload();
     }
     /**
      * Close the sync connection.

package/lib/client/sync/bucket/BucketStorageAdapter.d.ts

@@ -51,17 +51,27 @@ export declare enum PSInternalTable {
     OPLOG = "ps_oplog",
     UNTYPED = "ps_untyped"
 }
+export declare enum PowerSyncControlCommand {
+    PROCESS_TEXT_LINE = "line_text",
+    PROCESS_BSON_LINE = "line_binary",
+    STOP = "stop",
+    START = "start",
+    NOTIFY_TOKEN_REFRESHED = "refreshed_token",
+    NOTIFY_CRUD_UPLOAD_COMPLETED = "completed_upload"
+}
 export interface BucketStorageListener extends BaseListener {
     crudUpdate: () => void;
 }
 export interface BucketStorageAdapter extends BaseObserver<BucketStorageListener>, Disposable {
     init(): Promise<void>;
-    saveSyncData(batch: SyncDataBatch): Promise<void>;
+    saveSyncData(batch: SyncDataBatch, fixedKeyFormat?: boolean): Promise<void>;
     removeBuckets(buckets: string[]): Promise<void>;
     setTargetCheckpoint(checkpoint: Checkpoint): Promise<void>;
     startSession(): void;
     getBucketStates(): Promise<BucketState[]>;
     getBucketOperationProgress(): Promise<BucketOperationProgress>;
+    hasMigratedSubkeys(): Promise<boolean>;
+    migrateToFixedSubkeys(): Promise<void>;
     syncLocalDatabase(checkpoint: Checkpoint, priority?: number): Promise<{
         checkpointValid: boolean;
         ready: boolean;
@@ -72,17 +82,13 @@ export interface BucketStorageAdapter extends BaseObserver<BucketStorageListener
     getCrudBatch(limit?: number): Promise<CrudBatch | null>;
     hasCompletedSync(): Promise<boolean>;
     updateLocalTarget(cb: () => Promise<string>): Promise<boolean>;
-    /**
-     * Exposed for tests only.
-     */
-    autoCompact(): Promise<void>;
-    /**
-     * Exposed for tests only.
-     */
-    forceCompact(): Promise<void>;
     getMaxOpId(): string;
     /**
      * Get an unique client id.
      */
     getClientId(): Promise<string>;
+    /**
+     * Invokes the `powersync_control` function for the sync client.
+     */
+    control(op: PowerSyncControlCommand, payload: string | Uint8Array | null): Promise<string>;
 }

package/lib/client/sync/bucket/BucketStorageAdapter.js

@@ -6,3 +6,12 @@ export var PSInternalTable;
     PSInternalTable["OPLOG"] = "ps_oplog";
     PSInternalTable["UNTYPED"] = "ps_untyped";
 })(PSInternalTable || (PSInternalTable = {}));
+export var PowerSyncControlCommand;
+(function (PowerSyncControlCommand) {
+    PowerSyncControlCommand["PROCESS_TEXT_LINE"] = "line_text";
+    PowerSyncControlCommand["PROCESS_BSON_LINE"] = "line_binary";
+    PowerSyncControlCommand["STOP"] = "stop";
+    PowerSyncControlCommand["START"] = "start";
+    PowerSyncControlCommand["NOTIFY_TOKEN_REFRESHED"] = "refreshed_token";
+    PowerSyncControlCommand["NOTIFY_CRUD_UPLOAD_COMPLETED"] = "completed_upload";
+})(PowerSyncControlCommand || (PowerSyncControlCommand = {}));

package/lib/client/sync/bucket/CrudEntry.d.ts

@@ -30,6 +30,8 @@ type CrudEntryOutputJSON = {
     id: string;
     tx_id?: number;
     data?: Record<string, any>;
+    old?: Record<string, any>;
+    metadata?: string;
 };
 /**
  * A single client-side change.

package/lib/client/sync/bucket/CrudEntry.js

@@ -74,7 +74,9 @@ export class CrudEntry {
             type: this.table,
             id: this.id,
             tx_id: this.transactionId,
-            data: this.opData
+            data: this.opData,
+            old: this.previousValues,
+            metadata: this.metadata
         };
     }
     equals(entry) {
@@ -93,6 +95,15 @@ export class CrudEntry {
      * Generates an array for use in deep comparison operations
      */
     toComparisonArray() {
-        return [this.transactionId, this.clientId, this.op, this.table, this.id, this.opData];
+        return [
+            this.transactionId,
+            this.clientId,
+            this.op,
+            this.table,
+            this.id,
+            this.opData,
+            this.previousValues,
+            this.metadata
+        ];
     }
 }

package/lib/client/sync/bucket/OplogEntry.d.ts

@@ -7,17 +7,17 @@ export interface OplogEntryJSON {
     object_type?: string;
     op_id: string;
     op: OpTypeJSON;
-    subkey?: string | object;
+    subkey?: string;
 }
 export declare class OplogEntry {
     op_id: OpId;
     op: OpType;
     checksum: number;
-    subkey: string;
+    subkey?: string | undefined;
     object_type?: string | undefined;
     object_id?: string | undefined;
     data?: string | undefined;
     static fromRow(row: OplogEntryJSON): OplogEntry;
-    constructor(op_id: OpId, op: OpType, checksum: number, subkey: string, object_type?: string | undefined, object_id?: string | undefined, data?: string | undefined);
-    toJSON(): OplogEntryJSON;
+    constructor(op_id: OpId, op: OpType, checksum: number, subkey?: string | undefined, object_type?: string | undefined, object_id?: string | undefined, data?: string | undefined);
+    toJSON(fixedKeyEncoding?: boolean): OplogEntryJSON;
 }

package/lib/client/sync/bucket/OplogEntry.js

@@ -8,7 +8,7 @@ export class OplogEntry {
     object_id;
     data;
     static fromRow(row) {
-        return new OplogEntry(row.op_id, OpType.fromJSON(row.op), row.checksum, typeof row.subkey == 'string' ? row.subkey : JSON.stringify(row.subkey), row.object_type, row.object_id, row.data);
+        return new OplogEntry(row.op_id, OpType.fromJSON(row.op), row.checksum, row.subkey, row.object_type, row.object_id, row.data);
     }
     constructor(op_id, op, checksum, subkey, object_type, object_id, data) {
         this.op_id = op_id;
@@ -19,7 +19,7 @@ export class OplogEntry {
         this.object_id = object_id;
         this.data = data;
     }
-    toJSON() {
+    toJSON(fixedKeyEncoding = false) {
         return {
             op_id: this.op_id,
             op: this.op.toJSON(),
@@ -27,7 +27,9 @@ export class OplogEntry {
             object_id: this.object_id,
             checksum: this.checksum,
             data: this.data,
-            subkey: JSON.stringify(this.subkey)
+            // Older versions of the JS SDK used to always JSON.stringify here. That has always been wrong,
+            // but we need to migrate gradually to not break existing databases.
+            subkey: fixedKeyEncoding ? this.subkey : JSON.stringify(this.subkey)
         };
     }
 }

package/lib/client/sync/bucket/SqliteBucketStorage.d.ts

@@ -2,7 +2,7 @@ import { Mutex } from 'async-mutex';
 import { ILogger } from 'js-logger';
 import { DBAdapter, Transaction } from '../../../db/DBAdapter.js';
 import { BaseObserver } from '../../../utils/BaseObserver.js';
-import { BucketOperationProgress, BucketState, BucketStorageAdapter, BucketStorageListener, Checkpoint, SyncLocalDatabaseResult } from './BucketStorageAdapter.js';
+import { BucketOperationProgress, BucketState, BucketStorageAdapter, BucketStorageListener, Checkpoint, PowerSyncControlCommand, SyncLocalDatabaseResult } from './BucketStorageAdapter.js';
 import { CrudBatch } from './CrudBatch.js';
 import { CrudEntry } from './CrudEntry.js';
 import { SyncDataBatch } from './SyncDataBatch.js';
@@ -11,14 +11,9 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
     private mutex;
     private logger;
     tableNames: Set<string>;
-    private pendingBucketDeletes;
     private _hasCompletedSync;
     private updateListener;
     private _clientId?;
-    /**
-     * Count up, and do a compact on startup.
-     */
-    private compactCounter;
     constructor(db: DBAdapter, mutex: Mutex, logger?: ILogger);
     init(): Promise<void>;
     dispose(): Promise<void>;
@@ -31,7 +26,7 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
     startSession(): void;
     getBucketStates(): Promise<BucketState[]>;
     getBucketOperationProgress(): Promise<BucketOperationProgress>;
-    saveSyncData(batch: SyncDataBatch): Promise<void>;
+    saveSyncData(batch: SyncDataBatch, fixedKeyFormat?: boolean): Promise<void>;
     removeBuckets(buckets: string[]): Promise<void>;
     /**
      * Mark a bucket for deletion.
@@ -46,13 +41,6 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
      */
     private updateObjectsFromBuckets;
     validateChecksums(checkpoint: Checkpoint, priority: number | undefined): Promise<SyncLocalDatabaseResult>;
-    /**
-     * Force a compact, for tests.
-     */
-    forceCompact(): Promise<void>;
-    autoCompact(): Promise<void>;
-    private deletePendingBuckets;
-    private clearRemoveOps;
     updateLocalTarget(cb: () => Promise<string>): Promise<boolean>;
     nextCrudItem(): Promise<CrudEntry | undefined>;
     hasCrud(): Promise<boolean>;
@@ -68,4 +56,8 @@ export declare class SqliteBucketStorage extends BaseObserver<BucketStorageListe
      * Set a target checkpoint.
      */
     setTargetCheckpoint(checkpoint: Checkpoint): Promise<void>;
+    control(op: PowerSyncControlCommand, payload: string | Uint8Array | ArrayBuffer | null): Promise<string>;
+    hasMigratedSubkeys(): Promise<boolean>;
+    migrateToFixedSubkeys(): Promise<void>;
+    static _subkeyMigrationKey: string;
 }

package/lib/client/sync/bucket/SqliteBucketStorage.js

@@ -4,27 +4,20 @@ import { BaseObserver } from '../../../utils/BaseObserver.js';
 import { MAX_OP_ID } from '../../constants.js';
 import { PSInternalTable } from './BucketStorageAdapter.js';
 import { CrudEntry } from './CrudEntry.js';
-const COMPACT_OPERATION_INTERVAL = 1_000;
 export class SqliteBucketStorage extends BaseObserver {
     db;
     mutex;
     logger;
     tableNames;
-    pendingBucketDeletes;
     _hasCompletedSync;
     updateListener;
     _clientId;
-    /**
-     * Count up, and do a compact on startup.
-     */
-    compactCounter = COMPACT_OPERATION_INTERVAL;
     constructor(db, mutex, logger = Logger.get('SqliteBucketStorage')) {
         super();
         this.db = db;
         this.mutex = mutex;
         this.logger = logger;
         this._hasCompletedSync = false;
-        this.pendingBucketDeletes = true;
         this.tableNames = new Set();
         this.updateListener = db.registerListener({
             tablesUpdated: (update) => {
@@ -70,18 +63,15 @@ export class SqliteBucketStorage extends BaseObserver {
         const rows = await this.db.getAll('SELECT name, count_at_last, count_since_last FROM ps_buckets');
         return Object.fromEntries(rows.map((r) => [r.name, { atLast: r.count_at_last, sinceLast: r.count_since_last }]));
     }
-    async saveSyncData(batch) {
+    async saveSyncData(batch, fixedKeyFormat = false) {
         await this.writeTransaction(async (tx) => {
-            let count = 0;
             for (const b of batch.buckets) {
                 const result = await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', [
                     'save',
-                    JSON.stringify({ buckets: [b.toJSON()] })
+                    JSON.stringify({ buckets: [b.toJSON(fixedKeyFormat)] })
                 ]);
                 this.logger.debug('saveSyncData', JSON.stringify(result));
-                count += b.data.length;
             }
-            this.compactCounter += count;
         });
     }
     async removeBuckets(buckets) {
@@ -97,7 +87,6 @@ export class SqliteBucketStorage extends BaseObserver {
             await tx.execute('INSERT INTO powersync_operations(op, data) VALUES(?, ?)', ['delete_bucket', bucket]);
         });
         this.logger.debug('done deleting bucket');
-        this.pendingBucketDeletes = true;
     }
     async hasCompletedSync() {
         if (this._hasCompletedSync) {
@@ -138,7 +127,6 @@ export class SqliteBucketStorage extends BaseObserver {
             this.logger.debug('Not at a consistent checkpoint - cannot update local db');
             return { ready: false, checkpointValid: true };
         }
-        await this.forceCompact();
         return {
             ready: true,
             checkpointValid: true
@@ -209,36 +197,6 @@ export class SqliteBucketStorage extends BaseObserver {
             };
         }
     }
-    /**
-     * Force a compact, for tests.
-     */
-    async forceCompact() {
-        this.compactCounter = COMPACT_OPERATION_INTERVAL;
-        this.pendingBucketDeletes = true;
-        await this.autoCompact();
-    }
-    async autoCompact() {
-        await this.deletePendingBuckets();
-        await this.clearRemoveOps();
-    }
-    async deletePendingBuckets() {
-        if (this.pendingBucketDeletes !== false) {
-            await this.writeTransaction(async (tx) => {
-                await tx.execute('INSERT INTO powersync_operations(op, data) VALUES (?, ?)', ['delete_pending_buckets', '']);
-            });
-            // Executed once after start-up, and again when there are pending deletes.
-            this.pendingBucketDeletes = false;
-        }
-    }
-    async clearRemoveOps() {
-        if (this.compactCounter < COMPACT_OPERATION_INTERVAL) {
-            return;
-        }
-        await this.writeTransaction(async (tx) => {
-            await tx.execute('INSERT INTO powersync_operations(op, data) VALUES (?, ?)', ['clear_remove_ops', '']);
-        });
-        this.compactCounter = 0;
-    }
     async updateLocalTarget(cb) {
         const rs1 = await this.db.getAll("SELECT target_op FROM ps_buckets WHERE name = '$local' AND target_op = CAST(? as INTEGER)", [MAX_OP_ID]);
         if (!rs1.length) {
@@ -339,6 +297,28 @@ export class SqliteBucketStorage extends BaseObserver {
     async setTargetCheckpoint(checkpoint) {
         // No-op for now
     }
+    async control(op, payload) {
+        return await this.writeTransaction(async (tx) => {
+            const [[raw]] = await tx.executeRaw('SELECT powersync_control(?, ?)', [op, payload]);
+            return raw;
+        });
+    }
+    async hasMigratedSubkeys() {
+        const { r } = await this.db.get('SELECT EXISTS(SELECT * FROM ps_kv WHERE key = ?) as r', [
+            SqliteBucketStorage._subkeyMigrationKey
+        ]);
+        return r != 0;
+    }
+    async migrateToFixedSubkeys() {
+        await this.writeTransaction(async (tx) => {
+            await tx.execute('UPDATE ps_oplog SET key = powersync_remove_duplicate_key_encoding(key);');
+            await tx.execute('INSERT OR REPLACE INTO ps_kv (key, value) VALUES (?, ?);', [
+                SqliteBucketStorage._subkeyMigrationKey,
+                '1'
+            ]);
+        });
+    }
+    static _subkeyMigrationKey = 'powersync_js_migrated_subkeys';
 }
 function hasMatchingPriority(priority, bucket) {
     return bucket.priority != null && bucket.priority <= priority;

package/lib/client/sync/bucket/SyncDataBucket.d.ts

@@ -36,5 +36,5 @@ export declare class SyncDataBucket {
      * Use this for the next request.
      */
     next_after?: OpId | undefined);
-    toJSON(): SyncDataBucketJSON;
+    toJSON(fixedKeyEncoding?: boolean): SyncDataBucketJSON;
 }

package/lib/client/sync/bucket/SyncDataBucket.js

@@ -27,13 +27,13 @@ export class SyncDataBucket {
         this.after = after;
         this.next_after = next_after;
     }
-    toJSON() {
+    toJSON(fixedKeyEncoding = false) {
         return {
             bucket: this.bucket,
             has_more: this.has_more,
             after: this.after,
             next_after: this.next_after,
-            data: this.data.map((entry) => entry.toJSON())
+            data: this.data.map((entry) => entry.toJSON(fixedKeyEncoding))
         };
     }
 }

package/lib/client/sync/stream/AbstractRemote.d.ts

@@ -115,18 +115,30 @@ export declare abstract class AbstractRemote {
     }>;
     post(path: string, data: any, headers?: Record<string, string>): Promise<any>;
     get(path: string, headers?: Record<string, string>): Promise<any>;
-    postStreaming(path: string, data: any, headers?: Record<string, string>, signal?: AbortSignal): Promise<any>;
     /**
      * Provides a BSON implementation. The import nature of this varies depending on the platform
      */
     abstract getBSON(): Promise<BSONImplementation>;
     protected createSocket(url: string): WebSocket;
     /**
-     * Connects to the sync/stream websocket endpoint
+     * Connects to the sync/stream websocket endpoint and delivers sync lines by decoding the BSON events
+     * sent by the server.
      */
     socketStream(options: SocketSyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
     /**
-     * Connects to the sync/stream http endpoint
+     * Returns a data stream of sync line data.
+     *
+     * @param map Maps received payload frames to the typed event value.
+     * @param bson A BSON encoder and decoder. When set, the data stream will be requested with a BSON payload
+     * (required for compatibility with older sync services).
+     */
+    socketStreamRaw<T>(options: SocketSyncStreamOptions, map: (buffer: Uint8Array) => T, bson?: typeof BSON): Promise<DataStream<T>>;
+    /**
+     * Connects to the sync/stream http endpoint, parsing lines as JSON.
      */
     postStream(options: SyncStreamOptions): Promise<DataStream<StreamingSyncLine>>;
+    /**
+     * Connects to the sync/stream http endpoint, mapping and emitting each received string line.
+     */
+    postStreamRaw<T>(options: SyncStreamOptions, mapLine: (line: string) => T): Promise<DataStream<T>>;
 }