@powersync/common 1.32.0 → 1.33.1

package/lib/client/sync/stream/AbstractStreamingSyncImplementation.js

@@ -1,8 +1,10 @@
 import Logger from 'js-logger';
 import { SyncStatus } from '../../../db/crud/SyncStatus.js';
+import { FULL_SYNC_PRIORITY } from '../../../db/crud/SyncProgress.js';
 import { AbortOperation } from '../../../utils/AbortOperation.js';
 import { BaseObserver } from '../../../utils/BaseObserver.js';
 import { onAbortPromise, throttleLeadingTrailing } from '../../../utils/async.js';
+import { PowerSyncControlCommand } from '../bucket/BucketStorageAdapter.js';
 import { SyncDataBucket } from '../bucket/SyncDataBucket.js';
 import { FetchStrategy } from './AbstractRemote.js';
 import { isStreamingKeepalive, isStreamingSyncCheckpoint, isStreamingSyncCheckpointComplete, isStreamingSyncCheckpointDiff, isStreamingSyncCheckpointPartiallyComplete, isStreamingSyncData } from './streaming-sync-types.js';
@@ -16,6 +18,49 @@ export var SyncStreamConnectionMethod;
     SyncStreamConnectionMethod["HTTP"] = "http";
     SyncStreamConnectionMethod["WEB_SOCKET"] = "web-socket";
 })(SyncStreamConnectionMethod || (SyncStreamConnectionMethod = {}));
+export var SyncClientImplementation;
+(function (SyncClientImplementation) {
+    /**
+     * Decodes and handles sync lines received from the sync service in JavaScript.
+     *
+     * This is the default option.
+     *
+     * @deprecated Don't use {@link SyncClientImplementation.JAVASCRIPT} directly. Instead, use
+     * {@link DEFAULT_SYNC_CLIENT_IMPLEMENTATION} or omit the option. The explicit choice to use
+     * the JavaScript-based sync implementation will be removed from a future version of the SDK.
+     */
+    SyncClientImplementation["JAVASCRIPT"] = "js";
+    /**
+     * This implementation offloads the sync line decoding and handling into the PowerSync
+     * core extension.
+     *
+     * @experimental
+     * While this implementation is more performant than {@link SyncClientImplementation.JAVASCRIPT},
+     * it has seen less real-world testing and is marked as __experimental__ at the moment.
+     *
+     * ## Compatibility warning
+     *
+     * The Rust sync client stores sync data in a format that is slightly different than the one used
+     * by the old {@link JAVASCRIPT} implementation. When adopting the {@link RUST} client on existing
+     * databases, the PowerSync SDK will migrate the format automatically.
+     * Further, the {@link JAVASCRIPT} client in recent versions of the PowerSync JS SDK (starting from
+     * the version introducing {@link RUST} as an option) also supports the new format, so you can switch
+     * back to {@link JAVASCRIPT} later.
+     *
+     * __However__: Upgrading the SDK version, then adopting {@link RUST} as a sync client and later
+     * downgrading the SDK to an older version (necessarily using the JavaScript-based implementation then)
+     * can lead to sync issues.
+     */
+    SyncClientImplementation["RUST"] = "rust";
+})(SyncClientImplementation || (SyncClientImplementation = {}));
+/**
+ * The default {@link SyncClientImplementation} to use.
+ *
+ * Please use this field instead of {@link SyncClientImplementation.JAVASCRIPT} directly. A future version
+ * of the PowerSync SDK will enable {@link SyncClientImplementation.RUST} by default and remove the JavaScript
+ * option.
+ */
+export const DEFAULT_SYNC_CLIENT_IMPLEMENTATION = SyncClientImplementation.JAVASCRIPT;
 export const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
 export const DEFAULT_RETRY_DELAY_MS = 5000;
 export const DEFAULT_STREAMING_SYNC_OPTIONS = {
@@ -25,6 +70,7 @@ export const DEFAULT_STREAMING_SYNC_OPTIONS = {
 };
 export const DEFAULT_STREAM_CONNECTION_OPTIONS = {
     connectionMethod: SyncStreamConnectionMethod.WEB_SOCKET,
+    clientImplementation: DEFAULT_SYNC_CLIENT_IMPLEMENTATION,
     fetchStrategy: FetchStrategy.Buffered,
     params: {}
 };
@@ -40,6 +86,7 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
     crudUpdateListener;
     streamingSyncPromise;
     pendingCrudUpload;
+    notifyCompletedUploads;
     syncStatus;
     triggerCrudUpload;
     constructor(options) {
@@ -61,6 +108,7 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
             }
             this.pendingCrudUpload = new Promise((resolve) => {
                 this._uploadAllCrud().finally(() => {
+                    this.notifyCompletedUploads?.();
                     this.pendingCrudUpload = undefined;
                     resolve();
                 });
@@ -134,17 +182,17 @@ export class AbstractStreamingSyncImplementation extends BaseObserver {
                  */
                 let checkedCrudItem;
                 while (true) {
-                    this.updateSyncStatus({
-                        dataFlow: {
-                            uploading: true
-                        }
-                    });
                     try {
                         /**
                          * This is the first item in the FIFO CRUD queue.
                          */
                         const nextCrudItem = await this.options.adapter.nextCrudItem();
                         if (nextCrudItem) {
+                            this.updateSyncStatus({
+                                dataFlow: {
+                                    uploading: true
+                                }
+                            });
                             if (nextCrudItem.clientId == checkedCrudItem?.clientId) {
                                 // This will force a higher log level than exceptions which are caught here.
                                 this.logger.warn(`Potentially previously uploaded CRUD entries are still present in the upload queue.
@@ -199,23 +247,17 @@ The next upload iteration will be delayed.`);
         const controller = new AbortController();
         this.abortController = controller;
         this.streamingSyncPromise = this.streamingSync(this.abortController.signal, options);
-        // Return a promise that resolves when the connection status is updated
+        // Return a promise that resolves when the connection status is updated to indicate that we're connected.
         return new Promise((resolve) => {
             const disposer = this.registerListener({
-                statusUpdated: (update) => {
-                    // This is triggered as soon as a connection is read from
-                    if (typeof update.connected == 'undefined') {
-                        // only concern with connection updates
-                        return;
-                    }
-                    if (update.connected == false) {
-                        /**
-                         * This function does not reject if initial connect attempt failed.
-                         * Connected can be false if the connection attempt was aborted or if the initial connection
-                         * attempt failed.
-                         */
+                statusChanged: (status) => {
+                    if (status.dataFlowStatus.downloadError != null) {
                         this.logger.warn('Initial connect attempt did not successfully connect to server');
                     }
+                    else if (status.connecting) {
+                        // Still connecting.
+                        return;
+                    }
                     disposer();
                     resolve();
                 }
@@ -348,6 +390,31 @@ The next upload iteration will be delayed.`);
         }
         return [req, localDescriptions];
     }
+    /**
+     * Older versions of the JS SDK used to encode subkeys as JSON in {@link OplogEntry.toJSON}.
+     * Because subkeys are always strings, this leads to quotes being added around them in `ps_oplog`.
+     * While this is not a problem as long as it's done consistently, it causes issues when a database
+     * created by the JS SDK is used with other SDKs, or (more likely) when the new Rust sync client
+     * is enabled.
+     *
+     * So, we add a migration from the old key format (with quotes) to the new one (no quotes). The
+     * migration is only triggered when necessary (for now). The function returns whether the new format
+     * should be used, so that the JS SDK is able to write to updated databases.
+     *
+     * @param requireFixedKeyFormat Whether we require the new format or also support the old one.
+     *        The Rust client requires the new subkey format.
+     * @returns Whether the database is now using the new, fixed subkey format.
+     */
+    async requireKeyFormat(requireFixedKeyFormat) {
+        const hasMigrated = await this.options.adapter.hasMigratedSubkeys();
+        if (requireFixedKeyFormat && !hasMigrated) {
+            await this.options.adapter.migrateToFixedSubkeys();
+            return true;
+        }
+        else {
+            return hasMigrated;
+        }
+    }
     async streamingSyncIteration(signal, options) {
         await this.obtainLock({
             type: LockType.SYNC,
@@ -357,219 +424,397 @@ The next upload iteration will be delayed.`);
                     ...DEFAULT_STREAM_CONNECTION_OPTIONS,
                     ...(options ?? {})
                 };
-                this.logger.debug('Streaming sync iteration started');
-                this.options.adapter.startSession();
-                let [req, bucketMap] = await this.collectLocalBucketState();
-                // These are compared by reference
-                let targetCheckpoint = null;
-                let validatedCheckpoint = null;
-                let appliedCheckpoint = null;
-                const clientId = await this.options.adapter.getClientId();
-                if (signal.aborted) {
+                if (resolvedOptions.clientImplementation == SyncClientImplementation.JAVASCRIPT) {
+                    await this.legacyStreamingSyncIteration(signal, resolvedOptions);
+                }
+                else {
+                    await this.requireKeyFormat(true);
+                    await this.rustSyncIteration(signal, resolvedOptions);
+                }
+            }
+        });
+    }
+    async legacyStreamingSyncIteration(signal, resolvedOptions) {
+        this.logger.debug('Streaming sync iteration started');
+        this.options.adapter.startSession();
+        let [req, bucketMap] = await this.collectLocalBucketState();
+        // These are compared by reference
+        let targetCheckpoint = null;
+        let validatedCheckpoint = null;
+        let appliedCheckpoint = null;
+        const clientId = await this.options.adapter.getClientId();
+        const usingFixedKeyFormat = await this.requireKeyFormat(false);
+        this.logger.debug('Requesting stream from server');
+        const syncOptions = {
+            path: '/sync/stream',
+            abortSignal: signal,
+            data: {
+                buckets: req,
+                include_checksum: true,
+                raw_data: true,
+                parameters: resolvedOptions.params,
+                client_id: clientId
+            }
+        };
+        let stream;
+        if (resolvedOptions?.connectionMethod == SyncStreamConnectionMethod.HTTP) {
+            stream = await this.options.remote.postStream(syncOptions);
+        }
+        else {
+            stream = await this.options.remote.socketStream({
+                ...syncOptions,
+                ...{ fetchStrategy: resolvedOptions.fetchStrategy }
+            });
+        }
+        this.logger.debug('Stream established. Processing events');
+        while (!stream.closed) {
+            const line = await stream.read();
+            if (!line) {
+                // The stream has closed while waiting
+                return;
+            }
+            // A connection is active and messages are being received
+            if (!this.syncStatus.connected) {
+                // There is a connection now
+                Promise.resolve().then(() => this.triggerCrudUpload());
+                this.updateSyncStatus({
+                    connected: true
+                });
+            }
+            if (isStreamingSyncCheckpoint(line)) {
+                targetCheckpoint = line.checkpoint;
+                const bucketsToDelete = new Set(bucketMap.keys());
+                const newBuckets = new Map();
+                for (const checksum of line.checkpoint.buckets) {
+                    newBuckets.set(checksum.bucket, {
+                        name: checksum.bucket,
+                        priority: checksum.priority ?? FALLBACK_PRIORITY
+                    });
+                    bucketsToDelete.delete(checksum.bucket);
+                }
+                if (bucketsToDelete.size > 0) {
+                    this.logger.debug('Removing buckets', [...bucketsToDelete]);
+                }
+                bucketMap = newBuckets;
+                await this.options.adapter.removeBuckets([...bucketsToDelete]);
+                await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
+                await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
+            }
+            else if (isStreamingSyncCheckpointComplete(line)) {
+                const result = await this.applyCheckpoint(targetCheckpoint, signal);
+                if (result.endIteration) {
                     return;
                 }
-                this.logger.debug('Requesting stream from server');
-                const syncOptions = {
-                    path: '/sync/stream',
-                    abortSignal: signal,
-                    data: {
-                        buckets: req,
-                        include_checksum: true,
-                        raw_data: true,
-                        parameters: resolvedOptions.params,
-                        client_id: clientId
-                    }
-                };
-                let stream;
-                if (resolvedOptions?.connectionMethod == SyncStreamConnectionMethod.HTTP) {
-                    stream = await this.options.remote.postStream(syncOptions);
+                else if (result.applied) {
+                    appliedCheckpoint = targetCheckpoint;
+                }
+                validatedCheckpoint = targetCheckpoint;
+            }
+            else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
+                const priority = line.partial_checkpoint_complete.priority;
+                this.logger.debug('Partial checkpoint complete', priority);
+                const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint, priority);
+                if (!result.checkpointValid) {
+                    // This means checksums failed. Start again with a new checkpoint.
+                    // TODO: better back-off
+                    await new Promise((resolve) => setTimeout(resolve, 50));
+                    return;
+                }
+                else if (!result.ready) {
+                    // If we have pending uploads, we can't complete new checkpoints outside of priority 0.
+                    // We'll resolve this for a complete checkpoint.
                 }
                 else {
-                    stream = await this.options.remote.socketStream({
-                        ...syncOptions,
-                        ...{ fetchStrategy: resolvedOptions.fetchStrategy }
+                    // We'll keep on downloading, but can report that this priority is synced now.
+                    this.logger.debug('partial checkpoint validation succeeded');
+                    // All states with a higher priority can be deleted since this partial sync includes them.
+                    const priorityStates = this.syncStatus.priorityStatusEntries.filter((s) => s.priority <= priority);
+                    priorityStates.push({
+                        priority,
+                        lastSyncedAt: new Date(),
+                        hasSynced: true
+                    });
+                    this.updateSyncStatus({
+                        connected: true,
+                        priorityStatusEntries: priorityStates
                     });
                 }
-                this.logger.debug('Stream established. Processing events');
-                while (!stream.closed) {
-                    const line = await stream.read();
-                    if (!line) {
-                        // The stream has closed while waiting
-                        return;
-                    }
-                    // A connection is active and messages are being received
-                    if (!this.syncStatus.connected) {
-                        // There is a connection now
-                        Promise.resolve().then(() => this.triggerCrudUpload());
-                        this.updateSyncStatus({
-                            connected: true
-                        });
+            }
+            else if (isStreamingSyncCheckpointDiff(line)) {
+                // TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint
+                if (targetCheckpoint == null) {
+                    throw new Error('Checkpoint diff without previous checkpoint');
+                }
+                const diff = line.checkpoint_diff;
+                const newBuckets = new Map();
+                for (const checksum of targetCheckpoint.buckets) {
+                    newBuckets.set(checksum.bucket, checksum);
+                }
+                for (const checksum of diff.updated_buckets) {
+                    newBuckets.set(checksum.bucket, checksum);
+                }
+                for (const bucket of diff.removed_buckets) {
+                    newBuckets.delete(bucket);
+                }
+                const newCheckpoint = {
+                    last_op_id: diff.last_op_id,
+                    buckets: [...newBuckets.values()],
+                    write_checkpoint: diff.write_checkpoint
+                };
+                targetCheckpoint = newCheckpoint;
+                await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
+                bucketMap = new Map();
+                newBuckets.forEach((checksum, name) => bucketMap.set(name, {
+                    name: checksum.bucket,
+                    priority: checksum.priority ?? FALLBACK_PRIORITY
+                }));
+                const bucketsToDelete = diff.removed_buckets;
+                if (bucketsToDelete.length > 0) {
+                    this.logger.debug('Remove buckets', bucketsToDelete);
+                }
+                await this.options.adapter.removeBuckets(bucketsToDelete);
+                await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
+            }
+            else if (isStreamingSyncData(line)) {
+                const { data } = line;
+                const previousProgress = this.syncStatus.dataFlowStatus.downloadProgress;
+                let updatedProgress = null;
+                if (previousProgress) {
+                    updatedProgress = { ...previousProgress };
+                    const progressForBucket = updatedProgress[data.bucket];
+                    if (progressForBucket) {
+                        updatedProgress[data.bucket] = {
+                            ...progressForBucket,
+                            since_last: progressForBucket.since_last + data.data.length
+                        };
                     }
-                    if (isStreamingSyncCheckpoint(line)) {
-                        targetCheckpoint = line.checkpoint;
-                        const bucketsToDelete = new Set(bucketMap.keys());
-                        const newBuckets = new Map();
-                        for (const checksum of line.checkpoint.buckets) {
-                            newBuckets.set(checksum.bucket, {
-                                name: checksum.bucket,
-                                priority: checksum.priority ?? FALLBACK_PRIORITY
-                            });
-                            bucketsToDelete.delete(checksum.bucket);
-                        }
-                        if (bucketsToDelete.size > 0) {
-                            this.logger.debug('Removing buckets', [...bucketsToDelete]);
-                        }
-                        bucketMap = newBuckets;
-                        await this.options.adapter.removeBuckets([...bucketsToDelete]);
-                        await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
-                        await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
+                }
+                this.updateSyncStatus({
+                    dataFlow: {
+                        downloading: true,
+                        downloadProgress: updatedProgress
                     }
-                    else if (isStreamingSyncCheckpointComplete(line)) {
-                        const result = await this.applyCheckpoint(targetCheckpoint, signal);
-                        if (result.endIteration) {
-                            return;
-                        }
-                        else if (result.applied) {
-                            appliedCheckpoint = targetCheckpoint;
+                });
+                await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] }, usingFixedKeyFormat);
+            }
+            else if (isStreamingKeepalive(line)) {
+                const remaining_seconds = line.token_expires_in;
+                if (remaining_seconds == 0) {
+                    // Connection would be closed automatically right after this
+                    this.logger.debug('Token expiring; reconnect');
+                    /**
+                     * For a rare case where the backend connector does not update the token
+                     * (uses the same one), this should have some delay.
+                     */
+                    await this.delayRetry();
+                    return;
+                }
+                else if (remaining_seconds < 30) {
+                    this.logger.debug('Token will expire soon; reconnect');
+                    // Pre-emptively refresh the token
+                    this.options.remote.invalidateCredentials();
+                    return;
+                }
+                this.triggerCrudUpload();
+            }
+            else {
+                this.logger.debug('Sync complete');
+                if (targetCheckpoint === appliedCheckpoint) {
+                    this.updateSyncStatus({
+                        connected: true,
+                        lastSyncedAt: new Date(),
+                        priorityStatusEntries: [],
+                        dataFlow: {
+                            downloadError: undefined
                         }
-                        validatedCheckpoint = targetCheckpoint;
+                    });
+                }
+                else if (validatedCheckpoint === targetCheckpoint) {
+                    const result = await this.applyCheckpoint(targetCheckpoint, signal);
+                    if (result.endIteration) {
+                        return;
                     }
-                    else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
-                        const priority = line.partial_checkpoint_complete.priority;
-                        this.logger.debug('Partial checkpoint complete', priority);
-                        const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint, priority);
-                        if (!result.checkpointValid) {
-                            // This means checksums failed. Start again with a new checkpoint.
-                            // TODO: better back-off
-                            await new Promise((resolve) => setTimeout(resolve, 50));
-                            return;
-                        }
-                        else if (!result.ready) {
-                            // If we have pending uploads, we can't complete new checkpoints outside of priority 0.
-                            // We'll resolve this for a complete checkpoint.
-                        }
-                        else {
-                            // We'll keep on downloading, but can report that this priority is synced now.
-                            this.logger.debug('partial checkpoint validation succeeded');
-                            // All states with a higher priority can be deleted since this partial sync includes them.
-                            const priorityStates = this.syncStatus.priorityStatusEntries.filter((s) => s.priority <= priority);
-                            priorityStates.push({
-                                priority,
-                                lastSyncedAt: new Date(),
-                                hasSynced: true
-                            });
-                            this.updateSyncStatus({
-                                connected: true,
-                                priorityStatusEntries: priorityStates
-                            });
-                        }
+                    else if (result.applied) {
+                        appliedCheckpoint = targetCheckpoint;
                     }
-                    else if (isStreamingSyncCheckpointDiff(line)) {
-                        // TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint
-                        if (targetCheckpoint == null) {
-                            throw new Error('Checkpoint diff without previous checkpoint');
-                        }
-                        const diff = line.checkpoint_diff;
-                        const newBuckets = new Map();
-                        for (const checksum of targetCheckpoint.buckets) {
-                            newBuckets.set(checksum.bucket, checksum);
-                        }
-                        for (const checksum of diff.updated_buckets) {
-                            newBuckets.set(checksum.bucket, checksum);
-                        }
-                        for (const bucket of diff.removed_buckets) {
-                            newBuckets.delete(bucket);
-                        }
-                        const newCheckpoint = {
-                            last_op_id: diff.last_op_id,
-                            buckets: [...newBuckets.values()],
-                            write_checkpoint: diff.write_checkpoint
+                }
+            }
+        }
+        this.logger.debug('Stream input empty');
+        // Connection closed. Likely due to auth issue.
+        return;
+    }
+    async rustSyncIteration(signal, resolvedOptions) {
+        const syncImplementation = this;
+        const adapter = this.options.adapter;
+        const remote = this.options.remote;
+        let receivingLines = null;
+        let hadSyncLine = false;
+        const abortController = new AbortController();
+        signal.addEventListener('abort', () => abortController.abort());
+        // Pending sync lines received from the service, as well as local events that trigger a powersync_control
+        // invocation (local events include refreshed tokens and completed uploads).
+        // This is a single data stream so that we can handle all control calls from a single place.
+        let controlInvocations = null;
+        async function connect(instr) {
+            const syncOptions = {
+                path: '/sync/stream',
+                abortSignal: abortController.signal,
+                data: instr.request
+            };
+            if (resolvedOptions.connectionMethod == SyncStreamConnectionMethod.HTTP) {
+                controlInvocations = await remote.postStreamRaw(syncOptions, (line) => {
+                    if (typeof line == 'string') {
+                        return {
+                            command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
+                            payload: line
                         };
-                        targetCheckpoint = newCheckpoint;
-                        await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
-                        bucketMap = new Map();
-                        newBuckets.forEach((checksum, name) => bucketMap.set(name, {
-                            name: checksum.bucket,
-                            priority: checksum.priority ?? FALLBACK_PRIORITY
-                        }));
-                        const bucketsToDelete = diff.removed_buckets;
-                        if (bucketsToDelete.length > 0) {
-                            this.logger.debug('Remove buckets', bucketsToDelete);
-                        }
-                        await this.options.adapter.removeBuckets(bucketsToDelete);
-                        await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
                     }
-                    else if (isStreamingSyncData(line)) {
-                        const { data } = line;
-                        const previousProgress = this.syncStatus.dataFlowStatus.downloadProgress;
-                        let updatedProgress = null;
-                        if (previousProgress) {
-                            updatedProgress = { ...previousProgress };
-                            const progressForBucket = updatedProgress[data.bucket];
-                            if (progressForBucket) {
-                                updatedProgress[data.bucket] = {
-                                    ...progressForBucket,
-                                    sinceLast: Math.min(progressForBucket.sinceLast + data.data.length, progressForBucket.targetCount - progressForBucket.atLast)
-                                };
-                            }
-                        }
-                        this.updateSyncStatus({
-                            dataFlow: {
-                                downloading: true,
-                                downloadProgress: updatedProgress
-                            }
-                        });
-                        await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] });
+                    else {
+                        // Directly enqueued by us
+                        return line;
                     }
-                    else if (isStreamingKeepalive(line)) {
-                        const remaining_seconds = line.token_expires_in;
-                        if (remaining_seconds == 0) {
-                            // Connection would be closed automatically right after this
-                            this.logger.debug('Token expiring; reconnect');
-                            this.options.remote.invalidateCredentials();
-                            /**
-                             * For a rare case where the backend connector does not update the token
-                             * (uses the same one), this should have some delay.
-                             */
-                            await this.delayRetry();
-                            return;
-                        }
-                        else if (remaining_seconds < 30) {
-                            this.logger.debug('Token will expire soon; reconnect');
-                            // Pre-emptively refresh the token
-                            this.options.remote.invalidateCredentials();
-                            return;
-                        }
-                        this.triggerCrudUpload();
+                });
+            }
+            else {
+                controlInvocations = await remote.socketStreamRaw({
+                    ...syncOptions,
+                    fetchStrategy: resolvedOptions.fetchStrategy
+                }, (payload) => {
+                    if (payload instanceof Uint8Array) {
+                        return {
+                            command: PowerSyncControlCommand.PROCESS_BSON_LINE,
+                            payload: payload
+                        };
                     }
                     else {
-                        this.logger.debug('Sync complete');
-                        if (targetCheckpoint === appliedCheckpoint) {
-                            this.updateSyncStatus({
-                                connected: true,
-                                lastSyncedAt: new Date(),
-                                priorityStatusEntries: [],
-                                dataFlow: {
-                                    downloadError: undefined
-                                }
-                            });
-                        }
-                        else if (validatedCheckpoint === targetCheckpoint) {
-                            const result = await this.applyCheckpoint(targetCheckpoint, signal);
-                            if (result.endIteration) {
-                                return;
-                            }
-                            else if (result.applied) {
-                                appliedCheckpoint = targetCheckpoint;
-                            }
-                        }
+                        // Directly enqueued by us
+                        return payload;
+                    }
+                });
+            }
+            // The rust client will set connected: true after the first sync line because that's when it gets invoked, but
+            // we're already connected here and can report that.
+            syncImplementation.updateSyncStatus({ connected: true });
+            try {
+                while (!controlInvocations.closed) {
+                    const line = await controlInvocations.read();
+                    if (line == null) {
+                        return;
+                    }
+                    await control(line.command, line.payload);
+                    if (!hadSyncLine) {
+                        syncImplementation.triggerCrudUpload();
+                        hadSyncLine = true;
                     }
                 }
-                this.logger.debug('Stream input empty');
-                // Connection closed. Likely due to auth issue.
-                return;
             }
-        });
+            finally {
+                const activeInstructions = controlInvocations;
+                // We concurrently add events to the active data stream when e.g. a CRUD upload is completed or a token is
+                // refreshed. That would throw after closing (and we can't handle those events either way), so set this back
+                // to null.
+                controlInvocations = null;
+                await activeInstructions.close();
+            }
+        }
+        async function stop() {
+            await control(PowerSyncControlCommand.STOP);
+        }
+        async function control(op, payload) {
+            const rawResponse = await adapter.control(op, payload ?? null);
+            await handleInstructions(JSON.parse(rawResponse));
+        }
+        async function handleInstruction(instruction) {
+            if ('LogLine' in instruction) {
+                switch (instruction.LogLine.severity) {
+                    case 'DEBUG':
+                        syncImplementation.logger.debug(instruction.LogLine.line);
+                        break;
+                    case 'INFO':
+                        syncImplementation.logger.info(instruction.LogLine.line);
+                        break;
+                    case 'WARNING':
+                        syncImplementation.logger.warn(instruction.LogLine.line);
+                        break;
+                }
+            }
+            else if ('UpdateSyncStatus' in instruction) {
+                function coreStatusToJs(status) {
+                    return {
+                        priority: status.priority,
+                        hasSynced: status.has_synced ?? undefined,
+                        lastSyncedAt: status?.last_synced_at != null ? new Date(status.last_synced_at * 1000) : undefined
+                    };
+                }
+                const info = instruction.UpdateSyncStatus.status;
+                const coreCompleteSync = info.priority_status.find((s) => s.priority == FULL_SYNC_PRIORITY);
+                const completeSync = coreCompleteSync != null ? coreStatusToJs(coreCompleteSync) : null;
+                syncImplementation.updateSyncStatus({
+                    connected: info.connected,
+                    connecting: info.connecting,
+                    dataFlow: {
+                        downloading: info.downloading != null,
+                        downloadProgress: info.downloading?.buckets
+                    },
+                    lastSyncedAt: completeSync?.lastSyncedAt,
+                    hasSynced: completeSync?.hasSynced,
+                    priorityStatusEntries: info.priority_status.map(coreStatusToJs)
+                });
+            }
+            else if ('EstablishSyncStream' in instruction) {
+                if (receivingLines != null) {
+                    // Already connected, this shouldn't happen during a single iteration.
+                    throw 'Unexpected request to establish sync stream, already connected';
+                }
+                receivingLines = connect(instruction.EstablishSyncStream);
+            }
+            else if ('FetchCredentials' in instruction) {
+                if (instruction.FetchCredentials.did_expire) {
+                    remote.invalidateCredentials();
+                }
+                else {
+                    remote.invalidateCredentials();
+                    // Restart iteration after the credentials have been refreshed.
+                    remote.fetchCredentials().then((_) => {
+                        controlInvocations?.enqueueData({ command: PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED });
+                    }, (err) => {
+                        syncImplementation.logger.warn('Could not prefetch credentials', err);
+                    });
+                }
+            }
+            else if ('CloseSyncStream' in instruction) {
+                abortController.abort();
+            }
+            else if ('FlushFileSystem' in instruction) {
+                // Not necessary on JS platforms.
+            }
+            else if ('DidCompleteSync' in instruction) {
+                syncImplementation.updateSyncStatus({
+                    dataFlow: {
+                        downloadError: undefined
+                    }
+                });
+            }
+        }
+        async function handleInstructions(instructions) {
+            for (const instr of instructions) {
+                await handleInstruction(instr);
+            }
+        }
+        try {
+            await control(PowerSyncControlCommand.START, JSON.stringify({
+                parameters: resolvedOptions.params
+            }));
+            this.notifyCompletedUploads = () => {
+                controlInvocations?.enqueueData({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
+            };
+            await receivingLines;
+        }
+        finally {
+            this.notifyCompletedUploads = undefined;
+            await stop();
+        }
     }
     async updateSyncStatusForStartingCheckpoint(checkpoint) {
         const localProgress = await this.options.adapter.getBucketOperationProgress();
@@ -583,9 +828,9 @@ The next upload iteration will be delayed.`);
                 // The fallback priority doesn't matter here, but 3 is the one newer versions of the sync service
                 // will use by default.
                 priority: bucket.priority ?? 3,
-                atLast: atLast,
-                sinceLast: sinceLast,
-                targetCount: bucket.count ?? 0
+                at_last: atLast,
+                since_last: sinceLast,
+                target_count: bucket.count ?? 0
             };
             if (bucket.count != null && bucket.count < atLast + sinceLast) {
                 // Either due to a defrag / sync rule deploy or a compaction operation, the size
@@ -597,8 +842,8 @@ The next upload iteration will be delayed.`);
         if (invalidated) {
             for (const bucket in progress) {
                 const bucketProgress = progress[bucket];
-                bucketProgress.atLast = 0;
-                bucketProgress.sinceLast = 0;
+                bucketProgress.at_last = 0;
+                bucketProgress.since_last = 0;
             }
         }
         this.updateSyncStatus({