@powersync/common 1.51.0 → 1.53.0

package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts

@@ -1,35 +1,27 @@
 import Logger, { ILogger } from 'js-logger';
 
-import { InternalProgressInformation } from '../../../db/crud/SyncProgress.js';
 import { SyncStatus, SyncStatusOptions } from '../../../db/crud/SyncStatus.js';
 import { AbortOperation } from '../../../utils/AbortOperation.js';
 import { BaseListener, BaseObserver, BaseObserverInterface, Disposable } from '../../../utils/BaseObserver.js';
-import { DataStream } from '../../../utils/DataStream.js';
 import { throttleLeadingTrailing } from '../../../utils/async.js';
-import {
-  BucketChecksum,
-  BucketDescription,
-  BucketStorageAdapter,
-  Checkpoint,
-  PowerSyncControlCommand
-} from '../bucket/BucketStorageAdapter.js';
+import { BucketStorageAdapter, PowerSyncControlCommand } from '../bucket/BucketStorageAdapter.js';
 import { CrudEntry } from '../bucket/CrudEntry.js';
-import { SyncDataBucket } from '../bucket/SyncDataBucket.js';
 import { AbstractRemote, FetchStrategy, SyncStreamOptions } from './AbstractRemote.js';
-import { EstablishSyncStream, Instruction, coreStatusToJs } from './core-instruction.js';
 import {
-  BucketRequest,
-  CrudUploadNotification,
-  StreamingSyncLine,
-  StreamingSyncLineOrCrudUploadComplete,
-  StreamingSyncRequestParameterType,
-  isStreamingKeepalive,
-  isStreamingSyncCheckpoint,
-  isStreamingSyncCheckpointComplete,
-  isStreamingSyncCheckpointDiff,
-  isStreamingSyncCheckpointPartiallyComplete,
-  isStreamingSyncData
-} from './streaming-sync-types.js';
+  Instruction,
+  NonInterruptingInstruction,
+  coreStatusToJs,
+  isInterruptingInstruction
+} from './core-instruction.js';
+import {
+  doneResult,
+  injectable,
+  InjectableIterator,
+  map,
+  SimpleAsyncIterator,
+  valueResult
+} from '../../../utils/stream_transform.js';
+import { StreamingSyncRequestParameterType } from './JsonValue.js';
 
 export enum LockType {
   CRUD = 'crud',
@@ -42,35 +34,21 @@ export enum SyncStreamConnectionMethod {
 }
 
 export enum SyncClientImplementation {
-  /**
-   * Decodes and handles sync lines received from the sync service in JavaScript.
-   *
-   * This is the default option.
-   *
-   * @deprecated We recommend the {@link RUST} client implementation for all apps. If you have issues with
-   * the Rust client, please file an issue or reach out to us. The JavaScript client will be removed in a future
-   * version of the PowerSync SDK.
-   */
-  JAVASCRIPT = 'js',
   /**
    * This implementation offloads the sync line decoding and handling into the PowerSync
    * core extension.
    *
-   * This option is more performant than the {@link JAVASCRIPT} client, enabled by default and the
-   * recommended client implementation for all apps.
+   * This is the only option, as an older JavaScript client implementation has been removed from the SDK.
    *
    * ## Compatibility warning
    *
    * The Rust sync client stores sync data in a format that is slightly different than the one used
-   * by the old {@link JAVASCRIPT} implementation. When adopting the {@link RUST} client on existing
-   * databases, the PowerSync SDK will migrate the format automatically.
-   * Further, the {@link JAVASCRIPT} client in recent versions of the PowerSync JS SDK (starting from
-   * the version introducing {@link RUST} as an option) also supports the new format, so you can switch
-   * back to {@link JAVASCRIPT} later.
+   * by the old JavaScript client. When adopting the {@link RUST} client on existing databases, the PowerSync SDK will
+   * migrate the format automatically.
    *
-   * __However__: Upgrading the SDK version, then adopting {@link RUST} as a sync client and later
-   * downgrading the SDK to an older version (necessarily using the JavaScript-based implementation then)
-   * can lead to sync issues.
+   * SDK versions supporting both the JavaScript and the Rust client support both formats with the JavaScript client
+   * implementaiton. However, downgrading to an SDK version that only supports the JavaScript client would not be
+   * possible anymore. Problematic SDK versions have been released before 2025-06-09.
    */
   RUST = 'rust'
 }
@@ -130,11 +108,7 @@ export interface BaseConnectionOptions {
   appMetadata?: Record<string, string>;
 
   /**
-   * Whether to use a JavaScript implementation to handle received sync lines from the sync
-   * service, or whether this work should be offloaded to the PowerSync core extension.
-   *
-   * This defaults to the JavaScript implementation ({@link SyncClientImplementation.JAVASCRIPT})
-   * since the ({@link SyncClientImplementation.RUST}) implementation is experimental at the moment.
+   * @deprecated The Rust sync client is used unconditionally, so this option can't be configured.
    */
   clientImplementation?: SyncClientImplementation;
 
@@ -200,15 +174,14 @@ export interface StreamingSyncImplementation
    */
   disconnect(): Promise<void>;
   getWriteCheckpoint: () => Promise<string>;
-  hasCompletedSync: () => Promise<boolean>;
   isConnected: boolean;
-  lastSyncedAt?: Date;
   syncStatus: SyncStatus;
   triggerCrudUpload: () => void;
   waitForReady(): Promise<void>;
   waitForStatus(status: SyncStatusOptions): Promise<void>;
   waitUntilStatusMatches(predicate: (status: SyncStatus) => boolean): Promise<void>;
   updateSubscriptions(subscriptions: SubscribedStream[]): void;
+  markConnectionMayHaveChanged(): void;
 }
 
 export const DEFAULT_CRUD_UPLOAD_THROTTLE_MS = 1000;
@@ -246,16 +219,16 @@ export abstract class AbstractStreamingSyncImplementation
   extends BaseObserver<StreamingSyncImplementationListener>
   implements StreamingSyncImplementation
 {
-  protected _lastSyncedAt: Date | null;
   protected options: AbstractStreamingSyncImplementationOptions;
   protected abortController: AbortController | null;
   // In rare cases, mostly for tests, uploads can be triggered without being properly connected.
   // This allows ensuring that all upload processes can be aborted.
-  protected uploadAbortController: AbortController | null;
+  protected uploadAbortController: AbortController | undefined;
   protected crudUpdateListener?: () => void;
   protected streamingSyncPromise?: Promise<void>;
   protected logger: ILogger;
   private activeStreams: SubscribedStream[];
+  private connectionMayHaveChanged = false;
 
   private isUploadingCrud: boolean = false;
   private notifyCompletedUploads?: () => void;
@@ -302,7 +275,7 @@ export abstract class AbstractStreamingSyncImplementation
        * Match only the partial status options provided in the
        * matching status
        */
-      const matchPartialObject = (compA: object, compB: object) => {
+      const matchPartialObject = (compA: object, compB: any): any => {
         return Object.entries(compA).every(([key, value]) => {
           const comparisonBValue = compB[key];
           if (typeof value == 'object' && typeof comparisonBValue == 'object') {
@@ -352,10 +325,6 @@ export abstract class AbstractStreamingSyncImplementation
 
   abstract obtainLock<T>(lockOptions: LockOptions<T>): Promise<T>;
 
-  async hasCompletedSync() {
-    return this.options.adapter.hasCompletedSync();
-  }
-
   async getWriteCheckpoint(): Promise<string> {
     const clientId = await this.options.adapter.getClientId();
     let path = `/write-checkpoint2.json?client_id=${clientId}`;
@@ -426,7 +395,7 @@ The next upload iteration will be delayed.`);
             this.updateSyncStatus({
               dataFlow: {
                 uploading: false,
-                uploadError: ex
+                uploadError: ex as Error
               }
             });
             await this.delayRetry(controller.signal);
@@ -435,7 +404,7 @@ The next upload iteration will be delayed.`);
               break;
             }
             this.logger.debug(
-              `Caught exception when uploading. Upload will retry after a delay. Exception: ${ex.message}`
+              `Caught exception when uploading. Upload will retry after a delay. Exception: ${(ex as Error).message}`
             );
           } finally {
             this.updateSyncStatus({
@@ -445,7 +414,7 @@ The next upload iteration will be delayed.`);
             });
           }
         }
-        this.uploadAbortController = null;
+        this.uploadAbortController = undefined;
       }
     });
   }
@@ -572,13 +541,17 @@ The next upload iteration will be delayed.`);
           this.logger.warn(ex);
           shouldDelayRetry = false;
           // A disconnect was requested, we should not delay since there is no explicit retry
+        } else if (this.connectionMayHaveChanged && (ex as Error).message?.indexOf('No iteration is active') >= 0) {
+          this.connectionMayHaveChanged = false;
+          this.logger.info('Sync error after changed connection, retrying immediately');
+          shouldDelayRetry = false;
         } else {
           this.logger.error(ex);
         }
 
         this.updateSyncStatus({
           dataFlow: {
-            downloadError: ex
+            downloadError: ex as Error
           }
         });
       } finally {
@@ -607,18 +580,15 @@ The next upload iteration will be delayed.`);
     this.updateSyncStatus({ connected: false, connecting: false });
   }
 
-  private async collectLocalBucketState(): Promise<[BucketRequest[], Map<string, BucketDescription | null>]> {
-    const bucketEntries = await this.options.adapter.getBucketStates();
-    const req: BucketRequest[] = bucketEntries.map((entry) => ({
-      name: entry.bucket,
-      after: entry.op_id
-    }));
-    const localDescriptions = new Map<string, BucketDescription | null>();
-    for (const entry of bucketEntries) {
-      localDescriptions.set(entry.bucket, null);
-    }
+  markConnectionMayHaveChanged() {
+    // By setting this field, we'll immediately retry if the next sync event causes an error triggered by us not having
+    // an active sync iteration on the connection in use.
+    this.connectionMayHaveChanged = true;
 
-    return [req, localDescriptions];
+    // This triggers a `powersync_control` invocation if a sync iteration is currently active. This is a cheap call to
+    // make when no subscriptions have actually changed, we're mainly interested in this immediately throwing if no
+    // iteration is active. That allows us to reconnect ASAP, instead of having to wait for the next sync line.
+    this.handleActiveStreamsChange?.();
   }
 
   /**
@@ -668,271 +638,62 @@ The next upload iteration will be delayed.`);
             `Invalid appMetadata provided. Only string values are allowed. Invalid values: ${invalidMetadata.map(([key, value]) => `${key}: ${value}`).join(', ')}`
           );
         }
-        const clientImplementation = resolvedOptions.clientImplementation;
-        this.updateSyncStatus({ clientImplementation });
 
-        if (clientImplementation == SyncClientImplementation.JAVASCRIPT) {
-          await this.legacyStreamingSyncIteration(signal, resolvedOptions);
-          return null;
-        } else {
-          await this.requireKeyFormat(true);
-          return await this.rustSyncIteration(signal, resolvedOptions);
-        }
+        await this.requireKeyFormat(true);
+        return await this.rustSyncIteration(signal, resolvedOptions);
       }
     });
   }
 
-  private async legacyStreamingSyncIteration(signal: AbortSignal, resolvedOptions: RequiredPowerSyncConnectionOptions) {
-    const rawTables = resolvedOptions.serializedSchema?.raw_tables;
-    if (rawTables != null && rawTables.length) {
-      this.logger.warn('Raw tables require the Rust-based sync client. The JS client will ignore them.');
-    }
-    if (this.activeStreams.length) {
-      this.logger.error('Sync streams require `clientImplementation: SyncClientImplementation.RUST` when connecting.');
-    }
-
-    this.logger.debug('Streaming sync iteration started');
-    this.options.adapter.startSession();
-    let [req, bucketMap] = await this.collectLocalBucketState();
-
-    let targetCheckpoint: Checkpoint | null = null;
-    // A checkpoint that has been validated but not applied (e.g. due to pending local writes)
-    let pendingValidatedCheckpoint: Checkpoint | null = null;
-
-    const clientId = await this.options.adapter.getClientId();
-    const usingFixedKeyFormat = await this.requireKeyFormat(false);
-
-    this.logger.debug('Requesting stream from server');
-
-    const syncOptions: SyncStreamOptions = {
-      path: '/sync/stream',
-      abortSignal: signal,
-      data: {
-        buckets: req,
-        include_checksum: true,
-        raw_data: true,
-        parameters: resolvedOptions.params,
-        app_metadata: resolvedOptions.appMetadata,
-        client_id: clientId
-      }
-    };
-
-    let stream: DataStream<StreamingSyncLineOrCrudUploadComplete>;
-    if (resolvedOptions?.connectionMethod == SyncStreamConnectionMethod.HTTP) {
-      stream = await this.options.remote.postStreamRaw(syncOptions, (line: string | CrudUploadNotification) => {
-        if (typeof line == 'string') {
-          return JSON.parse(line) as StreamingSyncLine;
-        } else {
-          // Directly enqueued by us
-          return line;
-        }
-      });
-    } else {
-      const bson = await this.options.remote.getBSON();
-      stream = await this.options.remote.socketStreamRaw(
-        {
-          ...syncOptions,
-          ...{ fetchStrategy: resolvedOptions.fetchStrategy }
-        },
-        (payload: Uint8Array | CrudUploadNotification) => {
-          if (payload instanceof Uint8Array) {
-            return bson.deserialize(payload) as StreamingSyncLine;
-          } else {
-            // Directly enqueued by us
-            return payload;
-          }
-        },
-        bson
-      );
-    }
-
-    this.logger.debug('Stream established. Processing events');
-
-    this.notifyCompletedUploads = () => {
-      if (!stream.closed) {
-        stream.enqueueData({ crud_upload_completed: null });
-      }
-    };
-
-    while (!stream.closed) {
-      const line = await stream.read();
-      if (!line) {
-        // The stream has closed while waiting
-        return;
-      }
-
-      if ('crud_upload_completed' in line) {
-        if (pendingValidatedCheckpoint != null) {
-          const { applied, endIteration } = await this.applyCheckpoint(pendingValidatedCheckpoint);
-          if (applied) {
-            pendingValidatedCheckpoint = null;
-          } else if (endIteration) {
-            break;
-          }
-        }
-
-        continue;
-      }
+  private receiveSyncLines(data: {
+    options: SyncStreamOptions;
+    connection: RequiredPowerSyncConnectionOptions;
+  }): SimpleAsyncIterator<EnqueuedCommand> {
+    const { options, connection } = data;
+    const remote = this.options.remote;
 
-      // A connection is active and messages are being received
-      if (!this.syncStatus.connected) {
-        // There is a connection now
-        Promise.resolve().then(() => this.triggerCrudUpload());
-        this.updateSyncStatus({
-          connected: true
+    const openInner = async () => {
+      if (connection.connectionMethod == SyncStreamConnectionMethod.HTTP) {
+        return await remote.fetchStream(options);
+      } else {
+        return await this.options.remote.socketStreamRaw({
+          ...options,
+          ...{ fetchStrategy: connection.fetchStrategy }
         });
       }
+    };
 
-      if (isStreamingSyncCheckpoint(line)) {
-        targetCheckpoint = line.checkpoint;
-        // New checkpoint - existing validated checkpoint is no longer valid
-        pendingValidatedCheckpoint = null;
-        const bucketsToDelete = new Set<string>(bucketMap.keys());
-        const newBuckets = new Map<string, BucketDescription>();
-        for (const checksum of line.checkpoint.buckets) {
-          newBuckets.set(checksum.bucket, {
-            name: checksum.bucket,
-            priority: checksum.priority ?? FALLBACK_PRIORITY
+    let inner: SimpleAsyncIterator<string | Uint8Array> | undefined;
+    let done = false;
+
+    return {
+      async next(): Promise<IteratorResult<EnqueuedCommand>> {
+        if (done) {
+          return doneResult;
+        } else if (inner == null) {
+          inner = await openInner();
+          // We're connected here, so we can tell the core extension about it.
+          return valueResult<EnqueuedCommand>({
+            command: PowerSyncControlCommand.CONNECTION_STATE,
+            payload: 'established'
           });
-          bucketsToDelete.delete(checksum.bucket);
-        }
-        if (bucketsToDelete.size > 0) {
-          this.logger.debug('Removing buckets', [...bucketsToDelete]);
-        }
-        bucketMap = newBuckets;
-        await this.options.adapter.removeBuckets([...bucketsToDelete]);
-        await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
-        await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
-      } else if (isStreamingSyncCheckpointComplete(line)) {
-        const result = await this.applyCheckpoint(targetCheckpoint!);
-        if (result.endIteration) {
-          return;
-        } else if (!result.applied) {
-          // "Could not apply checkpoint due to local data". We need to retry after
-          // finishing uploads.
-          pendingValidatedCheckpoint = targetCheckpoint;
-        } else {
-          // Nothing to retry later. This would likely already be null from the last
-          // checksum or checksum_diff operation, but we make sure.
-          pendingValidatedCheckpoint = null;
-        }
-      } else if (isStreamingSyncCheckpointPartiallyComplete(line)) {
-        const priority = line.partial_checkpoint_complete.priority;
-        this.logger.debug('Partial checkpoint complete', priority);
-        const result = await this.options.adapter.syncLocalDatabase(targetCheckpoint!, priority);
-        if (!result.checkpointValid) {
-          // This means checksums failed. Start again with a new checkpoint.
-          // TODO: better back-off
-          await new Promise((resolve) => setTimeout(resolve, 50));
-          return;
-        } else if (!result.ready) {
-          // If we have pending uploads, we can't complete new checkpoints outside of priority 0.
-          // We'll resolve this for a complete checkpoint.
         } else {
-          // We'll keep on downloading, but can report that this priority is synced now.
-          this.logger.debug('partial checkpoint validation succeeded');
-
-          // All states with a higher priority can be deleted since this partial sync includes them.
-          const priorityStates = this.syncStatus.priorityStatusEntries.filter((s) => s.priority <= priority);
-          priorityStates.push({
-            priority,
-            lastSyncedAt: new Date(),
-            hasSynced: true
-          });
-
-          this.updateSyncStatus({
-            connected: true,
-            priorityStatusEntries: priorityStates
-          });
-        }
-      } else if (isStreamingSyncCheckpointDiff(line)) {
-        // TODO: It may be faster to just keep track of the diff, instead of the entire checkpoint
-        if (targetCheckpoint == null) {
-          throw new Error('Checkpoint diff without previous checkpoint');
-        }
-        // New checkpoint - existing validated checkpoint is no longer valid
-        pendingValidatedCheckpoint = null;
-        const diff = line.checkpoint_diff;
-        const newBuckets = new Map<string, BucketChecksum>();
-        for (const checksum of targetCheckpoint.buckets) {
-          newBuckets.set(checksum.bucket, checksum);
-        }
-        for (const checksum of diff.updated_buckets) {
-          newBuckets.set(checksum.bucket, checksum);
-        }
-        for (const bucket of diff.removed_buckets) {
-          newBuckets.delete(bucket);
-        }
-
-        const newCheckpoint: Checkpoint = {
-          last_op_id: diff.last_op_id,
-          buckets: [...newBuckets.values()],
-          write_checkpoint: diff.write_checkpoint
-        };
-        targetCheckpoint = newCheckpoint;
-        await this.updateSyncStatusForStartingCheckpoint(targetCheckpoint);
-
-        bucketMap = new Map();
-        newBuckets.forEach((checksum, name) =>
-          bucketMap.set(name, {
-            name: checksum.bucket,
-            priority: checksum.priority ?? FALLBACK_PRIORITY
-          })
-        );
-
-        const bucketsToDelete = diff.removed_buckets;
-        if (bucketsToDelete.length > 0) {
-          this.logger.debug('Remove buckets', bucketsToDelete);
-        }
-        await this.options.adapter.removeBuckets(bucketsToDelete);
-        await this.options.adapter.setTargetCheckpoint(targetCheckpoint);
-      } else if (isStreamingSyncData(line)) {
-        const { data } = line;
-        const previousProgress = this.syncStatus.dataFlowStatus.downloadProgress;
-        let updatedProgress: InternalProgressInformation | null = null;
-        if (previousProgress) {
-          updatedProgress = { ...previousProgress };
-          const progressForBucket = updatedProgress[data.bucket];
-          if (progressForBucket) {
-            updatedProgress[data.bucket] = {
-              ...progressForBucket,
-              since_last: progressForBucket.since_last + data.data.length
-            };
-          }
-        }
-
-        this.updateSyncStatus({
-          dataFlow: {
-            downloading: true,
-            downloadProgress: updatedProgress
+          const event = await inner.next();
+          if (event.done) {
+            done = true;
+            return valueResult<EnqueuedCommand>({ command: PowerSyncControlCommand.CONNECTION_STATE, payload: 'end' });
+          } else {
+            return valueResult<EnqueuedCommand>({
+              command:
+                typeof event.value == 'string'
+                  ? PowerSyncControlCommand.PROCESS_TEXT_LINE
+                  : PowerSyncControlCommand.PROCESS_BSON_LINE,
+              payload: event.value
+            });
           }
-        });
-        await this.options.adapter.saveSyncData({ buckets: [SyncDataBucket.fromRow(data)] }, usingFixedKeyFormat);
-      } else if (isStreamingKeepalive(line)) {
-        const remaining_seconds = line.token_expires_in;
-        if (remaining_seconds == 0) {
-          // Connection would be closed automatically right after this
-          this.logger.debug('Token expiring; reconnect');
-          /**
-           * For a rare case where the backend connector does not update the token
-           * (uses the same one), this should have some delay.
-           */
-          await this.delayRetry();
-          return;
-        } else if (remaining_seconds < 30) {
-          this.logger.debug('Token will expire soon; reconnect');
-          // Pre-emptively refresh the token
-          this.options.remote.invalidateCredentials();
-          return;
         }
-        this.triggerCrudUpload();
-      } else {
-        this.logger.debug('Received unknown sync line', line);
       }
-    }
-    this.logger.debug('Stream input empty');
-    // Connection closed. Likely due to auth issue.
-    return;
+    };
   }
 
   private async rustSyncIteration(
@@ -942,93 +703,41 @@ The next upload iteration will be delayed.`);
     const syncImplementation = this;
     const adapter = this.options.adapter;
     const remote = this.options.remote;
-    let receivingLines: Promise<void> | null = null;
-    let hadSyncLine = false;
     let hideDisconnectOnRestart = false;
+    let notifyTokenRefreshed: (() => void) | undefined;
 
     if (signal.aborted) {
       throw new AbortOperation('Connection request has been aborted');
     }
-    const abortController = new AbortController();
-    signal.addEventListener('abort', () => abortController.abort());
-
-    // Pending sync lines received from the service, as well as local events that trigger a powersync_control
-    // invocation (local events include refreshed tokens and completed uploads).
-    // This is a single data stream so that we can handle all control calls from a single place.
-    let controlInvocations: DataStream<EnqueuedCommand, Uint8Array | EnqueuedCommand> | null = null;
-
-    async function connect(instr: EstablishSyncStream) {
-      const syncOptions: SyncStreamOptions = {
-        path: '/sync/stream',
-        abortSignal: abortController.signal,
-        data: instr.request
-      };
 
-      if (resolvedOptions.connectionMethod == SyncStreamConnectionMethod.HTTP) {
-        controlInvocations = await remote.postStreamRaw(syncOptions, (line: string | EnqueuedCommand) => {
-          if (typeof line == 'string') {
-            return {
-              command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
-              payload: line
-            };
-          } else {
-            // Directly enqueued by us
-            return line;
-          }
-        });
-      } else {
-        controlInvocations = await remote.socketStreamRaw(
-          {
-            ...syncOptions,
-            fetchStrategy: resolvedOptions.fetchStrategy
-          },
-          (payload: Uint8Array | EnqueuedCommand) => {
-            if (payload instanceof Uint8Array) {
-              return {
-                command: PowerSyncControlCommand.PROCESS_BSON_LINE,
-                payload: payload
-              };
-            } else {
-              // Directly enqueued by us
-              return payload;
-            }
-          }
-        );
+    function startCommand() {
+      const options: any = {
+        parameters: resolvedOptions.params,
+        app_metadata: resolvedOptions.appMetadata,
+        active_streams: syncImplementation.activeStreams,
+        include_defaults: resolvedOptions.includeDefaultStreams
+      };
+      if (resolvedOptions.serializedSchema) {
+        options.schema = resolvedOptions.serializedSchema;
       }
 
-      // The rust client will set connected: true after the first sync line because that's when it gets invoked, but
-      // we're already connected here and can report that.
-      syncImplementation.updateSyncStatus({ connected: true });
-
-      try {
-        while (!controlInvocations.closed) {
-          const line = await controlInvocations.read();
-          if (line == null) {
-            return;
-          }
-
-          await control(line.command, line.payload);
-
-          if (!hadSyncLine) {
-            syncImplementation.triggerCrudUpload();
-            hadSyncLine = true;
-          }
-        }
-      } finally {
-        const activeInstructions = controlInvocations;
-        // We concurrently add events to the active data stream when e.g. a CRUD upload is completed or a token is
-        // refreshed. That would throw after closing (and we can't handle those events either way), so set this back
-        // to null.
-        controlInvocations = null;
-        await activeInstructions.close();
-      }
+      return invokePowerSyncControl(PowerSyncControlCommand.START, JSON.stringify(options));
     }
 
     async function stop() {
-      await control(PowerSyncControlCommand.STOP);
+      const instructions = await invokePowerSyncControl(PowerSyncControlCommand.STOP);
+      for (const instruction of instructions) {
+        // We don't need to handle interrupting instructions since we're unconditionally ending the sync iteration at
+        // this point.
+        if (isInterruptingInstruction(instruction)) continue;
+        await handleInstruction(instruction);
+      }
     }
 
-    async function control(op: PowerSyncControlCommand, payload?: Uint8Array | string) {
+    async function invokePowerSyncControl(
+      op: PowerSyncControlCommand,
+      payload?: Uint8Array | string
+    ): Promise<Instruction[]> {
       const rawResponse = await adapter.control(op, payload ?? null);
       const logger = syncImplementation.logger;
       logger.trace(
@@ -1038,10 +747,15 @@ The next upload iteration will be delayed.`);
         rawResponse
       );
 
-      await handleInstructions(JSON.parse(rawResponse));
+      if (op != PowerSyncControlCommand.STOP) {
+        // Evidently we have a working connection here, otherwise powersync_control would have failed.
+        syncImplementation.connectionMayHaveChanged = false;
+      }
+
+      return JSON.parse(rawResponse);
     }
 
-    async function handleInstruction(instruction: Instruction) {
+    async function handleInstruction(instruction: NonInterruptingInstruction) {
       if ('LogLine' in instruction) {
         switch (instruction.LogLine.severity) {
           case 'DEBUG':
@@ -1056,13 +770,6 @@ The next upload iteration will be delayed.`);
         }
       } else if ('UpdateSyncStatus' in instruction) {
         syncImplementation.updateSyncStatus(coreStatusToJs(instruction.UpdateSyncStatus.status));
-      } else if ('EstablishSyncStream' in instruction) {
-        if (receivingLines != null) {
-          // Already connected, this shouldn't happen during a single iteration.
-          throw 'Unexpected request to establish sync stream, already connected';
-        }
-
-        receivingLines = connect(instruction.EstablishSyncStream);
       } else if ('FetchCredentials' in instruction) {
         if (instruction.FetchCredentials.did_expire) {
           remote.invalidateCredentials();
@@ -1072,16 +779,13 @@ The next upload iteration will be delayed.`);
           // Restart iteration after the credentials have been refreshed.
           remote.fetchCredentials().then(
             (_) => {
-              controlInvocations?.enqueueData({ command: PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED });
+              notifyTokenRefreshed?.();
             },
             (err) => {
               syncImplementation.logger.warn('Could not prefetch credentials', err);
             }
           );
         }
-      } else if ('CloseSyncStream' in instruction) {
-        abortController.abort();
-        hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
       } else if ('FlushFileSystem' in instruction) {
         // Not necessary on JS platforms.
       } else if ('DidCompleteSync' in instruction) {
@@ -1093,119 +797,86 @@ The next upload iteration will be delayed.`);
       }
     }
 
-    async function handleInstructions(instructions: Instruction[]) {
-      for (const instr of instructions) {
-        await handleInstruction(instr);
-      }
-    }
-
     try {
-      const options: any = {
-        parameters: resolvedOptions.params,
-        app_metadata: resolvedOptions.appMetadata,
-        active_streams: this.activeStreams,
-        include_defaults: resolvedOptions.includeDefaultStreams
-      };
-      if (resolvedOptions.serializedSchema) {
-        options.schema = resolvedOptions.serializedSchema;
+      const defaultResult = { immediateRestart: false };
+      // Pending sync lines received from the service, as well as local events that trigger a powersync_control
+      // invocation (local events include refreshed tokens and completed uploads).
+      // This is a single data stream so that we can handle all control calls from a single place.
+      let controlInvocations: InjectableIterator<EnqueuedCommand> | null = null;
+
+      for (const startInstruction of await startCommand()) {
+        if ('EstablishSyncStream' in startInstruction) {
+          const syncOptions: SyncStreamOptions = {
+            path: '/sync/stream',
+            abortSignal: signal,
+            data: startInstruction.EstablishSyncStream.request
+          };
+
+          controlInvocations = injectable(
+            syncImplementation.receiveSyncLines({
+              options: syncOptions,
+              connection: resolvedOptions
+            })
+          );
+        } else if ('CloseSyncStream' in startInstruction) {
+          return defaultResult;
+        } else {
+          await handleInstruction(startInstruction);
+        }
       }
-
-      await control(PowerSyncControlCommand.START, JSON.stringify(options));
+      if (controlInvocations == null) return defaultResult;
 
       this.notifyCompletedUploads = () => {
-        if (controlInvocations && !controlInvocations?.closed) {
-          controlInvocations.enqueueData({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
-        }
+        controlInvocations.inject({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
       };
       this.handleActiveStreamsChange = () => {
-        if (controlInvocations && !controlInvocations?.closed) {
-          controlInvocations.enqueueData({
-            command: PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
-            payload: JSON.stringify(this.activeStreams)
-          });
-        }
+        controlInvocations.inject({
+          command: PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
+          payload: JSON.stringify(this.activeStreams)
+        });
       };
-      await receivingLines;
-    } finally {
-      this.notifyCompletedUploads = this.handleActiveStreamsChange = undefined;
-      await stop();
-    }
-
-    return { immediateRestart: hideDisconnectOnRestart };
-  }
-
-  private async updateSyncStatusForStartingCheckpoint(checkpoint: Checkpoint) {
-    const localProgress = await this.options.adapter.getBucketOperationProgress();
-    const progress: InternalProgressInformation = {};
-    let invalidated = false;
-
-    for (const bucket of checkpoint.buckets) {
-      const savedProgress = localProgress[bucket.bucket];
-      const atLast = savedProgress?.atLast ?? 0;
-      const sinceLast = savedProgress?.sinceLast ?? 0;
-
-      progress[bucket.bucket] = {
-        // The fallback priority doesn't matter here, but 3 is the one newer versions of the sync service
-        // will use by default.
-        priority: bucket.priority ?? 3,
-        at_last: atLast,
-        since_last: sinceLast,
-        target_count: bucket.count ?? 0
+      notifyTokenRefreshed = () => {
+        controlInvocations.inject({
+          command: PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED
+        });
       };
 
-      if (bucket.count != null && bucket.count < atLast + sinceLast) {
-        // Either due to a defrag / sync rule deploy or a compaction operation, the size
-        // of the bucket shrank so much that the local ops exceed the ops in the updated
-        // bucket. We can't prossibly report progress in this case (it would overshoot 100%).
-        invalidated = true;
-      }
-    }
-
-    if (invalidated) {
-      for (const bucket in progress) {
-        const bucketProgress = progress[bucket];
-        bucketProgress.at_last = 0;
-        bucketProgress.since_last = 0;
-      }
-    }
+      let hadSyncLine = false;
+      loop: while (true) {
+        const { done, value } = await controlInvocations.next();
+        if (done) break;
+
+        if (!hadSyncLine) {
+          // Trigger a local CRUD upload when the first sync line has been received, this allows uploading local changes
+          // that have been made while offline or disconnected.
+          if (
+            value.command == PowerSyncControlCommand.PROCESS_TEXT_LINE ||
+            value.command == PowerSyncControlCommand.PROCESS_BSON_LINE
+          ) {
+            hadSyncLine = true;
+            this.triggerCrudUpload?.();
+          }
+        }
 
-    this.updateSyncStatus({
-      dataFlow: {
-        downloading: true,
-        downloadProgress: progress
+        const instructions = await invokePowerSyncControl(value.command, value.payload);
+        for (const instruction of instructions) {
+          if ('EstablishSyncStream' in instruction) {
+            throw new Error('Received EstablishSyncStream while already connected.');
+          } else if ('CloseSyncStream' in instruction) {
+            hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
+            break loop;
+          } else {
+            await handleInstruction(instruction);
+          }
+        }
       }
-    });
-  }
-
-  private async applyCheckpoint(checkpoint: Checkpoint) {
-    let result = await this.options.adapter.syncLocalDatabase(checkpoint);
-
-    if (!result.checkpointValid) {
-      this.logger.debug(`Checksum mismatch in checkpoint ${checkpoint.last_op_id}, will reconnect`);
-      // This means checksums failed. Start again with a new checkpoint.
-      // TODO: better back-off
-      await new Promise((resolve) => setTimeout(resolve, 50));
-      return { applied: false, endIteration: true };
-    } else if (!result.ready) {
-      this.logger.debug(
-        `Could not apply checkpoint ${checkpoint.last_op_id} due to local data. We will retry applying the checkpoint after that upload is completed.`
-      );
-
-      return { applied: false, endIteration: false };
+    } finally {
+      this.notifyCompletedUploads = this.handleActiveStreamsChange = undefined;
+      notifyTokenRefreshed = undefined;
+      await stop();
     }
 
-    this.logger.debug(`Applied checkpoint ${checkpoint.last_op_id}`, checkpoint);
-    this.updateSyncStatus({
-      connected: true,
-      lastSyncedAt: new Date(),
-      dataFlow: {
-        downloading: false,
-        downloadProgress: null,
-        downloadError: undefined
-      }
-    });
-
-    return { applied: true, endIteration: false };
+    return { immediateRestart: hideDisconnectOnRestart };
   }
 
   protected updateSyncStatus(options: SyncStatusOptions) {