@powersync/common 1.50.0 → 1.52.0

package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts

@@ -4,7 +4,6 @@ import { InternalProgressInformation } from '../../../db/crud/SyncProgress.js';
 import { SyncStatus, SyncStatusOptions } from '../../../db/crud/SyncStatus.js';
 import { AbortOperation } from '../../../utils/AbortOperation.js';
 import { BaseListener, BaseObserver, BaseObserverInterface, Disposable } from '../../../utils/BaseObserver.js';
-import { DataStream } from '../../../utils/DataStream.js';
 import { throttleLeadingTrailing } from '../../../utils/async.js';
 import {
   BucketChecksum,
@@ -19,7 +18,6 @@ import { AbstractRemote, FetchStrategy, SyncStreamOptions } from './AbstractRemo
 import { EstablishSyncStream, Instruction, coreStatusToJs } from './core-instruction.js';
 import {
   BucketRequest,
-  CrudUploadNotification,
   StreamingSyncLine,
   StreamingSyncLineOrCrudUploadComplete,
   StreamingSyncRequestParameterType,
@@ -30,6 +28,15 @@ import {
   isStreamingSyncCheckpointPartiallyComplete,
   isStreamingSyncData
 } from './streaming-sync-types.js';
+import {
+  extractBsonObjects,
+  extractJsonLines,
+  injectable,
+  InjectableIterator,
+  map,
+  SimpleAsyncIterator
+} from '../../../utils/stream_transform.js';
+import type { BSON } from 'bson';
 
 export enum LockType {
   CRUD = 'crud',
@@ -682,6 +689,27 @@ The next upload iteration will be delayed.`);
     });
   }
 
+  private async receiveSyncLines(data: {
+    options: SyncStreamOptions;
+    connection: RequiredPowerSyncConnectionOptions;
+    bson?: typeof BSON;
+  }): Promise<SimpleAsyncIterator<Uint8Array | string>> {
+    const { options, connection, bson } = data;
+    const remote = this.options.remote;
+
+    if (connection.connectionMethod == SyncStreamConnectionMethod.HTTP) {
+      return await remote.fetchStream(options);
+    } else {
+      return await this.options.remote.socketStreamRaw(
+        {
+          ...options,
+          ...{ fetchStrategy: connection.fetchStrategy }
+        },
+        bson
+      );
+    }
+  }
+
   private async legacyStreamingSyncIteration(signal: AbortSignal, resolvedOptions: RequiredPowerSyncConnectionOptions) {
     const rawTables = resolvedOptions.serializedSchema?.raw_tables;
     if (rawTables != null && rawTables.length) {
@@ -717,46 +745,31 @@ The next upload iteration will be delayed.`);
       }
     };
 
-    let stream: DataStream<StreamingSyncLineOrCrudUploadComplete>;
-    if (resolvedOptions?.connectionMethod == SyncStreamConnectionMethod.HTTP) {
-      stream = await this.options.remote.postStreamRaw(syncOptions, (line: string | CrudUploadNotification) => {
+    const bson = await this.options.remote.getBSON();
+    const source = await this.receiveSyncLines({
+      options: syncOptions,
+      connection: resolvedOptions,
+      bson
+    });
+    const stream: InjectableIterator<StreamingSyncLineOrCrudUploadComplete> = injectable(
+      map(source, (line) => {
         if (typeof line == 'string') {
           return JSON.parse(line) as StreamingSyncLine;
         } else {
-          // Directly enqueued by us
-          return line;
+          return bson.deserialize(line) as StreamingSyncLine;
         }
-      });
-    } else {
-      const bson = await this.options.remote.getBSON();
-      stream = await this.options.remote.socketStreamRaw(
-        {
-          ...syncOptions,
-          ...{ fetchStrategy: resolvedOptions.fetchStrategy }
-        },
-        (payload: Uint8Array | CrudUploadNotification) => {
-          if (payload instanceof Uint8Array) {
-            return bson.deserialize(payload) as StreamingSyncLine;
-          } else {
-            // Directly enqueued by us
-            return payload;
-          }
-        },
-        bson
-      );
-    }
+      })
+    );
 
     this.logger.debug('Stream established. Processing events');
 
     this.notifyCompletedUploads = () => {
-      if (!stream.closed) {
-        stream.enqueueData({ crud_upload_completed: null });
-      }
+      stream.inject({ crud_upload_completed: null });
     };
 
-    while (!stream.closed) {
-      const line = await stream.read();
-      if (!line) {
+    while (true) {
+      const { value: line, done } = await stream.next();
+      if (done) {
         // The stream has closed while waiting
         return;
       }
@@ -942,6 +955,11 @@ The next upload iteration will be delayed.`);
     const syncImplementation = this;
     const adapter = this.options.adapter;
     const remote = this.options.remote;
+    const controller = new AbortController();
+    const abort = () => {
+      return controller.abort(signal.reason);
+    };
+    signal.addEventListener('abort', abort);
     let receivingLines: Promise<void> | null = null;
     let hadSyncLine = false;
     let hideDisconnectOnRestart = false;
@@ -949,64 +967,53 @@ The next upload iteration will be delayed.`);
     if (signal.aborted) {
       throw new AbortOperation('Connection request has been aborted');
     }
-    const abortController = new AbortController();
-    signal.addEventListener('abort', () => abortController.abort());
 
     // Pending sync lines received from the service, as well as local events that trigger a powersync_control
     // invocation (local events include refreshed tokens and completed uploads).
     // This is a single data stream so that we can handle all control calls from a single place.
-    let controlInvocations: DataStream<EnqueuedCommand, Uint8Array | EnqueuedCommand> | null = null;
+    let controlInvocations: InjectableIterator<EnqueuedCommand> | null = null;
 
     async function connect(instr: EstablishSyncStream) {
       const syncOptions: SyncStreamOptions = {
         path: '/sync/stream',
-        abortSignal: abortController.signal,
+        abortSignal: controller.signal,
         data: instr.request
       };
 
-      if (resolvedOptions.connectionMethod == SyncStreamConnectionMethod.HTTP) {
-        controlInvocations = await remote.postStreamRaw(syncOptions, (line: string | EnqueuedCommand) => {
-          if (typeof line == 'string') {
-            return {
-              command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
-              payload: line
-            };
-          } else {
-            // Directly enqueued by us
-            return line;
-          }
-        });
-      } else {
-        controlInvocations = await remote.socketStreamRaw(
-          {
-            ...syncOptions,
-            fetchStrategy: resolvedOptions.fetchStrategy
-          },
-          (payload: Uint8Array | EnqueuedCommand) => {
-            if (payload instanceof Uint8Array) {
+      controlInvocations = injectable(
+        map(
+          await syncImplementation.receiveSyncLines({
+            options: syncOptions,
+            connection: resolvedOptions
+          }),
+          (line) => {
+            if (typeof line == 'string') {
               return {
-                command: PowerSyncControlCommand.PROCESS_BSON_LINE,
-                payload: payload
+                command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
+                payload: line
               };
             } else {
-              // Directly enqueued by us
-              return payload;
+              return {
+                command: PowerSyncControlCommand.PROCESS_BSON_LINE,
+                payload: line
+              };
             }
           }
-        );
-      }
+        )
+      );
 
       // The rust client will set connected: true after the first sync line because that's when it gets invoked, but
       // we're already connected here and can report that.
       syncImplementation.updateSyncStatus({ connected: true });
 
       try {
-        while (!controlInvocations.closed) {
-          const line = await controlInvocations.read();
-          if (line == null) {
-            return;
+        while (true) {
+          let event = await controlInvocations.next();
+          if (event.done) {
+            break;
           }
 
+          const line = event.value;
           await control(line.command, line.payload);
 
           if (!hadSyncLine) {
@@ -1015,12 +1022,8 @@ The next upload iteration will be delayed.`);
           }
         }
       } finally {
-        const activeInstructions = controlInvocations;
-        // We concurrently add events to the active data stream when e.g. a CRUD upload is completed or a token is
-        // refreshed. That would throw after closing (and we can't handle those events either way), so set this back
-        // to null.
-        controlInvocations = null;
-        await activeInstructions.close();
+        abort();
+        signal.removeEventListener('abort', abort);
       }
     }
 
@@ -1072,7 +1075,7 @@ The next upload iteration will be delayed.`);
           // Restart iteration after the credentials have been refreshed.
           remote.fetchCredentials().then(
             (_) => {
-              controlInvocations?.enqueueData({ command: PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED });
+              controlInvocations?.inject({ command: PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED });
             },
             (err) => {
               syncImplementation.logger.warn('Could not prefetch credentials', err);
@@ -1080,7 +1083,7 @@ The next upload iteration will be delayed.`);
           );
         }
       } else if ('CloseSyncStream' in instruction) {
-        abortController.abort();
+        controller.abort();
         hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
       } else if ('FlushFileSystem' in instruction) {
         // Not necessary on JS platforms.
@@ -1113,17 +1116,13 @@ The next upload iteration will be delayed.`);
       await control(PowerSyncControlCommand.START, JSON.stringify(options));
 
       this.notifyCompletedUploads = () => {
-        if (controlInvocations && !controlInvocations?.closed) {
-          controlInvocations.enqueueData({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
-        }
+        controlInvocations?.inject({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
       };
       this.handleActiveStreamsChange = () => {
-        if (controlInvocations && !controlInvocations?.closed) {
-          controlInvocations.enqueueData({
-            command: PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
-            payload: JSON.stringify(this.activeStreams)
-          });
-        }
+        controlInvocations?.inject({
+          command: PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
+          payload: JSON.stringify(this.activeStreams)
+        });
       };
       await receivingLines;
     } finally {

package/src/index.ts

@@ -59,9 +59,9 @@ export * from './client/watched/WatchedQuery.js';
 export * from './utils/AbortOperation.js';
 export * from './utils/BaseObserver.js';
 export * from './utils/ControlledExecutor.js';
-export * from './utils/DataStream.js';
 export * from './utils/Logger.js';
 export * from './utils/mutex.js';
 export * from './utils/parseQuery.js';
+export type { SimpleAsyncIterator } from './utils/stream_transform.js';
 
 export * from './types/types.js';

package/src/utils/async.ts

@@ -1,14 +1,3 @@
-/**
- * A ponyfill for `Symbol.asyncIterator` that is compatible with the
- * [recommended polyfill](https://github.com/Azure/azure-sdk-for-js/blob/%40azure/core-asynciterator-polyfill_1.0.2/sdk/core/core-asynciterator-polyfill/src/index.ts#L4-L6)
- * we recommend for React Native.
- *
- * As long as we use this symbol (instead of `for await` and `async *`) in this package, we can be compatible with async
- * iterators without requiring them.
- */
-export const symbolAsyncIterator: typeof Symbol.asyncIterator =
-  Symbol.asyncIterator ?? Symbol.for('Symbol.asyncIterator');
-
 /**
  * Throttle a function to be called at most once every "wait" milliseconds,
  * on the trailing edge.

package/src/utils/mutex.ts

@@ -1,21 +1,32 @@
+import { Queue } from './queue.js';
+
 export type UnlockFn = () => void;
 
 /**
- * An asynchronous mutex implementation.
+ * An asynchronous semaphore implementation with associated items per lease.
  *
  * @internal This class is meant to be used in PowerSync SDKs only, and is not part of the public API.
  */
-export class Mutex {
-  private inCriticalSection = false;
+export class Semaphore<T> {
+  // Available items that are not currently assigned to a waiter.
+  private readonly available: Queue<T>;
 
+  readonly size: number;
   // Linked list of waiters. We don't expect the wait list to become particularly large, and this allows removing
   // aborted waiters from the middle of the list efficiently.
-  private firstWaiter?: MutexWaitNode;
-  private lastWaiter?: MutexWaitNode;
+  private firstWaiter?: SemaphoreWaitNode<T>;
+  private lastWaiter?: SemaphoreWaitNode<T>;
 
-  private addWaiter(onAcquire: () => void): MutexWaitNode {
-    const node: MutexWaitNode = {
+  constructor(elements: Iterable<T>) {
+    this.available = new Queue(elements);
+    this.size = this.available.length;
+  }
+
+  private addWaiter(requestedItems: number, onAcquire: () => void): SemaphoreWaitNode<T> {
+    const node: SemaphoreWaitNode<T> = {
       isActive: true,
+      acquiredItems: [],
+      remainingItems: requestedItems,
       onAcquire,
       prev: this.lastWaiter
     };
@@ -30,7 +41,7 @@ export class Mutex {
     return node;
   }
 
-  private deactivateWaiter(waiter: MutexWaitNode) {
+  private deactivateWaiter(waiter: SemaphoreWaitNode<T>) {
     const { prev, next } = waiter;
     waiter.isActive = false;
 
@@ -40,77 +51,129 @@ export class Mutex {
     if (waiter == this.lastWaiter) this.lastWaiter = prev;
   }
 
-  acquire(abort?: AbortSignal): Promise<UnlockFn> {
+  private requestPermits(amount: number, abort?: AbortSignal): Promise<{ items: T[]; release: UnlockFn }> {
+    if (amount <= 0 || amount > this.size) {
+      throw new Error(`Invalid amount of items requested (${amount}), must be between 1 and ${this.size}`);
+    }
+
     return new Promise((resolve, reject) => {
       function rejectAborted() {
-        reject(abort?.reason ?? new Error('Mutex acquire aborted'));
+        reject(abort?.reason ?? new Error('Semaphore acquire aborted'));
       }
       if (abort?.aborted) {
         return rejectAborted();
       }
 
-      let holdsMutex = false;
+      let waiter: SemaphoreWaitNode<T>;
 
       const markCompleted = () => {
-        if (!holdsMutex) return;
-        holdsMutex = false;
+        const items = waiter.acquiredItems;
+        waiter.acquiredItems = []; // Avoid releasing items twice.
+
+        for (const element of items) {
+          // Give to next waiter, if possible.
+          const nextWaiter = this.firstWaiter;
+          if (nextWaiter) {
+            nextWaiter.acquiredItems.push(element);
+            nextWaiter.remainingItems--;
+            if (nextWaiter.remainingItems == 0) {
+              nextWaiter.onAcquire();
+            }
+          } else {
+            // No pending waiter, return lease into pool.
+            this.available.addLast(element);
+          }
+        }
+      };
+
+      const onAbort = () => {
+        abort?.removeEventListener('abort', onAbort);
 
-        const waiter = this.firstWaiter;
-        if (waiter) {
+        if (waiter.isActive) {
           this.deactivateWaiter(waiter);
-          // Still in critical section, but owned by next waiter now.
-          waiter.onAcquire();
-        } else {
-          this.inCriticalSection = false;
+          rejectAborted();
         }
       };
 
-      if (!this.inCriticalSection) {
-        this.inCriticalSection = true;
-        holdsMutex = true;
-        return resolve(markCompleted);
-      } else {
-        let node: MutexWaitNode;
+      const resolvePromise = () => {
+        this.deactivateWaiter(waiter);
+        abort?.removeEventListener('abort', onAbort);
 
-        const onAbort = () => {
-          abort?.removeEventListener('abort', onAbort);
+        const items = waiter.acquiredItems;
+        resolve({ items, release: markCompleted });
+      };
 
-          if (node.isActive) {
-            this.deactivateWaiter(node);
-            rejectAborted();
-          }
-        };
+      waiter = this.addWaiter(amount, resolvePromise);
 
-        node = this.addWaiter(() => {
-          abort?.removeEventListener('abort', onAbort);
-          holdsMutex = true;
-          resolve(markCompleted);
-        });
+      // If there are items in the pool that haven't been assigned, we can pull them into this waiter. Note that this is
+      // only the case if we're the first waiter (otherwise, items would have been assigned to an earlier waiter).
+      while (!this.available.isEmpty && waiter.remainingItems > 0) {
+        waiter.acquiredItems.push(this.available.removeFirst());
+        waiter.remainingItems--;
+      }
 
-        abort?.addEventListener('abort', onAbort);
+      if (waiter.remainingItems == 0) {
+        return resolvePromise();
       }
+
+      abort?.addEventListener('abort', onAbort);
     });
   }
 
-  async runExclusive<T>(fn: () => PromiseLike<T> | T, abort?: AbortSignal): Promise<T> {
-    const returnMutex = await this.acquire(abort);
+  /**
+   * Requests a single item from the pool.
+   *
+   * The returned `release` callback must be invoked to return the item into the pool.
+   */
+  async requestOne(abort?: AbortSignal): Promise<{ item: T; release: UnlockFn }> {
+    const { items, release } = await this.requestPermits(1, abort);
+    return { release, item: items[0] };
+  }
 
-    try {
-      return await fn();
-    } finally {
-      returnMutex();
-    }
+  /**
+   * Requests access to all items from the pool.
+   *
+   * The returned `release` callback must be invoked to return items into the pool.
+   */
+  requestAll(abort?: AbortSignal): Promise<{ items: T[]; release: UnlockFn }> {
+    return this.requestPermits(this.size, abort);
   }
 }
 
-interface MutexWaitNode {
+interface SemaphoreWaitNode<T> {
   /**
    * Whether the waiter is currently active (not aborted and not fullfilled).
    */
   isActive: boolean;
+  acquiredItems: T[];
+  remainingItems: number;
   onAcquire: () => void;
-  prev?: MutexWaitNode;
-  next?: MutexWaitNode;
+  prev?: SemaphoreWaitNode<T>;
+  next?: SemaphoreWaitNode<T>;
+}
+
+/**
+ * An asynchronous mutex implementation.
+ *
+ * @internal This class is meant to be used in PowerSync SDKs only, and is not part of the public API.
+ */
+export class Mutex {
+  private inner = new Semaphore([null]);
+
+  async acquire(abort?: AbortSignal): Promise<UnlockFn> {
+    const { release } = await this.inner.requestOne(abort);
+    return release;
+  }
+
+  async runExclusive<T>(fn: () => PromiseLike<T> | T, abort?: AbortSignal): Promise<T> {
+    const returnMutex = await this.acquire(abort);
+
+    try {
+      return await fn();
+    } finally {
+      returnMutex();
+    }
+  }
 }
 
 /**

package/src/utils/queue.ts

@@ -0,0 +1,48 @@
+/**
+ * A simple fixed-capacity queue implementation.
+ *
+ * Unlike a naive queue implemented by `array.push()` and `array.shift()`, this avoids moving array elements around
+ * and is `O(1)` for {@link addLast} and {@link removeFirst}.
+ */
+export class Queue<T> {
+  private table: (T | undefined)[];
+  // Index of the first element in the table.
+  private head: number;
+  // Amount of items currently in the queue.
+  private _length: number;
+
+  constructor(initialItems: Iterable<T>) {
+    this.table = [...initialItems];
+    this.head = 0;
+    this._length = this.table.length;
+  }
+
+  get isEmpty(): boolean {
+    return this.length == 0;
+  }
+
+  get length(): number {
+    return this._length;
+  }
+
+  removeFirst(): T {
+    if (this.isEmpty) {
+      throw new Error('Queue is empty');
+    }
+
+    const result = this.table[this.head] as T;
+    this._length--;
+    this.table[this.head] = undefined;
+    this.head = (this.head + 1) % this.table.length;
+    return result;
+  }
+
+  addLast(element: T) {
+    if (this.length == this.table.length) {
+      throw new Error('Queue is full');
+    }
+
+    this.table[(this.head + this._length) % this.table.length] = element;
+    this._length++;
+  }
+}