@powersync/common 1.51.0 → 1.52.0

package/src/client/sync/stream/AbstractStreamingSyncImplementation.ts

@@ -4,7 +4,6 @@ import { InternalProgressInformation } from '../../../db/crud/SyncProgress.js';
 import { SyncStatus, SyncStatusOptions } from '../../../db/crud/SyncStatus.js';
 import { AbortOperation } from '../../../utils/AbortOperation.js';
 import { BaseListener, BaseObserver, BaseObserverInterface, Disposable } from '../../../utils/BaseObserver.js';
-import { DataStream } from '../../../utils/DataStream.js';
 import { throttleLeadingTrailing } from '../../../utils/async.js';
 import {
   BucketChecksum,
@@ -19,7 +18,6 @@ import { AbstractRemote, FetchStrategy, SyncStreamOptions } from './AbstractRemo
 import { EstablishSyncStream, Instruction, coreStatusToJs } from './core-instruction.js';
 import {
   BucketRequest,
-  CrudUploadNotification,
   StreamingSyncLine,
   StreamingSyncLineOrCrudUploadComplete,
   StreamingSyncRequestParameterType,
@@ -30,6 +28,15 @@ import {
   isStreamingSyncCheckpointPartiallyComplete,
   isStreamingSyncData
 } from './streaming-sync-types.js';
+import {
+  extractBsonObjects,
+  extractJsonLines,
+  injectable,
+  InjectableIterator,
+  map,
+  SimpleAsyncIterator
+} from '../../../utils/stream_transform.js';
+import type { BSON } from 'bson';
 
 export enum LockType {
   CRUD = 'crud',
@@ -682,6 +689,27 @@ The next upload iteration will be delayed.`);
     });
   }
 
+  private async receiveSyncLines(data: {
+    options: SyncStreamOptions;
+    connection: RequiredPowerSyncConnectionOptions;
+    bson?: typeof BSON;
+  }): Promise<SimpleAsyncIterator<Uint8Array | string>> {
+    const { options, connection, bson } = data;
+    const remote = this.options.remote;
+
+    if (connection.connectionMethod == SyncStreamConnectionMethod.HTTP) {
+      return await remote.fetchStream(options);
+    } else {
+      return await this.options.remote.socketStreamRaw(
+        {
+          ...options,
+          ...{ fetchStrategy: connection.fetchStrategy }
+        },
+        bson
+      );
+    }
+  }
+
   private async legacyStreamingSyncIteration(signal: AbortSignal, resolvedOptions: RequiredPowerSyncConnectionOptions) {
     const rawTables = resolvedOptions.serializedSchema?.raw_tables;
     if (rawTables != null && rawTables.length) {
@@ -717,46 +745,31 @@ The next upload iteration will be delayed.`);
       }
     };
 
-    let stream: DataStream<StreamingSyncLineOrCrudUploadComplete>;
-    if (resolvedOptions?.connectionMethod == SyncStreamConnectionMethod.HTTP) {
-      stream = await this.options.remote.postStreamRaw(syncOptions, (line: string | CrudUploadNotification) => {
+    const bson = await this.options.remote.getBSON();
+    const source = await this.receiveSyncLines({
+      options: syncOptions,
+      connection: resolvedOptions,
+      bson
+    });
+    const stream: InjectableIterator<StreamingSyncLineOrCrudUploadComplete> = injectable(
+      map(source, (line) => {
         if (typeof line == 'string') {
           return JSON.parse(line) as StreamingSyncLine;
         } else {
-          // Directly enqueued by us
-          return line;
+          return bson.deserialize(line) as StreamingSyncLine;
         }
-      });
-    } else {
-      const bson = await this.options.remote.getBSON();
-      stream = await this.options.remote.socketStreamRaw(
-        {
-          ...syncOptions,
-          ...{ fetchStrategy: resolvedOptions.fetchStrategy }
-        },
-        (payload: Uint8Array | CrudUploadNotification) => {
-          if (payload instanceof Uint8Array) {
-            return bson.deserialize(payload) as StreamingSyncLine;
-          } else {
-            // Directly enqueued by us
-            return payload;
-          }
-        },
-        bson
-      );
-    }
+      })
+    );
 
     this.logger.debug('Stream established. Processing events');
 
     this.notifyCompletedUploads = () => {
-      if (!stream.closed) {
-        stream.enqueueData({ crud_upload_completed: null });
-      }
+      stream.inject({ crud_upload_completed: null });
     };
 
-    while (!stream.closed) {
-      const line = await stream.read();
-      if (!line) {
+    while (true) {
+      const { value: line, done } = await stream.next();
+      if (done) {
         // The stream has closed while waiting
         return;
       }
@@ -942,6 +955,11 @@ The next upload iteration will be delayed.`);
     const syncImplementation = this;
     const adapter = this.options.adapter;
     const remote = this.options.remote;
+    const controller = new AbortController();
+    const abort = () => {
+      return controller.abort(signal.reason);
+    };
+    signal.addEventListener('abort', abort);
     let receivingLines: Promise<void> | null = null;
     let hadSyncLine = false;
     let hideDisconnectOnRestart = false;
@@ -949,64 +967,53 @@ The next upload iteration will be delayed.`);
     if (signal.aborted) {
       throw new AbortOperation('Connection request has been aborted');
     }
-    const abortController = new AbortController();
-    signal.addEventListener('abort', () => abortController.abort());
 
     // Pending sync lines received from the service, as well as local events that trigger a powersync_control
     // invocation (local events include refreshed tokens and completed uploads).
     // This is a single data stream so that we can handle all control calls from a single place.
-    let controlInvocations: DataStream<EnqueuedCommand, Uint8Array | EnqueuedCommand> | null = null;
+    let controlInvocations: InjectableIterator<EnqueuedCommand> | null = null;
 
     async function connect(instr: EstablishSyncStream) {
       const syncOptions: SyncStreamOptions = {
         path: '/sync/stream',
-        abortSignal: abortController.signal,
+        abortSignal: controller.signal,
         data: instr.request
       };
 
-      if (resolvedOptions.connectionMethod == SyncStreamConnectionMethod.HTTP) {
-        controlInvocations = await remote.postStreamRaw(syncOptions, (line: string | EnqueuedCommand) => {
-          if (typeof line == 'string') {
-            return {
-              command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
-              payload: line
-            };
-          } else {
-            // Directly enqueued by us
-            return line;
-          }
-        });
-      } else {
-        controlInvocations = await remote.socketStreamRaw(
-          {
-            ...syncOptions,
-            fetchStrategy: resolvedOptions.fetchStrategy
-          },
-          (payload: Uint8Array | EnqueuedCommand) => {
-            if (payload instanceof Uint8Array) {
+      controlInvocations = injectable(
+        map(
+          await syncImplementation.receiveSyncLines({
+            options: syncOptions,
+            connection: resolvedOptions
+          }),
+          (line) => {
+            if (typeof line == 'string') {
               return {
-                command: PowerSyncControlCommand.PROCESS_BSON_LINE,
-                payload: payload
+                command: PowerSyncControlCommand.PROCESS_TEXT_LINE,
+                payload: line
               };
             } else {
-              // Directly enqueued by us
-              return payload;
+              return {
+                command: PowerSyncControlCommand.PROCESS_BSON_LINE,
+                payload: line
+              };
             }
           }
-        );
-      }
+        )
+      );
 
       // The rust client will set connected: true after the first sync line because that's when it gets invoked, but
       // we're already connected here and can report that.
       syncImplementation.updateSyncStatus({ connected: true });
 
       try {
-        while (!controlInvocations.closed) {
-          const line = await controlInvocations.read();
-          if (line == null) {
-            return;
+        while (true) {
+          let event = await controlInvocations.next();
+          if (event.done) {
+            break;
           }
 
+          const line = event.value;
           await control(line.command, line.payload);
 
           if (!hadSyncLine) {
@@ -1015,12 +1022,8 @@ The next upload iteration will be delayed.`);
           }
         }
       } finally {
-        const activeInstructions = controlInvocations;
-        // We concurrently add events to the active data stream when e.g. a CRUD upload is completed or a token is
-        // refreshed. That would throw after closing (and we can't handle those events either way), so set this back
-        // to null.
-        controlInvocations = null;
-        await activeInstructions.close();
+        abort();
+        signal.removeEventListener('abort', abort);
       }
     }
 
@@ -1072,7 +1075,7 @@ The next upload iteration will be delayed.`);
           // Restart iteration after the credentials have been refreshed.
           remote.fetchCredentials().then(
             (_) => {
-              controlInvocations?.enqueueData({ command: PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED });
+              controlInvocations?.inject({ command: PowerSyncControlCommand.NOTIFY_TOKEN_REFRESHED });
             },
             (err) => {
               syncImplementation.logger.warn('Could not prefetch credentials', err);
@@ -1080,7 +1083,7 @@ The next upload iteration will be delayed.`);
           );
         }
       } else if ('CloseSyncStream' in instruction) {
-        abortController.abort();
+        controller.abort();
         hideDisconnectOnRestart = instruction.CloseSyncStream.hide_disconnect;
       } else if ('FlushFileSystem' in instruction) {
         // Not necessary on JS platforms.
@@ -1113,17 +1116,13 @@ The next upload iteration will be delayed.`);
       await control(PowerSyncControlCommand.START, JSON.stringify(options));
 
       this.notifyCompletedUploads = () => {
-        if (controlInvocations && !controlInvocations?.closed) {
-          controlInvocations.enqueueData({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
-        }
+        controlInvocations?.inject({ command: PowerSyncControlCommand.NOTIFY_CRUD_UPLOAD_COMPLETED });
       };
       this.handleActiveStreamsChange = () => {
-        if (controlInvocations && !controlInvocations?.closed) {
-          controlInvocations.enqueueData({
-            command: PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
-            payload: JSON.stringify(this.activeStreams)
-          });
-        }
+        controlInvocations?.inject({
+          command: PowerSyncControlCommand.UPDATE_SUBSCRIPTIONS,
+          payload: JSON.stringify(this.activeStreams)
+        });
       };
       await receivingLines;
     } finally {

package/src/index.ts

@@ -59,9 +59,9 @@ export * from './client/watched/WatchedQuery.js';
 export * from './utils/AbortOperation.js';
 export * from './utils/BaseObserver.js';
 export * from './utils/ControlledExecutor.js';
-export * from './utils/DataStream.js';
 export * from './utils/Logger.js';
 export * from './utils/mutex.js';
 export * from './utils/parseQuery.js';
+export type { SimpleAsyncIterator } from './utils/stream_transform.js';
 
 export * from './types/types.js';

package/src/utils/async.ts

@@ -1,14 +1,3 @@
-/**
- * A ponyfill for `Symbol.asyncIterator` that is compatible with the
- * [recommended polyfill](https://github.com/Azure/azure-sdk-for-js/blob/%40azure/core-asynciterator-polyfill_1.0.2/sdk/core/core-asynciterator-polyfill/src/index.ts#L4-L6)
- * we recommend for React Native.
- *
- * As long as we use this symbol (instead of `for await` and `async *`) in this package, we can be compatible with async
- * iterators without requiring them.
- */
-export const symbolAsyncIterator: typeof Symbol.asyncIterator =
-  Symbol.asyncIterator ?? Symbol.for('Symbol.asyncIterator');
-
 /**
  * Throttle a function to be called at most once every "wait" milliseconds,
  * on the trailing edge.

package/src/utils/stream_transform.ts

@@ -0,0 +1,252 @@
+/**
+ * An async iterator that can't be cancelled.
+ *
+ * To keep data flow simple, we always pass an explicit cancellation token when subscribing to async streams. Once the
+ * {@link AbortSignal} aborts, iterators are supposed to clean up and then emit a final `{done: true}` event. This means
+ * that there's no way to distinguish between streams that have completed normally and streams that have been cancelled,
+ * but that is acceptable for our uses of this.
+ */
+export type SimpleAsyncIterator<T> = Pick<AsyncIterator<T>, 'next'>;
+
+export const doneResult: IteratorReturnResult<any> = { done: true, value: undefined };
+
+export function valueResult<T>(value: T) {
+  return { done: false, value };
+}
+
+/**
+ * A variant of {@link Array.map} for async iterators.
+ */
+export function map<T1, T2>(source: SimpleAsyncIterator<T1>, map: (source: T1) => T2): SimpleAsyncIterator<T2> {
+  return {
+    next: async () => {
+      const value = await source.next();
+      if (value.done) {
+        return value;
+      } else {
+        return { value: map(value.value) };
+      }
+    }
+  };
+}
+
+export interface InjectableIterator<T> extends SimpleAsyncIterator<T> {
+  inject(event: T): void;
+}
+
+/**
+ * Expands a source async iterator by allowing to inject events asynchronously.
+ *
+ * The resulting iterator will emit all events from its source. Additionally though, events can be injected. These
+ * events are dropped once the main iterator completes, but are otherwise forwarded.
+ *
+ * The iterator completes when its source completes, and it supports backpressure by only calling `next()` on the source
+ * in response to a `next()` call from downstream if no pending injected events can be dispatched.
+ */
+export function injectable<T>(source: SimpleAsyncIterator<T>): InjectableIterator<T> {
+  type Waiter = { resolve: (t: IteratorResult<T>) => void; reject: (e: unknown) => void };
+
+  let sourceIsDone = false;
+  let waiter: Waiter | undefined = undefined; // An active, waiting next() call.
+  // A pending upstream event that couldn't be dispatched because inject() has been called before it was resolved.
+  let pendingSourceEvent: ((w: Waiter) => void) | null = null;
+
+  let pendingInjectedEvents: T[] = [];
+
+  const consumeWaiter = () => {
+    const pending = waiter;
+    waiter = undefined;
+    return pending;
+  };
+
+  const fetchFromSource = () => {
+    const resolveWaiter = (propagate: (w: Waiter) => void) => {
+      const active = consumeWaiter();
+      if (active) {
+        propagate(active);
+      } else {
+        pendingSourceEvent = propagate;
+      }
+    };
+
+    const nextFromSource = source.next();
+    nextFromSource.then(
+      (value) => {
+        sourceIsDone = value.done == true;
+        resolveWaiter((w) => w.resolve(value));
+      },
+      (error) => {
+        resolveWaiter((w) => w.reject(error));
+      }
+    );
+  };
+
+  return {
+    next: () => {
+      return new Promise((resolve, reject) => {
+        // First priority: Dispatch ready upstream events.
+        if (sourceIsDone) {
+          return resolve(doneResult);
+        }
+        if (pendingSourceEvent) {
+          pendingSourceEvent({ resolve, reject });
+          pendingSourceEvent = null;
+          return;
+        }
+
+        // Second priority: Dispatch injected events
+        if (pendingInjectedEvents.length) {
+          return resolve(valueResult(pendingInjectedEvents.shift()!));
+        }
+
+        // Nothing pending? Fetch from source
+        waiter = { resolve, reject };
+        return fetchFromSource();
+      });
+    },
+    inject: (event) => {
+      const pending = consumeWaiter();
+      if (pending != null) {
+        pending.resolve(valueResult(event));
+      } else {
+        pendingInjectedEvents.push(event);
+      }
+    }
+  };
+}
+
+/**
+ * Splits a byte stream at line endings, emitting each line as a string.
+ */
+export function extractJsonLines(
+  source: SimpleAsyncIterator<Uint8Array>,
+  decoder: TextDecoder
+): SimpleAsyncIterator<string> {
+  let buffer = '';
+  const pendingLines: string[] = [];
+  let isFinalEvent = false;
+
+  return {
+    next: async () => {
+      while (true) {
+        if (isFinalEvent) {
+          return doneResult;
+        }
+
+        {
+          const first = pendingLines.shift();
+          if (first) {
+            return { done: false, value: first };
+          }
+        }
+
+        const { done, value } = await source.next();
+        if (done) {
+          const remaining = buffer.trim();
+          if (remaining.length != 0) {
+            isFinalEvent = true;
+            return { done: false, value: remaining };
+          }
+
+          return doneResult;
+        }
+
+        const data = decoder.decode(value, { stream: true });
+        buffer += data;
+
+        const lines = buffer.split('\n');
+        for (let i = 0; i < lines.length - 1; i++) {
+          const l = lines[i].trim();
+          if (l.length > 0) {
+            pendingLines.push(l);
+          }
+        }
+
+        buffer = lines[lines.length - 1];
+      }
+    }
+  };
+}
+
+/**
+ * Splits a concatenated stream of BSON objects by emitting individual objects.
+ */
+export function extractBsonObjects(source: SimpleAsyncIterator<Uint8Array>): SimpleAsyncIterator<Uint8Array> {
+  // Fully read but not emitted yet.
+  const completedObjects: Uint8Array[] = [];
+
+  // Whether source has returned { done: true }. We do the same once completed objects have been emitted.
+  let isDone = false;
+
+  const lengthBuffer = new DataView(new ArrayBuffer(4));
+  let objectBody: Uint8Array | null = null;
+  // If we're parsing the length field, a number between 1 and 4 (inclusive) describing remaining bytes in the header.
+  // If we're consuming a document, the bytes remaining.
+  let remainingLength = 4;
+
+  return {
+    async next(): Promise<IteratorResult<Uint8Array>> {
+      while (true) {
+        // Before fetching new data from upstream, return completed objects.
+        if (completedObjects.length) {
+          return valueResult(completedObjects.shift()!);
+        }
+        if (isDone) {
+          return doneResult;
+        }
+
+        const upstreamEvent = await source.next();
+        if (upstreamEvent.done) {
+          isDone = true;
+          if (objectBody || remainingLength != 4) {
+            throw new Error('illegal end of stream in BSON object');
+          }
+          return doneResult;
+        }
+
+        const chunk = upstreamEvent.value;
+        for (let i = 0; i < chunk.length; ) {
+          const availableInData = chunk.length - i;
+
+          if (objectBody) {
+            // We're in the middle of reading a BSON document.
+            const bytesToRead = Math.min(availableInData, remainingLength);
+            const copySource = new Uint8Array(chunk.buffer, chunk.byteOffset + i, bytesToRead);
+            objectBody.set(copySource, objectBody.length - remainingLength);
+            i += bytesToRead;
+            remainingLength -= bytesToRead;
+
+            if (remainingLength == 0) {
+              completedObjects.push(objectBody);
+
+              // Prepare to read another document, starting with its length
+              objectBody = null;
+              remainingLength = 4;
+            }
+          } else {
+            // Copy up to 4 bytes into lengthBuffer, depending on how many we still need.
+            const bytesToRead = Math.min(availableInData, remainingLength);
+            for (let j = 0; j < bytesToRead; j++) {
+              lengthBuffer.setUint8(4 - remainingLength + j, chunk[i + j]);
+            }
+            i += bytesToRead;
+            remainingLength -= bytesToRead;
+
+            if (remainingLength == 0) {
+              // Transition from reading length header to reading document. Subtracting 4 because the length of the
+              // header is included in length.
+              const length = lengthBuffer.getInt32(0, true /* little endian */);
+              remainingLength = length - 4;
+              if (remainingLength < 1) {
+                throw new Error(`invalid length for bson: ${length}`);
+              }
+
+              objectBody = new Uint8Array(length);
+              new DataView(objectBody.buffer).setInt32(0, length, true);
+            }
+          }
+        }
+      }
+    }
+  };
+}

package/lib/utils/DataStream.d.ts

@@ -1,62 +0,0 @@
-import { ILogger } from 'js-logger';
-import { BaseListener, BaseObserver } from './BaseObserver.js';
-export type DataStreamOptions<ParsedData, SourceData> = {
-    mapLine?: (line: SourceData) => ParsedData;
-    /**
-     * Close the stream if any consumer throws an error
-     */
-    closeOnError?: boolean;
-    pressure?: {
-        highWaterMark?: number;
-        lowWaterMark?: number;
-    };
-    logger?: ILogger;
-};
-export type DataStreamCallback<Data extends any = any> = (data: Data) => Promise<void>;
-export interface DataStreamListener<Data extends any = any> extends BaseListener {
-    data: (data: Data) => Promise<void>;
-    closed: () => void;
-    error: (error: Error) => void;
-    highWater: () => Promise<void>;
-    lowWater: () => Promise<void>;
-}
-export declare const DEFAULT_PRESSURE_LIMITS: {
-    highWater: number;
-    lowWater: number;
-};
-/**
- * A very basic implementation of a data stream with backpressure support which does not use
- * native JS streams or async iterators.
- * This is handy for environments such as React Native which need polyfills for the above.
- */
-export declare class DataStream<ParsedData, SourceData = any> extends BaseObserver<DataStreamListener<ParsedData>> {
-    protected options?: DataStreamOptions<ParsedData, SourceData> | undefined;
-    dataQueue: SourceData[];
-    protected isClosed: boolean;
-    protected processingPromise: Promise<void> | null;
-    protected notifyDataAdded: (() => void) | null;
-    protected logger: ILogger;
-    protected mapLine: (line: SourceData) => ParsedData;
-    constructor(options?: DataStreamOptions<ParsedData, SourceData> | undefined);
-    get highWatermark(): number;
-    get lowWatermark(): number;
-    get closed(): boolean;
-    close(): Promise<void>;
-    /**
-     * Enqueues data for the consumers to read
-     */
-    enqueueData(data: SourceData): void;
-    /**
-     * Reads data once from the data stream
-     * @returns a Data payload or Null if the stream closed.
-     */
-    read(): Promise<ParsedData | null>;
-    /**
-     * Executes a callback for each data item in the stream
-     */
-    forEach(callback: DataStreamCallback<ParsedData>): () => void;
-    protected processQueue(): Promise<void> | undefined;
-    protected hasDataReader(): boolean;
-    protected _processQueue(): Promise<void>;
-    protected iterateAsyncErrored(cb: (l: Partial<DataStreamListener<ParsedData>>) => Promise<void>): Promise<void>;
-}