@powersync/service-core 0.0.2

package/src/sync/BroadcastIterable.ts

@@ -0,0 +1,161 @@
+import { AbortError } from 'ix/aborterror.js';
+import { LastValueSink } from './LastValueSink.js';
+
+export type IterableSource<T> = (signal: AbortSignal) => AsyncIterable<T>;
+
+/**
+ * Loosely based on Dart's broadcast streams:
+ * https://api.flutter.dev/flutter/dart-async/StreamController/StreamController.broadcast.html
+ *
+ * 1. Exposes an AsyncIterable interface.
+ * 2. Allows multiple concurrent subscribers.
+ * 3. Any new subscriber gets new events as they are published.
+ * 4. The source iterable is only created once there are subscribers.
+ *
+ * One notable difference: The last value is buffered and immediately returned for any new subscribers.
+ *
+ * When all subscribers are stopped, this source iterable is stopped.
+ *
+ * Any error on the source is passed on to all subscribers. This generally stops all
+ * subscribers. Once a new subscriber is then added, a new source is started.
+ *
+ * Slow subscribers will skip events.
+ *
+ * Implementation note: It is possible to do this using:
+ * 1. rxjs `share` for multicasting and auto-starting + stopping the source.
+ * 2. rxjs `BehaviorSubject` to keep the last value.
+ * 3. Use `LastValueSink` to convert to an AsyncIterable.
+ */
+export class BroadcastIterable<T> implements AsyncIterable<T> {
+  private last: T | undefined = undefined;
+  private subscribers: Set<LastValueSink<T>> | undefined = undefined;
+  private abortController: AbortController | undefined = undefined;
+
+  constructor(private source: IterableSource<T>) {}
+
+  private start(sink: LastValueSink<T>) {
+    const abortController = new AbortController();
+    const listeners = new Set<LastValueSink<T>>();
+    listeners.add(sink);
+
+    this.abortController = abortController;
+    this.subscribers = listeners;
+
+    this.loop(abortController, listeners);
+  }
+
+  private async loop(abortController: AbortController, sinks: Set<LastValueSink<T>>) {
+    try {
+      for await (let doc of this.source(abortController.signal)) {
+        if (abortController.signal.aborted || sinks.size == 0) {
+          throw new AbortError();
+        }
+        this.last = doc;
+        for (let sink of sinks) {
+          sink.next(doc);
+        }
+      }
+
+      // End of stream
+      for (let sink of sinks) {
+        sink.complete();
+      }
+    } catch (e) {
+      // Just in case the error is not from the source
+      abortController.abort();
+
+      for (let listener of sinks) {
+        listener.error(e);
+      }
+    } finally {
+      // Clear state, so that a new subscription may be started
+      if (this.subscribers === sinks) {
+        this.subscribers = undefined;
+        this.abortController = undefined;
+        this.last = undefined;
+      }
+    }
+  }
+
+  private removeSink(listener: LastValueSink<T>) {
+    this.subscribers?.delete(listener);
+    if (this.subscribers?.size == 0) {
+      // This is not immediate - there may be a delay until it is fully stopped,
+      // depending on the underlying source.
+      this.abortController?.abort();
+      this.last = undefined;
+      this.subscribers = undefined;
+      this.abortController = undefined;
+    }
+  }
+
+  private addSink(listener: LastValueSink<T>) {
+    if (this.subscribers == null) {
+      this.start(listener);
+    } else {
+      this.subscribers.add(listener);
+    }
+  }
+
+  async *[Symbol.asyncIterator](signal?: AbortSignal): AsyncIterator<T> {
+    const sink = new LastValueSink(this.last);
+    this.addSink(sink);
+    try {
+      yield* sink.withSignal(signal);
+    } finally {
+      this.removeSink(sink);
+    }
+  }
+
+  get active() {
+    return this.subscribers != null;
+  }
+}
+
+// For reference, this is an alternative implementation using rxjs.
+// It still relies on LastValueSink.
+//
+// import { BehaviorSubject, Observable, filter, from, share } from 'rxjs';
+//
+// export class RxBroadcastIterable<T> implements AsyncIterable<T> {
+//   private observable: Observable<T>;
+//
+//   constructor(source: IterableSource<T>) {
+//     const obsSource = new Observable<T>((subscriber) => {
+//       const controller = new AbortController();
+//       const s = source(controller.signal);
+//       const inner = from(s);
+//       const subscription = inner.subscribe(subscriber);
+//       subscription.add({
+//         unsubscribe() {
+//           controller.abort();
+//         }
+//       });
+//       return subscription;
+//     });
+//
+//     const obs = obsSource.pipe(
+//       share({ connector: () => new BehaviorSubject(undefined as T) }),
+//       filter((v) => v != undefined)
+//     );
+//     this.observable = obs;
+//   }
+//
+//   async *[Symbol.asyncIterator](signal?: AbortSignal): AsyncIterator<T> {
+//     if (signal?.aborted) {
+//       return;
+//     }
+//     const sink = new LastValueSink<T>(undefined);
+//     const subscription = this.observable.subscribe(sink);
+//
+//     try {
+//       yield* sink.withSignal(signal);
+//     } finally {
+//       subscription.unsubscribe();
+//     }
+//   }
+//
+//   get active() {
+//     return false;
+//   }
+// }

package/src/sync/LastValueSink.ts

@@ -0,0 +1,100 @@
+import { AbortError } from 'ix/aborterror.js';
+
+/**
+ * An AsyncIterable that allows pushing values.
+ *
+ * If the consumer is slower than the source, only the last value is buffered.
+ *
+ * Similar to ix AsyncSink, except that we only keep the last value.
+ */
+export class LastValueSink<T> implements AsyncIterable<T> {
+  buffer: NextResult<T> | undefined;
+  nextResolve: undefined | (() => void);
+
+  constructor(initial: T | undefined) {
+    if (initial != null) {
+      this.buffer = { value: initial, done: false, error: undefined };
+    }
+  }
+
+  next(value: T) {
+    this.push({
+      value,
+      done: false,
+      error: undefined
+    });
+  }
+
+  complete() {
+    this.push({
+      value: undefined,
+      done: true,
+      error: undefined
+    });
+  }
+
+  error(e: any) {
+    this.push({
+      value: undefined,
+      done: true,
+      error: e
+    });
+  }
+
+  private push(r: NextResult<T>) {
+    if (this.buffer?.done) {
+      return;
+    }
+    this.buffer = r;
+    this.nextResolve?.();
+  }
+
+  async *withSignal(signal?: AbortSignal): AsyncIterable<T> {
+    if (!signal) {
+      yield* this;
+      return;
+    }
+
+    if (signal?.aborted) {
+      throw new AbortError();
+    }
+
+    const onAbort = () => {
+      this.error(new AbortError());
+    };
+    signal?.addEventListener('abort', onAbort);
+
+    try {
+      yield* this;
+    } finally {
+      signal?.removeEventListener('abort', onAbort);
+    }
+  }
+
+  async *[Symbol.asyncIterator](): AsyncIterator<T> {
+    while (true) {
+      if (this.buffer == null) {
+        const promise = new Promise<void>((resolve) => {
+          this.nextResolve = resolve;
+        });
+        await promise;
+      }
+      const n = this.buffer!;
+      this.buffer = undefined;
+
+      if (n.error) {
+        throw n.error;
+      } else if (n.done) {
+        return;
+      } else {
+        yield n.value!;
+      }
+    }
+  }
+}
+
+interface NextResult<T> {
+  value: T | undefined;
+  done: boolean;
+  error: any;
+}

package/src/sync/merge.ts

@@ -0,0 +1,200 @@
+import { throwIfAborted } from 'ix/aborterror.js';
+import { AsyncIterableX } from 'ix/asynciterable/index.js';
+import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
+import { safeRace } from './safeRace.js';
+
+/**
+ * Merge multiple source AsyncIterables into one output AsyncIterable.
+ *
+ * The results from all iterables are interleaved.
+ *
+ * When any iterable is done, the output stops.
+ *
+ * @param source The source AsyncIterables.
+ * @param signal Optional signal to abort iteration.
+ * @returns The merged AsyncIterable.
+ */
+export function mergeAsyncIterables<T>(source: AsyncIterable<T>[], signal?: AbortSignal): AsyncIterable<T> {
+  return mergeAsyncIterablesNew(source, signal);
+  // return mergeAsyncIterablesOld(source, signal);
+}
+
+export function mergeAsyncIterablesNew<T>(source: AsyncIterable<T>[], signal?: AbortSignal): AsyncIterable<T> {
+  return new MergedAsyncIterable(source, signal);
+}
+
+export function mergeAsyncIterablesOld<T>(source: AsyncIterable<T>[], signal?: AbortSignal): AsyncIterable<T> {
+  return wrapWithAbort(new FixedMergeAsyncIterable(source, { race: true }), signal);
+}
+
+const NEVER_PROMISE = new Promise<never>(() => {});
+
+type MergeResult<T> = { value: T; index: number };
+
+function wrapPromiseWithIndex<T>(promise: Promise<T>, index: number) {
+  return promise.then((value) => ({ value, index })) as Promise<MergeResult<T>>;
+}
+
+/**
+ * Re-implementation of FixedMergeAsyncIterable, without using async generators.
+ *
+ * The functionality should be the same in most cases.
+ */
+class MergedAsyncIterable<T> implements AsyncIterable<T> {
+  private _source: AsyncIterable<T>[];
+  private _signal?: AbortSignal;
+
+  constructor(source: AsyncIterable<T>[], signal?: AbortSignal) {
+    this._source = source;
+    this._signal = signal;
+  }
+
+  [Symbol.asyncIterator](): AsyncIterator<T> {
+    const signal = this._signal;
+    throwIfAborted(signal);
+
+    const length = this._source.length;
+    const iterators = new Array<AsyncIterator<T>>(length);
+    const nexts = new Array<Promise<MergeResult<IteratorResult<T>>> | null>(length);
+
+    for (let i = 0; i < length; i++) {
+      const iterator = wrapWithAbort(this._source[i], signal)[Symbol.asyncIterator]();
+      iterators[i] = iterator;
+      nexts[i] = null;
+    }
+
+    const returnIterators = async (_value?: any): Promise<IteratorResult<T>> => {
+      for (let iter of iterators) {
+        // Do not wait for return() to complete, since it may block
+        iter.return?.();
+      }
+      return { value: undefined, done: true };
+    };
+
+    return {
+      next: async (): Promise<IteratorResult<T>> => {
+        for (let i = 0; i < length; i++) {
+          if (nexts[i] == null) {
+            nexts[i] = wrapPromiseWithIndex(iterators[i].next(), i);
+          }
+        }
+
+        try {
+          // IMPORTANT! safeRace here acts like Promise.race, but avoids major memory leaks.
+          const {
+            value: { done, value },
+            index
+          } = (await safeRace(nexts as Array<Promise<MergeResult<IteratorResult<T>>>>))!;
+          if (done) {
+            // One of the source iterators is done - return them all
+            await returnIterators();
+
+            return {
+              value: undefined,
+              done: true
+            };
+          } else {
+            // Consume the result, which will cause the next one to be requested on the
+            // next iteration.
+            nexts[index] = null;
+
+            return {
+              value: value,
+              done: false
+            };
+          }
+        } catch (e) {
+          // One of the source iterators raised an error - return all others
+          // and propagate the error
+          await returnIterators();
+          throw e;
+        }
+      },
+      return: () => {
+        return returnIterators();
+      },
+      throw: async (e) => {
+        for (let iter of iterators) {
+          // Do not wait for throw() to complete, since it may block
+          iter.throw?.(e);
+        }
+        throw e;
+      }
+    };
+  }
+}
+
+/**
+ * Loosely based on IxJS merge, but with some changes:
+ * 1. Fix an issue with uncaught errors.
+ *      Partially fixed: https://github.com/ReactiveX/IxJS/pull/354
+ *      Essentially, we only want to call next() on inner iterators when next() is called on the outer one.
+ * 2. Call return() on all inner iterators when the outer one returns.
+ * 3. Returning when the first iterator returns.
+ *
+ * https://github.com/ReactiveX/IxJS/blob/f07b7ef4095120f1ef21a4023030c75b36335cd1/src/asynciterable/merge.ts
+ */
+export class FixedMergeAsyncIterable<T> extends AsyncIterableX<T> {
+  private _source: AsyncIterable<T>[];
+  private _race: boolean;
+
+  /**
+   *
+   * @param source
+   * @param options Specify `race: true` to stop the iterator when any inner one returns, instead of waiting for all.
+   */
+  constructor(source: AsyncIterable<T>[], options?: { race?: boolean }) {
+    super();
+    this._source = source;
+    this._race = options?.race ?? false;
+  }
+
+  async *[Symbol.asyncIterator](signal?: AbortSignal): AsyncIterator<T> {
+    throwIfAborted(signal);
+    const length = this._source.length;
+    const iterators = new Array<AsyncIterator<T>>(length);
+    const nexts = new Array<Promise<MergeResult<IteratorResult<T>>>>(length);
+    let active = length;
+    for (let i = 0; i < length; i++) {
+      const iterator = wrapWithAbort(this._source[i], signal)[Symbol.asyncIterator]();
+      iterators[i] = iterator;
+      nexts[i] = wrapPromiseWithIndex(iterator.next(), i);
+    }
+
+    try {
+      while (active > 0) {
+        // IMPORTANT! safeRace here acts like Promise.race, but avoids major memory leaks.
+        const {
+          value: { done, value },
+          index
+        } = await safeRace(nexts);
+        if (done) {
+          nexts[index] = NEVER_PROMISE;
+          active--;
+          if (this._race) {
+            break;
+          }
+        } else {
+          const iterator$ = iterators[index];
+          nexts[index] = NEVER_PROMISE;
+          try {
+            yield value;
+          } catch (e) {
+            // iter.throw() was called on the merged iterator
+            // Propagate the error to the source iterators
+            for (let iter of iterators) {
+              iter.throw?.(e);
+            }
+            throw e;
+          }
+          nexts[index] = wrapPromiseWithIndex(iterator$.next(), index);
+        }
+      }
+    } finally {
+      for (let iter of iterators) {
+        // This may be an early return - return all inner iterators
+        iter.return?.();
+      }
+    }
+  }
+}

package/src/sync/safeRace.ts

@@ -0,0 +1,99 @@
+// Source: https://github.com/ReactiveX/IxJS/blob/ff1cef03d91a4a7c527fa8c8dbdac025a652c236/src/util/safeRace.ts
+
+/*
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>
+*/
+
+// see: https://github.com/nodejs/node/issues/17469#issuecomment-685216777
+// see: https://github.com/ReactiveX/IxJS/pull/323
+
+function isPrimitive(value: unknown): boolean {
+  return value === null || (typeof value !== 'object' && typeof value !== 'function');
+}
+
+// Keys are the values passed to race, values are a record of data containing a
+// set of deferreds and whether the value has settled.
+const wm = new WeakMap<any, any>();
+export function safeRace<T>(contenders: Promise<T>[]): Promise<T> {
+  let deferred: any;
+  const result = new Promise<T>((resolve, reject) => {
+    deferred = { resolve, reject };
+    for (const contender of contenders) {
+      if (isPrimitive(contender)) {
+        // If the contender is a primitive, attempting to use it as a key in the
+        // weakmap would throw an error. Luckily, it is safe to call
+        // `Promise.resolve(contender).then` on a primitive value multiple times
+        // because the promise fulfills immediately.
+        Promise.resolve(contender).then(resolve, reject);
+        continue;
+      }
+
+      let record = wm.get(contender);
+      if (record === undefined) {
+        record = { deferreds: new Set([deferred]), settled: false };
+        wm.set(contender, record);
+        // This call to `then` happens once for the lifetime of the value.
+        Promise.resolve(contender).then(
+          (value) => {
+            // eslint-disable-next-line no-shadow
+            for (const { resolve } of record.deferreds) {
+              resolve(value);
+            }
+
+            record.deferreds.clear();
+            record.settled = true;
+          },
+          (err) => {
+            // eslint-disable-next-line no-shadow
+            for (const { reject } of record.deferreds) {
+              reject(err);
+            }
+
+            record.deferreds.clear();
+            record.settled = true;
+          }
+        );
+      } else if (record.settled) {
+        // If the value has settled, it is safe to call
+        // `Promise.resolve(contender).then` on it.
+        Promise.resolve(contender).then(resolve, reject);
+      } else {
+        record.deferreds.add(deferred);
+      }
+    }
+  });
+
+  // The finally callback executes when any value settles, preventing any of
+  // the unresolved values from retaining a reference to the resolved value.
+  return result.finally(() => {
+    for (const contender of contenders) {
+      if (!isPrimitive(contender)) {
+        const record = wm.get(contender);
+        record.deferreds.delete(deferred);
+      }
+    }
+  });
+}

package/src/sync/sync-index.ts

@@ -0,0 +1,6 @@
+export * from './BroadcastIterable.js';
+export * from './LastValueSink.js';
+export * from './merge.js';
+export * from './safeRace.js';
+export * from './sync.js';
+export * from './util.js';