@powersync/service-core 1.9.0 → 1.10.0

package/src/streams/Demultiplexer.ts

@@ -0,0 +1,165 @@
+import { AbortError } from 'ix/aborterror.js';
+import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
+import { LastValueSink } from './LastValueSink.js';
+
+export interface DemultiplexerValue<T> {
+  /**
+   * The key used for demultiplexing, for example the user id.
+   */
+  key: string;
+  /**
+   * The stream value.
+   */
+  value: T;
+}
+
+export interface DemultiplexerSource<T> {
+  /**
+   * The async iterator providing a stream of values.
+   */
+  iterator: AsyncIterable<DemultiplexerValue<T>>;
+
+  /**
+   * Fetches the first value for a given key.
+   *
+   * This is used to get an initial value for each subscription.
+   */
+  getFirstValue(key: string): Promise<T>;
+}
+
+export type DemultiplexerSourceFactory<T> = (signal: AbortSignal) => DemultiplexerSource<T>;
+
+/**
+ * Takes a multiplexed stream (e.g. a changestream covering many individual users),
+ * and allows subscribing to individual streams.
+ *
+ * The source subscription is lazy:
+ * 1. We only start subscribing when there is a downstream subscriber.
+ * 2. When all downstream subscriptions have ended, we end the source subscription.
+ *
+ * For each subscriber, if backpressure builds up, we only keep the _last_ value.
+ */
+export class Demultiplexer<T> {
+  private subscribers: Map<string, Set<LastValueSink<T>>> | undefined = undefined;
+  private abortController: AbortController | undefined = undefined;
+  private currentSource: DemultiplexerSource<T> | undefined = undefined;
+
+  constructor(private source: DemultiplexerSourceFactory<T>) {}
+
+  private start(filter: string, sink: LastValueSink<T>) {
+    const abortController = new AbortController();
+    const listeners = new Map();
+    listeners.set(filter, new Set([sink]));
+
+    this.abortController = abortController;
+    this.subscribers = listeners;
+
+    const source = this.source(abortController.signal);
+    this.currentSource = source;
+    this.loop(source, abortController, listeners);
+    return source;
+  }
+
+  private async loop(
+    source: DemultiplexerSource<T>,
+    abortController: AbortController,
+    sinks: Map<string, Set<LastValueSink<T>>>
+  ) {
+    try {
+      for await (let doc of source.iterator) {
+        if (abortController.signal.aborted || sinks.size == 0) {
+          throw new AbortError();
+        }
+        const key = doc.key;
+        const keySinks = sinks.get(key);
+        if (keySinks == null) {
+          continue;
+        }
+
+        for (let sink of keySinks) {
+          sink.write(doc.value);
+        }
+      }
+
+      // End of stream
+      for (let keySinks of sinks.values()) {
+        for (let sink of keySinks) {
+          sink.end();
+        }
+      }
+    } catch (e) {
+      // Just in case the error is not from the source
+      abortController.abort();
+
+      for (let keySinks of sinks.values()) {
+        for (let sink of keySinks) {
+          sink.error(e);
+        }
+      }
+    } finally {
+      // Clear state, so that a new subscription may be started
+      if (this.subscribers === sinks) {
+        this.subscribers = undefined;
+        this.abortController = undefined;
+        this.currentSource = undefined;
+      }
+    }
+  }
+
+  private removeSink(key: string, sink: LastValueSink<T>) {
+    const existing = this.subscribers?.get(key);
+    if (existing == null) {
+      return;
+    }
+    existing.delete(sink);
+    if (existing.size == 0) {
+      this.subscribers!.delete(key);
+    }
+
+    if (this.subscribers?.size == 0) {
+      // This is not immediate - there may be a delay until it is fully stopped,
+      // depending on the underlying source.
+      this.abortController?.abort();
+      this.subscribers = undefined;
+      this.abortController = undefined;
+      this.currentSource = undefined;
+    }
+  }
+
+  private addSink(key: string, sink: LastValueSink<T>) {
+    if (this.currentSource == null) {
+      return this.start(key, sink);
+    } else {
+      const existing = this.subscribers!.get(key);
+      if (existing != null) {
+        existing.add(sink);
+      } else {
+        this.subscribers!.set(key, new Set([sink]));
+      }
+      return this.currentSource;
+    }
+  }
+
+  /**
+   * Subscribe to a specific stream.
+   *
+   * @param key The key used for demultiplexing, e.g. user id.
+   * @param signal
+   */
+  async *subscribe(key: string, signal: AbortSignal): AsyncIterable<T> {
+    const sink = new LastValueSink<T>(undefined);
+    // Important that we register the sink before calling getFirstValue().
+    const source = this.addSink(key, sink);
+    try {
+      const firstValue = await source.getFirstValue(key);
+      yield firstValue;
+      yield* sink.withSignal(signal);
+    } finally {
+      this.removeSink(key, sink);
+    }
+  }
+
+  get active() {
+    return this.subscribers != null;
+  }
+}

package/src/{sync → streams}/LastValueSink.ts

@@ -17,7 +17,7 @@ export class LastValueSink<T> implements AsyncIterable<T> {
     }
   }
 
-  next(value: T) {
+  write(value: T) {
     this.push({
       value,
       done: false,
@@ -25,7 +25,7 @@ export class LastValueSink<T> implements AsyncIterable<T> {
     });
   }
 
-  complete() {
+  end() {
     this.push({
       value: undefined,
       done: true,

package/src/{sync → streams}/merge.ts

@@ -1,7 +1,7 @@
 import { throwIfAborted } from 'ix/aborterror.js';
 import { AsyncIterableX } from 'ix/asynciterable/index.js';
 import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
-import { safeRace } from './safeRace.js';
+import { safeRace } from '../sync/safeRace.js';
 
 /**
  * Merge multiple source AsyncIterables into one output AsyncIterable.

package/src/streams/streams-index.ts

@@ -0,0 +1,4 @@
+export * from './merge.js';
+export * from './Demultiplexer.js';
+export * from './LastValueSink.js';
+export * from './BroadcastIterable.js';

package/src/sync/BucketChecksumState.ts

@@ -4,9 +4,11 @@ import * as storage from '../storage/storage-index.js';
 import * as util from '../util/util-index.js';
 
 import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
+import { JSONBig } from '@powersync/service-jsonbig';
 import { BucketParameterQuerier } from '@powersync/service-sync-rules/src/BucketParameterQuerier.js';
 import { BucketSyncState } from './sync.js';
 import { SyncContext } from './SyncContext.js';
+import { getIntersection, hasIntersection } from './util.js';
 
 export interface BucketChecksumStateOptions {
   syncContext: SyncContext;
@@ -68,10 +70,9 @@ export class BucketChecksumState {
     const storage = this.bucketStorage;
 
     const update = await this.parameterState.getCheckpointUpdate(next);
-    if (update == null) {
+    if (update == null && this.lastWriteCheckpoint == writeCheckpoint) {
       return null;
     }
-
     const { buckets: allBuckets, updatedBuckets } = update;
 
     let dataBucketsNew = new Map<string, BucketSyncState>();
@@ -90,7 +91,7 @@ export class BucketChecksumState {
     }
 
     let checksumMap: util.ChecksumMap;
-    if (updatedBuckets != null) {
+    if (updatedBuckets != INVALIDATE_ALL_BUCKETS) {
       if (this.lastChecksums == null) {
         throw new ServiceAssertionError(`Bucket diff received without existing checksums`);
       }
@@ -113,9 +114,11 @@ export class BucketChecksumState {
         }
       }
 
-      let updatedChecksums = await storage.getChecksums(base.checkpoint, checksumLookups);
-      for (let [bucket, value] of updatedChecksums.entries()) {
-        newChecksums.set(bucket, value);
+      if (checksumLookups.length > 0) {
+        let updatedChecksums = await storage.getChecksums(base.checkpoint, checksumLookups);
+        for (let [bucket, value] of updatedChecksums.entries()) {
+          newChecksums.set(bucket, value);
+        }
       }
       checksumMap = newChecksums;
     } else {
@@ -123,6 +126,7 @@ export class BucketChecksumState {
       const bucketList = [...dataBucketsNew.keys()];
       checksumMap = await storage.getChecksums(base.checkpoint, bucketList);
     }
+
     // Subset of buckets for which there may be new data in this batch.
     let bucketsToFetch: BucketDescription[];
 
@@ -247,6 +251,8 @@ export class BucketChecksumState {
   }
 }
 
+const INVALIDATE_ALL_BUCKETS = Symbol('INVALIDATE_ALL_BUCKETS');
+
 export interface CheckpointUpdate {
   /**
    * All buckets forming part of the checkpoint.
@@ -258,7 +264,7 @@ export interface CheckpointUpdate {
    *
    * If null, assume that any bucket in `buckets` may have been updated.
    */
-  updatedBuckets: Set<string> | null;
+  updatedBuckets: Set<string> | typeof INVALIDATE_ALL_BUCKETS;
 }
 
 export class BucketParameterState {
@@ -268,6 +274,10 @@ export class BucketParameterState {
   public readonly syncParams: RequestParameters;
   private readonly querier: BucketParameterQuerier;
   private readonly staticBuckets: Map<string, BucketDescription>;
+  private cachedDynamicBuckets: BucketDescription[] | null = null;
+  private cachedDynamicBucketSet: Set<string> | null = null;
+
+  private readonly lookups: Set<string>;
 
   constructor(
     context: SyncContext,
@@ -282,21 +292,18 @@ export class BucketParameterState {
 
     this.querier = syncRules.getBucketParameterQuerier(this.syncParams);
     this.staticBuckets = new Map<string, BucketDescription>(this.querier.staticBuckets.map((b) => [b.bucket, b]));
+    this.lookups = new Set<string>(this.querier.parameterQueryLookups.map((l) => JSONBig.stringify(l.values)));
   }
 
-  async getCheckpointUpdate(checkpoint: storage.StorageCheckpointUpdate): Promise<CheckpointUpdate | null> {
+  async getCheckpointUpdate(checkpoint: storage.StorageCheckpointUpdate): Promise<CheckpointUpdate> {
     const querier = this.querier;
-    let update: CheckpointUpdate | null;
+    let update: CheckpointUpdate;
     if (querier.hasDynamicBuckets) {
       update = await this.getCheckpointUpdateDynamic(checkpoint);
     } else {
       update = await this.getCheckpointUpdateStatic(checkpoint);
     }
 
-    if (update == null) {
-      return null;
-    }
-
     if (update.buckets.length > this.context.maxParameterQueryResults) {
       // TODO: Limit number of results even before we get to this point
       // This limit applies _before_ we get the unique set
@@ -318,32 +325,18 @@ export class BucketParameterState {
   /**
    * For static buckets, we can keep track of which buckets have been updated.
    */
-  private async getCheckpointUpdateStatic(
-    checkpoint: storage.StorageCheckpointUpdate
-  ): Promise<CheckpointUpdate | null> {
+  private async getCheckpointUpdateStatic(checkpoint: storage.StorageCheckpointUpdate): Promise<CheckpointUpdate> {
     const querier = this.querier;
     const update = checkpoint.update;
 
     if (update.invalidateDataBuckets) {
       return {
         buckets: querier.staticBuckets,
-        updatedBuckets: null
+        updatedBuckets: INVALIDATE_ALL_BUCKETS
       };
     }
 
-    let updatedBuckets = new Set<string>();
-
-    for (let bucket of update.updatedDataBuckets ?? []) {
-      if (this.staticBuckets.has(bucket)) {
-        updatedBuckets.add(bucket);
-      }
-    }
-
-    if (updatedBuckets.size == 0) {
-      // No change - skip this checkpoint
-      return null;
-    }
-
+    const updatedBuckets = new Set<string>(getIntersection(this.staticBuckets, update.updatedDataBuckets));
     return {
       buckets: querier.staticBuckets,
       updatedBuckets
@@ -353,44 +346,67 @@ export class BucketParameterState {
   /**
    * For dynamic buckets, we need to re-query the list of buckets every time.
    */
-  private async getCheckpointUpdateDynamic(
-    checkpoint: storage.StorageCheckpointUpdate
-  ): Promise<CheckpointUpdate | null> {
+  private async getCheckpointUpdateDynamic(checkpoint: storage.StorageCheckpointUpdate): Promise<CheckpointUpdate> {
     const querier = this.querier;
     const storage = this.bucketStorage;
     const staticBuckets = querier.staticBuckets;
     const update = checkpoint.update;
 
-    let hasChange = false;
-    if (update.invalidateDataBuckets || update.updatedDataBuckets?.length > 0) {
-      hasChange = true;
-    } else if (update.invalidateParameterBuckets) {
-      hasChange = true;
+    let hasParameterChange = false;
+    let invalidateDataBuckets = false;
+    // If hasParameterChange == true, then invalidateDataBuckets = true
+    // If invalidateDataBuckets == true, we ignore updatedBuckets
+    let updatedBuckets = new Set<string>();
+
+    if (update.invalidateDataBuckets) {
+      invalidateDataBuckets = true;
+    }
+
+    if (update.invalidateParameterBuckets) {
+      hasParameterChange = true;
     } else {
-      for (let bucket of update.updatedParameterBucketDefinitions ?? []) {
-        if (querier.dynamicBucketDefinitions.has(bucket)) {
-          hasChange = true;
-          break;
-        }
+      if (hasIntersection(this.lookups, update.updatedParameterLookups)) {
+        // This is a very coarse re-check of all queries
+        hasParameterChange = true;
       }
     }
 
-    if (!hasChange) {
-      return null;
-    }
+    let dynamicBuckets: BucketDescription[];
+    if (hasParameterChange || this.cachedDynamicBuckets == null || this.cachedDynamicBucketSet == null) {
+      dynamicBuckets = await querier.queryDynamicBucketDescriptions({
+        getParameterSets(lookups) {
+          return storage.getParameterSets(checkpoint.base.checkpoint, lookups);
+        }
+      });
+      this.cachedDynamicBuckets = dynamicBuckets;
+      this.cachedDynamicBucketSet = new Set<string>(dynamicBuckets.map((b) => b.bucket));
+      invalidateDataBuckets = true;
+    } else {
+      dynamicBuckets = this.cachedDynamicBuckets;
 
-    const dynamicBuckets = await querier.queryDynamicBucketDescriptions({
-      getParameterSets(lookups) {
-        return storage.getParameterSets(checkpoint.base.checkpoint, lookups);
+      if (!invalidateDataBuckets) {
+        for (let bucket of getIntersection(this.staticBuckets, update.updatedDataBuckets)) {
+          updatedBuckets.add(bucket);
+        }
+        for (let bucket of getIntersection(this.cachedDynamicBucketSet, update.updatedDataBuckets)) {
+          updatedBuckets.add(bucket);
+        }
       }
-    });
+    }
     const allBuckets = [...staticBuckets, ...dynamicBuckets];
 
-    return {
-      buckets: allBuckets,
-      // We cannot track individual bucket updates for dynamic lookups yet
-      updatedBuckets: null
-    };
+    if (invalidateDataBuckets) {
+      return {
+        buckets: allBuckets,
+        // We cannot track individual bucket updates for dynamic lookups yet
+        updatedBuckets: INVALIDATE_ALL_BUCKETS
+      };
+    } else {
+      return {
+        buckets: allBuckets,
+        updatedBuckets: updatedBuckets
+      };
+    }
   }
 }
 

package/src/sync/sync-index.ts

@@ -1,6 +1,3 @@
-export * from './BroadcastIterable.js';
-export * from './LastValueSink.js';
-export * from './merge.js';
 export * from './RequestTracker.js';
 export * from './safeRace.js';
 export * from './sync.js';

package/src/sync/sync.ts

@@ -9,7 +9,7 @@ import * as util from '../util/util-index.js';
 
 import { logger } from '@powersync/lib-services-framework';
 import { BucketChecksumState } from './BucketChecksumState.js';
-import { mergeAsyncIterables } from './merge.js';
+import { mergeAsyncIterables } from '../streams/streams-index.js';
 import { acquireSemaphoreAbortable, settledPromise, tokenStream, TokenStreamOptions } from './util.js';
 import { SyncContext } from './SyncContext.js';
 import { RequestTracker } from './RequestTracker.js';
@@ -105,7 +105,7 @@ async function* streamResponseInner(
       after: BigInt(bucket.after)
     }))
   });
-  const stream = bucketStorage.watchWriteCheckpoint({
+  const stream = bucketStorage.watchCheckpointChanges({
     user_id: checkpointUserId,
     signal
   });

package/src/sync/util.ts

@@ -1,8 +1,8 @@
 import * as timers from 'timers/promises';
 
+import { SemaphoreInterface } from 'async-mutex';
 import * as util from '../util/util-index.js';
 import { RequestTracker } from './RequestTracker.js';
-import { SemaphoreInterface } from 'async-mutex';
 
 export type TokenStreamOptions = {
   /**
@@ -153,3 +153,36 @@ export function settledPromise<T>(promise: Promise<T>): Promise<PromiseSettledRe
     }
   );
 }
+
+export type MapOrSet<T> = Map<T, any> | Set<T>;
+
+/**
+ * Check if two sets have any element(s) in common.
+ */
+export function hasIntersection<T>(a: MapOrSet<T>, b: MapOrSet<T>) {
+  for (let _ of getIntersection(a, b)) {
+    return true;
+  }
+  return false;
+}
+
+/**
+ * Return the intersection of two sets or maps.
+ */
+export function* getIntersection<T>(a: MapOrSet<T>, b: MapOrSet<T>): IterableIterator<T> {
+  // Iterate over the smaller set to reduce the number of lookups
+  if (a.size < b.size) {
+    for (let key of a.keys()) {
+      if (b.has(key)) {
+        yield key;
+      }
+    }
+    return false;
+  } else {
+    for (let key of b.keys()) {
+      if (a.has(key)) {
+        yield key;
+      }
+    }
+  }
+}

package/test/src/broadcast_iterable.test.ts

@@ -1,14 +1,14 @@
-import { BroadcastIterable, IterableSource } from '@/sync/BroadcastIterable.js';
+import { BroadcastIterable, IterableSource } from '@/streams/BroadcastIterable.js';
 import { AsyncIterableX, interval } from 'ix/asynciterable/index.js';
 import { delayEach } from 'ix/asynciterable/operators/delayeach.js';
 import { take } from 'ix/asynciterable/operators/take.js';
 import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
 import { toArray } from 'ix/asynciterable/toarray.js';
 import * as timers from 'timers/promises';
-import { describe, expect, test } from 'vitest';
+import { describe, expect, it } from 'vitest';
 
 describe('BroadcastIterable', () => {
-  test('should iterate', async () => {
+  it('should iterate', async () => {
     const range = AsyncIterableX.from([1, 2, 3]);
     const broadcast = new BroadcastIterable(() => range);
 
@@ -17,7 +17,7 @@ describe('BroadcastIterable', () => {
     expect(broadcast.active).toBe(false);
   });
 
-  test('should skip values if sink is slow', async () => {
+  it('should skip values if sink is slow', async () => {
     const range = AsyncIterableX.from([1, 2, 3]);
     const broadcast = new BroadcastIterable(() => range);
 
@@ -30,7 +30,7 @@ describe('BroadcastIterable', () => {
     expect(broadcast.active).toBe(false);
   });
 
-  test('should abort', async () => {
+  it('should abort', async () => {
     const range = AsyncIterableX.from([1, 2, 3]);
     let recordedSignal: AbortSignal | undefined;
     const broadcast = new BroadcastIterable((signal) => {
@@ -46,7 +46,7 @@ describe('BroadcastIterable', () => {
     expect(recordedSignal!.aborted).toEqual(true);
   });
 
-  test('should handle indefinite sources', async () => {
+  it('should handle indefinite sources', async () => {
     const source: IterableSource<number> = (signal) => {
       return wrapWithAbort(interval(1), signal);
     };
@@ -65,7 +65,7 @@ describe('BroadcastIterable', () => {
     expect(broadcast.active).toBe(false);
   });
 
-  test('should handle multiple subscribers', async () => {
+  it('should handle multiple subscribers', async () => {
     let sourceIndex = 0;
     const source = async function* (signal: AbortSignal) {
       // Test value out by 1000 means it may have used the wrong iteration of the source
@@ -111,7 +111,7 @@ describe('BroadcastIterable', () => {
     expect(results3[4]).toBeLessThan(2145);
   });
 
-  test('should handle errors on multiple subscribers', async () => {
+  it('should handle errors on multiple subscribers', async () => {
     let sourceIndex = 0;
     const source = async function* (signal: AbortSignal) {
       // Test value out by 1000 means it may have used the wrong iteration of the source