@powersync/service-core 1.9.0 → 1.10.0

package/test/src/demultiplexer.test.ts

@@ -0,0 +1,205 @@
+// Vitest Unit Tests
+import { Demultiplexer, DemultiplexerSource, DemultiplexerSourceFactory, DemultiplexerValue } from '@/index.js';
+import { delayEach } from 'ix/asynciterable/operators/delayeach.js';
+import { take } from 'ix/asynciterable/operators/take.js';
+import { toArray } from 'ix/asynciterable/toarray.js';
+import * as timers from 'node:timers/promises';
+import { describe, expect, it } from 'vitest';
+
+describe('Demultiplexer', () => {
+  it('should start subscription lazily and provide first value', async () => {
+    const mockSource: DemultiplexerSourceFactory<string> = (signal: AbortSignal) => {
+      const iterator = (async function* (): AsyncIterable<DemultiplexerValue<string>> {})();
+      return {
+        iterator,
+        getFirstValue: async (key: string) => `first-${key}`
+      };
+    };
+
+    const demux = new Demultiplexer(mockSource);
+    const signal = new AbortController().signal;
+
+    const iter = demux.subscribe('user1', signal)[Symbol.asyncIterator]();
+    const result = await iter.next();
+    expect(result.value).toBe('first-user1');
+  });
+
+  it('should handle multiple subscribers to the same key', async () => {
+    const iter = (async function* () {
+      yield { key: 'user1', value: 'value1' };
+      yield { key: 'user1', value: 'value2' };
+    })();
+    const source: DemultiplexerSource<string> = {
+      iterator: iter,
+      getFirstValue: async (key: string) => `first-${key}`
+    };
+
+    const demux = new Demultiplexer(() => source);
+    const signal = new AbortController().signal;
+
+    const iter1 = demux.subscribe('user1', signal)[Symbol.asyncIterator]();
+    const iter2 = demux.subscribe('user1', signal)[Symbol.asyncIterator]();
+
+    // Due to only keeping the last value, some values are skipped
+    expect(await iter1.next()).toEqual({ value: 'first-user1', done: false });
+    expect(await iter1.next()).toEqual({ value: 'value1', done: false });
+    expect(await iter1.next()).toEqual({ value: undefined, done: true });
+
+    expect(await iter2.next()).toEqual({ value: 'first-user1', done: false });
+    expect(await iter2.next()).toEqual({ value: undefined, done: true });
+  });
+
+  it('should handle multiple subscribers to the same key (2)', async () => {
+    const p1 = Promise.withResolvers<void>();
+    const p2 = Promise.withResolvers<void>();
+    const p3 = Promise.withResolvers<void>();
+
+    const iter = (async function* () {
+      await p1.promise;
+      yield { key: 'user1', value: 'value1' };
+      await p2.promise;
+      yield { key: 'user1', value: 'value2' };
+      await p3.promise;
+    })();
+
+    const source: DemultiplexerSource<string> = {
+      iterator: iter,
+      getFirstValue: async (key: string) => `first-${key}`
+    };
+
+    const demux = new Demultiplexer(() => source);
+    const signal = new AbortController().signal;
+
+    const iter1 = demux.subscribe('user1', signal)[Symbol.asyncIterator]();
+    const iter2 = demux.subscribe('user1', signal)[Symbol.asyncIterator]();
+
+    // Due to only keeping the last value, some values are skilled
+    expect(await iter1.next()).toEqual({ value: 'first-user1', done: false });
+    expect(await iter2.next()).toEqual({ value: 'first-user1', done: false });
+    p1.resolve();
+
+    expect(await iter1.next()).toEqual({ value: 'value1', done: false });
+    expect(await iter2.next()).toEqual({ value: 'value1', done: false });
+    p2.resolve();
+
+    expect(await iter1.next()).toEqual({ value: 'value2', done: false });
+    p3.resolve();
+
+    expect(await iter1.next()).toEqual({ value: undefined, done: true });
+    expect(await iter2.next()).toEqual({ value: undefined, done: true });
+  });
+
+  it('should handle multiple subscribers to different keys', async () => {
+    const p1 = Promise.withResolvers<void>();
+    const p2 = Promise.withResolvers<void>();
+    const p3 = Promise.withResolvers<void>();
+
+    const iter = (async function* () {
+      await p1.promise;
+      yield { key: 'user1', value: 'value1' };
+      await p2.promise;
+      yield { key: 'user2', value: 'value2' };
+      await p3.promise;
+    })();
+
+    const source: DemultiplexerSource<string> = {
+      iterator: iter,
+      getFirstValue: async (key: string) => `first-${key}`
+    };
+
+    const demux = new Demultiplexer(() => source);
+    const signal = new AbortController().signal;
+
+    const iter1 = demux.subscribe('user1', signal)[Symbol.asyncIterator]();
+    const iter2 = demux.subscribe('user2', signal)[Symbol.asyncIterator]();
+
+    // Due to only keeping the last value, some values are skilled
+    expect(await iter1.next()).toEqual({ value: 'first-user1', done: false });
+    expect(await iter2.next()).toEqual({ value: 'first-user2', done: false });
+    p1.resolve();
+
+    expect(await iter1.next()).toEqual({ value: 'value1', done: false });
+    p2.resolve();
+
+    expect(await iter2.next()).toEqual({ value: 'value2', done: false });
+    p3.resolve();
+
+    expect(await iter1.next()).toEqual({ value: undefined, done: true });
+    expect(await iter2.next()).toEqual({ value: undefined, done: true });
+  });
+
+  it('should abort', async () => {
+    const iter = (async function* () {
+      yield { key: 'user1', value: 'value1' };
+      yield { key: 'user1', value: 'value2' };
+    })();
+
+    const source: DemultiplexerSource<string> = {
+      iterator: iter,
+      getFirstValue: async (key: string) => `first-${key}`
+    };
+
+    const demux = new Demultiplexer(() => source);
+    const controller = new AbortController();
+
+    const iter1 = demux.subscribe('user1', controller.signal)[Symbol.asyncIterator]();
+
+    expect(await iter1.next()).toEqual({ value: 'first-user1', done: false });
+    controller.abort();
+
+    await expect(iter1.next()).rejects.toThrow('The operation has been aborted');
+  });
+
+  it('should handle errors on multiple subscribers', async () => {
+    let sourceIndex = 0;
+    const sourceFn = async function* (signal: AbortSignal): AsyncIterable<DemultiplexerValue<number>> {
+      // Test value out by 1000 means it may have used the wrong iteration of the source
+      const base = (sourceIndex += 1000);
+      const abortedPromise = new Promise((resolve) => {
+        signal.addEventListener('abort', resolve, { once: true });
+      });
+      for (let i = 0; !signal.aborted; i++) {
+        if (base + i == 1005) {
+          throw new Error('simulated failure');
+        }
+        yield { key: 'u1', value: base + i };
+        await Promise.race([abortedPromise, timers.setTimeout(1)]);
+      }
+      // Test value out by 100 means this wasn't reached
+      sourceIndex += 100;
+    };
+
+    const sourceFactory: DemultiplexerSourceFactory<number> = (signal) => {
+      const source: DemultiplexerSource<number> = {
+        iterator: sourceFn(signal),
+        getFirstValue: async (key: string) => -1
+      };
+      return source;
+    };
+    const demux = new Demultiplexer(sourceFactory);
+
+    const controller = new AbortController();
+
+    const delayed1 = delayEach(9)(demux.subscribe('u1', controller.signal));
+    const delayed2 = delayEach(10)(demux.subscribe('u1', controller.signal));
+    expect(demux.active).toBe(false);
+    const results1Promise = toArray(take(5)(delayed1)) as Promise<number[]>;
+    const results2Promise = toArray(take(5)(delayed2)) as Promise<number[]>;
+
+    const [r1, r2] = await Promise.allSettled([results1Promise, results2Promise]);
+
+    expect(r1).toEqual({ status: 'rejected', reason: new Error('simulated failure') });
+    expect(r2).toEqual({ status: 'rejected', reason: new Error('simulated failure') });
+
+    expect(demux.active).toBe(false);
+
+    // This starts a new source
+    const delayed3 = delayEach(10)(demux.subscribe('u1', controller.signal));
+    const results3 = await toArray(take(6)(delayed3));
+    expect(results3.length).toEqual(6);
+    expect(results3[0]).toEqual(-1); // Initial value
+    // There should be approximately 10ms between each value, but we allow for some slack
+    expect(results3[5]).toBeGreaterThan(2005);
+    expect(results3[5]).toBeLessThan(2200);
+  });
+});

package/test/src/merge_iterable.test.ts

@@ -1,4 +1,4 @@
-import { mergeAsyncIterablesNew, mergeAsyncIterablesOld } from '@/sync/merge.js';
+import { mergeAsyncIterablesNew, mergeAsyncIterablesOld } from '@/streams/merge.js';
 import * as timers from 'timers/promises';
 import { describe, expect, test } from 'vitest';
 

package/test/src/routes/probes.integration.test.ts

@@ -1,10 +1,10 @@
-import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
-import Fastify, { FastifyInstance } from 'fastify';
 import { container } from '@powersync/lib-services-framework';
-import * as auth from '../../../src/routes/auth.js';
-import * as system from '../../../src/system/system-index.js';
+import Fastify, { FastifyInstance } from 'fastify';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
 import { configureFastifyServer } from '../../../src/index.js';
+import * as auth from '../../../src/routes/auth.js';
 import { ProbeRoutes } from '../../../src/routes/endpoints/probes.js';
+import * as system from '../../../src/system/system-index.js';
 
 vi.mock('@powersync/lib-services-framework', async () => {
   const actual = (await vi.importActual('@powersync/lib-services-framework')) as any;
@@ -25,7 +25,7 @@ describe('Probe Routes Integration', () => {
 
   beforeEach(async () => {
     app = Fastify();
-    mockSystem = { routerEngine: {} } as system.ServiceContext;
+    mockSystem = { routerEngine: {}, replicationEngine: {} } as system.ServiceContext;
     await configureFastifyServer(app, { service_context: mockSystem });
     await app.ready();
   });

package/test/src/routes/probes.test.ts

@@ -1,6 +1,6 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
 import { container } from '@powersync/lib-services-framework';
-import { startupCheck, livenessCheck, readinessCheck } from '../../../src/routes/endpoints/probes.js';
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { livenessCheck, readinessCheck, startupCheck } from '../../../src/routes/endpoints/probes.js';
 
 // Mock the container
 vi.mock('@powersync/lib-services-framework', () => ({
@@ -83,6 +83,7 @@ describe('Probe Routes', () => {
   });
 
   describe('livenessCheck', () => {
+    const mockedContext = { context: { service_context: { replicationEngine: {} } } } as any;
     it('has the correct route definitions', () => {
       expect(livenessCheck.path).toBe('/probes/liveness');
       expect(livenessCheck.method).toBe('GET');
@@ -97,7 +98,7 @@ describe('Probe Routes', () => {
 
       vi.mocked(container.probes.state).mockReturnValue(mockState);
 
-      const response = await livenessCheck.handler();
+      const response = await livenessCheck.handler(mockedContext);
 
       expect(response.status).toBe(200);
       expect(response.data).toEqual(mockState);
@@ -112,7 +113,7 @@ describe('Probe Routes', () => {
 
       vi.mocked(container.probes.state).mockReturnValue(mockState);
 
-      const response = await livenessCheck.handler();
+      const response = await livenessCheck.handler(mockedContext);
 
       expect(response.status).toBe(400);
       expect(response.data).toEqual(mockState);

package/test/src/sync/BucketChecksumState.test.ts

@@ -8,7 +8,8 @@ import {
   SyncContext,
   WatchFilterEvent
 } from '@/index.js';
-import { RequestParameters, SqliteJsonRow, SqliteJsonValue, SqlSyncRules } from '@powersync/service-sync-rules';
+import { JSONBig } from '@powersync/service-jsonbig';
+import { RequestParameters, SqliteJsonRow, ParameterLookup, SqlSyncRules } from '@powersync/service-sync-rules';
 import { describe, expect, test } from 'vitest';
 
 describe('BucketChecksumState', () => {
@@ -97,9 +98,9 @@ bucket_definitions:
       base: { checkpoint: 2n, lsn: '2' },
       writeCheckpoint: null,
       update: {
-        updatedDataBuckets: ['global[]'],
+        updatedDataBuckets: new Set(['global[]']),
         invalidateDataBuckets: false,
-        updatedParameterBucketDefinitions: [],
+        updatedParameterLookups: new Set(),
         invalidateParameterBuckets: false
       }
     }))!;
@@ -200,7 +201,7 @@ bucket_definitions:
       writeCheckpoint: null,
       update: {
         ...CHECKPOINT_INVALIDATE_ALL,
-        updatedDataBuckets: ['global[1]', 'global[2]'],
+        updatedDataBuckets: new Set(['global[1]', 'global[2]']),
         invalidateDataBuckets: false
       }
     }))!;
@@ -293,7 +294,7 @@ bucket_definitions:
         // Invalidate the state for global[1] - will only re-check the single bucket.
         // This is essentially inconsistent state, but is the simplest way to test that
         // the filter is working.
-        updatedDataBuckets: ['global[1]'],
+        updatedDataBuckets: new Set(['global[1]']),
         invalidateDataBuckets: false
       }
     }))!;
@@ -420,7 +421,7 @@ bucket_definitions:
       update: {
         ...CHECKPOINT_INVALIDATE_ALL,
         invalidateDataBuckets: false,
-        updatedDataBuckets: ['global[1]']
+        updatedDataBuckets: new Set(['global[1]'])
       }
     }))!;
     expect(line2.checkpointLine).toEqual({
@@ -474,10 +475,10 @@ bucket_definitions:
 
     storage.getParameterSets = async (
       checkpoint: InternalOpId,
-      lookups: SqliteJsonValue[][]
+      lookups: ParameterLookup[]
     ): Promise<SqliteJsonRow[]> => {
       expect(checkpoint).toEqual(1n);
-      expect(lookups).toEqual([['by_project', '1', 'u1']]);
+      expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]);
       return [{ id: 1 }, { id: 2 }];
     };
 
@@ -519,10 +520,10 @@ bucket_definitions:
 
     storage.getParameterSets = async (
       checkpoint: InternalOpId,
-      lookups: SqliteJsonValue[][]
+      lookups: ParameterLookup[]
     ): Promise<SqliteJsonRow[]> => {
       expect(checkpoint).toEqual(2n);
-      expect(lookups).toEqual([['by_project', '1', 'u1']]);
+      expect(lookups).toEqual([ParameterLookup.normalized('by_project', '1', ['u1'])]);
       return [{ id: 1 }, { id: 2 }, { id: 3 }];
     };
 
@@ -532,8 +533,8 @@ bucket_definitions:
       writeCheckpoint: null,
       update: {
         invalidateDataBuckets: false,
-        updatedDataBuckets: [],
-        updatedParameterBucketDefinitions: ['by_project'],
+        updatedDataBuckets: new Set(),
+        updatedParameterLookups: new Set([JSONBig.stringify(['by_project', '1', 'u1'])]),
         invalidateParameterBuckets: false
       }
     }))!;
@@ -580,7 +581,7 @@ class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
     );
   }
 
-  async getParameterSets(checkpoint: InternalOpId, lookups: SqliteJsonValue[][]): Promise<SqliteJsonRow[]> {
+  async getParameterSets(checkpoint: InternalOpId, lookups: ParameterLookup[]): Promise<SqliteJsonRow[]> {
     throw new Error('Method not implemented.');
   }
 }

package/test/src/util.test.ts

@@ -0,0 +1,48 @@
+import { getIntersection, hasIntersection } from '@/index.js';
+import { describe, expect, test } from 'vitest';
+
+describe('utils', () => {
+  function testInstersection(a: Set<any>, b: Set<any>, expected: boolean) {
+    expect(hasIntersection(a, b)).toBe(expected);
+    expect(hasIntersection(b, a)).toBe(expected);
+    const mapA = new Map([...a].map((v) => [v, 1]));
+    const mapB = new Map([...b].map((v) => [v, 1]));
+    expect(hasIntersection(mapA, b)).toBe(expected);
+    expect(hasIntersection(mapB, a)).toBe(expected);
+    expect(hasIntersection(mapA, mapB)).toBe(expected);
+  }
+
+  test('hasIntersection', async () => {
+    testInstersection(new Set(['a']), new Set(['a']), true);
+    testInstersection(new Set(['a', 'b', 'c']), new Set(['a', 'b', 'c']), true);
+    testInstersection(new Set(['a', 'b', 'c']), new Set(['d', 'e']), false);
+    testInstersection(new Set(['a', 'b', 'c']), new Set(['d', 'c', 'e']), true);
+    testInstersection(new Set(['a', 'b', 'c']), new Set(['c', 'e']), true);
+    testInstersection(new Set(['a', 'b', 'c', 2]), new Set([1, 2, 3]), true);
+    testInstersection(new Set(['a', 'b', 'c', 4]), new Set([1, 2, 3]), false);
+    testInstersection(new Set([]), new Set([1, 2, 3]), false);
+    testInstersection(new Set([]), new Set([]), false);
+  });
+
+  function testGetIntersection(a: Set<any>, b: Set<any>, expected: any[]) {
+    expect([...getIntersection(a, b)]).toEqual(expected);
+    expect([...getIntersection(b, a)]).toEqual(expected);
+    const mapA = new Map([...a].map((v) => [v, 1]));
+    const mapB = new Map([...b].map((v) => [v, 1]));
+    expect([...getIntersection(mapA, b)]).toEqual(expected);
+    expect([...getIntersection(mapB, a)]).toEqual(expected);
+    expect([...getIntersection(mapA, mapB)]).toEqual(expected);
+  }
+
+  test('getIntersection', async () => {
+    testGetIntersection(new Set(['a']), new Set(['a']), ['a']);
+    testGetIntersection(new Set(['a', 'b', 'c']), new Set(['a', 'b', 'c']), ['a', 'b', 'c']);
+    testGetIntersection(new Set(['a', 'b', 'c']), new Set(['d', 'e']), []);
+    testGetIntersection(new Set(['a', 'b', 'c']), new Set(['d', 'c', 'e']), ['c']);
+    testGetIntersection(new Set(['a', 'b', 'c']), new Set(['c', 'e']), ['c']);
+    testGetIntersection(new Set(['a', 'b', 'c', 2]), new Set([1, 2, 3]), [2]);
+    testGetIntersection(new Set(['a', 'b', 'c', 4]), new Set([1, 2, 3]), []);
+    testGetIntersection(new Set([]), new Set([1, 2, 3]), []);
+    testGetIntersection(new Set([]), new Set([]), []);
+  });
+});