@powersync/service-core 1.23.1 → 1.23.2

package/test/src/checksum_cache.test.ts

@@ -482,4 +482,26 @@ describe('checksum cache', function () {
     expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_123]);
     expect(await cache.getChecksums(1234n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_1234]);
   });
+
+  it('should refresh cached checksums after the configured TTL', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = new ChecksumCache({
+      fetchChecksums: async (batch) => {
+        lookups.push(batch);
+        return fetchTestChecksums(batch);
+      },
+      ttlMs: 1
+    });
+
+    expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_123]);
+    expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_123]);
+
+    await new Promise((resolve) => setTimeout(resolve, 1001)); // Needs to be >1s
+
+    expect(await cache.getChecksums(123n, [{ bucket: 'test', source: DUMMY_SOURCE }])).toEqual([TEST_123]);
+    expect(lookups.map(removeLookupSources)).toEqual([
+      [{ bucket: 'test', end: 123n }],
+      [{ bucket: 'test', end: 123n }]
+    ]);
+  });
 });

package/test/src/config.test.ts

@@ -70,6 +70,48 @@ describe('Config', () => {
     expect(config.api_parameters.max_buckets_per_connection).toBe(1);
   });
 
+  it('should resolve checksum cache TTL from API parameters', async () => {
+    const yamlConfig = /* yaml */ `
+      # PowerSync config
+      replication:
+        connections: []
+      storage:
+        type: mongodb
+      api:
+        parameters:
+          bucket_count_cache_ttl_minutes: 15
+    `;
+
+    const collector = new CompoundConfigCollector();
+
+    const config = await collector.collectConfig({
+      config_base64: Buffer.from(yamlConfig, 'utf-8').toString('base64')
+    });
+
+    expect(config.api_parameters.bucket_count_cache_ttl_minutes).toBe(15);
+  });
+
+  it('should reject fractional checksum cache TTL minutes', async () => {
+    const yamlConfig = /* yaml */ `
+      # PowerSync config
+      replication:
+        connections: []
+      storage:
+        type: mongodb
+      api:
+        parameters:
+          bucket_count_cache_ttl_minutes: 1.5
+    `;
+
+    const collector = new CompoundConfigCollector();
+
+    await expect(
+      collector.collectConfig({
+        config_base64: Buffer.from(yamlConfig, 'utf-8').toString('base64')
+      })
+    ).rejects.toThrow('api.parameters.bucket_count_cache_ttl_minutes must be a positive integer');
+  });
+
   it('should throw YAML validation error for invalid base64 config', {}, async () => {
     const yamlConfig = /* yaml */ `
       # PowerSync config

package/test/src/sync/BucketChecksumState.test.ts

@@ -137,6 +137,7 @@ bucket_definitions:
         priority: 3
       }
     ]);
+    expect(line.bucketDataRequestHint).toEqual('bulk');
     // This is the bucket data to be fetched
     expect(bucketStarts(line.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 0n]]));
 
@@ -170,6 +171,45 @@ bucket_definitions:
       }
     });
     expect(bucketStarts(line2.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 1n]]));
+    expect(line2.bucketDataRequestHint).toEqual('incremental');
+  });
+
+  test('unfinished bulk bucket does not make checkpoint diff bulk', async () => {
+    const storage = new MockBucketChecksumStateStorage();
+    storage.updateTestChecksum({ bucket: '1#global[]', checksum: 1, count: 1 });
+
+    const state = new BucketChecksumState({
+      syncContext,
+      syncRequest,
+      tokenPayload,
+      syncRules: SYNC_RULES_GLOBAL,
+      bucketStorage: storage
+    });
+
+    const line = (await state.buildNextCheckpointLine({
+      base: storage.makeCheckpoint(1n),
+      writeCheckpoint: null,
+      update: CHECKPOINT_INVALIDATE_ALL
+    }))!;
+    line.advance();
+    line.updateBucketPosition({ bucket: '1#global[]', nextAfter: 1n, hasMore: true });
+
+    storage.updateTestChecksum({ bucket: '1#global[]', checksum: 2, count: 2 });
+
+    const line2 = (await state.buildNextCheckpointLine({
+      base: storage.makeCheckpoint(2n),
+      writeCheckpoint: null,
+      update: {
+        updatedDataBuckets: new Set(['1#global[]']),
+        invalidateDataBuckets: false,
+        updatedParameterLookups: new Set(),
+        invalidateParameterBuckets: false
+      }
+    }))!;
+    line2.advance();
+
+    expect(line2.bucketDataRequestHint).toEqual('incremental');
+    expect(bucketStarts(line2.getFilteredBucketPositions())).toEqual(new Map([['1#global[]', 1n]]));
   });
 
   test('global bucket with initial state', async () => {
@@ -1240,7 +1280,7 @@ class MockBucketChecksumStateStorage implements BucketChecksumStateStorage {
     this.filter?.({ invalidate: true });
   }
 
-  async getChecksums(_checkpoint: InternalOpId, buckets: { bucket: string }[]): Promise<ChecksumMap> {
+  async getChecksums(_checkpoint: ReplicationCheckpoint, buckets: { bucket: string }[]): Promise<ChecksumMap> {
     return new Map<string, BucketChecksum>(
       buckets.map(({ bucket }) => {
         const checksum = this.state.get(bucket);

package/test/src/util/checkpointing.test.ts

@@ -0,0 +1,170 @@
+import { createWriteCheckpoint, WriteCheckpointBatcher } from '@/index.js';
+import { describe, expect, test, vi } from 'vitest';
+
+function deferred<T = void>() {
+  return Promise.withResolvers<T>();
+}
+
+async function waitForAsyncWork() {
+  await new Promise((resolve) => setTimeout(resolve, 0));
+  await new Promise((resolve) => setImmediate(resolve));
+}
+
+function createStorage() {
+  let nextCheckpoint = 1n;
+  const bucketStorage = {
+    createManagedWriteCheckpoints: vi.fn(async (checkpoints: { user_id: string }[]) => {
+      return new Map(checkpoints.map((checkpoint) => [checkpoint.user_id, nextCheckpoint++]));
+    })
+  };
+
+  const storage = {
+    getActiveSyncConfig: vi.fn(async () => ({ storage: bucketStorage }))
+  };
+
+  return { bucketStorage, storage };
+}
+
+function createBatcher(api: any, storage: any) {
+  return new WriteCheckpointBatcher(
+    () => api,
+    () => storage
+  );
+}
+
+describe('write checkpoint batching', () => {
+  test('coalesces same-turn requests and dispatches queued requests as capacity becomes available', async () => {
+    const gates = [deferred(), deferred()];
+    const { bucketStorage, storage } = createStorage();
+    const api = {
+      createReplicationHead: vi.fn(async (callback: (head: string) => Promise<unknown>) => {
+        const batch = api.createReplicationHead.mock.calls.length;
+        const result = await callback(`head-${batch}`);
+        await gates[batch - 1].promise;
+        return result;
+      })
+    };
+    const batcher = createBatcher(api, storage);
+
+    const firstBatch = [
+      createWriteCheckpoint({
+        userId: 'user-a',
+        clientId: 'client-1',
+        batcher
+      }),
+      createWriteCheckpoint({
+        userId: 'user-b',
+        clientId: undefined,
+        batcher
+      })
+    ];
+    await waitForAsyncWork();
+
+    expect(api.createReplicationHead).toHaveBeenCalledTimes(1);
+    expect(bucketStorage.createManagedWriteCheckpoints).toHaveBeenNthCalledWith(1, [
+      { user_id: 'user-a/client-1', heads: { '1': 'head-1' } },
+      { user_id: 'user-b', heads: { '1': 'head-1' } }
+    ]);
+
+    const secondBatch = [
+      createWriteCheckpoint({ userId: 'user-c', clientId: 'client-3', batcher }),
+      createWriteCheckpoint({ userId: 'user-d', clientId: undefined, batcher }),
+      createWriteCheckpoint({ userId: 'user-e', clientId: undefined, batcher })
+    ];
+
+    await waitForAsyncWork();
+
+    expect(api.createReplicationHead).toHaveBeenCalledTimes(2);
+    expect(bucketStorage.createManagedWriteCheckpoints).toHaveBeenNthCalledWith(2, [
+      { user_id: 'user-c/client-3', heads: { '1': 'head-2' } },
+      { user_id: 'user-d', heads: { '1': 'head-2' } },
+      { user_id: 'user-e', heads: { '1': 'head-2' } }
+    ]);
+
+    gates[0].resolve();
+    gates[1].resolve();
+    await expect(Promise.all([...firstBatch, ...secondBatch])).resolves.toEqual([
+      { writeCheckpoint: '1', replicationHead: 'head-1' },
+      { writeCheckpoint: '2', replicationHead: 'head-1' },
+      { writeCheckpoint: '3', replicationHead: 'head-2' },
+      { writeCheckpoint: '4', replicationHead: 'head-2' },
+      { writeCheckpoint: '5', replicationHead: 'head-2' }
+    ]);
+  });
+
+  test('allows three executing batches and queues later requests until one completes', async () => {
+    const gates = [deferred(), deferred(), deferred(), deferred()];
+    const { storage } = createStorage();
+    const api = {
+      createReplicationHead: vi.fn(async (callback: (head: string) => Promise<unknown>) => {
+        const batch = api.createReplicationHead.mock.calls.length;
+        const result = await callback(`head-${batch}`);
+        await gates[batch - 1].promise;
+        return result;
+      })
+    };
+    const batcher = createBatcher(api, storage);
+
+    const first = createWriteCheckpoint({
+      userId: 'user-a',
+      clientId: undefined,
+      batcher
+    });
+    await waitForAsyncWork();
+
+    const second = createWriteCheckpoint({
+      userId: 'user-b',
+      clientId: undefined,
+      batcher
+    });
+    await waitForAsyncWork();
+
+    const third = createWriteCheckpoint({
+      userId: 'user-c',
+      clientId: undefined,
+      batcher
+    });
+    await waitForAsyncWork();
+
+    expect(api.createReplicationHead).toHaveBeenCalledTimes(3);
+
+    const fourth = createWriteCheckpoint({
+      userId: 'user-d',
+      clientId: undefined,
+      batcher
+    });
+    await waitForAsyncWork();
+
+    expect(api.createReplicationHead).toHaveBeenCalledTimes(3);
+
+    gates[0].resolve();
+    await first;
+    await waitForAsyncWork();
+
+    expect(api.createReplicationHead).toHaveBeenCalledTimes(4);
+
+    gates[1].resolve();
+    gates[2].resolve();
+    gates[3].resolve();
+
+    await expect(Promise.all([second, third, fourth])).resolves.toEqual([
+      { writeCheckpoint: '2', replicationHead: 'head-2' },
+      { writeCheckpoint: '3', replicationHead: 'head-3' },
+      { writeCheckpoint: '4', replicationHead: 'head-4' }
+    ]);
+  });
+
+  test('passes batch errors through without retrying', async () => {
+    const error = new Error('source unavailable');
+    const { storage } = createStorage();
+    const api = {
+      createReplicationHead: vi.fn(async () => {
+        throw error;
+      })
+    };
+    const batcher = createBatcher(api, storage);
+
+    await expect(createWriteCheckpoint({ userId: 'user-a', clientId: undefined, batcher })).rejects.toBe(error);
+    expect(api.createReplicationHead).toHaveBeenCalledTimes(1);
+  });
+});