@powersync/service-core 0.1.3 → 0.2.1

package/test/src/checksum_cache.test.ts

@@ -0,0 +1,436 @@
+import { describe, expect, it } from 'vitest';
+import { BucketChecksum, OpId } from '@/util/protocol-types.js';
+import * as crypto from 'node:crypto';
+import { addBucketChecksums } from '@/util/util-index.js';
+import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum } from '@/storage/ChecksumCache.js';
+
+/**
+ * Create a deterministic BucketChecksum based on the bucket name and checkpoint for testing purposes.
+ */
+function testHash(bucket: string, checkpoint: OpId) {
+  const key = `${checkpoint}/${bucket}`;
+  const hash = crypto.createHash('sha256').update(key).digest().readInt32LE(0);
+  return hash;
+}
+
+function testPartialHash(request: FetchPartialBucketChecksum): BucketChecksum {
+  if (request.start) {
+    const a = testHash(request.bucket, request.start);
+    const b = testHash(request.bucket, request.end);
+    return addBucketChecksums(
+      {
+        bucket: request.bucket,
+        checksum: b,
+        count: Number(request.end)
+      },
+      {
+        // Subtract a
+        bucket: request.bucket,
+        checksum: -a,
+        count: -Number(request.start)
+      }
+    );
+  } else {
+    return {
+      bucket: request.bucket,
+      checksum: testHash(request.bucket, request.end),
+      count: Number(request.end)
+    };
+  }
+}
+
+const TEST_123 = {
+  bucket: 'test',
+  count: 123,
+  checksum: 1104081737
+};
+
+const TEST_1234 = {
+  bucket: 'test',
+  count: 1234,
+  checksum: -1593864957
+};
+
+const TEST2_123 = {
+  bucket: 'test2',
+  count: 123,
+  checksum: 1741377449
+};
+
+const TEST3_123 = {
+  bucket: 'test3',
+  count: 123,
+  checksum: -2085080402
+};
+
+function fetchTestChecksums(batch: FetchPartialBucketChecksum[]) {
+  return new Map(
+    batch.map((v) => {
+      return [v.bucket, testPartialHash(v)];
+    })
+  );
+}
+
+describe('checksum cache', function () {
+  const factory = (fetch: FetchChecksums) => {
+    return new ChecksumCache({ fetchChecksums: fetch });
+  };
+
+  it('should handle a sequential lookups (a)', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      return fetchTestChecksums(batch);
+    });
+
+    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+
+    expect(await cache.getChecksums('1234', ['test'])).toEqual([TEST_1234]);
+
+    expect(await cache.getChecksums('123', ['test2'])).toEqual([TEST2_123]);
+
+    expect(lookups).toEqual([
+      [{ bucket: 'test', end: '123' }],
+      // This should use the previous lookup
+      [{ bucket: 'test', start: '123', end: '1234' }],
+      [{ bucket: 'test2', end: '123' }]
+    ]);
+  });
+
+  it('should handle a sequential lookups (b)', async function () {
+    // Reverse order of the above
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      return fetchTestChecksums(batch);
+    });
+
+    expect(await cache.getChecksums('123', ['test2'])).toEqual([TEST2_123]);
+
+    expect(await cache.getChecksums('1234', ['test'])).toEqual([TEST_1234]);
+
+    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+
+    expect(lookups).toEqual([
+      // With this order, there is no option for a partial lookup
+      [{ bucket: 'test2', end: '123' }],
+      [{ bucket: 'test', end: '1234' }],
+      [{ bucket: 'test', end: '123' }]
+    ]);
+  });
+
+  it('should handle a concurrent lookups (a)', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      return fetchTestChecksums(batch);
+    });
+
+    const p1 = cache.getChecksums('123', ['test']);
+    const p2 = cache.getChecksums('1234', ['test']);
+    const p3 = cache.getChecksums('123', ['test2']);
+
+    expect(await p1).toEqual([TEST_123]);
+    expect(await p2).toEqual([TEST_1234]);
+    expect(await p3).toEqual([TEST2_123]);
+
+    // Concurrent requests, so we can't do a partial lookup for 123 -> 1234
+    expect(lookups).toEqual([
+      [{ bucket: 'test', end: '123' }],
+      [{ bucket: 'test', end: '1234' }],
+      [{ bucket: 'test2', end: '123' }]
+    ]);
+  });
+
+  it('should handle a concurrent lookups (b)', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      return fetchTestChecksums(batch);
+    });
+
+    const p1 = cache.getChecksums('123', ['test']);
+    const p2 = cache.getChecksums('123', ['test']);
+
+    expect(await p1).toEqual([TEST_123]);
+
+    expect(await p2).toEqual([TEST_123]);
+
+    // The lookup should be deduplicated, even though it's in progress
+    expect(lookups).toEqual([[{ bucket: 'test', end: '123' }]]);
+  });
+
+  it('should handle serial + concurrent lookups', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      return fetchTestChecksums(batch);
+    });
+
+    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+
+    const p2 = cache.getChecksums('1234', ['test']);
+    const p3 = cache.getChecksums('1234', ['test']);
+
+    expect(await p2).toEqual([TEST_1234]);
+    expect(await p3).toEqual([TEST_1234]);
+
+    expect(lookups).toEqual([
+      [{ bucket: 'test', end: '123' }],
+      // This lookup is deduplicated
+      [{ bucket: 'test', start: '123', end: '1234' }]
+    ]);
+  });
+
+  it('should handle multiple buckets', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      return fetchTestChecksums(batch);
+    });
+
+    expect(await cache.getChecksums('123', ['test', 'test2'])).toEqual([TEST_123, TEST2_123]);
+
+    expect(lookups).toEqual([
+      [
+        // Both lookups in the same request
+        { bucket: 'test', end: '123' },
+        { bucket: 'test2', end: '123' }
+      ]
+    ]);
+  });
+
+  it('should handle multiple buckets with partial caching (a)', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      return fetchTestChecksums(batch);
+    });
+
+    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+    expect(await cache.getChecksums('123', ['test', 'test2'])).toEqual([TEST_123, TEST2_123]);
+
+    expect(lookups).toEqual([
+      // Request 1
+      [{ bucket: 'test', end: '123' }],
+      // Request 2
+      [{ bucket: 'test2', end: '123' }]
+    ]);
+  });
+
+  it('should handle multiple buckets with partial caching (b)', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      return fetchTestChecksums(batch);
+    });
+
+    const a = cache.getChecksums('123', ['test', 'test2']);
+    const b = cache.getChecksums('123', ['test2', 'test3']);
+
+    expect(await a).toEqual([TEST_123, TEST2_123]);
+    expect(await b).toEqual([TEST2_123, TEST3_123]);
+
+    expect(lookups).toEqual([
+      // Request A
+      [
+        { bucket: 'test', end: '123' },
+        { bucket: 'test2', end: '123' }
+      ],
+      // Request B (re-uses the checksum for test2 from request a)
+      [{ bucket: 'test3', end: '123' }]
+    ]);
+  });
+
+  it('should handle out-of-order requests', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      return fetchTestChecksums(batch);
+    });
+
+    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+
+    expect(await cache.getChecksums('125', ['test'])).toEqual([
+      {
+        bucket: 'test',
+        checksum: -1865121912,
+        count: 125
+      }
+    ]);
+
+    expect(await cache.getChecksums('124', ['test'])).toEqual([
+      {
+        bucket: 'test',
+        checksum: 1887460431,
+        count: 124
+      }
+    ]);
+    expect(lookups).toEqual([
+      [{ bucket: 'test', end: '123' }],
+      [{ bucket: 'test', start: '123', end: '125' }],
+      [{ bucket: 'test', start: '123', end: '124' }]
+    ]);
+  });
+
+  it('should handle errors', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const TEST_ERROR = new Error('Simulated error');
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      if (lookups.length == 1) {
+        throw new Error('Simulated error');
+      }
+      return fetchTestChecksums(batch);
+    });
+
+    const a = cache.getChecksums('123', ['test', 'test2']);
+    const b = cache.getChecksums('123', ['test2', 'test3']);
+
+    await expect(a).rejects.toEqual(TEST_ERROR);
+    await expect(b).rejects.toEqual(TEST_ERROR);
+
+    const a2 = cache.getChecksums('123', ['test', 'test2']);
+    const b2 = cache.getChecksums('123', ['test2', 'test3']);
+
+    expect(await a2).toEqual([TEST_123, TEST2_123]);
+    expect(await b2).toEqual([TEST2_123, TEST3_123]);
+
+    expect(lookups).toEqual([
+      // Request A (fails)
+      [
+        { bucket: 'test', end: '123' },
+        { bucket: 'test2', end: '123' }
+      ],
+      // Request B (re-uses the checksum for test2 from request a)
+      // Even thought the full request fails, this batch succeeds
+      [{ bucket: 'test3', end: '123' }],
+      // Retry request A
+      [
+        { bucket: 'test', end: '123' },
+        { bucket: 'test2', end: '123' }
+      ]
+    ]);
+  });
+
+  it('should handle missing checksums (a)', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      return fetchTestChecksums(batch.filter((b) => b.bucket != 'test'));
+    });
+
+    expect(await cache.getChecksums('123', ['test'])).toEqual([{ bucket: 'test', checksum: 0, count: 0 }]);
+    expect(await cache.getChecksums('123', ['test', 'test2'])).toEqual([
+      { bucket: 'test', checksum: 0, count: 0 },
+      TEST2_123
+    ]);
+  });
+
+  it('should handle missing checksums (b)', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      return fetchTestChecksums(batch.filter((b) => b.bucket != 'test' || b.end != '123'));
+    });
+
+    expect(await cache.getChecksums('123', ['test'])).toEqual([{ bucket: 'test', checksum: 0, count: 0 }]);
+    expect(await cache.getChecksums('1234', ['test'])).toEqual([
+      {
+        bucket: 'test',
+        checksum: 1597020602,
+        count: 1111
+      }
+    ]);
+
+    expect(lookups).toEqual([[{ bucket: 'test', end: '123' }], [{ bucket: 'test', start: '123', end: '1234' }]]);
+  });
+
+  it('should use maxSize', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = new ChecksumCache({
+      fetchChecksums: async (batch) => {
+        lookups.push(batch);
+        return fetchTestChecksums(batch);
+      },
+      maxSize: 2
+    });
+
+    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+    expect(await cache.getChecksums('124', ['test'])).toEqual([
+      {
+        bucket: 'test',
+        checksum: 1887460431,
+        count: 124
+      }
+    ]);
+
+    expect(await cache.getChecksums('125', ['test'])).toEqual([
+      {
+        bucket: 'test',
+        checksum: -1865121912,
+        count: 125
+      }
+    ]);
+    expect(await cache.getChecksums('126', ['test'])).toEqual([
+      {
+        bucket: 'test',
+        checksum: -1720007310,
+        count: 126
+      }
+    ]);
+    expect(await cache.getChecksums('124', ['test'])).toEqual([
+      {
+        bucket: 'test',
+        checksum: 1887460431,
+        count: 124
+      }
+    ]);
+    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+
+    expect(lookups).toEqual([
+      [{ bucket: 'test', end: '123' }],
+      [{ bucket: 'test', start: '123', end: '124' }],
+      [{ bucket: 'test', start: '124', end: '125' }],
+      [{ bucket: 'test', start: '125', end: '126' }],
+      [{ bucket: 'test', end: '124' }],
+      [{ bucket: 'test', end: '123' }]
+    ]);
+  });
+
+  it('should handle concurrent requests greater than cache size', async function () {
+    // This will not be cached efficiently, but we test that we don't get errors at least.
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = new ChecksumCache({
+      fetchChecksums: async (batch) => {
+        lookups.push(batch);
+        return fetchTestChecksums(batch);
+      },
+      maxSize: 2
+    });
+
+    const p3 = cache.getChecksums('123', ['test3']);
+    const p4 = cache.getChecksums('123', ['test4']);
+    const p1 = cache.getChecksums('123', ['test']);
+    const p2 = cache.getChecksums('123', ['test2']);
+
+    expect(await p1).toEqual([TEST_123]);
+    expect(await p2).toEqual([TEST2_123]);
+    expect(await p3).toEqual([TEST3_123]);
+    expect(await p4).toEqual([
+      {
+        bucket: 'test4',
+        checksum: 1004797863,
+        count: 123
+      }
+    ]);
+
+    // The lookup should be deduplicated, even though it's in progress
+    expect(lookups).toEqual([
+      [{ bucket: 'test3', end: '123' }],
+      [{ bucket: 'test4', end: '123' }],
+      [{ bucket: 'test', end: '123' }],
+      [{ bucket: 'test2', end: '123' }]
+    ]);
+  });
+});

package/test/src/data_storage.test.ts

@@ -252,7 +252,7 @@ bucket_definitions:
       { op: 'REMOVE', object_id: 'test1', checksum: c2 }
     ]);
 
-    const checksums = await storage.getChecksums(checkpoint, ['global[]']);
+    const checksums = [...(await storage.getChecksums(checkpoint, ['global[]'])).values()];
     expect(checksums).toEqual([
       {
         bucket: 'global[]',
@@ -599,7 +599,7 @@ bucket_definitions:
       { op: 'REMOVE', object_id: 'test1', checksum: c2 }
     ]);
 
-    const checksums = await storage.getChecksums(checkpoint, ['global[]']);
+    const checksums = [...(await storage.getChecksums(checkpoint, ['global[]'])).values()];
     expect(checksums).toEqual([
       {
         bucket: 'global[]',
@@ -713,7 +713,7 @@ bucket_definitions:
       { op: 'REMOVE', object_id: 'test1', checksum: c2 }
     ]);
 
-    const checksums = await storage.getChecksums(checkpoint, ['global[]']);
+    const checksums = [...(await storage.getChecksums(checkpoint, ['global[]'])).values()];
     expect(checksums).toEqual([
       {
         bucket: 'global[]',

package/test/src/large_batch.test.ts

@@ -50,7 +50,7 @@ function defineBatchTests(factory: StorageFactory) {
       const duration = Date.now() - start;
       const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
       const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
-      expect(checksum[0].count).toEqual(operation_count);
+      expect(checksum.get('global[]')!.count).toEqual(operation_count);
       const perSecond = Math.round((operation_count / duration) * 1000);
       console.log(`${operation_count} ops in ${duration}ms ${perSecond} ops/s. ${used}MB heap`);
     }),
@@ -101,7 +101,7 @@ function defineBatchTests(factory: StorageFactory) {
         const checkpoint = await context.getCheckpoint({ timeout: 100_000 });
         const duration = Date.now() - start;
         const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
-        expect(checksum[0].count).toEqual(operation_count);
+        expect(checksum.get('global[]')!.count).toEqual(operation_count);
         const perSecond = Math.round((operation_count / duration) * 1000);
         console.log(`${operation_count} ops in ${duration}ms ${perSecond} ops/s.`);
         printMemoryUsage();
@@ -157,7 +157,7 @@ function defineBatchTests(factory: StorageFactory) {
       const duration = Date.now() - start;
       const used = Math.round(process.memoryUsage().heapUsed / 1024 / 1024);
       const checksum = await context.storage!.getChecksums(checkpoint, ['global[]']);
-      expect(checksum[0].count).toEqual(operationCount);
+      expect(checksum.get('global[]')!.count).toEqual(operationCount);
       const perSecond = Math.round((operationCount / duration) * 1000);
       // This number depends on the test machine, so we keep the test significantly
       // lower than expected numbers.
@@ -174,7 +174,7 @@ function defineBatchTests(factory: StorageFactory) {
       const truncateDuration = Date.now() - truncateStart;
 
       const checksum2 = await context.storage!.getChecksums(checkpoint2, ['global[]']);
-      const truncateCount = checksum2[0].count - checksum[0].count;
+      const truncateCount = checksum2.get('global[]')!.count - checksum.get('global[]')!.count;
       expect(truncateCount).toEqual(numTransactions * perTransaction);
       const truncatePerSecond = Math.round((truncateCount / truncateDuration) * 1000);
       console.log(`Truncated ${truncateCount} ops in ${truncateDuration}ms ${truncatePerSecond} ops/s. ${used}MB heap`);

package/test/src/sync_rules.test.ts

@@ -100,27 +100,29 @@ bucket_definitions:
     const rules = SqlSyncRules.fromYaml(`
 bucket_definitions:
   mybucket:
-    parameters: SELECT token_parameters.user_id
+    parameters: SELECT token_parameters.user_id, user_parameters.device_id
     data:
-      - SELECT id, description FROM assets WHERE assets.user_id = bucket.user_id AND NOT assets.archived
+      - SELECT id, description FROM assets WHERE assets.user_id = bucket.user_id AND assets.device_id = bucket.device_id AND NOT assets.archived
     `);
     const bucket = rules.bucket_descriptors[0];
-    expect(bucket.bucket_parameters).toEqual(['user_id']);
+    expect(bucket.bucket_parameters).toEqual(['user_id', 'device_id']);
     const param_query = bucket.global_parameter_queries[0];
-    expect(param_query.bucket_parameters).toEqual(['user_id']);
-    expect(rules.getStaticBucketIds(normalizeTokenParameters({ user_id: 'user1' }))).toEqual(['mybucket["user1"]']);
+    expect(param_query.bucket_parameters).toEqual(['user_id', 'device_id']);
+    expect(rules.getStaticBucketIds(normalizeTokenParameters({ user_id: 'user1' }, { device_id: 'device1' }))).toEqual([
+      'mybucket["user1","device1"]'
+    ]);
 
     const data_query = bucket.data_queries[0];
-    expect(data_query.bucket_parameters).toEqual(['user_id']);
+    expect(data_query.bucket_parameters).toEqual(['user_id', 'device_id']);
     expect(
       rules.evaluateRow({
         sourceTable: ASSETS,
-        record: { id: 'asset1', description: 'test', user_id: 'user1' }
+        record: { id: 'asset1', description: 'test', user_id: 'user1', device_id: 'device1' }
       })
     ).toEqual([
       {
         ruleId: '1',
-        bucket: 'mybucket["user1"]',
+        bucket: 'mybucket["user1","device1"]',
         id: 'asset1',
         data: {
           id: 'asset1',
@@ -132,7 +134,7 @@ bucket_definitions:
     expect(
       rules.evaluateRow({
         sourceTable: ASSETS,
-        record: { id: 'asset1', description: 'test', user_id: 'user1', archived: 1 }
+        record: { id: 'asset1', description: 'test', user_id: 'user1', archived: 1, device_id: 'device1' }
       })
     ).toEqual([]);
   });