@powersync/service-core 0.0.0-dev-20241001150040 → 0.0.0-dev-20241002111637

package/test/src/__snapshots__/sync.test.ts.snap

@@ -56,7 +56,7 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
       "data": [
         {
           "checksum": 1859363232n,
-          "data": "{\\"id\\":\\"t1\\",\\"description\\":\\"Test 1b\\"}",
+          "data": "{"id":"t1","description":"Test 1b"}",
           "object_id": "t1",
           "object_type": "test",
           "op": "PUT",
@@ -65,7 +65,7 @@ exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
         },
         {
           "checksum": 3028503153n,
-          "data": "{\\"id\\":\\"t2\\",\\"description\\":\\"Test 2b\\"}",
+          "data": "{"id":"t2","description":"Test 2b"}",
           "object_id": "t2",
           "object_type": "test",
           "op": "PUT",
@@ -146,7 +146,7 @@ exports[`sync - mongodb > sync global data 1`] = `
       "data": [
         {
           "checksum": 920318466n,
-          "data": "{\\"id\\":\\"t1\\",\\"description\\":\\"Test 1\\"}",
+          "data": "{"id":"t1","description":"Test 1"}",
           "object_id": "t1",
           "object_type": "test",
           "op": "PUT",
@@ -155,7 +155,7 @@ exports[`sync - mongodb > sync global data 1`] = `
         },
         {
           "checksum": 3280762209n,
-          "data": "{\\"id\\":\\"t2\\",\\"description\\":\\"Test 2\\"}",
+          "data": "{"id":"t2","description":"Test 2"}",
           "object_id": "t2",
           "object_type": "test",
           "op": "PUT",
@@ -199,7 +199,7 @@ exports[`sync - mongodb > sync legacy non-raw data 1`] = `
           "checksum": 3442149460n,
           "data": {
             "description": "Test
-\\"string\\"",
+"string"",
             "id": "t1",
             "large_num": 12345678901234567890n,
           },
@@ -268,7 +268,7 @@ exports[`sync - mongodb > sync updates to global data 2`] = `
       "data": [
         {
           "checksum": 920318466n,
-          "data": "{\\"id\\":\\"t1\\",\\"description\\":\\"Test 1\\"}",
+          "data": "{"id":"t1","description":"Test 1"}",
           "object_id": "t1",
           "object_type": "test",
           "op": "PUT",
@@ -311,7 +311,7 @@ exports[`sync - mongodb > sync updates to global data 3`] = `
       "data": [
         {
           "checksum": 3280762209n,
-          "data": "{\\"id\\":\\"t2\\",\\"description\\":\\"Test 2\\"}",
+          "data": "{"id":"t2","description":"Test 2"}",
           "object_id": "t2",
           "object_type": "test",
           "op": "PUT",

package/test/src/checksum_cache.test.ts

@@ -1,6 +1,6 @@
-import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum } from '@/storage/ChecksumCache.js';
-import { BucketChecksum, OpId } from '@/util/protocol-types.js';
-import { addBucketChecksums } from '@/util/util-index.js';
+import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum, PartialChecksum } from '@/storage/ChecksumCache.js';
+import { OpId } from '@/util/protocol-types.js';
+import { addChecksums } from '@/util/util-index.js';
 import * as crypto from 'node:crypto';
 import { describe, expect, it } from 'vitest';
 
@@ -13,28 +13,22 @@ function testHash(bucket: string, checkpoint: OpId) {
   return hash;
 }
 
-function testPartialHash(request: FetchPartialBucketChecksum): BucketChecksum {
+function testPartialHash(request: FetchPartialBucketChecksum): PartialChecksum {
   if (request.start) {
     const a = testHash(request.bucket, request.start);
     const b = testHash(request.bucket, request.end);
-    return addBucketChecksums(
-      {
-        bucket: request.bucket,
-        checksum: b,
-        count: Number(request.end)
-      },
-      {
-        // Subtract a
-        bucket: request.bucket,
-        checksum: -a,
-        count: -Number(request.start)
-      }
-    );
+    return {
+      bucket: request.bucket,
+      partialCount: Number(request.end) - Number(request.start),
+      partialChecksum: addChecksums(b, -a),
+      isFullChecksum: false
+    };
   } else {
     return {
       bucket: request.bucket,
-      checksum: testHash(request.bucket, request.end),
-      count: Number(request.end)
+      partialChecksum: testHash(request.bucket, request.end),
+      partialCount: Number(request.end),
+      isFullChecksum: true
     };
   }
 }
@@ -433,4 +427,17 @@ describe('checksum cache', function () {
       [{ bucket: 'test2', end: '123' }]
     ]);
   });
+
+  it('should handle CLEAR/isFullChecksum checksums', async function () {
+    let lookups: FetchPartialBucketChecksum[][] = [];
+    const cache = factory(async (batch) => {
+      lookups.push(batch);
+      // This forces a `isFullChecksum: true` result
+      delete batch[0].start;
+      return fetchTestChecksums(batch);
+    });
+
+    expect(await cache.getChecksums('123', ['test'])).toEqual([TEST_123]);
+    expect(await cache.getChecksums('1234', ['test'])).toEqual([TEST_1234]);
+  });
 });

package/test/src/compacting.test.ts

@@ -61,6 +61,7 @@ bucket_definitions:
 
     const batchBefore = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
     const dataBefore = batchBefore.batch.data;
+    const checksumBefore = await storage.getChecksums(checkpoint, ['global[]']);
 
     expect(dataBefore).toMatchObject([
       {
@@ -87,6 +88,7 @@ bucket_definitions:
 
     const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
     const dataAfter = batchAfter.batch.data;
+    const checksumAfter = await storage.getChecksums(checkpoint, ['global[]']);
 
     expect(batchAfter.targetOp).toEqual(3n);
     expect(dataAfter).toMatchObject([
@@ -109,6 +111,8 @@ bucket_definitions:
       }
     ]);
 
+    expect(checksumBefore.get('global[]')).toEqual(checksumAfter.get('global[]'));
+
     validateCompactedBucket(dataBefore, dataAfter);
   });
 
@@ -163,6 +167,7 @@ bucket_definitions:
 
     const batchBefore = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
     const dataBefore = batchBefore.batch.data;
+    const checksumBefore = await storage.getChecksums(checkpoint, ['global[]']);
 
     expect(dataBefore).toMatchObject([
       {
@@ -195,6 +200,7 @@ bucket_definitions:
 
     const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
     const dataAfter = batchAfter.batch.data;
+    const checksumAfter = await storage.getChecksums(checkpoint, ['global[]']);
 
     expect(batchAfter.targetOp).toEqual(4n);
     expect(dataAfter).toMatchObject([
@@ -210,7 +216,82 @@ bucket_definitions:
         op_id: '4'
       }
     ]);
+    expect(checksumBefore.get('global[]')).toEqual(checksumAfter.get('global[]'));
 
     validateCompactedBucket(dataBefore, dataAfter);
   });
+
+  test('compacting (3)', async () => {
+    const sync_rules = testRules(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+    `);
+
+    const storage = (await factory()).getInstance(sync_rules);
+
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't1'
+        },
+        afterReplicaId: 't1'
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't2'
+        },
+        afterReplicaId: 't2'
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'delete',
+        before: {
+          id: 't1'
+        },
+        beforeReplicaId: 't1'
+      });
+    });
+
+    const checkpoint1 = result!.flushed_op;
+    const checksumBefore = await storage.getChecksums(checkpoint1, ['global[]']);
+
+    const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'delete',
+        before: {
+          id: 't2'
+        },
+        beforeReplicaId: 't2'
+      });
+    });
+    const checkpoint2 = result2!.flushed_op;
+
+    await storage.compact(compactOptions);
+
+    const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint2, new Map([['global[]', '0']])));
+    const dataAfter = batchAfter.batch.data;
+    const checksumAfter = await storage.getChecksums(checkpoint2, ['global[]']);
+
+    expect(batchAfter.targetOp).toEqual(4n);
+    expect(dataAfter).toMatchObject([
+      {
+        checksum: 1874612650,
+        op: 'CLEAR',
+        op_id: '4'
+      }
+    ]);
+    expect(checksumAfter.get('global[]')).toEqual({
+      bucket: 'global[]',
+      count: 1,
+      checksum: 1874612650
+    });
+  });
 }