@powersync/service-core 0.4.2 → 0.5.1

package/test/src/bucket_validation.test.ts

@@ -0,0 +1,142 @@
+import { OplogEntry } from '@/util/protocol-types.js';
+import { describe, expect, test } from 'vitest';
+import { reduceBucket, validateBucket } from './bucket_validation.js';
+
+// This tests the reduceBucket function.
+// While this function is not used directly in the service implementation,
+// it is an important part of validating consistency in other tests.
+describe('bucket validation', () => {
+  const ops1: OplogEntry[] = [
+    {
+      op_id: '1',
+      op: 'PUT',
+      object_type: 'test',
+      object_id: 't1',
+      checksum: 2634521662,
+      subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
+      data: '{"id":"t1"}'
+    },
+    {
+      op_id: '2',
+      op: 'PUT',
+      object_type: 'test',
+      object_id: 't2',
+      checksum: 4243212114,
+      subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
+      data: '{"id":"t2"}'
+    },
+    {
+      op_id: '3',
+      op: 'REMOVE',
+      object_type: 'test',
+      object_id: 't1',
+      checksum: 4228978084,
+      subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
+      data: null
+    },
+    {
+      op_id: '4',
+      op: 'PUT',
+      object_type: 'test',
+      object_id: 't2',
+      checksum: 4243212114,
+      subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
+      data: '{"id":"t2"}'
+    }
+  ];
+
+  test('reduce 1', () => {
+    expect(reduceBucket(ops1)).toEqual([
+      {
+        checksum: -1778190028,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 4243212114,
+        data: '{"id":"t2"}',
+        object_id: 't2',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '4',
+        subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    expect(reduceBucket(reduceBucket(ops1))).toEqual([
+      {
+        checksum: -1778190028,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 4243212114,
+        data: '{"id":"t2"}',
+        object_id: 't2',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '4',
+        subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    validateBucket(ops1);
+  });
+
+  test('reduce 2', () => {
+    const bucket: OplogEntry[] = [
+      ...ops1,
+
+      {
+        checksum: 93784613,
+        op: 'CLEAR',
+        op_id: '5'
+      },
+      {
+        checksum: 5133378,
+        data: '{"id":"t3"}',
+        object_id: 't3',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '11',
+        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ];
+
+    expect(reduceBucket(bucket)).toEqual([
+      {
+        checksum: 93784613,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 5133378,
+        data: '{"id":"t3"}',
+        object_id: 't3',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '11',
+        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    expect(reduceBucket(reduceBucket(bucket))).toEqual([
+      {
+        checksum: 93784613,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 5133378,
+        data: '{"id":"t3"}',
+        object_id: 't3',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '11',
+        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    validateBucket(bucket);
+  });
+});

package/test/src/bucket_validation.ts

@@ -0,0 +1,116 @@
+import { OplogEntry } from '@/util/protocol-types.js';
+import { addChecksums } from '@/util/utils.js';
+import { expect } from 'vitest';
+
+/**
+ * Reduce a bucket to the final state as stored on the client.
+ *
+ * This keeps the final state for each row as a PUT operation.
+ *
+ * All other operations are replaced with a single CLEAR operation,
+ * summing their checksums, and using a 0 as an op_id.
+ *
+ * This is the function $r(B)$, as described in /docs/bucket-properties.md.
+ */
+export function reduceBucket(operations: OplogEntry[]) {
+  let rowState = new Map<string, OplogEntry>();
+  let otherChecksum = 0;
+
+  for (let op of operations) {
+    const key = rowKey(op);
+    if (op.op == 'PUT') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.set(key, op);
+    } else if (op.op == 'REMOVE') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.delete(key);
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else if (op.op == 'CLEAR') {
+      rowState.clear();
+      otherChecksum = op.checksum as number;
+    } else if (op.op == 'MOVE') {
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else {
+      throw new Error(`Unknown operation ${op.op}`);
+    }
+  }
+
+  const puts = [...rowState.values()].sort((a, b) => {
+    return Number(BigInt(a.op_id) - BigInt(b.op_id));
+  });
+
+  let finalState: OplogEntry[] = [
+    // Special operation to indiciate the checksum remainder
+    { op_id: '0', op: 'CLEAR', checksum: otherChecksum },
+    ...puts
+  ];
+
+  return finalState;
+}
+
+function rowKey(entry: OplogEntry) {
+  return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
+}
+
+/**
+ * Validate this property, as described in /docs/bucket-properties.md:
+ *
+ * $r(B_{[..id_n]}) = r(r(B_{[..id_i]}) \cup B_{[id_{i+1}..id_n]}) \;\forall\; i \in [1..n]$
+ *
+ * We test that a client syncing the entire bucket in one go (left side of the equation),
+ * ends up with the same result as another client syncing up to operation id_i, then sync
+ * the rest.
+ */
+export function validateBucket(bucket: OplogEntry[]) {
+  const r1 = reduceBucket(bucket);
+  for (let i = 0; i <= bucket.length; i++) {
+    const r2 = reduceBucket(bucket.slice(0, i + 1));
+    const b3 = bucket.slice(i + 1);
+    const r3 = r2.concat(b3);
+    const r4 = reduceBucket(r3);
+    expect(r4).toEqual(r1);
+  }
+
+  // This is the same check, just implemented differently
+  validateCompactedBucket(bucket, bucket);
+}
+
+/**
+ * Validate these properties for a bucket $B$ and its compacted version $B'$,:
+ * as described in /docs/bucket-properties.md:
+ *
+ * 1. $r(B) = r(B')$
+ * 2. $r(B_{[..c]}) = r(r(B_{[..c_i]}) \cup B'_{[c_i+1..c]}) \;\forall\; c_i \in B$
+ *
+ * The first one is that the result of syncing the original bucket is the same as
+ * syncing the compacted bucket.
+ *
+ * The second property is that result of syncing the entire original bucket, is the same
+ * as syncing any partial version of that (up to op $c_i$), and then continue syncing
+ * using the compacted bucket.
+ */
+export function validateCompactedBucket(bucket: OplogEntry[], compacted: OplogEntry[]) {
+  // r(B_{[..c]})
+  const r1 = reduceBucket(bucket);
+  // r(B) = r(B')
+  expect(reduceBucket(compacted)).toEqual(r1);
+
+  for (let i = 0; i < bucket.length; i++) {
+    // r(B_{[..c_i]})
+    const r2 = reduceBucket(bucket.slice(0, i + 1));
+    const c_i = BigInt(bucket[i].op_id);
+    // B'_{[c_i+1..c]}
+    const b3 = compacted.filter((op) => BigInt(op.op_id) > c_i);
+    // r(B_{[..c_i]}) \cup B'_{[c_i+1..c]}
+    const r3 = r2.concat(b3);
+    // r(r(B_{[..c_i]}) \cup B'_{[c_i+1..c]})
+    const r4 = reduceBucket(r3);
+    expect(r4).toEqual(r1);
+  }
+}

package/test/src/compacting.test.ts

@@ -0,0 +1,207 @@
+import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { describe, expect, test } from 'vitest';
+import { makeTestTable, MONGO_STORAGE_FACTORY } from './util.js';
+import { oneFromAsync } from './wal_stream_utils.js';
+import { MongoCompactOptions } from '@/storage/mongo/MongoCompactor.js';
+import { reduceBucket, validateCompactedBucket, validateBucket } from './bucket_validation.js';
+
+const TEST_TABLE = makeTestTable('test', ['id']);
+
+// Test with the default options - large batch sizes
+describe('compacting buckets - default options', () => compactTests({}));
+
+// Also test with the miniumum batch sizes, forcing usage of multiple batches internally
+describe('compacting buckets - batched', () =>
+  compactTests({ clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1 }));
+
+function compactTests(compactOptions: MongoCompactOptions) {
+  const factory = MONGO_STORAGE_FACTORY;
+
+  test('compacting (1)', async () => {
+    const sync_rules = SqlSyncRules.fromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+    `);
+
+    const storage = (await factory()).getInstance({ id: 1, sync_rules, slot_name: 'test' });
+
+    const result = await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't1'
+        }
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't2'
+        }
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'update',
+        after: {
+          id: 't2'
+        }
+      });
+    });
+
+    const checkpoint = result!.flushed_op;
+
+    const batchBefore = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
+    const dataBefore = batchBefore.batch.data;
+
+    expect(dataBefore).toMatchObject([
+      {
+        checksum: 2634521662,
+        object_id: 't1',
+        op: 'PUT',
+        op_id: '1'
+      },
+      {
+        checksum: 4243212114,
+        object_id: 't2',
+        op: 'PUT',
+        op_id: '2'
+      },
+      {
+        checksum: 4243212114,
+        object_id: 't2',
+        op: 'PUT',
+        op_id: '3'
+      }
+    ]);
+
+    await storage.compact(compactOptions);
+
+    const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
+    const dataAfter = batchAfter.batch.data;
+
+    expect(batchAfter.targetOp).toEqual(3n);
+    expect(dataAfter).toMatchObject([
+      {
+        checksum: 2634521662,
+        object_id: 't1',
+        op: 'PUT',
+        op_id: '1'
+      },
+      {
+        checksum: 4243212114,
+        op: 'MOVE',
+        op_id: '2'
+      },
+      {
+        checksum: 4243212114,
+        object_id: 't2',
+        op: 'PUT',
+        op_id: '3'
+      }
+    ]);
+
+    validateCompactedBucket(dataBefore, dataAfter);
+  });
+
+  test('compacting (2)', async () => {
+    const sync_rules = SqlSyncRules.fromYaml(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+    `);
+
+    const storage = (await factory()).getInstance({ id: 1, sync_rules, slot_name: 'test' });
+
+    const result = await storage.startBatch({}, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't1'
+        }
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't2'
+        }
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'delete',
+        before: {
+          id: 't1'
+        }
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'update',
+        after: {
+          id: 't2'
+        }
+      });
+    });
+
+    const checkpoint = result!.flushed_op;
+
+    const batchBefore = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
+    const dataBefore = batchBefore.batch.data;
+
+    expect(dataBefore).toMatchObject([
+      {
+        checksum: 2634521662,
+        object_id: 't1',
+        op: 'PUT',
+        op_id: '1'
+      },
+      {
+        checksum: 4243212114,
+        object_id: 't2',
+        op: 'PUT',
+        op_id: '2'
+      },
+      {
+        checksum: 4228978084,
+        object_id: 't1',
+        op: 'REMOVE',
+        op_id: '3'
+      },
+      {
+        checksum: 4243212114,
+        object_id: 't2',
+        op: 'PUT',
+        op_id: '4'
+      }
+    ]);
+
+    await storage.compact(compactOptions);
+
+    const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
+    const dataAfter = batchAfter.batch.data;
+
+    expect(batchAfter.targetOp).toEqual(4n);
+    expect(dataAfter).toMatchObject([
+      {
+        checksum: -1778190028,
+        op: 'CLEAR',
+        op_id: '3'
+      },
+      {
+        checksum: 4243212114,
+        object_id: 't2',
+        op: 'PUT',
+        op_id: '4'
+      }
+    ]);
+
+    validateCompactedBucket(dataBefore, dataAfter);
+  });
+}

package/test/src/data_storage.test.ts

@@ -1,26 +1,8 @@
 import { RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
 import { describe, expect, test } from 'vitest';
-import { SourceTable } from '../../src/storage/SourceTable.js';
-import { hashData } from '../../src/util/utils.js';
-import { MONGO_STORAGE_FACTORY, StorageFactory } from './util.js';
-import { SyncBucketData } from '../../src/util/protocol-types.js';
 import { BucketDataBatchOptions } from '../../src/storage/BucketStorage.js';
-import { fromAsync } from './wal_stream_utils.js';
-
-function makeTestTable(name: string, columns?: string[] | undefined) {
-  const relId = hashData('table', name, (columns ?? ['id']).join(','));
-  const id = new bson.ObjectId('6544e3899293153fa7b38331');
-  return new SourceTable(
-    id,
-    SourceTable.DEFAULT_TAG,
-    relId,
-    SourceTable.DEFAULT_SCHEMA,
-    name,
-    (columns ?? ['id']).map((column) => ({ name: column, typeOid: 25 })),
-    true
-  );
-}
+import { getBatchData, getBatchMeta, makeTestTable, MONGO_STORAGE_FACTORY, StorageFactory } from './util.js';
+import { fromAsync, oneFromAsync } from './wal_stream_utils.js';
 
 const TEST_TABLE = makeTestTable('test', ['id']);
 
@@ -236,7 +218,7 @@ bucket_definitions:
     const checkpoint = result!.flushed_op;
 
     const batch = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
-    const data = batch[0].data.map((d) => {
+    const data = batch[0].batch.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id,
@@ -504,7 +486,7 @@ bucket_definitions:
     });
     const checkpoint = result!.flushed_op;
     const batch = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
-    const data = batch[0].data.map((d) => {
+    const data = batch[0].batch.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id
@@ -568,7 +550,7 @@ bucket_definitions:
     const checkpoint = result!.flushed_op;
 
     const batch = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
-    const data = batch[0].data.map((d) => {
+    const data = batch[0].batch.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id,
@@ -680,7 +662,7 @@ bucket_definitions:
 
     const batch = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
 
-    const data = batch[0].data.map((d) => {
+    const data = batch[0].batch.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id,
@@ -855,7 +837,7 @@ bucket_definitions:
     const checkpoint2 = result2!.flushed_op;
 
     const batch = await fromAsync(storage.getBucketDataBatch(checkpoint2, new Map([['global[]', checkpoint1]])));
-    const data = batch[0].data.map((d) => {
+    const data = batch[0].batch.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id,
@@ -939,7 +921,7 @@ bucket_definitions:
     const checkpoint3 = result3!.flushed_op;
 
     const batch = await fromAsync(storage.getBucketDataBatch(checkpoint3, new Map([['global[]', checkpoint1]])));
-    const data = batch[0].data.map((d) => {
+    const data = batch[0].batch.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id,
@@ -1031,7 +1013,7 @@ bucket_definitions:
     const checkpoint3 = result3!.flushed_op;
 
     const batch = await fromAsync(storage.getBucketDataBatch(checkpoint3, new Map([['global[]', checkpoint1]])));
-    const data = batch[0].data.map((d) => {
+    const data = batch[0].batch.data.map((d) => {
       return {
         op: d.op,
         object_id: d.object_id,
@@ -1133,7 +1115,7 @@ bucket_definitions:
     });
 
     const batch2 = await fromAsync(
-      storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].next_after]]), options)
+      storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].batch.next_after]]), options)
     );
     expect(getBatchData(batch2)).toEqual([
       { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1795508474 },
@@ -1146,7 +1128,7 @@ bucket_definitions:
     });
 
     const batch3 = await fromAsync(
-      storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].next_after]]), options)
+      storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].batch.next_after]]), options)
     );
     expect(getBatchData(batch3)).toEqual([]);
     expect(getBatchMeta(batch3)).toEqual(null);
@@ -1223,7 +1205,7 @@ bucket_definitions:
     });
 
     const batch2 = await fromAsync(
-      storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].next_after]]), options)
+      storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].batch.next_after]]), options)
     );
     expect(getBatchData(batch2)).toEqual([{ op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }]);
     expect(getBatchMeta(batch2)).toEqual({
@@ -1233,7 +1215,7 @@ bucket_definitions:
     });
 
     const batch3 = await fromAsync(
-      storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].next_after]]), options)
+      storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].batch.next_after]]), options)
     );
     expect(getBatchData(batch3)).toEqual([{ op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }]);
     expect(getBatchMeta(batch3)).toEqual({
@@ -1270,7 +1252,9 @@ bucket_definitions:
 
     const checkpoint = result!.flushed_op;
 
-    const batch1 = await fromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]), { limit: 4 }));
+    const batch1 = await oneFromAsync(
+      storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]), { limit: 4 })
+    );
 
     expect(getBatchData(batch1)).toEqual([
       { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
@@ -1285,8 +1269,8 @@ bucket_definitions:
       next_after: '4'
     });
 
-    const batch2 = await fromAsync(
-      storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].next_after]]), {
+    const batch2 = await oneFromAsync(
+      storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1.batch.next_after]]), {
         limit: 4
       })
     );
@@ -1302,7 +1286,7 @@ bucket_definitions:
     });
 
     const batch3 = await fromAsync(
-      storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].next_after]]), {
+      storage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2.batch.next_after]]), {
         limit: 4
       })
     );
@@ -1311,28 +1295,3 @@ bucket_definitions:
     expect(getBatchMeta(batch3)).toEqual(null);
   });
 }
-
-function getBatchData(batch: SyncBucketData[]) {
-  if (batch.length == 0) {
-    return [];
-  }
-  return batch[0].data.map((d) => {
-    return {
-      op_id: d.op_id,
-      op: d.op,
-      object_id: d.object_id,
-      checksum: d.checksum
-    };
-  });
-}
-
-function getBatchMeta(batch: SyncBucketData[]) {
-  if (batch.length == 0) {
-    return null;
-  }
-  return {
-    has_more: batch[0].has_more,
-    after: batch[0].after,
-    next_after: batch[0].next_after
-  };
-}