@powersync/service-core 0.0.0-dev-20240718134716 → 0.0.0-dev-20240918082156

package/test/src/bucket_validation.test.ts

@@ -0,0 +1,142 @@
+import { OplogEntry } from '@/util/protocol-types.js';
+import { describe, expect, test } from 'vitest';
+import { reduceBucket, validateBucket } from './bucket_validation.js';
+
+// This tests the reduceBucket function.
+// While this function is not used directly in the service implementation,
+// it is an important part of validating consistency in other tests.
+describe('bucket validation', () => {
+  const ops1: OplogEntry[] = [
+    {
+      op_id: '1',
+      op: 'PUT',
+      object_type: 'test',
+      object_id: 't1',
+      checksum: 2634521662,
+      subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
+      data: '{"id":"t1"}'
+    },
+    {
+      op_id: '2',
+      op: 'PUT',
+      object_type: 'test',
+      object_id: 't2',
+      checksum: 4243212114,
+      subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
+      data: '{"id":"t2"}'
+    },
+    {
+      op_id: '3',
+      op: 'REMOVE',
+      object_type: 'test',
+      object_id: 't1',
+      checksum: 4228978084,
+      subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
+      data: null
+    },
+    {
+      op_id: '4',
+      op: 'PUT',
+      object_type: 'test',
+      object_id: 't2',
+      checksum: 4243212114,
+      subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
+      data: '{"id":"t2"}'
+    }
+  ];
+
+  test('reduce 1', () => {
+    expect(reduceBucket(ops1)).toEqual([
+      {
+        checksum: -1778190028,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 4243212114,
+        data: '{"id":"t2"}',
+        object_id: 't2',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '4',
+        subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    expect(reduceBucket(reduceBucket(ops1))).toEqual([
+      {
+        checksum: -1778190028,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 4243212114,
+        data: '{"id":"t2"}',
+        object_id: 't2',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '4',
+        subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    validateBucket(ops1);
+  });
+
+  test('reduce 2', () => {
+    const bucket: OplogEntry[] = [
+      ...ops1,
+
+      {
+        checksum: 93784613,
+        op: 'CLEAR',
+        op_id: '5'
+      },
+      {
+        checksum: 5133378,
+        data: '{"id":"t3"}',
+        object_id: 't3',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '11',
+        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ];
+
+    expect(reduceBucket(bucket)).toEqual([
+      {
+        checksum: 93784613,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 5133378,
+        data: '{"id":"t3"}',
+        object_id: 't3',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '11',
+        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    expect(reduceBucket(reduceBucket(bucket))).toEqual([
+      {
+        checksum: 93784613,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 5133378,
+        data: '{"id":"t3"}',
+        object_id: 't3',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '11',
+        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    validateBucket(bucket);
+  });
+});

package/test/src/bucket_validation.ts

@@ -0,0 +1,116 @@
+import { OplogEntry } from '@/util/protocol-types.js';
+import { addChecksums } from '@/util/utils.js';
+import { expect } from 'vitest';
+
+/**
+ * Reduce a bucket to the final state as stored on the client.
+ *
+ * This keeps the final state for each row as a PUT operation.
+ *
+ * All other operations are replaced with a single CLEAR operation,
+ * summing their checksums, and using a 0 as an op_id.
+ *
+ * This is the function $r(B)$, as described in /docs/bucket-properties.md.
+ */
+export function reduceBucket(operations: OplogEntry[]) {
+  let rowState = new Map<string, OplogEntry>();
+  let otherChecksum = 0;
+
+  for (let op of operations) {
+    const key = rowKey(op);
+    if (op.op == 'PUT') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.set(key, op);
+    } else if (op.op == 'REMOVE') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.delete(key);
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else if (op.op == 'CLEAR') {
+      rowState.clear();
+      otherChecksum = op.checksum as number;
+    } else if (op.op == 'MOVE') {
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else {
+      throw new Error(`Unknown operation ${op.op}`);
+    }
+  }
+
+  const puts = [...rowState.values()].sort((a, b) => {
+    return Number(BigInt(a.op_id) - BigInt(b.op_id));
+  });
+
+  let finalState: OplogEntry[] = [
+    // Special operation to indiciate the checksum remainder
+    { op_id: '0', op: 'CLEAR', checksum: otherChecksum },
+    ...puts
+  ];
+
+  return finalState;
+}
+
+function rowKey(entry: OplogEntry) {
+  return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
+}
+
+/**
+ * Validate this property, as described in /docs/bucket-properties.md:
+ *
+ * $r(B_{[..id_n]}) = r(r(B_{[..id_i]}) \cup B_{[id_{i+1}..id_n]}) \;\forall\; i \in [1..n]$
+ *
+ * We test that a client syncing the entire bucket in one go (left side of the equation),
+ * ends up with the same result as another client syncing up to operation id_i, then sync
+ * the rest.
+ */
+export function validateBucket(bucket: OplogEntry[]) {
+  const r1 = reduceBucket(bucket);
+  for (let i = 0; i <= bucket.length; i++) {
+    const r2 = reduceBucket(bucket.slice(0, i + 1));
+    const b3 = bucket.slice(i + 1);
+    const r3 = r2.concat(b3);
+    const r4 = reduceBucket(r3);
+    expect(r4).toEqual(r1);
+  }
+
+  // This is the same check, just implemented differently
+  validateCompactedBucket(bucket, bucket);
+}
+
+/**
+ * Validate these properties for a bucket $B$ and its compacted version $B'$,:
+ * as described in /docs/bucket-properties.md:
+ *
+ * 1. $r(B) = r(B')$
+ * 2. $r(B_{[..c]}) = r(r(B_{[..c_i]}) \cup B'_{[c_i+1..c]}) \;\forall\; c_i \in B$
+ *
+ * The first one is that the result of syncing the original bucket is the same as
+ * syncing the compacted bucket.
+ *
+ * The second property is that result of syncing the entire original bucket, is the same
+ * as syncing any partial version of that (up to op $c_i$), and then continue syncing
+ * using the compacted bucket.
+ */
+export function validateCompactedBucket(bucket: OplogEntry[], compacted: OplogEntry[]) {
+  // r(B_{[..c]})
+  const r1 = reduceBucket(bucket);
+  // r(B) = r(B')
+  expect(reduceBucket(compacted)).toEqual(r1);
+
+  for (let i = 0; i < bucket.length; i++) {
+    // r(B_{[..c_i]})
+    const r2 = reduceBucket(bucket.slice(0, i + 1));
+    const c_i = BigInt(bucket[i].op_id);
+    // B'_{[c_i+1..c]}
+    const b3 = compacted.filter((op) => BigInt(op.op_id) > c_i);
+    // r(B_{[..c_i]}) \cup B'_{[c_i+1..c]}
+    const r3 = r2.concat(b3);
+    // r(r(B_{[..c_i]}) \cup B'_{[c_i+1..c]})
+    const r4 = reduceBucket(r3);
+    expect(r4).toEqual(r1);
+  }
+}

package/test/src/checksum_cache.test.ts

@@ -1,8 +1,8 @@
-import { describe, expect, it } from 'vitest';
+import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum } from '@/storage/ChecksumCache.js';
 import { BucketChecksum, OpId } from '@/util/protocol-types.js';
-import * as crypto from 'node:crypto';
 import { addBucketChecksums } from '@/util/util-index.js';
-import { ChecksumCache, FetchChecksums, FetchPartialBucketChecksum } from '@/storage/ChecksumCache.js';
+import * as crypto from 'node:crypto';
+import { describe, expect, it } from 'vitest';
 
 /**
  * Create a deterministic BucketChecksum based on the bucket name and checkpoint for testing purposes.

package/test/src/compacting.test.ts

@@ -0,0 +1,216 @@
+import { MongoCompactOptions } from '@/storage/mongo/MongoCompactor.js';
+import { SqlSyncRules } from '@powersync/service-sync-rules';
+import { describe, expect, test } from 'vitest';
+import { validateCompactedBucket } from './bucket_validation.js';
+import { oneFromAsync } from './stream_utils.js';
+import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules, ZERO_LSN } from './util.js';
+import { ParseSyncRulesOptions, PersistedSyncRulesContent, StartBatchOptions } from '@/storage/BucketStorage.js';
+import { getUuidReplicaIdentityBson } from '@/util/util-index.js';
+
+const TEST_TABLE = makeTestTable('test', ['id']);
+
+// Test with the default options - large batch sizes
+describe('compacting buckets - default options', () => compactTests({}));
+
+// Also test with the miniumum batch sizes, forcing usage of multiple batches internally
+describe('compacting buckets - batched', () =>
+  compactTests({ clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1 }));
+
+function compactTests(compactOptions: MongoCompactOptions) {
+  const factory = MONGO_STORAGE_FACTORY;
+
+  test('compacting (1)', async () => {
+    const sync_rules = testRules(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+    `);
+
+    const storage = (await factory()).getInstance(sync_rules);
+
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't1'
+        },
+        afterReplicaId: rid('t1')
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't2'
+        },
+        afterReplicaId: rid('t2')
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'update',
+        after: {
+          id: 't2'
+        },
+        afterReplicaId: rid('t2')
+      });
+    });
+
+    const checkpoint = result!.flushed_op;
+
+    const batchBefore = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
+    const dataBefore = batchBefore.batch.data;
+
+    expect(dataBefore).toMatchObject([
+      {
+        checksum: 2634521662,
+        object_id: 't1',
+        op: 'PUT',
+        op_id: '1'
+      },
+      {
+        checksum: 4243212114,
+        object_id: 't2',
+        op: 'PUT',
+        op_id: '2'
+      },
+      {
+        checksum: 4243212114,
+        object_id: 't2',
+        op: 'PUT',
+        op_id: '3'
+      }
+    ]);
+
+    await storage.compact(compactOptions);
+
+    const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
+    const dataAfter = batchAfter.batch.data;
+
+    expect(batchAfter.targetOp).toEqual(3n);
+    expect(dataAfter).toMatchObject([
+      {
+        checksum: 2634521662,
+        object_id: 't1',
+        op: 'PUT',
+        op_id: '1'
+      },
+      {
+        checksum: 4243212114,
+        op: 'MOVE',
+        op_id: '2'
+      },
+      {
+        checksum: 4243212114,
+        object_id: 't2',
+        op: 'PUT',
+        op_id: '3'
+      }
+    ]);
+
+    validateCompactedBucket(dataBefore, dataAfter);
+  });
+
+  test('compacting (2)', async () => {
+    const sync_rules = testRules(`
+bucket_definitions:
+  global:
+    data: [select * from test]
+    `);
+
+    const storage = (await factory()).getInstance(sync_rules);
+
+    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't1'
+        },
+        afterReplicaId: rid('t1')
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'insert',
+        after: {
+          id: 't2'
+        },
+        afterReplicaId: rid('t2')
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'delete',
+        before: {
+          id: 't1'
+        },
+        beforeReplicaId: rid('t1')
+      });
+
+      await batch.save({
+        sourceTable: TEST_TABLE,
+        tag: 'update',
+        after: {
+          id: 't2'
+        },
+        afterReplicaId: rid('t2')
+      });
+    });
+
+    const checkpoint = result!.flushed_op;
+
+    const batchBefore = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
+    const dataBefore = batchBefore.batch.data;
+
+    expect(dataBefore).toMatchObject([
+      {
+        checksum: 2634521662,
+        object_id: 't1',
+        op: 'PUT',
+        op_id: '1'
+      },
+      {
+        checksum: 4243212114,
+        object_id: 't2',
+        op: 'PUT',
+        op_id: '2'
+      },
+      {
+        checksum: 4228978084,
+        object_id: 't1',
+        op: 'REMOVE',
+        op_id: '3'
+      },
+      {
+        checksum: 4243212114,
+        object_id: 't2',
+        op: 'PUT',
+        op_id: '4'
+      }
+    ]);
+
+    await storage.compact(compactOptions);
+
+    const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
+    const dataAfter = batchAfter.batch.data;
+
+    expect(batchAfter.targetOp).toEqual(4n);
+    expect(dataAfter).toMatchObject([
+      {
+        checksum: -1778190028,
+        op: 'CLEAR',
+        op_id: '3'
+      },
+      {
+        checksum: 4243212114,
+        object_id: 't2',
+        op: 'PUT',
+        op_id: '4'
+      }
+    ]);
+
+    validateCompactedBucket(dataBefore, dataAfter);
+  });
+}