@powersync/service-core 0.0.0-dev-20241128134723 → 0.0.0-dev-20241219091224

package/test/src/bucket_validation.test.ts

@@ -1,142 +0,0 @@
-import { OplogEntry } from '@/util/protocol-types.js';
-import { describe, expect, test } from 'vitest';
-import { reduceBucket, validateBucket } from './bucket_validation.js';
-
-// This tests the reduceBucket function.
-// While this function is not used directly in the service implementation,
-// it is an important part of validating consistency in other tests.
-describe('bucket validation', () => {
-  const ops1: OplogEntry[] = [
-    {
-      op_id: '1',
-      op: 'PUT',
-      object_type: 'test',
-      object_id: 't1',
-      checksum: 2634521662,
-      subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
-      data: '{"id":"t1"}'
-    },
-    {
-      op_id: '2',
-      op: 'PUT',
-      object_type: 'test',
-      object_id: 't2',
-      checksum: 4243212114,
-      subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
-      data: '{"id":"t2"}'
-    },
-    {
-      op_id: '3',
-      op: 'REMOVE',
-      object_type: 'test',
-      object_id: 't1',
-      checksum: 4228978084,
-      subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
-      data: null
-    },
-    {
-      op_id: '4',
-      op: 'PUT',
-      object_type: 'test',
-      object_id: 't2',
-      checksum: 4243212114,
-      subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
-      data: '{"id":"t2"}'
-    }
-  ];
-
-  test('reduce 1', () => {
-    expect(reduceBucket(ops1)).toEqual([
-      {
-        checksum: -1778190028,
-        op: 'CLEAR',
-        op_id: '0'
-      },
-      {
-        checksum: 4243212114,
-        data: '{"id":"t2"}',
-        object_id: 't2',
-        object_type: 'test',
-        op: 'PUT',
-        op_id: '4',
-        subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee'
-      }
-    ]);
-
-    expect(reduceBucket(reduceBucket(ops1))).toEqual([
-      {
-        checksum: -1778190028,
-        op: 'CLEAR',
-        op_id: '0'
-      },
-      {
-        checksum: 4243212114,
-        data: '{"id":"t2"}',
-        object_id: 't2',
-        object_type: 'test',
-        op: 'PUT',
-        op_id: '4',
-        subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee'
-      }
-    ]);
-
-    validateBucket(ops1);
-  });
-
-  test('reduce 2', () => {
-    const bucket: OplogEntry[] = [
-      ...ops1,
-
-      {
-        checksum: 93784613,
-        op: 'CLEAR',
-        op_id: '5'
-      },
-      {
-        checksum: 5133378,
-        data: '{"id":"t3"}',
-        object_id: 't3',
-        object_type: 'test',
-        op: 'PUT',
-        op_id: '11',
-        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
-      }
-    ];
-
-    expect(reduceBucket(bucket)).toEqual([
-      {
-        checksum: 93784613,
-        op: 'CLEAR',
-        op_id: '0'
-      },
-      {
-        checksum: 5133378,
-        data: '{"id":"t3"}',
-        object_id: 't3',
-        object_type: 'test',
-        op: 'PUT',
-        op_id: '11',
-        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
-      }
-    ]);
-
-    expect(reduceBucket(reduceBucket(bucket))).toEqual([
-      {
-        checksum: 93784613,
-        op: 'CLEAR',
-        op_id: '0'
-      },
-      {
-        checksum: 5133378,
-        data: '{"id":"t3"}',
-        object_id: 't3',
-        object_type: 'test',
-        op: 'PUT',
-        op_id: '11',
-        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
-      }
-    ]);
-
-    validateBucket(bucket);
-  });
-});

package/test/src/bucket_validation.ts

@@ -1,116 +0,0 @@
-import { OplogEntry } from '@/util/protocol-types.js';
-import { addChecksums } from '@/util/utils.js';
-import { expect } from 'vitest';
-
-/**
- * Reduce a bucket to the final state as stored on the client.
- *
- * This keeps the final state for each row as a PUT operation.
- *
- * All other operations are replaced with a single CLEAR operation,
- * summing their checksums, and using a 0 as an op_id.
- *
- * This is the function $r(B)$, as described in /docs/bucket-properties.md.
- */
-export function reduceBucket(operations: OplogEntry[]) {
-  let rowState = new Map<string, OplogEntry>();
-  let otherChecksum = 0;
-
-  for (let op of operations) {
-    const key = rowKey(op);
-    if (op.op == 'PUT') {
-      const existing = rowState.get(key);
-      if (existing) {
-        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
-      }
-      rowState.set(key, op);
-    } else if (op.op == 'REMOVE') {
-      const existing = rowState.get(key);
-      if (existing) {
-        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
-      }
-      rowState.delete(key);
-      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
-    } else if (op.op == 'CLEAR') {
-      rowState.clear();
-      otherChecksum = op.checksum as number;
-    } else if (op.op == 'MOVE') {
-      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
-    } else {
-      throw new Error(`Unknown operation ${op.op}`);
-    }
-  }
-
-  const puts = [...rowState.values()].sort((a, b) => {
-    return Number(BigInt(a.op_id) - BigInt(b.op_id));
-  });
-
-  let finalState: OplogEntry[] = [
-    // Special operation to indiciate the checksum remainder
-    { op_id: '0', op: 'CLEAR', checksum: otherChecksum },
-    ...puts
-  ];
-
-  return finalState;
-}
-
-function rowKey(entry: OplogEntry) {
-  return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
-}
-
-/**
- * Validate this property, as described in /docs/bucket-properties.md:
- *
- * $r(B_{[..id_n]}) = r(r(B_{[..id_i]}) \cup B_{[id_{i+1}..id_n]}) \;\forall\; i \in [1..n]$
- *
- * We test that a client syncing the entire bucket in one go (left side of the equation),
- * ends up with the same result as another client syncing up to operation id_i, then sync
- * the rest.
- */
-export function validateBucket(bucket: OplogEntry[]) {
-  const r1 = reduceBucket(bucket);
-  for (let i = 0; i <= bucket.length; i++) {
-    const r2 = reduceBucket(bucket.slice(0, i + 1));
-    const b3 = bucket.slice(i + 1);
-    const r3 = r2.concat(b3);
-    const r4 = reduceBucket(r3);
-    expect(r4).toEqual(r1);
-  }
-
-  // This is the same check, just implemented differently
-  validateCompactedBucket(bucket, bucket);
-}
-
-/**
- * Validate these properties for a bucket $B$ and its compacted version $B'$,:
- * as described in /docs/bucket-properties.md:
- *
- * 1. $r(B) = r(B')$
- * 2. $r(B_{[..c]}) = r(r(B_{[..c_i]}) \cup B'_{[c_i+1..c]}) \;\forall\; c_i \in B$
- *
- * The first one is that the result of syncing the original bucket is the same as
- * syncing the compacted bucket.
- *
- * The second property is that result of syncing the entire original bucket, is the same
- * as syncing any partial version of that (up to op $c_i$), and then continue syncing
- * using the compacted bucket.
- */
-export function validateCompactedBucket(bucket: OplogEntry[], compacted: OplogEntry[]) {
-  // r(B_{[..c]})
-  const r1 = reduceBucket(bucket);
-  // r(B) = r(B')
-  expect(reduceBucket(compacted)).toEqual(r1);
-
-  for (let i = 0; i < bucket.length; i++) {
-    // r(B_{[..c_i]})
-    const r2 = reduceBucket(bucket.slice(0, i + 1));
-    const c_i = BigInt(bucket[i].op_id);
-    // B'_{[c_i+1..c]}
-    const b3 = compacted.filter((op) => BigInt(op.op_id) > c_i);
-    // r(B_{[..c_i]}) \cup B'_{[c_i+1..c]}
-    const r3 = r2.concat(b3);
-    // r(r(B_{[..c_i]}) \cup B'_{[c_i+1..c]})
-    const r4 = reduceBucket(r3);
-    expect(r4).toEqual(r1);
-  }
-}

package/test/src/compacting.test.ts

@@ -1,295 +0,0 @@
-import { SaveOperationTag } from '@/storage/BucketStorage.js';
-import { MongoCompactOptions } from '@/storage/mongo/MongoCompactor.js';
-import { describe, expect, test } from 'vitest';
-import { validateCompactedBucket } from './bucket_validation.js';
-import { oneFromAsync } from './stream_utils.js';
-import { BATCH_OPTIONS, makeTestTable, MONGO_STORAGE_FACTORY, rid, testRules } from './util.js';
-
-const TEST_TABLE = makeTestTable('test', ['id']);
-
-// Test with the default options - large batch sizes
-describe('compacting buckets - default options', () => compactTests({}));
-
-// Also test with the miniumum batch sizes, forcing usage of multiple batches internally
-describe('compacting buckets - batched', () =>
-  compactTests({ clearBatchLimit: 2, moveBatchLimit: 1, moveBatchQueryLimit: 1 }));
-
-function compactTests(compactOptions: MongoCompactOptions) {
-  const factory = MONGO_STORAGE_FACTORY;
-
-  test('compacting (1)', async () => {
-    const sync_rules = testRules(`
-bucket_definitions:
-  global:
-    data: [select * from test]
-    `);
-
-    const storage = (await factory()).getInstance(sync_rules);
-
-    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
-      await batch.save({
-        sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
-        after: {
-          id: 't1'
-        },
-        afterReplicaId: rid('t1')
-      });
-
-      await batch.save({
-        sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
-        after: {
-          id: 't2'
-        },
-        afterReplicaId: rid('t2')
-      });
-
-      await batch.save({
-        sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.UPDATE,
-        after: {
-          id: 't2'
-        },
-        afterReplicaId: rid('t2')
-      });
-    });
-
-    const checkpoint = result!.flushed_op;
-
-    const batchBefore = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
-    const dataBefore = batchBefore.batch.data;
-    const checksumBefore = await storage.getChecksums(checkpoint, ['global[]']);
-
-    expect(dataBefore).toMatchObject([
-      {
-        checksum: 2634521662,
-        object_id: 't1',
-        op: 'PUT',
-        op_id: '1'
-      },
-      {
-        checksum: 4243212114,
-        object_id: 't2',
-        op: 'PUT',
-        op_id: '2'
-      },
-      {
-        checksum: 4243212114,
-        object_id: 't2',
-        op: 'PUT',
-        op_id: '3'
-      }
-    ]);
-
-    await storage.compact(compactOptions);
-
-    const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
-    const dataAfter = batchAfter.batch.data;
-    const checksumAfter = await storage.getChecksums(checkpoint, ['global[]']);
-
-    expect(batchAfter.targetOp).toEqual(3n);
-    expect(dataAfter).toMatchObject([
-      {
-        checksum: 2634521662,
-        object_id: 't1',
-        op: 'PUT',
-        op_id: '1'
-      },
-      {
-        checksum: 4243212114,
-        op: 'MOVE',
-        op_id: '2'
-      },
-      {
-        checksum: 4243212114,
-        object_id: 't2',
-        op: 'PUT',
-        op_id: '3'
-      }
-    ]);
-
-    expect(checksumBefore.get('global[]')).toEqual(checksumAfter.get('global[]'));
-
-    validateCompactedBucket(dataBefore, dataAfter);
-  });
-
-  test('compacting (2)', async () => {
-    const sync_rules = testRules(`
-bucket_definitions:
-  global:
-    data: [select * from test]
-    `);
-
-    const storage = (await factory()).getInstance(sync_rules);
-
-    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
-      await batch.save({
-        sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
-        after: {
-          id: 't1'
-        },
-        afterReplicaId: rid('t1')
-      });
-
-      await batch.save({
-        sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
-        after: {
-          id: 't2'
-        },
-        afterReplicaId: rid('t2')
-      });
-
-      await batch.save({
-        sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.DELETE,
-        before: {
-          id: 't1'
-        },
-        beforeReplicaId: rid('t1')
-      });
-
-      await batch.save({
-        sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.UPDATE,
-        after: {
-          id: 't2'
-        },
-        afterReplicaId: rid('t2')
-      });
-    });
-
-    const checkpoint = result!.flushed_op;
-
-    const batchBefore = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
-    const dataBefore = batchBefore.batch.data;
-    const checksumBefore = await storage.getChecksums(checkpoint, ['global[]']);
-
-    expect(dataBefore).toMatchObject([
-      {
-        checksum: 2634521662,
-        object_id: 't1',
-        op: 'PUT',
-        op_id: '1'
-      },
-      {
-        checksum: 4243212114,
-        object_id: 't2',
-        op: 'PUT',
-        op_id: '2'
-      },
-      {
-        checksum: 4228978084,
-        object_id: 't1',
-        op: 'REMOVE',
-        op_id: '3'
-      },
-      {
-        checksum: 4243212114,
-        object_id: 't2',
-        op: 'PUT',
-        op_id: '4'
-      }
-    ]);
-
-    await storage.compact(compactOptions);
-
-    const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']])));
-    const dataAfter = batchAfter.batch.data;
-    const checksumAfter = await storage.getChecksums(checkpoint, ['global[]']);
-
-    expect(batchAfter.targetOp).toEqual(4n);
-    expect(dataAfter).toMatchObject([
-      {
-        checksum: -1778190028,
-        op: 'CLEAR',
-        op_id: '3'
-      },
-      {
-        checksum: 4243212114,
-        object_id: 't2',
-        op: 'PUT',
-        op_id: '4'
-      }
-    ]);
-    expect(checksumBefore.get('global[]')).toEqual(checksumAfter.get('global[]'));
-
-    validateCompactedBucket(dataBefore, dataAfter);
-  });
-
-  test('compacting (3)', async () => {
-    const sync_rules = testRules(`
-bucket_definitions:
-  global:
-    data: [select * from test]
-    `);
-
-    const storage = (await factory()).getInstance(sync_rules);
-
-    const result = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
-      await batch.save({
-        sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
-        after: {
-          id: 't1'
-        },
-        afterReplicaId: 't1'
-      });
-
-      await batch.save({
-        sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.INSERT,
-        after: {
-          id: 't2'
-        },
-        afterReplicaId: 't2'
-      });
-
-      await batch.save({
-        sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.DELETE,
-        before: {
-          id: 't1'
-        },
-        beforeReplicaId: 't1'
-      });
-    });
-
-    const checkpoint1 = result!.flushed_op;
-    const checksumBefore = await storage.getChecksums(checkpoint1, ['global[]']);
-
-    const result2 = await storage.startBatch(BATCH_OPTIONS, async (batch) => {
-      await batch.save({
-        sourceTable: TEST_TABLE,
-        tag: SaveOperationTag.DELETE,
-        before: {
-          id: 't2'
-        },
-        beforeReplicaId: 't2'
-      });
-    });
-    const checkpoint2 = result2!.flushed_op;
-
-    await storage.compact(compactOptions);
-
-    const batchAfter = await oneFromAsync(storage.getBucketDataBatch(checkpoint2, new Map([['global[]', '0']])));
-    const dataAfter = batchAfter.batch.data;
-    const checksumAfter = await storage.getChecksums(checkpoint2, ['global[]']);
-
-    expect(batchAfter.targetOp).toEqual(4n);
-    expect(dataAfter).toMatchObject([
-      {
-        checksum: 1874612650,
-        op: 'CLEAR',
-        op_id: '4'
-      }
-    ]);
-    expect(checksumAfter.get('global[]')).toEqual({
-      bucket: 'global[]',
-      count: 1,
-      checksum: 1874612650
-    });
-  });
-}