@powersync/service-core 0.4.1 → 0.5.0

package/src/sync/sync.ts

@@ -11,6 +11,7 @@ import { logger } from '@powersync/lib-services-framework';
 import { Metrics } from '../metrics/Metrics.js';
 import { mergeAsyncIterables } from './merge.js';
 import { TokenStreamOptions, tokenStream } from './util.js';
+import { RequestTracker } from './RequestTracker.js';
 
 /**
  * Maximum number of connections actively fetching data.
@@ -28,12 +29,14 @@ export interface SyncStreamParameters {
    */
   signal?: AbortSignal;
   tokenStreamOptions?: Partial<TokenStreamOptions>;
+
+  tracker: RequestTracker;
 }
 
 export async function* streamResponse(
   options: SyncStreamParameters
 ): AsyncIterable<util.StreamingSyncLine | string | null> {
-  const { storage, params, syncParams, token, tokenStreamOptions, signal } = options;
+  const { storage, params, syncParams, token, tokenStreamOptions, tracker, signal } = options;
   // We also need to be able to abort, so we create our own controller.
   const controller = new AbortController();
   if (signal) {
@@ -49,7 +52,7 @@ export async function* streamResponse(
     }
   }
   const ki = tokenStream(token, controller.signal, tokenStreamOptions);
-  const stream = streamResponseInner(storage, params, syncParams, controller.signal);
+  const stream = streamResponseInner(storage, params, syncParams, tracker, controller.signal);
   // Merge the two streams, and abort as soon as one of the streams end.
   const merged = mergeAsyncIterables([stream, ki], controller.signal);
 
@@ -72,6 +75,7 @@ async function* streamResponseInner(
   storage: storage.BucketStorageFactory,
   params: util.StreamingSyncRequest,
   syncParams: RequestParameters,
+  tracker: RequestTracker,
   signal: AbortSignal
 ): AsyncGenerator<util.StreamingSyncLine | string | null> {
   // Bucket state of bucket id -> op_id.
@@ -109,6 +113,11 @@ async function* streamResponseInner(
     });
 
     if (allBuckets.length > 1000) {
+      logger.error(`Too many buckets`, {
+        checkpoint,
+        user_id: syncParams.user_id,
+        buckets: allBuckets.length
+      });
       // TODO: Limit number of buckets even before we get to this point
       throw new Error(`Too many buckets: ${allBuckets.length}`);
     }
@@ -137,11 +146,18 @@ async function* streamResponseInner(
       }
       bucketsToFetch = diff.updatedBuckets.map((c) => c.bucket);
 
-      let message = `Updated checkpoint: ${checkpoint} | write: ${writeCheckpoint} | `;
+      let message = `Updated checkpoint: ${checkpoint} | `;
+      message += `write: ${writeCheckpoint} | `;
       message += `buckets: ${allBuckets.length} | `;
       message += `updated: ${limitedBuckets(diff.updatedBuckets, 20)} | `;
-      message += `removed: ${limitedBuckets(diff.removedBuckets, 20)} | `;
-      logger.info(message);
+      message += `removed: ${limitedBuckets(diff.removedBuckets, 20)}`;
+      logger.info(message, {
+        checkpoint,
+        user_id: syncParams.user_id,
+        buckets: allBuckets.length,
+        updated: diff.updatedBuckets.length,
+        removed: diff.removedBuckets.length
+      });
 
       const checksum_line: util.StreamingSyncCheckpointDiff = {
         checkpoint_diff: {
@@ -156,7 +172,7 @@ async function* streamResponseInner(
     } else {
       let message = `New checkpoint: ${checkpoint} | write: ${writeCheckpoint} | `;
       message += `buckets: ${allBuckets.length} ${limitedBuckets(allBuckets, 20)}`;
-      logger.info(message);
+      logger.info(message, { checkpoint, user_id: syncParams.user_id, buckets: allBuckets.length });
       bucketsToFetch = allBuckets;
       const checksum_line: util.StreamingSyncCheckpoint = {
         checkpoint: {
@@ -172,7 +188,16 @@ async function* streamResponseInner(
 
     // This incrementally updates dataBuckets with each individual bucket position.
     // At the end of this, we can be sure that all buckets have data up to the checkpoint.
-    yield* bucketDataInBatches({ storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, signal });
+    yield* bucketDataInBatches({
+      storage,
+      checkpoint,
+      bucketsToFetch,
+      dataBuckets,
+      raw_data,
+      binary_data,
+      signal,
+      tracker
+    });
 
     await new Promise((resolve) => setTimeout(resolve, 10));
   }
@@ -186,6 +211,7 @@ interface BucketDataRequest {
   dataBuckets: Map<string, string>;
   raw_data: boolean | undefined;
   binary_data: boolean | undefined;
+  tracker: RequestTracker;
   signal: AbortSignal;
 }
 
@@ -221,11 +247,19 @@ async function* bucketDataInBatches(request: BucketDataRequest) {
   }
 }
 
+interface BucketDataBatchResult {
+  done: boolean;
+  data: any;
+}
+
 /**
  * Extracted as a separate internal function just to avoid memory leaks.
  */
-async function* bucketDataBatch(request: BucketDataRequest) {
-  const { storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, signal } = request;
+async function* bucketDataBatch(request: BucketDataRequest): AsyncGenerator<BucketDataBatchResult, void> {
+  const { storage, checkpoint, bucketsToFetch, dataBuckets, raw_data, binary_data, tracker, signal } = request;
+
+  const checkpointOp = BigInt(checkpoint);
+  let checkpointInvalidated = false;
 
   const [_, release] = await syncSemaphore.acquire();
   try {
@@ -236,13 +270,16 @@ async function* bucketDataBatch(request: BucketDataRequest) {
 
     let has_more = false;
 
-    for await (let r of data) {
+    for await (let { batch: r, targetOp } of data) {
       if (signal.aborted) {
         return;
       }
       if (r.has_more) {
         has_more = true;
       }
+      if (targetOp != null && targetOp > checkpointOp) {
+        checkpointInvalidated = true;
+      }
       if (r.data.length == 0) {
         continue;
       }
@@ -272,18 +309,25 @@ async function* bucketDataBatch(request: BucketDataRequest) {
         // iterator memory in case if large data sent.
         yield { data: null, done: false };
       }
-      Metrics.getInstance().operations_synced_total.add(r.data.length);
+      tracker.addOperationsSynced(r.data.length);
 
       dataBuckets.set(r.bucket, r.next_after);
     }
 
     if (!has_more) {
-      const line: util.StreamingSyncCheckpointComplete = {
-        checkpoint_complete: {
-          last_op_id: checkpoint
-        }
-      };
-      yield { data: line, done: true };
+      if (checkpointInvalidated) {
+        // Checkpoint invalidated by a CLEAR or MOVE op.
+        // Don't send the checkpoint_complete line in this case.
+        // More data should be available immediately for a new checkpoint.
+        yield { data: null, done: true };
+      } else {
+        const line: util.StreamingSyncCheckpointComplete = {
+          checkpoint_complete: {
+            last_op_id: checkpoint
+          }
+        };
+        yield { data: line, done: true };
+      }
     }
   } finally {
     release();

package/src/sync/util.ts

@@ -2,6 +2,7 @@ import * as timers from 'timers/promises';
 
 import * as util from '../util/util-index.js';
 import { Metrics } from '../metrics/Metrics.js';
+import { RequestTracker } from './RequestTracker.js';
 
 export type TokenStreamOptions = {
   /**
@@ -89,10 +90,13 @@ export async function* ndjson(iterator: AsyncIterable<string | null | Record<str
   }
 }
 
-export async function* transformToBytesTracked(iterator: AsyncIterable<string>): AsyncGenerator<Buffer> {
+export async function* transformToBytesTracked(
+  iterator: AsyncIterable<string>,
+  tracker: RequestTracker
+): AsyncGenerator<Buffer> {
   for await (let data of iterator) {
     const encoded = Buffer.from(data, 'utf8');
-    Metrics.getInstance().data_synced_bytes.add(encoded.length);
+    tracker.addDataSynced(encoded.length);
     yield encoded;
   }
 }

package/test/src/__snapshots__/sync.test.ts.snap

@@ -1,5 +1,90 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
+exports[`sync - mongodb > compacting data - invalidate checkpoint 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": -93886621,
+          "count": 2,
+        },
+      ],
+      "last_op_id": "2",
+      "write_checkpoint": undefined,
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > compacting data - invalidate checkpoint 2`] = `
+[
+  {
+    "data": {
+      "after": "0",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": -93886621n,
+          "op": "CLEAR",
+          "op_id": "2",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_diff": {
+      "last_op_id": "4",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "mybucket[]",
+          "checksum": 499012468,
+          "count": 4,
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "2",
+      "bucket": "mybucket[]",
+      "data": [
+        {
+          "checksum": 1859363232n,
+          "data": "{\\"id\\":\\"t1\\",\\"description\\":\\"Test 1b\\"}",
+          "object_id": "t1",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "3",
+          "subkey": "6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423",
+        },
+        {
+          "checksum": 3028503153n,
+          "data": "{\\"id\\":\\"t2\\",\\"description\\":\\"Test 2b\\"}",
+          "object_id": "t2",
+          "object_type": "test",
+          "op": "PUT",
+          "op_id": "4",
+          "subkey": "6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee",
+        },
+      ],
+      "has_more": false,
+      "next_after": "4",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "4",
+    },
+  },
+]
+`;
+
 exports[`sync - mongodb > expired token 1`] = `
 [
   {

package/test/src/bucket_validation.test.ts

@@ -0,0 +1,142 @@
+import { OplogEntry } from '@/util/protocol-types.js';
+import { describe, expect, test } from 'vitest';
+import { reduceBucket, validateBucket } from './bucket_validation.js';
+
+// This tests the reduceBucket function.
+// While this function is not used directly in the service implementation,
+// it is an important part of validating consistency in other tests.
+describe('bucket validation', () => {
+  const ops1: OplogEntry[] = [
+    {
+      op_id: '1',
+      op: 'PUT',
+      object_type: 'test',
+      object_id: 't1',
+      checksum: 2634521662,
+      subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
+      data: '{"id":"t1"}'
+    },
+    {
+      op_id: '2',
+      op: 'PUT',
+      object_type: 'test',
+      object_id: 't2',
+      checksum: 4243212114,
+      subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
+      data: '{"id":"t2"}'
+    },
+    {
+      op_id: '3',
+      op: 'REMOVE',
+      object_type: 'test',
+      object_id: 't1',
+      checksum: 4228978084,
+      subkey: '6544e3899293153fa7b38331/117ab485-4b42-58a2-ab32-0053a22c3423',
+      data: null
+    },
+    {
+      op_id: '4',
+      op: 'PUT',
+      object_type: 'test',
+      object_id: 't2',
+      checksum: 4243212114,
+      subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee',
+      data: '{"id":"t2"}'
+    }
+  ];
+
+  test('reduce 1', () => {
+    expect(reduceBucket(ops1)).toEqual([
+      {
+        checksum: -1778190028,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 4243212114,
+        data: '{"id":"t2"}',
+        object_id: 't2',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '4',
+        subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    expect(reduceBucket(reduceBucket(ops1))).toEqual([
+      {
+        checksum: -1778190028,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 4243212114,
+        data: '{"id":"t2"}',
+        object_id: 't2',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '4',
+        subkey: '6544e3899293153fa7b38331/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    validateBucket(ops1);
+  });
+
+  test('reduce 2', () => {
+    const bucket: OplogEntry[] = [
+      ...ops1,
+
+      {
+        checksum: 93784613,
+        op: 'CLEAR',
+        op_id: '5'
+      },
+      {
+        checksum: 5133378,
+        data: '{"id":"t3"}',
+        object_id: 't3',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '11',
+        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ];
+
+    expect(reduceBucket(bucket)).toEqual([
+      {
+        checksum: 93784613,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 5133378,
+        data: '{"id":"t3"}',
+        object_id: 't3',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '11',
+        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    expect(reduceBucket(reduceBucket(bucket))).toEqual([
+      {
+        checksum: 93784613,
+        op: 'CLEAR',
+        op_id: '0'
+      },
+      {
+        checksum: 5133378,
+        data: '{"id":"t3"}',
+        object_id: 't3',
+        object_type: 'test',
+        op: 'PUT',
+        op_id: '11',
+        subkey: '6544e3899293153fa7b38333/ec27c691-b47a-5d92-927a-9944feb89eee'
+      }
+    ]);
+
+    validateBucket(bucket);
+  });
+});

package/test/src/bucket_validation.ts

@@ -0,0 +1,116 @@
+import { OplogEntry } from '@/util/protocol-types.js';
+import { addChecksums } from '@/util/utils.js';
+import { expect } from 'vitest';
+
+/**
+ * Reduce a bucket to the final state as stored on the client.
+ *
+ * This keeps the final state for each row as a PUT operation.
+ *
+ * All other operations are replaced with a single CLEAR operation,
+ * summing their checksums, and using a 0 as an op_id.
+ *
+ * This is the function $r(B)$, as described in /docs/bucket-properties.md.
+ */
+export function reduceBucket(operations: OplogEntry[]) {
+  let rowState = new Map<string, OplogEntry>();
+  let otherChecksum = 0;
+
+  for (let op of operations) {
+    const key = rowKey(op);
+    if (op.op == 'PUT') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.set(key, op);
+    } else if (op.op == 'REMOVE') {
+      const existing = rowState.get(key);
+      if (existing) {
+        otherChecksum = addChecksums(otherChecksum, existing.checksum as number);
+      }
+      rowState.delete(key);
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else if (op.op == 'CLEAR') {
+      rowState.clear();
+      otherChecksum = op.checksum as number;
+    } else if (op.op == 'MOVE') {
+      otherChecksum = addChecksums(otherChecksum, op.checksum as number);
+    } else {
+      throw new Error(`Unknown operation ${op.op}`);
+    }
+  }
+
+  const puts = [...rowState.values()].sort((a, b) => {
+    return Number(BigInt(a.op_id) - BigInt(b.op_id));
+  });
+
+  let finalState: OplogEntry[] = [
+    // Special operation to indiciate the checksum remainder
+    { op_id: '0', op: 'CLEAR', checksum: otherChecksum },
+    ...puts
+  ];
+
+  return finalState;
+}
+
+function rowKey(entry: OplogEntry) {
+  return `${entry.object_type}/${entry.object_id}/${entry.subkey}`;
+}
+
+/**
+ * Validate this property, as described in /docs/bucket-properties.md:
+ *
+ * $r(B_{[..id_n]}) = r(r(B_{[..id_i]}) \cup B_{[id_{i+1}..id_n]}) \;\forall\; i \in [1..n]$
+ *
+ * We test that a client syncing the entire bucket in one go (left side of the equation),
+ * ends up with the same result as another client syncing up to operation id_i, then sync
+ * the rest.
+ */
+export function validateBucket(bucket: OplogEntry[]) {
+  const r1 = reduceBucket(bucket);
+  for (let i = 0; i <= bucket.length; i++) {
+    const r2 = reduceBucket(bucket.slice(0, i + 1));
+    const b3 = bucket.slice(i + 1);
+    const r3 = r2.concat(b3);
+    const r4 = reduceBucket(r3);
+    expect(r4).toEqual(r1);
+  }
+
+  // This is the same check, just implemented differently
+  validateCompactedBucket(bucket, bucket);
+}
+
+/**
+ * Validate these properties for a bucket $B$ and its compacted version $B'$,:
+ * as described in /docs/bucket-properties.md:
+ *
+ * 1. $r(B) = r(B')$
+ * 2. $r(B_{[..c]}) = r(r(B_{[..c_i]}) \cup B'_{[c_i+1..c]}) \;\forall\; c_i \in B$
+ *
+ * The first one is that the result of syncing the original bucket is the same as
+ * syncing the compacted bucket.
+ *
+ * The second property is that result of syncing the entire original bucket, is the same
+ * as syncing any partial version of that (up to op $c_i$), and then continue syncing
+ * using the compacted bucket.
+ */
+export function validateCompactedBucket(bucket: OplogEntry[], compacted: OplogEntry[]) {
+  // r(B_{[..c]})
+  const r1 = reduceBucket(bucket);
+  // r(B) = r(B')
+  expect(reduceBucket(compacted)).toEqual(r1);
+
+  for (let i = 0; i < bucket.length; i++) {
+    // r(B_{[..c_i]})
+    const r2 = reduceBucket(bucket.slice(0, i + 1));
+    const c_i = BigInt(bucket[i].op_id);
+    // B'_{[c_i+1..c]}
+    const b3 = compacted.filter((op) => BigInt(op.op_id) > c_i);
+    // r(B_{[..c_i]}) \cup B'_{[c_i+1..c]}
+    const r3 = r2.concat(b3);
+    // r(r(B_{[..c_i]}) \cup B'_{[c_i+1..c]})
+    const r4 = reduceBucket(r3);
+    expect(r4).toEqual(r1);
+  }
+}