@powersync/service-module-mongodb-storage 0.7.1 → 0.8.0

package/src/storage/implementation/PersistedBatch.ts

@@ -11,6 +11,7 @@ import { PowerSyncMongo } from './db.js';
 import {
   BucketDataDocument,
   BucketParameterDocument,
+  BucketStateDocument,
   CurrentBucket,
   CurrentDataDocument,
   SourceKey
@@ -48,6 +49,7 @@ export class PersistedBatch {
   bucketData: mongo.AnyBulkWriteOperation<BucketDataDocument>[] = [];
   bucketParameters: mongo.AnyBulkWriteOperation<BucketParameterDocument>[] = [];
   currentData: mongo.AnyBulkWriteOperation<CurrentDataDocument>[] = [];
+  bucketStates: Map<string, BucketStateUpdate> = new Map();
 
   /**
    * For debug logging only.
@@ -66,6 +68,19 @@ export class PersistedBatch {
     this.currentSize = writtenSize;
   }
 
+  private incrementBucket(bucket: string, op_id: InternalOpId) {
+    let existingState = this.bucketStates.get(bucket);
+    if (existingState) {
+      existingState.lastOp = op_id;
+      existingState.incrementCount += 1;
+    } else {
+      this.bucketStates.set(bucket, {
+        lastOp: op_id,
+        incrementCount: 1
+      });
+    }
+  }
+
   saveBucketData(options: {
     op_seq: MongoIdSequence;
     sourceKey: storage.ReplicaId;
@@ -120,6 +135,7 @@ export class PersistedBatch {
           }
         }
       });
+      this.incrementBucket(k.bucket, op_id);
     }
 
     for (let bd of remaining_buckets.values()) {
@@ -147,6 +163,7 @@ export class PersistedBatch {
         }
       });
       this.currentSize += 200;
+      this.incrementBucket(bd.bucket, op_id);
     }
   }
 
@@ -277,6 +294,14 @@ export class PersistedBatch {
       });
     }
 
+    if (this.bucketStates.size > 0) {
+      await db.bucket_state.bulkWrite(this.getBucketStateUpdates(), {
+        session,
+        // Per-bucket operation - order doesn't matter
+        ordered: false
+      });
+    }
+
     const duration = performance.now() - startAt;
     logger.info(
       `powersync_${this.group_id} Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${
@@ -287,7 +312,34 @@ export class PersistedBatch {
     this.bucketData = [];
     this.bucketParameters = [];
     this.currentData = [];
+    this.bucketStates.clear();
     this.currentSize = 0;
     this.debugLastOpId = null;
   }
+
+  private getBucketStateUpdates(): mongo.AnyBulkWriteOperation<BucketStateDocument>[] {
+    return Array.from(this.bucketStates.entries()).map(([bucket, state]) => {
+      return {
+        updateOne: {
+          filter: {
+            _id: {
+              g: this.group_id,
+              b: bucket
+            }
+          },
+          update: {
+            $set: {
+              last_op: state.lastOp
+            }
+          },
+          upsert: true
+        }
+      } satisfies mongo.AnyBulkWriteOperation<BucketStateDocument>;
+    });
+  }
+}
+
+interface BucketStateUpdate {
+  lastOp: InternalOpId;
+  incrementCount: number;
 }

package/src/storage/implementation/db.ts

@@ -6,6 +6,7 @@ import { MongoStorageConfig } from '../../types/types.js';
 import {
   BucketDataDocument,
   BucketParameterDocument,
+  BucketStateDocument,
   CurrentDataDocument,
   CustomWriteCheckpointDocument,
   IdSequenceDocument,
@@ -33,6 +34,7 @@ export class PowerSyncMongo {
   readonly write_checkpoints: mongo.Collection<WriteCheckpointDocument>;
   readonly instance: mongo.Collection<InstanceDocument>;
   readonly locks: mongo.Collection<lib_mongo.locks.Lock>;
+  readonly bucket_state: mongo.Collection<BucketStateDocument>;
 
   readonly client: mongo.MongoClient;
   readonly db: mongo.Db;
@@ -55,6 +57,7 @@ export class PowerSyncMongo {
     this.write_checkpoints = db.collection('write_checkpoints');
     this.instance = db.collection('instance');
     this.locks = this.db.collection('locks');
+    this.bucket_state = this.db.collection('bucket_state');
   }
 
   /**
@@ -70,6 +73,8 @@ export class PowerSyncMongo {
     await this.write_checkpoints.deleteMany({});
     await this.instance.deleteOne({});
     await this.locks.deleteMany({});
+    await this.bucket_state.deleteMany({});
+    await this.custom_write_checkpoints.deleteMany({});
   }
 
   /**

package/src/storage/implementation/models.ts

@@ -75,6 +75,24 @@ export interface SourceTableDocument {
   snapshot_done: boolean | undefined;
 }
 
+/**
+ * Record the state of each bucket.
+ *
+ * Right now, this is just used to track when buckets are updated, for efficient incremental sync.
+ * In the future, this could be used to track operation counts, both for diagnostic purposes, and for
+ * determining when a compact and/or defragment could be beneficial.
+ *
+ * Note: There is currently no migration to populate this collection from existing data - it is only
+ * populated by new updates.
+ */
+export interface BucketStateDocument {
+  _id: {
+    g: number;
+    b: string;
+  };
+  last_op: bigint;
+}
+
 export interface IdSequenceDocument {
   _id: string;
   op_id: bigint;

package/test/src/__snapshots__/storage_sync.test.ts.snap

@@ -357,6 +357,74 @@ exports[`sync - mongodb > sync legacy non-raw data 1`] = `
 ]
 `;
 
+exports[`sync - mongodb > sync updates to data query only 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [
+        {
+          "bucket": "by_user["user1"]",
+          "checksum": 0,
+          "count": 0,
+          "priority": 3,
+        },
+      ],
+      "last_op_id": "1",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > sync updates to data query only 2`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "2",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "by_user["user1"]",
+          "checksum": 1418351250,
+          "count": 1,
+          "priority": 3,
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "by_user["user1"]",
+      "data": [
+        {
+          "checksum": 1418351250n,
+          "data": "{"id":"list1","user_id":"user1","name":"User 1"}",
+          "object_id": "list1",
+          "object_type": "lists",
+          "op": "PUT",
+          "op_id": "2",
+          "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8",
+        },
+      ],
+      "has_more": false,
+      "next_after": "2",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
 exports[`sync - mongodb > sync updates to global data 1`] = `
 [
   {
@@ -468,3 +536,106 @@ exports[`sync - mongodb > sync updates to global data 3`] = `
   },
 ]
 `;
+
+exports[`sync - mongodb > sync updates to parameter query + data 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [],
+      "last_op_id": "0",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "0",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > sync updates to parameter query + data 2`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "2",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "by_user["user1"]",
+          "checksum": 1418351250,
+          "count": 1,
+          "priority": 3,
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "data": {
+      "after": "0",
+      "bucket": "by_user["user1"]",
+      "data": [
+        {
+          "checksum": 1418351250n,
+          "data": "{"id":"list1","user_id":"user1","name":"User 1"}",
+          "object_id": "list1",
+          "object_type": "lists",
+          "op": "PUT",
+          "op_id": "1",
+          "subkey": "0ffb7b58-d14d-5efa-be6c-c8eda74ab7a8",
+        },
+      ],
+      "has_more": false,
+      "next_after": "1",
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "2",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > sync updates to parameter query only 1`] = `
+[
+  {
+    "checkpoint": {
+      "buckets": [],
+      "last_op_id": "0",
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "0",
+    },
+  },
+]
+`;
+
+exports[`sync - mongodb > sync updates to parameter query only 2`] = `
+[
+  {
+    "checkpoint_diff": {
+      "last_op_id": "1",
+      "removed_buckets": [],
+      "updated_buckets": [
+        {
+          "bucket": "by_user["user1"]",
+          "checksum": 0,
+          "count": 0,
+          "priority": 3,
+        },
+      ],
+      "write_checkpoint": undefined,
+    },
+  },
+  {
+    "checkpoint_complete": {
+      "last_op_id": "1",
+    },
+  },
+]
+`;