@powersync/service-module-mongodb-storage 0.10.3 → 0.11.0

package/dist/storage/implementation/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA2B;IAClE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,OAAkC,EAAE,EAAE;IACnF,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,EAAE;QACpC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAEzC,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAE1E,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA2B;IAClE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE;QACxC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC;AAEF,MAAM,UAAU,sBAAsB,CAAC,OAA4B,EAAE,IAAoB;IACvF,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;IACxE,CAAC;IACD,IAAK,OAAe,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QACzC,OAAe,CAAC,YAAY,GAAG,IAAI,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;IACzE,CAAC;AACH,CAAC"}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-mongodb-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/index.d.ts",
-  "version": "0.10.3",
+  "version": "0.11.0",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -27,15 +27,15 @@
     "lru-cache": "^10.2.2",
     "ts-codec": "^1.3.0",
     "uuid": "^11.1.0",
-    "@powersync/lib-service-mongodb": "0.6.1",
-    "@powersync/lib-services-framework": "0.7.0",
-    "@powersync/service-core": "1.13.3",
+    "@powersync/lib-service-mongodb": "0.6.3",
+    "@powersync/lib-services-framework": "0.7.2",
+    "@powersync/service-core": "1.14.0",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-sync-rules": "0.27.0",
-    "@powersync/service-types": "0.12.0"
+    "@powersync/service-sync-rules": "0.28.0",
+    "@powersync/service-types": "0.12.1"
   },
   "devDependencies": {
-    "@powersync/service-core-tests": "0.10.3"
+    "@powersync/service-core-tests": "0.11.0"
   },
   "scripts": {
     "build": "tsc -b",

package/src/storage/implementation/MongoBucketBatch.ts

@@ -16,6 +16,7 @@ import {
   BucketStorageMarkRecordUnavailable,
   deserializeBson,
   InternalOpId,
+  isCompleteRow,
   SaveOperationTag,
   storage,
   utils
@@ -49,6 +50,7 @@ export interface MongoBucketBatchOptions {
   lastCheckpointLsn: string | null;
   keepaliveOp: InternalOpId | null;
   noCheckpointBeforeLsn: string;
+  resumeFromLsn: string | null;
   storeCurrentData: boolean;
   /**
    * Set to true for initial replication.
@@ -99,6 +101,20 @@ export class MongoBucketBatch
    */
   public last_flushed_op: InternalOpId | null = null;
 
+  /**
+   * lastCheckpointLsn is the last consistent commit.
+   *
+   * While that is generally a "safe" point to resume from, there are cases where we may want to resume from a different point:
+   * 1. After an initial snapshot, we don't have a consistent commit yet, but need to resume from the snapshot LSN.
+   * 2. If "no_checkpoint_before_lsn" is set far in advance, it may take a while to reach that point. We
+   *    may want to resume at incremental points before that.
+   *
+   * This is set when creating the batch, but may not be updated afterwards.
+   */
+  public resumeFromLsn: string | null = null;
+
+  private needsActivation = true;
+
   constructor(options: MongoBucketBatchOptions) {
     super();
     this.logger = options.logger ?? defaultLogger;
@@ -107,6 +123,7 @@ export class MongoBucketBatch
     this.group_id = options.groupId;
     this.last_checkpoint_lsn = options.lastCheckpointLsn;
     this.no_checkpoint_before_lsn = options.noCheckpointBeforeLsn;
+    this.resumeFromLsn = options.resumeFromLsn;
     this.session = this.client.startSession();
     this.slot_name = options.slotName;
     this.sync_rules = options.syncRules;
@@ -332,7 +349,7 @@ export class MongoBucketBatch
         // Not an error if we re-apply a transaction
         existing_buckets = [];
         existing_lookups = [];
-        if (this.storeCurrentData) {
+        if (!isCompleteRow(this.storeCurrentData, after!)) {
           if (this.markRecordUnavailable != null) {
             // This will trigger a "resnapshot" of the record.
             // This is not relevant if storeCurrentData is false, since we'll get the full row
@@ -685,6 +702,7 @@ export class MongoBucketBatch
 
     if (!createEmptyCheckpoints && this.persisted_op == null) {
       // Nothing to commit - also return true
+      await this.autoActivate(lsn);
       return true;
     }
 
@@ -729,12 +747,65 @@ export class MongoBucketBatch
       },
       { session: this.session }
     );
+    await this.autoActivate(lsn);
     await this.db.notifyCheckpoint();
     this.persisted_op = null;
     this.last_checkpoint_lsn = lsn;
     return true;
   }
 
+  /**
+   * Switch from processing -> active if relevant.
+   *
+   * Called on new commits.
+   */
+  private async autoActivate(lsn: string) {
+    if (!this.needsActivation) {
+      return;
+    }
+
+    // Activate the batch, so it can start processing.
+    // This is done automatically when the first save() is called.
+
+    const session = this.session;
+    let activated = false;
+    await session.withTransaction(async () => {
+      const doc = await this.db.sync_rules.findOne({ _id: this.group_id }, { session });
+      if (doc && doc.state == 'PROCESSING') {
+        await this.db.sync_rules.updateOne(
+          {
+            _id: this.group_id
+          },
+          {
+            $set: {
+              state: storage.SyncRuleState.ACTIVE
+            }
+          },
+          { session }
+        );
+
+        await this.db.sync_rules.updateMany(
+          {
+            _id: { $ne: this.group_id },
+            state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
+          },
+          {
+            $set: {
+              state: storage.SyncRuleState.STOP
+            }
+          },
+          { session }
+        );
+        activated = true;
+      }
+    });
+    if (activated) {
+      this.logger.info(`Activated new sync rules at ${lsn}`);
+      await this.db.notifyCheckpoint();
+    }
+    this.needsActivation = false;
+  }
+
   async keepalive(lsn: string): Promise<boolean> {
     if (this.last_checkpoint_lsn != null && lsn <= this.last_checkpoint_lsn) {
       // No-op
@@ -782,13 +853,14 @@ export class MongoBucketBatch
       },
       { session: this.session }
     );
+    await this.autoActivate(lsn);
     await this.db.notifyCheckpoint();
     this.last_checkpoint_lsn = lsn;
 
     return true;
   }
 
-  async setSnapshotLsn(lsn: string): Promise<void> {
+  async setResumeLsn(lsn: string): Promise<void> {
     const update: Partial<SyncRuleDocument> = {
       snapshot_lsn: lsn
     };

package/src/storage/implementation/MongoParameterCompactor.ts

@@ -0,0 +1,105 @@
+import { logger } from '@powersync/lib-services-framework';
+import { bson, CompactOptions, InternalOpId } from '@powersync/service-core';
+import { LRUCache } from 'lru-cache';
+import { PowerSyncMongo } from './db.js';
+import { mongo } from '@powersync/lib-service-mongodb';
+import { BucketParameterDocument } from './models.js';
+
+/**
+ * Compacts parameter lookup data (the bucket_parameters collection).
+ *
+ * This scans through the entire collection to find data to compact.
+ *
+ * For background, see the `/docs/parameters-lookups.md` file.
+ */
+export class MongoParameterCompactor {
+  constructor(
+    private db: PowerSyncMongo,
+    private group_id: number,
+    private checkpoint: InternalOpId,
+    private options: CompactOptions
+  ) {}
+
+  async compact() {
+    logger.info(`Compacting parameters for group ${this.group_id} up to checkpoint ${this.checkpoint}`);
+    // This is the currently-active checkpoint.
+    // We do not remove any data that may be used by this checkpoint.
+    // snapshot queries ensure that if any clients are still using older checkpoints, they would
+    // not be affected by this compaction.
+    const checkpoint = this.checkpoint;
+
+    // Index on {'key.g': 1, lookup: 1, _id: 1}
+    // In theory, we could let MongoDB do more of the work here, by grouping by (key, lookup)
+    // in MongoDB already. However, that risks running into cases where MongoDB needs to process
+    // very large amounts of data before returning results, which could lead to timeouts.
+    const cursor = this.db.bucket_parameters.find(
+      {
+        'key.g': this.group_id
+      },
+      {
+        sort: { lookup: 1, _id: 1 },
+        batchSize: 10_000,
+        projection: { _id: 1, key: 1, lookup: 1, bucket_parameters: 1 }
+      }
+    );
+
+    // The index doesn't cover sorting by key, so we keep our own cache of the last seen key.
+    let lastByKey = new LRUCache<string, InternalOpId>({
+      max: this.options.compactParameterCacheLimit ?? 10_000
+    });
+    let removeIds: InternalOpId[] = [];
+    let removeDeleted: mongo.AnyBulkWriteOperation<BucketParameterDocument>[] = [];
+
+    const flush = async (force: boolean) => {
+      if (removeIds.length >= 1000 || (force && removeIds.length > 0)) {
+        const results = await this.db.bucket_parameters.deleteMany({ _id: { $in: removeIds } });
+        logger.info(`Removed ${results.deletedCount} (${removeIds.length}) superseded parameter entries`);
+        removeIds = [];
+      }
+
+      if (removeDeleted.length > 10 || (force && removeDeleted.length > 0)) {
+        const results = await this.db.bucket_parameters.bulkWrite(removeDeleted);
+        logger.info(`Removed ${results.deletedCount} (${removeDeleted.length}) deleted parameter entries`);
+        removeDeleted = [];
+      }
+    };
+
+    while (await cursor.hasNext()) {
+      const batch = cursor.readBufferedDocuments();
+      for (let doc of batch) {
+        if (doc._id >= checkpoint) {
+          continue;
+        }
+        const uniqueKey = (
+          bson.serialize({
+            k: doc.key,
+            l: doc.lookup
+          }) as Buffer
+        ).toString('base64');
+        const previous = lastByKey.get(uniqueKey);
+        if (previous != null && previous < doc._id) {
+          // We have a newer entry for the same key, so we can remove the old one.
+          removeIds.push(previous);
+        }
+        lastByKey.set(uniqueKey, doc._id);
+
+        if (doc.bucket_parameters?.length == 0) {
+          // This is a delete operation, so we can remove it completely.
+          // For this we cannot remove the operation itself only: There is a possibility that
+          // there is still an earlier operation with the same key and lookup, that we don't have
+          // in the cache due to cache size limits. So we need to explicitly remove all earlier operations.
+          removeDeleted.push({
+            deleteMany: {
+              filter: { 'key.g': doc.key.g, lookup: doc.lookup, _id: { $lte: doc._id }, key: doc.key }
+            }
+          });
+        }
+      }
+
+      await flush(false);
+    }
+
+    await flush(true);
+    logger.info('Parameter compaction completed');
+  }
+}

package/src/storage/implementation/MongoStorageProvider.ts

@@ -1,6 +1,6 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
-import { storage } from '@powersync/service-core';
+import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
 import { MongoStorageConfig } from '../../types/types.js';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
 import { PowerSyncMongo } from './db.js';
@@ -23,6 +23,7 @@ export class MongoStorageProvider implements storage.BucketStorageProvider {
 
     const decodedConfig = MongoStorageConfig.decode(storage as any);
     const client = lib_mongo.db.createMongoClient(decodedConfig, {
+      powersyncVersion: POWERSYNC_VERSION,
       maxPoolSize: resolvedConfig.storage.max_pool_size ?? 8
     });