@powersync/service-module-mongodb-storage 0.6.2 → 0.7.1

package/dist/storage/implementation/util.js

@@ -66,7 +66,7 @@ export async function readSingleBatch(cursor) {
 export function mapOpEntry(row) {
     if (row.op == 'PUT' || row.op == 'REMOVE') {
         return {
-            op_id: utils.timestampToOpId(row._id.o),
+            op_id: utils.internalToExternalOpId(row._id.o),
             op: row.op,
             object_type: row.table,
             object_id: row.row_id,
@@ -78,7 +78,7 @@ export function mapOpEntry(row) {
     else {
         // MOVE, CLEAR
         return {
-            op_id: utils.timestampToOpId(row._id.o),
+            op_id: utils.internalToExternalOpId(row._id.o),
             op: row.op,
             checksum: Number(row.checksum)
         };

package/dist/storage/implementation/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA2B;IAClE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,OAAkC,EAAE,EAAE;IACnF,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,EAAE;QACpC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA2B;IAClE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,OAAkC,EAAE,EAAE;IACnF,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,EAAE;QACpC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC"}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-mongodb-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/index.d.ts",
-  "version": "0.6.2",
+  "version": "0.7.1",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -28,15 +28,15 @@
     "lru-cache": "^10.2.2",
     "uuid": "^9.0.1",
     "@powersync/lib-services-framework": "0.5.3",
-    "@powersync/service-core": "1.7.2",
+    "@powersync/service-core": "1.8.1",
     "@powersync/service-jsonbig": "0.17.10",
-    "@powersync/service-sync-rules": "0.24.0",
-    "@powersync/service-types": "0.8.0",
-    "@powersync/lib-service-mongodb": "0.4.3"
+    "@powersync/service-sync-rules": "0.24.1",
+    "@powersync/service-types": "0.9.0",
+    "@powersync/lib-service-mongodb": "0.5.1"
   },
   "devDependencies": {
     "@types/uuid": "^9.0.4",
-    "@powersync/service-core-tests": "0.6.1"
+    "@powersync/service-core-tests": "0.7.1"
   },
   "scripts": {
     "build": "tsc -b",

package/src/storage/implementation/MongoBucketBatch.ts

@@ -11,7 +11,7 @@ import {
   ReplicationAssertionError,
   ServiceError
 } from '@powersync/lib-services-framework';
-import { deserializeBson, SaveOperationTag, storage, utils } from '@powersync/service-core';
+import { deserializeBson, InternalOpId, SaveOperationTag, storage, utils } from '@powersync/service-core';
 import * as timers from 'node:timers/promises';
 import { PowerSyncMongo } from './db.js';
 import { CurrentBucket, CurrentDataDocument, SourceKey, SyncRuleDocument } from './models.js';
@@ -24,7 +24,7 @@ import { idPrefixFilter } from './util.js';
 /**
  * 15MB
  */
-const MAX_ROW_SIZE = 15 * 1024 * 1024;
+export const MAX_ROW_SIZE = 15 * 1024 * 1024;
 
 // Currently, we can only have a single flush() at a time, since it locks the op_id sequence.
 // While the MongoDB transaction retry mechanism handles this okay, using an in-process Mutex
@@ -39,7 +39,7 @@ export interface MongoBucketBatchOptions {
   groupId: number;
   slotName: string;
   lastCheckpointLsn: string | null;
-  keepaliveOp: string | null;
+  keepaliveOp: InternalOpId | null;
   noCheckpointBeforeLsn: string;
   storeCurrentData: boolean;
   /**
@@ -77,12 +77,12 @@ export class MongoBucketBatch
 
   private no_checkpoint_before_lsn: string;
 
-  private persisted_op: bigint | null = null;
+  private persisted_op: InternalOpId | null = null;
 
   /**
    * For tests only - not for persistence logic.
    */
-  public last_flushed_op: bigint | null = null;
+  public last_flushed_op: InternalOpId | null = null;
 
   constructor(options: MongoBucketBatchOptions) {
     super();
@@ -98,9 +98,7 @@ export class MongoBucketBatch
     this.skipExistingRows = options.skipExistingRows;
     this.batch = new OperationBatch();
 
-    if (options.keepaliveOp) {
-      this.persisted_op = BigInt(options.keepaliveOp);
-    }
+    this.persisted_op = options.keepaliveOp ?? null;
   }
 
   addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void {
@@ -135,7 +133,7 @@ export class MongoBucketBatch
       return null;
     }
 
-    let last_op: bigint | null = null;
+    let last_op: InternalOpId | null = null;
     let resumeBatch: OperationBatch | null = null;
 
     await this.withReplicationTransaction(`Flushing ${batch.length} ops`, async (session, opSeq) => {
@@ -153,7 +151,7 @@ export class MongoBucketBatch
 
     this.persisted_op = last_op;
     this.last_flushed_op = last_op;
-    return { flushed_op: String(last_op) };
+    return { flushed_op: last_op };
   }
 
   private async replicateBatch(
@@ -776,22 +774,23 @@ export class MongoBucketBatch
   async truncate(sourceTables: storage.SourceTable[]): Promise<storage.FlushedResult | null> {
     await this.flush();
 
-    let last_op: bigint | null = null;
+    let last_op: InternalOpId | null = null;
     for (let table of sourceTables) {
       last_op = await this.truncateSingle(table);
     }
 
     if (last_op) {
       this.persisted_op = last_op;
+      return {
+        flushed_op: last_op
+      };
+    } else {
+      return null;
     }
-
-    return {
-      flushed_op: String(last_op!)
-    };
   }
 
-  async truncateSingle(sourceTable: storage.SourceTable): Promise<bigint> {
-    let last_op: bigint | null = null;
+  async truncateSingle(sourceTable: storage.SourceTable): Promise<InternalOpId> {
+    let last_op: InternalOpId | null = null;
 
     // To avoid too large transactions, we limit the amount of data we delete per transaction.
     // Since we don't use the record data here, we don't have explicit size limits per batch.

package/src/storage/implementation/MongoCompactor.ts

@@ -1,6 +1,6 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import { logger, ReplicationAssertionError } from '@powersync/lib-services-framework';
-import { storage, utils } from '@powersync/service-core';
+import { InternalOpId, storage, utils } from '@powersync/service-core';
 
 import { PowerSyncMongo } from './db.js';
 import { BucketDataDocument, BucketDataKey } from './models.js';
@@ -12,7 +12,7 @@ interface CurrentBucketState {
   /**
    * Rows seen in the bucket, with the last op_id of each.
    */
-  seen: Map<string, bigint>;
+  seen: Map<string, InternalOpId>;
   /**
    * Estimated memory usage of the seen Map.
    */
@@ -21,7 +21,7 @@ interface CurrentBucketState {
   /**
    * Last (lowest) seen op_id that is not a PUT.
    */
-  lastNotPut: bigint | null;
+  lastNotPut: InternalOpId | null;
 
   /**
    * Number of REMOVE/MOVE operations seen since lastNotPut.
@@ -274,7 +274,7 @@ export class MongoCompactor {
    * @param bucket bucket name
    * @param op op_id of the last non-PUT operation, which will be converted to CLEAR.
    */
-  private async clearBucket(bucket: string, op: bigint) {
+  private async clearBucket(bucket: string, op: InternalOpId) {
     const opFilter = {
       _id: {
         $gte: {

package/src/storage/implementation/MongoStorageProvider.ts

@@ -22,7 +22,9 @@ export class MongoStorageProvider implements storage.BucketStorageProvider {
     }
 
     const decodedConfig = MongoStorageConfig.decode(storage as any);
-    const client = lib_mongo.db.createMongoClient(decodedConfig);
+    const client = lib_mongo.db.createMongoClient(decodedConfig, {
+      maxPoolSize: resolvedConfig.storage.max_pool_size ?? 8
+    });
 
     const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
     const factory = new MongoBucketStorage(database, {

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -12,6 +12,9 @@ import {
   CHECKPOINT_INVALIDATE_ALL,
   CheckpointChanges,
   GetCheckpointChangesOptions,
+  InternalOpId,
+  internalToExternalOpId,
+  ProtocolOpId,
   ReplicationCheckpoint,
   SourceTable,
   storage,
@@ -119,7 +122,7 @@ export class MongoSyncBucketStorage
       }
     );
     return {
-      checkpoint: utils.timestampToOpId(doc?.last_checkpoint ?? 0n),
+      checkpoint: doc?.last_checkpoint ?? 0n,
       lsn: doc?.last_checkpoint_lsn ?? null
     };
   }
@@ -143,7 +146,7 @@ export class MongoSyncBucketStorage
       slotName: this.slot_name,
       lastCheckpointLsn: checkpoint_lsn,
       noCheckpointBeforeLsn: doc?.no_checkpoint_before ?? options.zeroLSN,
-      keepaliveOp: doc?.keepalive_op ?? null,
+      keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null,
       storeCurrentData: options.storeCurrentData,
       skipExistingRows: options.skipExistingRows ?? false
     });
@@ -152,7 +155,7 @@ export class MongoSyncBucketStorage
     await callback(batch);
     await batch.flush();
     if (batch.last_flushed_op) {
-      return { flushed_op: String(batch.last_flushed_op) };
+      return { flushed_op: batch.last_flushed_op };
     } else {
       return null;
     }
@@ -249,7 +252,7 @@ export class MongoSyncBucketStorage
     return result!;
   }
 
-  async getParameterSets(checkpoint: utils.OpId, lookups: SqliteJsonValue[][]): Promise<SqliteJsonRow[]> {
+  async getParameterSets(checkpoint: utils.InternalOpId, lookups: SqliteJsonValue[][]): Promise<SqliteJsonRow[]> {
     const lookupFilter = lookups.map((lookup) => {
       return storage.serializeLookup(lookup);
     });
@@ -259,7 +262,7 @@ export class MongoSyncBucketStorage
           $match: {
             'key.g': this.group_id,
             lookup: { $in: lookupFilter },
-            _id: { $lte: BigInt(checkpoint) }
+            _id: { $lte: checkpoint }
           }
         },
         {
@@ -284,8 +287,8 @@ export class MongoSyncBucketStorage
   }
 
   async *getBucketDataBatch(
-    checkpoint: utils.OpId,
-    dataBuckets: Map<string, string>,
+    checkpoint: utils.InternalOpId,
+    dataBuckets: Map<string, InternalOpId>,
     options?: storage.BucketDataBatchOptions
   ): AsyncIterable<storage.SyncBucketDataBatch> {
     if (dataBuckets.size == 0) {
@@ -293,14 +296,17 @@ export class MongoSyncBucketStorage
     }
     let filters: mongo.Filter<BucketDataDocument>[] = [];
 
-    const end = checkpoint ? BigInt(checkpoint) : new bson.MaxKey();
+    if (checkpoint == null) {
+      throw new ServiceAssertionError('checkpoint is null');
+    }
+    const end = checkpoint;
     for (let [name, start] of dataBuckets.entries()) {
       filters.push({
         _id: {
           $gt: {
             g: this.group_id,
             b: name,
-            o: BigInt(start)
+            o: start
           },
           $lte: {
             g: this.group_id,
@@ -347,7 +353,7 @@ export class MongoSyncBucketStorage
 
     let batchSize = 0;
     let currentBatch: utils.SyncBucketData | null = null;
-    let targetOp: bigint | null = null;
+    let targetOp: InternalOpId | null = null;
 
     // Ordered by _id, meaning buckets are grouped together
     for (let rawData of data) {
@@ -355,7 +361,7 @@ export class MongoSyncBucketStorage
       const bucket = row._id.b;
 
       if (currentBatch == null || currentBatch.bucket != bucket || batchSize >= sizeLimit) {
-        let start: string | undefined = undefined;
+        let start: ProtocolOpId | undefined = undefined;
         if (currentBatch != null) {
           if (currentBatch.bucket == bucket) {
             currentBatch.has_more = true;
@@ -369,9 +375,12 @@ export class MongoSyncBucketStorage
           targetOp = null;
         }
 
-        start ??= dataBuckets.get(bucket);
         if (start == null) {
-          throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`);
+          const startOpId = dataBuckets.get(bucket);
+          if (startOpId == null) {
+            throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`);
+          }
+          start = internalToExternalOpId(startOpId);
         }
         currentBatch = {
           bucket,
@@ -406,7 +415,7 @@ export class MongoSyncBucketStorage
     }
   }
 
-  async getChecksums(checkpoint: utils.OpId, buckets: string[]): Promise<utils.ChecksumMap> {
+  async getChecksums(checkpoint: utils.InternalOpId, buckets: string[]): Promise<utils.ChecksumMap> {
     return this.checksumCache.getChecksumMap(checkpoint, buckets);
   }
 
@@ -638,7 +647,7 @@ export class MongoSyncBucketStorage
 
   private makeActiveCheckpoint(doc: SyncRuleCheckpointState | null) {
     return {
-      checkpoint: utils.timestampToOpId(doc?.last_checkpoint ?? 0n),
+      checkpoint: doc?.last_checkpoint ?? 0n,
       lsn: doc?.last_checkpoint_lsn ?? null
     };
   }
@@ -755,7 +764,7 @@ export class MongoSyncBucketStorage
    */
   async *watchWriteCheckpoint(options: WatchWriteCheckpointOptions): AsyncIterable<storage.StorageCheckpointUpdate> {
     const { user_id, signal } = options;
-    let lastCheckpoint: utils.OpId | null = null;
+    let lastCheckpoint: utils.InternalOpId | null = null;
     let lastWriteCheckpoint: bigint | null = null;
 
     const iter = wrapWithAbort(this.sharedIter, signal);

package/src/storage/implementation/PersistedBatch.ts

@@ -4,8 +4,8 @@ import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules
 import * as bson from 'bson';
 
 import { logger } from '@powersync/lib-services-framework';
-import { storage, utils } from '@powersync/service-core';
-import { currentBucketKey } from './MongoBucketBatch.js';
+import { InternalOpId, storage, utils } from '@powersync/service-core';
+import { currentBucketKey, MAX_ROW_SIZE } from './MongoBucketBatch.js';
 import { MongoIdSequence } from './MongoIdSequence.js';
 import { PowerSyncMongo } from './db.js';
 import {
@@ -52,7 +52,7 @@ export class PersistedBatch {
   /**
    * For debug logging only.
    */
-  debugLastOpId: bigint | null = null;
+  debugLastOpId: InternalOpId | null = null;
 
   /**
    * Very rough estimate of transaction size.
@@ -83,11 +83,20 @@ export class PersistedBatch {
 
     for (const k of options.evaluated) {
       const key = currentBucketKey(k);
-      remaining_buckets.delete(key);
 
       // INSERT
       const recordData = JSONBig.stringify(k.data);
       const checksum = utils.hashData(k.table, k.id, recordData);
+      if (recordData.length > MAX_ROW_SIZE) {
+        // In many cases, the raw data size would have been too large already. But there are cases where
+        // the BSON size is small enough, but the JSON size is too large.
+        // In these cases, we can't store the data, so we skip it, or generate a REMOVE operation if the row
+        // was synced previously.
+        logger.error(`powersync_${this.group_id} Row ${key} too large: ${recordData.length} bytes. Removing.`);
+        continue;
+      }
+
+      remaining_buckets.delete(key);
       this.currentSize += recordData.length + 200;
 
       const op_id = options.op_seq.next();

package/src/storage/implementation/db.ts

@@ -82,6 +82,6 @@ export class PowerSyncMongo {
   }
 }
 
-export function createPowerSyncMongo(config: MongoStorageConfig) {
-  return new PowerSyncMongo(lib_mongo.createMongoClient(config), { database: config.database });
+export function createPowerSyncMongo(config: MongoStorageConfig, options?: lib_mongo.MongoConnectionOptions) {
+  return new PowerSyncMongo(lib_mongo.createMongoClient(config, options), { database: config.database });
 }

package/src/storage/implementation/util.ts

@@ -74,7 +74,7 @@ export async function readSingleBatch<T>(cursor: mongo.FindCursor<T>): Promise<{
 export function mapOpEntry(row: BucketDataDocument): utils.OplogEntry {
   if (row.op == 'PUT' || row.op == 'REMOVE') {
     return {
-      op_id: utils.timestampToOpId(row._id.o),
+      op_id: utils.internalToExternalOpId(row._id.o),
       op: row.op,
       object_type: row.table,
       object_id: row.row_id,
@@ -86,7 +86,7 @@ export function mapOpEntry(row: BucketDataDocument): utils.OplogEntry {
     // MOVE, CLEAR
 
     return {
-      op_id: utils.timestampToOpId(row._id.o),
+      op_id: utils.internalToExternalOpId(row._id.o),
       op: row.op,
       checksum: Number(row.checksum)
     };

package/test/src/storage_sync.test.ts

@@ -74,7 +74,7 @@ describe('sync - mongodb', () => {
     const options: storage.BucketDataBatchOptions = {};
 
     const batch1 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', '0']]), options)
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', 0n]]), options)
     );
     expect(test_utils.getBatchData(batch1)).toEqual([
       { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
@@ -87,7 +87,7 @@ describe('sync - mongodb', () => {
     });
 
     const batch2 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch1[0].batch.next_after]]), options)
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', BigInt(batch1[0].batch.next_after)]]), options)
     );
     expect(test_utils.getBatchData(batch2)).toEqual([
       { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }
@@ -99,7 +99,7 @@ describe('sync - mongodb', () => {
     });
 
     const batch3 = await test_utils.fromAsync(
-      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', batch2[0].batch.next_after]]), options)
+      bucketStorage.getBucketDataBatch(checkpoint, new Map([['global[]', BigInt(batch2[0].batch.next_after)]]), options)
     );
     expect(test_utils.getBatchData(batch3)).toEqual([
       { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }