@powersync/service-module-mongodb-storage 0.6.0 → 0.6.2

package/dist/storage/implementation/OperationBatch.js

@@ -31,6 +31,9 @@ export class OperationBatch {
     shouldFlush() {
         return this.batch.length >= MAX_BATCH_COUNT || this.currentSize > MAX_RECORD_BATCH_SIZE;
     }
+    hasData() {
+        return this.length > 0;
+    }
     /**
      *
      * @param sizes Map of source key to estimated size of the current_data document, or undefined if current_data is not persisted.

package/dist/storage/implementation/OperationBatch.js.map

@@ -1 +1 @@
-{"version":3,"file":"OperationBatch.js","sourceRoot":"","sources":["../../../src/storage/implementation/OperationBatch.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD;;GAEG;AACH,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B;;GAEG;AACH,MAAM,qBAAqB,GAAG,SAAS,CAAC;AAExC;;GAEG;AACH,MAAM,2BAA2B,GAAG,UAAU,CAAC;AAE/C;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IACzB,KAAK,GAAsB,EAAE,CAAC;IAC9B,WAAW,GAAW,CAAC,CAAC;IAExB,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,IAAI,CAAC,EAAmB;QACtB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpB,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,aAAa,CAAC;IACvC,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,eAAe,IAAI,IAAI,CAAC,WAAW,GAAG,qBAAqB,CAAC;IAC1F,CAAC;IAED;;;;OAIG;IACH,CAAC,OAAO,CAAC,KAAsC;QAC7C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,KAAK,CAAC;YACjB,OAAO;QACT,CAAC;QACD,IAAI,YAAY,GAAsB,EAAE,CAAC;QACzC,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,KAAK,IAAI,EAAE,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,EAAE,CAAC,iBAAiB,CAAC;YACjC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,gBAAgB,GAAG,IAAI,GAAG,2BAA2B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrF,MAAM,YAAY,CAAC;gBACnB,YAAY,GAAG,EAAE,CAAC;gBAClB,gBAAgB,GAAG,CAAC,CAAC;YACvB,CAAC;YACD,gBAAgB,IAAI,IAAI,CAAC;YACzB,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC;QACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,YAAY,CAAC;QACrB,CAAC;IACH,CAAC;CACF;AAED,MAAM,OAAO,eAAe;IAOE;IANZ,OAAO,CAA2B;IAClC,QAAQ,CAAoB;IAC5B,iBAAiB,CAAS;IAC1B,gBAAgB,CAAgB;IAChC,aAAa,CAAS;IAEtC,YAA4B,MAA2B;QAA3B,WAAM,GAAN,MAAM,CAAqB;QACrD,MAAM,OAAO,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC;QAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,eAAe,IAAI,MAAM,CAAC,cAAc,CAAC;QACjE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QACnE,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAElF,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACtF,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAoB,EAAE,EAAqB;IAClE,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QACjC,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,EAAE,CAAC;IACxC,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAK,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC/F,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,MAAsC;IAC7D,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QACxB,+CAA+C;QAC/C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,CAAC;aAAM,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YACvC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
+{"version":3,"file":"OperationBatch.js","sourceRoot":"","sources":["../../../src/storage/implementation/OperationBatch.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAElD;;GAEG;AACH,MAAM,eAAe,GAAG,IAAI,CAAC;AAE7B;;GAEG;AACH,MAAM,qBAAqB,GAAG,SAAS,CAAC;AAExC;;GAEG;AACH,MAAM,2BAA2B,GAAG,UAAU,CAAC;AAE/C;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IACzB,KAAK,GAAsB,EAAE,CAAC;IAC9B,WAAW,GAAW,CAAC,CAAC;IAExB,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,IAAI,CAAC,EAAmB;QACtB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACpB,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,aAAa,CAAC;IACvC,CAAC;IAED,WAAW;QACT,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,eAAe,IAAI,IAAI,CAAC,WAAW,GAAG,qBAAqB,CAAC;IAC1F,CAAC;IAED,OAAO;QACL,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;IACzB,CAAC;IAED;;;;OAIG;IACH,CAAC,OAAO,CAAC,KAAsC;QAC7C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,MAAM,IAAI,CAAC,KAAK,CAAC;YACjB,OAAO;QACT,CAAC;QACD,IAAI,YAAY,GAAsB,EAAE,CAAC;QACzC,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,KAAK,IAAI,EAAE,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,GAAG,GAAG,EAAE,CAAC,iBAAiB,CAAC;YACjC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACjC,IAAI,gBAAgB,GAAG,IAAI,GAAG,2BAA2B,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACrF,MAAM,YAAY,CAAC;gBACnB,YAAY,GAAG,EAAE,CAAC;gBAClB,gBAAgB,GAAG,CAAC,CAAC;YACvB,CAAC;YACD,gBAAgB,IAAI,IAAI,CAAC;YACzB,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACxB,CAAC;QACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,YAAY,CAAC;QACrB,CAAC;IACH,CAAC;CACF;AAED,MAAM,OAAO,eAAe;IAOE;IANZ,OAAO,CAA2B;IAClC,QAAQ,CAAoB;IAC5B,iBAAiB,CAAS;IAC1B,gBAAgB,CAAgB;IAChC,aAAa,CAAS;IAEtC,YAA4B,MAA2B;QAA3B,WAAM,GAAN,MAAM,CAAqB;QACrD,MAAM,OAAO,GAAG,MAAM,CAAC,cAAc,IAAI,IAAI,CAAC;QAC9C,MAAM,QAAQ,GAAG,MAAM,CAAC,eAAe,IAAI,MAAM,CAAC,cAAc,CAAC;QACjE,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,iBAAiB,GAAG,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,QAAQ,CAAC,CAAC;QACnE,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAElF,IAAI,CAAC,aAAa,GAAG,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACtF,CAAC;CACF;AAED;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,KAAoB,EAAE,EAAqB;IAClE,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,IAAI,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;QACjC,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,EAAE,CAAC;IACxC,CAAC;SAAM,CAAC;QACN,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAK,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC/F,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,eAAe,CAAC,MAAsC;IAC7D,IAAI,MAAM,IAAI,IAAI,EAAE,CAAC;QACnB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,IAAI,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;QACxB,+CAA+C;QAC/C,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,CAAC,CAAC;QACZ,CAAC;aAAM,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;YACpC,IAAI,IAAI,KAAK,CAAC,MAAM,CAAC;QACvB,CAAC;aAAM,IAAI,KAAK,YAAY,UAAU,EAAE,CAAC;YACvC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC;QAC3B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}

package/dist/storage/implementation/PersistedBatch.js

@@ -2,7 +2,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
 import { logger } from '@powersync/lib-services-framework';
 import { storage, utils } from '@powersync/service-core';
 import { currentBucketKey } from './MongoBucketBatch.js';
-import { replicaIdToSubkey, safeBulkWrite } from './util.js';
+import { replicaIdToSubkey } from './util.js';
 /**
  * Maximum size of operations we write in a single transaction.
  *
@@ -191,29 +191,30 @@ export class PersistedBatch {
             this.bucketParameters.length >= MAX_TRANSACTION_DOC_COUNT);
     }
     async flush(db, session) {
+        const startAt = performance.now();
         if (this.bucketData.length > 0) {
-            // calculate total size
-            await safeBulkWrite(db.bucket_data, this.bucketData, {
+            await db.bucket_data.bulkWrite(this.bucketData, {
                 session,
                 // inserts only - order doesn't matter
                 ordered: false
             });
         }
         if (this.bucketParameters.length > 0) {
-            await safeBulkWrite(db.bucket_parameters, this.bucketParameters, {
+            await db.bucket_parameters.bulkWrite(this.bucketParameters, {
                 session,
                 // inserts only - order doesn't matter
                 ordered: false
             });
         }
         if (this.currentData.length > 0) {
-            await safeBulkWrite(db.current_data, this.currentData, {
+            await db.current_data.bulkWrite(this.currentData, {
                 session,
                 // may update and delete data within the same batch - order matters
                 ordered: true
             });
         }
-        logger.info(`powersync_${this.group_id} Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${this.currentData.length} updates, ${Math.round(this.currentSize / 1024)}kb. Last op_id: ${this.debugLastOpId}`);
+        const duration = performance.now() - startAt;
+        logger.info(`powersync_${this.group_id} Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${this.currentData.length} updates, ${Math.round(this.currentSize / 1024)}kb in ${duration.toFixed(0)}ms. Last op_id: ${this.debugLastOpId}`);
         this.bucketData = [];
         this.bucketParameters = [];
         this.currentData = [];

package/dist/storage/implementation/PersistedBatch.js.map

@@ -1 +1 @@
-{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../src/storage/implementation/PersistedBatch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAIrD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAUzD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAE7D;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IAgBf;IAfV,UAAU,GAAsD,EAAE,CAAC;IACnE,gBAAgB,GAA2D,EAAE,CAAC;IAC9E,WAAW,GAAuD,EAAE,CAAC;IAErE;;OAEG;IACH,aAAa,GAAkB,IAAI,CAAC;IAEpC;;OAEG;IACH,WAAW,GAAG,CAAC,CAAC;IAEhB,YACU,QAAgB,EACxB,WAAmB;QADX,aAAQ,GAAR,QAAQ,CAAQ;QAGxB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED,cAAc,CAAC,OAMd;QACC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAyB,CAAC;QAC3D,KAAK,IAAI,CAAC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QAE3F,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,SAAS;YACT,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3D,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAE5C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,CAAC,CAAC,MAAM;4BACX,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,KAAK;wBACT,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,MAAM,EAAE,CAAC,CAAC,EAAE;wBACZ,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,KAAK,IAAI,EAAE,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,SAAS;YAET,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,EAAE,CAAC,MAAM;4BACZ,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,QAAQ;wBACZ,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,EAAE,CAAC,KAAK;wBACf,MAAM,EAAE,EAAE,CAAC,EAAE;wBACb,QAAQ,EAAE,SAAS;wBACnB,IAAI,EAAE,IAAI;qBACX;iBACF;aACF,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,IAMjB;QACC,yCAAyC;QACzC,qEAAqE;QACrE,0HAA0H;QAC1H,2DAA2D;QAC3D,8GAA8G;QAC9G,+BAA+B;QAC/B,6CAA6C;QAC7C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;QACzD,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,wBAAwB;QACxB,KAAK,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACzC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,SAAS;wBACjB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;qBAC5C;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;QAED,kDAAkD;QAClD,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,MAAM;wBACd,iBAAiB,EAAE,EAAE;qBACtB;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,EAAa;QAC7B,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACpB;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,iBAAiB,CAAC,EAAa,EAAE,MAAoC;QACnE,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;gBACnB,MAAM,EAAE;oBACN,IAAI,EAAE,MAAM;iBACb;gBACD,MAAM,EAAE,IAAI;aACb;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC;IAED,sBAAsB;QACpB,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,yBAAyB;YACnD,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,yBAAyB;YACpD,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAkB,EAAE,OAA4B;QAC1D,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,uBAAuB;YACvB,MAAM,aAAa,CAAC,EAAE,CAAC,WAAW,EAAE,IAAI,CAAC,UAAU,EAAE;gBACnD,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,aAAa,CAAC,EAAE,CAAC,iBAAiB,EAAE,IAAI,CAAC,gBAAgB,EAAE;gBAC/D,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,aAAa,CAAC,EAAE,CAAC,YAAY,EAAE,IAAI,CAAC,WAAW,EAAE;gBACrD,OAAO;gBACP,mEAAmE;gBACnE,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,IAAI,CACT,aAAa,IAAI,CAAC,QAAQ,YAAY,IAAI,CAAC,UAAU,CAAC,MAAM,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,MAC5F,IAAI,CAAC,WAAW,CAAC,MACnB,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,mBAAmB,IAAI,CAAC,aAAa,EAAE,CACxF,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;CACF"}
+{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../src/storage/implementation/PersistedBatch.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AAIrD,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AAUzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAE9C;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC;;;;;GAKG;AACH,MAAM,OAAO,cAAc;IAgBf;IAfV,UAAU,GAAsD,EAAE,CAAC;IACnE,gBAAgB,GAA2D,EAAE,CAAC;IAC9E,WAAW,GAAuD,EAAE,CAAC;IAErE;;OAEG;IACH,aAAa,GAAkB,IAAI,CAAC;IAEpC;;OAEG;IACH,WAAW,GAAG,CAAC,CAAC;IAEhB,YACU,QAAgB,EACxB,WAAmB;QADX,aAAQ,GAAR,QAAQ,CAAQ;QAGxB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;IACjC,CAAC;IAED,cAAc,CAAC,OAMd;QACC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAyB,CAAC;QAC3D,KAAK,IAAI,CAAC,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,SAAS,GAAG,KAAK,CAAC,UAAU,CAAC,iBAAiB,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;QAE3F,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAClC,MAAM,GAAG,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAChC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,SAAS;YACT,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;YAC7C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3D,IAAI,CAAC,WAAW,IAAI,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAE5C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,CAAC,CAAC,MAAM;4BACX,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,KAAK;wBACT,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,CAAC,CAAC,KAAK;wBACd,MAAM,EAAE,CAAC,CAAC,EAAE;wBACZ,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,UAAU;qBACjB;iBACF;aACF,CAAC,CAAC;QACL,CAAC;QAED,KAAK,IAAI,EAAE,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC1C,SAAS;YAET,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;gBACnB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,EAAE,CAAC,MAAM;4BACZ,CAAC,EAAE,KAAK;yBACT;wBACD,EAAE,EAAE,QAAQ;wBACZ,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;wBAC9B,UAAU,EAAE,OAAO,CAAC,SAAS;wBAC7B,KAAK,EAAE,EAAE,CAAC,KAAK;wBACf,MAAM,EAAE,EAAE,CAAC,EAAE;wBACb,QAAQ,EAAE,SAAS;wBACnB,IAAI,EAAE,IAAI;qBACX;iBACF;aACF,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,IAMjB;QACC,yCAAyC;QACzC,qEAAqE;QACrE,0HAA0H;QAC1H,2DAA2D;QAC3D,8GAA8G;QAC9G,+BAA+B;QAC/B,6CAA6C;QAC7C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAEnD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAuB,CAAC;QACzD,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACpC,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACjD,CAAC;QAED,wBAAwB;QACxB,KAAK,IAAI,MAAM,IAAI,SAAS,EAAE,CAAC;YAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACzD,MAAM,GAAG,GAAG,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;YACzC,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAE9B,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,SAAS;wBACjB,iBAAiB,EAAE,MAAM,CAAC,iBAAiB;qBAC5C;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;QAED,kDAAkD;QAClD,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACjC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAC3B,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBACzB,SAAS,EAAE;oBACT,QAAQ,EAAE;wBACR,GAAG,EAAE,KAAK;wBACV,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,WAAW,CAAC,EAAE;4BACjB,CAAC,EAAE,SAAS;yBACb;wBACD,MAAM,EAAE,MAAM;wBACd,iBAAiB,EAAE,EAAE;qBACtB;iBACF;aACF,CAAC,CAAC;YAEH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,iBAAiB,CAAC,EAAa;QAC7B,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;aACpB;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC;IACzB,CAAC;IAED,iBAAiB,CAAC,EAAa,EAAE,MAAoC;QACnE,MAAM,EAAE,GAAqD;YAC3D,SAAS,EAAE;gBACT,MAAM,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE;gBACnB,MAAM,EAAE;oBACN,IAAI,EAAE,MAAM;iBACb;gBACD,MAAM,EAAE,IAAI;aACb;SACF,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC1B,IAAI,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC;IACzD,CAAC;IAED,sBAAsB;QACpB,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,UAAU,CAAC,MAAM,IAAI,yBAAyB;YACnD,IAAI,CAAC,WAAW,CAAC,MAAM,IAAI,yBAAyB;YACpD,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,EAAkB,EAAE,OAA4B;QAC1D,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/B,MAAM,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,EAAE;gBAC9C,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,IAAI,CAAC,gBAAgB,EAAE;gBAC1D,OAAO;gBACP,sCAAsC;gBACtC,OAAO,EAAE,KAAK;aACf,CAAC,CAAC;QACL,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,EAAE,CAAC,YAAY,CAAC,SAAS,CAAC,IAAI,CAAC,WAAW,EAAE;gBAChD,OAAO;gBACP,mEAAmE;gBACnE,OAAO,EAAE,IAAI;aACd,CAAC,CAAC;QACL,CAAC;QAED,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAC7C,MAAM,CAAC,IAAI,CACT,aAAa,IAAI,CAAC,QAAQ,YAAY,IAAI,CAAC,UAAU,CAAC,MAAM,MAAM,IAAI,CAAC,gBAAgB,CAAC,MAAM,MAC5F,IAAI,CAAC,WAAW,CAAC,MACnB,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,mBAAmB,IAAI,CAAC,aAAa,EAAE,CACpH,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC5B,CAAC;CACF"}

package/dist/storage/implementation/util.d.ts

@@ -30,17 +30,3 @@ export declare const createMongoClient: (url: string, options?: mongo.MongoClien
  * Helper for unit tests
  */
 export declare const connectMongoForTests: (url: string, isCI: boolean) => PowerSyncMongo;
-/**
- * MongoDB bulkWrite internally splits the operations into batches
- * so that no batch exceeds 16MB. However, there are cases where
- * the batch size is very close to 16MB, where additional metadata
- * on the server pushes it over the limit, resulting in this error
- * from the server:
- *
- * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
- *
- * We work around the issue by doing our own batching, limiting the
- * batch size to 15MB. This does add additional overhead with
- * BSON.calculateObjectSize.
- */
-export declare function safeBulkWrite<T extends mongo.Document>(collection: mongo.Collection<T>, operations: mongo.AnyBulkWriteOperation<T>[], options: mongo.BulkWriteOptions): Promise<void>;

package/dist/storage/implementation/util.js

@@ -114,42 +114,4 @@ export const connectMongoForTests = (url, isCI) => {
     });
     return new PowerSyncMongo(client);
 };
-/**
- * MongoDB bulkWrite internally splits the operations into batches
- * so that no batch exceeds 16MB. However, there are cases where
- * the batch size is very close to 16MB, where additional metadata
- * on the server pushes it over the limit, resulting in this error
- * from the server:
- *
- * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
- *
- * We work around the issue by doing our own batching, limiting the
- * batch size to 15MB. This does add additional overhead with
- * BSON.calculateObjectSize.
- */
-export async function safeBulkWrite(collection, operations, options) {
-    // Must be below 16MB.
-    // We could probably go a little closer, but 15MB is a safe threshold.
-    const BULK_WRITE_LIMIT = 15 * 1024 * 1024;
-    let batch = [];
-    let currentSize = 0;
-    // Estimated overhead per operation, should be smaller in reality.
-    const keySize = 8;
-    for (let op of operations) {
-        const bsonSize = mongo.BSON.calculateObjectSize(op, {
-            checkKeys: false,
-            ignoreUndefined: true
-        }) + keySize;
-        if (batch.length > 0 && currentSize + bsonSize > BULK_WRITE_LIMIT) {
-            await collection.bulkWrite(batch, options);
-            currentSize = 0;
-            batch = [];
-        }
-        batch.push(op);
-        currentSize += bsonSize;
-    }
-    if (batch.length > 0) {
-        await collection.bulkWrite(batch, options);
-    }
-}
 //# sourceMappingURL=util.js.map

package/dist/storage/implementation/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA2B;IAClE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,OAAkC,EAAE,EAAE;IACnF,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,EAAE;QACpC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC;AAEF;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,UAA+B,EAC/B,UAA4C,EAC5C,OAA+B;IAE/B,sBAAsB;IACtB,sEAAsE;IACtE,MAAM,gBAAgB,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC;IAE1C,IAAI,KAAK,GAAqC,EAAE,CAAC;IACjD,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,kEAAkE;IAClE,MAAM,OAAO,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;QAC1B,MAAM,QAAQ,GACZ,KAAK,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,EAAE;YACjC,SAAS,EAAE,KAAK;YAChB,eAAe,EAAE,IAAI;SACf,CAAC,GAAG,OAAO,CAAC;QACtB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,GAAG,QAAQ,GAAG,gBAAgB,EAAE,CAAC;YAClE,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC3C,WAAW,GAAG,CAAC,CAAC;YAChB,KAAK,GAAG,EAAE,CAAC;QACb,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,WAAW,IAAI,QAAQ,CAAC;IAC1B,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,UAAU,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC7C,CAAC;AACH,CAAC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../../src/storage/implementation/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAGzC,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA2B;IAClE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAoB,EAAE,EAAqB;IAC3E,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,KAAK,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACtD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,GAAW,EAAE,OAAkC,EAAE,EAAE;IACnF,OAAO,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;AAC7C,CAAC,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,EAAE;QACpC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC"}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-mongodb-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/index.d.ts",
-  "version": "0.6.0",
+  "version": "0.6.2",
   "main": "dist/index.js",
   "license": "FSL-1.1-Apache-2.0",
   "type": "module",
@@ -22,21 +22,21 @@
     }
   },
   "dependencies": {
-    "bson": "^6.8.0",
+    "bson": "^6.10.3",
     "ts-codec": "^1.3.0",
     "ix": "^5.0.0",
     "lru-cache": "^10.2.2",
     "uuid": "^9.0.1",
-    "@powersync/lib-services-framework": "0.5.2",
-    "@powersync/service-core": "1.7.0",
+    "@powersync/lib-services-framework": "0.5.3",
+    "@powersync/service-core": "1.7.2",
     "@powersync/service-jsonbig": "0.17.10",
     "@powersync/service-sync-rules": "0.24.0",
     "@powersync/service-types": "0.8.0",
-    "@powersync/lib-service-mongodb": "0.4.2"
+    "@powersync/lib-service-mongodb": "0.4.3"
   },
   "devDependencies": {
     "@types/uuid": "^9.0.4",
-    "@powersync/service-core-tests": "0.5.0"
+    "@powersync/service-core-tests": "0.6.1"
   },
   "scripts": {
     "build": "tsc -b",

package/src/storage/MongoBucketStorage.ts

@@ -1,6 +1,6 @@
 import { SqlSyncRules } from '@powersync/service-sync-rules';
 
-import { storage } from '@powersync/service-core';
+import { GetIntanceOptions, storage } from '@powersync/service-core';
 
 import { BaseObserver, ErrorCode, logger, ServiceError } from '@powersync/lib-services-framework';
 import { v4 as uuid } from 'uuid';
@@ -44,13 +44,15 @@ export class MongoBucketStorage
     // No-op
   }
 
-  getInstance(options: storage.PersistedSyncRulesContent): MongoSyncBucketStorage {
-    let { id, slot_name } = options;
+  getInstance(syncRules: storage.PersistedSyncRulesContent, options?: GetIntanceOptions): MongoSyncBucketStorage {
+    let { id, slot_name } = syncRules;
     if ((typeof id as any) == 'bigint') {
       id = Number(id);
     }
-    const storage = new MongoSyncBucketStorage(this, id, options, slot_name);
-    this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
+    const storage = new MongoSyncBucketStorage(this, id, syncRules, slot_name);
+    if (!options?.skipLifecycleHooks) {
+      this.iterateListeners((cb) => cb.syncStorageCreated?.(storage));
+    }
     storage.registerListener({
       batchStarted: (batch) => {
         batch.registerListener({
@@ -95,13 +97,11 @@ export class MongoBucketStorage
     }
   }
 
-  async slotRemoved(slot_name: string) {
+  async restartReplication(sync_rules_group_id: number) {
     const next = await this.getNextSyncRulesContent();
     const active = await this.getActiveSyncRulesContent();
 
-    // In both the below cases, we create a new sync rules instance.
-    // The current one will continue erroring until the next one has finished processing.
-    if (next != null && next.slot_name == slot_name) {
+    if (next != null && next.id == sync_rules_group_id) {
       // We need to redo the "next" sync rules
       await this.updateSyncRules({
         content: next.sync_rules_content,
@@ -119,14 +119,17 @@ export class MongoBucketStorage
           }
         }
       );
-    } else if (next == null && active?.slot_name == slot_name) {
+    } else if (next == null && active?.id == sync_rules_group_id) {
       // Slot removed for "active" sync rules, while there is no "next" one.
       await this.updateSyncRules({
         content: active.sync_rules_content,
         validate: false
       });
 
-      // Pro-actively stop replicating
+      // In this case we keep the old one as active for clients, so that that existing clients
+      // can still get the latest data while we replicate the new ones.
+      // It will however not replicate anymore.
+
       await this.db.sync_rules.updateOne(
         {
           _id: active.id,
@@ -134,7 +137,21 @@ export class MongoBucketStorage
         },
         {
           $set: {
-            state: storage.SyncRuleState.STOP
+            state: storage.SyncRuleState.ERRORED
+          }
+        }
+      );
+    } else if (next != null && active?.id == sync_rules_group_id) {
+      // Already have next sync rules, but need to stop replicating the active one.
+
+      await this.db.sync_rules.updateOne(
+        {
+          _id: active.id,
+          state: storage.SyncRuleState.ACTIVE
+        },
+        {
+          $set: {
+            state: storage.SyncRuleState.ERRORED
           }
         }
       );
@@ -211,7 +228,7 @@ export class MongoBucketStorage
   async getActiveSyncRulesContent(): Promise<MongoPersistedSyncRulesContent | null> {
     const doc = await this.db.sync_rules.findOne(
       {
-        state: storage.SyncRuleState.ACTIVE
+        state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
       },
       { sort: { _id: -1 }, limit: 1 }
     );
@@ -249,7 +266,7 @@ export class MongoBucketStorage
   async getReplicatingSyncRules(): Promise<storage.PersistedSyncRulesContent[]> {
     const docs = await this.db.sync_rules
       .find({
-        $or: [{ state: storage.SyncRuleState.ACTIVE }, { state: storage.SyncRuleState.PROCESSING }]
+        state: { $in: [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE] }
       })
       .toArray();
 

package/src/storage/implementation/MongoBucketBatch.ts

@@ -270,7 +270,7 @@ export class MongoBucketBatch
       }
     }
 
-    return resumeBatch;
+    return resumeBatch?.hasData() ? resumeBatch : null;
   }
 
   private saveOperation(

package/src/storage/implementation/MongoCompactor.ts

@@ -5,7 +5,6 @@ import { storage, utils } from '@powersync/service-core';
 import { PowerSyncMongo } from './db.js';
 import { BucketDataDocument, BucketDataKey } from './models.js';
 import { cacheKey } from './OperationBatch.js';
-import { safeBulkWrite } from './util.js';
 
 interface CurrentBucketState {
   /** Bucket name */
@@ -33,14 +32,7 @@ interface CurrentBucketState {
 /**
  * Additional options, primarily for testing.
  */
-export interface MongoCompactOptions extends storage.CompactOptions {
-  /** Minimum of 2 */
-  clearBatchLimit?: number;
-  /** Minimum of 1 */
-  moveBatchLimit?: number;
-  /** Minimum of 1 */
-  moveBatchQueryLimit?: number;
-}
+export interface MongoCompactOptions extends storage.CompactOptions {}
 
 const DEFAULT_CLEAR_BATCH_LIMIT = 5000;
 const DEFAULT_MOVE_BATCH_LIMIT = 2000;
@@ -265,7 +257,7 @@ export class MongoCompactor {
   private async flush() {
     if (this.updates.length > 0) {
       logger.info(`Compacting ${this.updates.length} ops`);
-      await safeBulkWrite(this.db.bucket_data, this.updates, {
+      await this.db.bucket_data.bulkWrite(this.updates, {
         // Order is not important.
         // Since checksums are not affected, these operations can happen in any order,
         // and it's fine if the operations are partially applied.

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -13,6 +13,7 @@ import {
   CheckpointChanges,
   GetCheckpointChangesOptions,
   ReplicationCheckpoint,
+  SourceTable,
   storage,
   utils,
   WatchWriteCheckpointOptions
@@ -23,7 +24,14 @@ import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
 import * as timers from 'timers/promises';
 import { MongoBucketStorage } from '../MongoBucketStorage.js';
 import { PowerSyncMongo } from './db.js';
-import { BucketDataDocument, BucketDataKey, SourceKey, SyncRuleCheckpointState, SyncRuleDocument } from './models.js';
+import {
+  BucketDataDocument,
+  BucketDataKey,
+  SourceKey,
+  SourceTableDocument,
+  SyncRuleCheckpointState,
+  SyncRuleDocument
+} from './models.js';
 import { MongoBucketBatch } from './MongoBucketBatch.js';
 import { MongoCompactor } from './MongoCompactor.js';
 import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
@@ -163,17 +171,17 @@ export class MongoSyncBucketStorage
     let result: storage.ResolveTableResult | null = null;
     await this.db.client.withSession(async (session) => {
       const col = this.db.source_tables;
-      let doc = await col.findOne(
-        {
-          group_id: group_id,
-          connection_id: connection_id,
-          relation_id: objectId,
-          schema_name: schema,
-          table_name: table,
-          replica_id_columns2: columns
-        },
-        { session }
-      );
+      let filter: Partial<SourceTableDocument> = {
+        group_id: group_id,
+        connection_id: connection_id,
+        schema_name: schema,
+        table_name: table,
+        replica_id_columns2: columns
+      };
+      if (objectId != null) {
+        filter.relation_id = objectId;
+      }
+      let doc = await col.findOne(filter, { session });
       if (doc == null) {
         doc = {
           _id: new bson.ObjectId(),
@@ -202,31 +210,40 @@ export class MongoSyncBucketStorage
       sourceTable.syncData = options.sync_rules.tableSyncsData(sourceTable);
       sourceTable.syncParameters = options.sync_rules.tableSyncsParameters(sourceTable);
 
+      let dropTables: storage.SourceTable[] = [];
+      // Detect tables that are either renamed, or have different replica_id_columns
+      let truncateFilter = [{ schema_name: schema, table_name: table }] as any[];
+      if (objectId != null) {
+        // Only detect renames if the source uses relation ids.
+        truncateFilter.push({ relation_id: objectId });
+      }
       const truncate = await col
         .find(
           {
             group_id: group_id,
             connection_id: connection_id,
             _id: { $ne: doc._id },
-            $or: [{ relation_id: objectId }, { schema_name: schema, table_name: table }]
+            $or: truncateFilter
           },
           { session }
         )
         .toArray();
+      dropTables = truncate.map(
+        (doc) =>
+          new storage.SourceTable(
+            doc._id,
+            connection_tag,
+            doc.relation_id,
+            doc.schema_name,
+            doc.table_name,
+            doc.replica_id_columns2?.map((c) => ({ name: c.name, typeOid: c.type_oid, type: c.type })) ?? [],
+            doc.snapshot_done ?? true
+          )
+      );
+
       result = {
         table: sourceTable,
-        dropTables: truncate.map(
-          (doc) =>
-            new storage.SourceTable(
-              doc._id,
-              connection_tag,
-              doc.relation_id ?? 0,
-              doc.schema_name,
-              doc.table_name,
-              doc.replica_id_columns2?.map((c) => ({ name: c.name, typeOid: c.type_oid, type: c.type })) ?? [],
-              doc.snapshot_done ?? true
-            )
-        )
+        dropTables: dropTables
       };
     });
     return result!;
@@ -313,13 +330,7 @@ export class MongoSyncBucketStorage
         // 1. We can calculate the document size accurately without serializing again.
         // 2. We can delay parsing the results until it's needed.
         // We manually use bson.deserialize below
-        raw: true,
-
-        // Since we're using raw: true and parsing ourselves later, we don't need bigint
-        // support here.
-        // Disabling due to https://jira.mongodb.org/browse/NODE-6165, and the fact that this
-        // is one of our most common queries.
-        useBigInt64: false
+        raw: true
       }
     ) as unknown as mongo.FindCursor<Buffer>;
 
@@ -593,7 +604,7 @@ export class MongoSyncBucketStorage
           await this.db.sync_rules.updateMany(
             {
               _id: { $ne: this.group_id },
-              state: storage.SyncRuleState.ACTIVE
+              state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
             },
             {
               $set: {
@@ -646,7 +657,7 @@ export class MongoSyncBucketStorage
       doc = await this.db.sync_rules.findOne(
         {
           _id: syncRulesId,
-          state: storage.SyncRuleState.ACTIVE
+          state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
         },
         {
           session,
@@ -717,7 +728,7 @@ export class MongoSyncBucketStorage
         // Irrelevant update
         continue;
       }
-      if (doc.state != storage.SyncRuleState.ACTIVE) {
+      if (doc.state != storage.SyncRuleState.ACTIVE && doc.state != storage.SyncRuleState.ERRORED) {
         // Sync rules have changed - abort and restart.
         // Should this error instead?
         break;

package/src/storage/implementation/MongoWriteCheckpointAPI.ts

@@ -1,7 +1,6 @@
 import * as framework from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
 import { PowerSyncMongo } from './db.js';
-import { safeBulkWrite } from './util.js';
 
 export type MongoCheckpointAPIOptions = {
   db: PowerSyncMongo;
@@ -127,8 +126,7 @@ export async function batchCreateCustomWriteCheckpoints(
     return;
   }
 
-  await safeBulkWrite(
-    db.custom_write_checkpoints,
+  await db.custom_write_checkpoints.bulkWrite(
     checkpoints.map((checkpointOptions) => ({
       updateOne: {
         filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },

package/src/storage/implementation/OperationBatch.ts

@@ -41,6 +41,10 @@ export class OperationBatch {
     return this.batch.length >= MAX_BATCH_COUNT || this.currentSize > MAX_RECORD_BATCH_SIZE;
   }
 
+  hasData() {
+    return this.length > 0;
+  }
+
   /**
    *
    * @param sizes Map of source key to estimated size of the current_data document, or undefined if current_data is not persisted.

package/src/storage/implementation/PersistedBatch.ts

@@ -15,7 +15,7 @@ import {
   CurrentDataDocument,
   SourceKey
 } from './models.js';
-import { replicaIdToSubkey, safeBulkWrite } from './util.js';
+import { replicaIdToSubkey } from './util.js';
 
 /**
  * Maximum size of operations we write in a single transaction.
@@ -245,33 +245,34 @@ export class PersistedBatch {
   }
 
   async flush(db: PowerSyncMongo, session: mongo.ClientSession) {
+    const startAt = performance.now();
     if (this.bucketData.length > 0) {
-      // calculate total size
-      await safeBulkWrite(db.bucket_data, this.bucketData, {
+      await db.bucket_data.bulkWrite(this.bucketData, {
         session,
         // inserts only - order doesn't matter
         ordered: false
       });
     }
     if (this.bucketParameters.length > 0) {
-      await safeBulkWrite(db.bucket_parameters, this.bucketParameters, {
+      await db.bucket_parameters.bulkWrite(this.bucketParameters, {
         session,
         // inserts only - order doesn't matter
         ordered: false
       });
     }
     if (this.currentData.length > 0) {
-      await safeBulkWrite(db.current_data, this.currentData, {
+      await db.current_data.bulkWrite(this.currentData, {
         session,
         // may update and delete data within the same batch - order matters
         ordered: true
       });
     }
 
+    const duration = performance.now() - startAt;
     logger.info(
       `powersync_${this.group_id} Flushed ${this.bucketData.length} + ${this.bucketParameters.length} + ${
         this.currentData.length
-      } updates, ${Math.round(this.currentSize / 1024)}kb. Last op_id: ${this.debugLastOpId}`
+      } updates, ${Math.round(this.currentSize / 1024)}kb in ${duration.toFixed(0)}ms. Last op_id: ${this.debugLastOpId}`
     );
 
     this.bucketData = [];

package/src/storage/implementation/util.ts

@@ -124,48 +124,3 @@ export const connectMongoForTests = (url: string, isCI: boolean) => {
   });
   return new PowerSyncMongo(client);
 };
-
-/**
- * MongoDB bulkWrite internally splits the operations into batches
- * so that no batch exceeds 16MB. However, there are cases where
- * the batch size is very close to 16MB, where additional metadata
- * on the server pushes it over the limit, resulting in this error
- * from the server:
- *
- * > MongoBulkWriteError: BSONObj size: 16814023 (0x1008FC7) is invalid. Size must be between 0 and 16793600(16MB) First element: insert: "bucket_data"
- *
- * We work around the issue by doing our own batching, limiting the
- * batch size to 15MB. This does add additional overhead with
- * BSON.calculateObjectSize.
- */
-export async function safeBulkWrite<T extends mongo.Document>(
-  collection: mongo.Collection<T>,
-  operations: mongo.AnyBulkWriteOperation<T>[],
-  options: mongo.BulkWriteOptions
-) {
-  // Must be below 16MB.
-  // We could probably go a little closer, but 15MB is a safe threshold.
-  const BULK_WRITE_LIMIT = 15 * 1024 * 1024;
-
-  let batch: mongo.AnyBulkWriteOperation<T>[] = [];
-  let currentSize = 0;
-  // Estimated overhead per operation, should be smaller in reality.
-  const keySize = 8;
-  for (let op of operations) {
-    const bsonSize =
-      mongo.BSON.calculateObjectSize(op, {
-        checkKeys: false,
-        ignoreUndefined: true
-      } as any) + keySize;
-    if (batch.length > 0 && currentSize + bsonSize > BULK_WRITE_LIMIT) {
-      await collection.bulkWrite(batch, options);
-      currentSize = 0;
-      batch = [];
-    }
-    batch.push(op);
-    currentSize += bsonSize;
-  }
-  if (batch.length > 0) {
-    await collection.bulkWrite(batch, options);
-  }
-}