@powersync/service-module-mongodb-storage 0.12.7 → 0.12.8

package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map

@@ -1 +1 @@
-{"version":3,"file":"MongoWriteCheckpointAPI.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoWriteCheckpointAPI.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAA6C,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAS7F,MAAM,OAAO,uBAAuB;IACzB,EAAE,CAAiB;IACpB,KAAK,CAA8B;IAE3C,YAAY,OAAkC;QAC5C,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,sBAAsB,CAAC,IAAiC;QACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,4BAA4B,CAAC,UAAiD;QAClF,IAAI,IAAI,CAAC,mBAAmB,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;YACrE,MAAM,IAAI,SAAS,CAAC,qBAAqB,CACvC,qGAAqG,IAAI,CAAC,mBAAmB,GAAG,CACjI,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,gBAAgB,CAC1D;YACE,OAAO,EAAE,OAAO;SACjB,EACD;YACE,IAAI,EAAE;gBACJ,IAAI;gBACJ,gBAAgB,EAAE,IAAI;aACvB;YACD,IAAI,EAAE;gBACJ,SAAS,EAAE,EAAE;aACd;SACF,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAC1C,CAAC;QACF,OAAO,GAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,OAA2C;QACnE,QAAQ,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACjC,KAAK,OAAO,CAAC,mBAAmB,CAAC,MAAM;gBACrC,IAAI,KAAK,IAAI,eAAe,IAAI,OAAO,EAAE,CAAC;oBACxC,MAAM,IAAI,SAAS,CAAC,qBAAqB,CAAC,iEAAiE,CAAC,CAAC;gBAC/G,CAAC;gBACD,OAAO,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;YACjD,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO;gBACtC,IAAI,KAAK,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;oBAChC,MAAM,IAAI,SAAS,CAAC,qBAAqB,CACvC,qEAAqE,CACtE,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,OAAoC;QAClE,QAAQ,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACjC,KAAK,OAAO,CAAC,mBAAmB,CAAC,MAAM;gBACrC,OAAO,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,CAAC;YACvD,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO;gBACtC,OAAO,IAAI,CAAC,gCAAgC,CAAC,OAAO,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAES,KAAK,CAAC,yBAAyB,CAAC,OAA6C;QACrF,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAC3C,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC;YACzE,OAAO;YACP,aAAa;SACd,CAAC,CAAC;QACH,OAAO,mBAAmB,EAAE,UAAU,IAAI,IAAI,CAAC;IACjD,CAAC;IAES,KAAK,CAAC,0BAA0B,CAAC,OAA8C;QACvF,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACnC,4EAA4E;QAC5E,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,oDAAoD;YACpD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC;YAClE,OAAO,EAAE,OAAO;YAChB,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;SACxB,CAAC,CAAC;QACH,OAAO,mBAAmB,EAAE,SAAS,IAAI,IAAI,CAAC;IAChD,CAAC;IAEO,KAAK,CAAC,gCAAgC,CAAC,OAAoC;QACjF,MAAM,KAAK,GAAG,IAAI,CAAC;QACnB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aAC5C,IAAI,CACH;YACE,gBAAgB,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,EAAE;SACxF,EACD;YACE,KAAK,EAAE,KAAK,GAAG,CAAC;YAChB,SAAS,EAAE,KAAK,GAAG,CAAC;YACpB,WAAW,EAAE,IAAI;SAClB,CACF;aACA,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;QAE1C,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC1D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;gBACtB,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QAED,OAAO;YACL,0BAA0B,EAAE,UAAU;YACtC,uBAAuB;SACxB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,+BAA+B,CAAC,OAAoC;QAChF,MAAM,KAAK,GAAG,IAAI,CAAC;QACnB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,wBAAwB;aACnD,IAAI,CACH;YACE,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE;SAC3F,EACD;YACE,KAAK,EAAE,KAAK,GAAG,CAAC;YAChB,SAAS,EAAE,KAAK,GAAG,CAAC;YACpB,WAAW,EAAE,IAAI;SAClB,CACF;aACA,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;QAE1C,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC1D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;gBACtB,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAED,OAAO;YACL,0BAA0B,EAAE,UAAU;YACtC,uBAAuB;SACxB,CAAC;IACJ,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,iCAAiC,CACrD,EAAkB,EAClB,OAA4B,EAC5B,WAAmD,EACnD,IAAkB;IAElB,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC5B,OAAO;IACT,CAAC;IAED,MAAM,EAAE,CAAC,wBAAwB,CAAC,SAAS,CACzC,WAAW,CAAC,GAAG,CAAC,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;QACtC,SAAS,EAAE;YACT,MAAM,EAAE,EAAE,OAAO,EAAE,iBAAiB,CAAC,OAAO,EAAE,aAAa,EAAE,iBAAiB,CAAC,aAAa,EAAE;YAC9F,MAAM,EAAE;gBACN,IAAI,EAAE;oBACJ,UAAU,EAAE,iBAAiB,CAAC,UAAU;oBACxC,aAAa,EAAE,iBAAiB,CAAC,aAAa;oBAC9C,KAAK,EAAE,IAAI;iBACZ;aACF;YACD,MAAM,EAAE,IAAI;SACb;KACF,CAAC,CAAC,EACH,EAAE,OAAO,EAAE,CACZ,CAAC;AACJ,CAAC"}
+{"version":3,"file":"MongoWriteCheckpointAPI.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoWriteCheckpointAPI.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,SAAS,MAAM,mCAAmC,CAAC;AAC/D,OAAO,EAA6C,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAS7F,MAAM,OAAO,uBAAuB;IACzB,EAAE,CAAiB;IACpB,KAAK,CAA8B;IAE3C,YAAY,OAAkC;QAC5C,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC;IAC5B,CAAC;IAED,IAAI,mBAAmB;QACrB,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAED,sBAAsB,CAAC,IAAiC;QACtD,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,4BAA4B,CAAC,UAAiD;QAClF,IAAI,IAAI,CAAC,mBAAmB,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO,EAAE,CAAC;YACrE,MAAM,IAAI,SAAS,CAAC,qBAAqB,CACvC,qGAAqG,IAAI,CAAC,mBAAmB,GAAG,CACjI,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,UAAU,CAAC;QAC5C,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,gBAAgB,CAC1D;YACE,OAAO,EAAE,OAAO;SACjB,EACD;YACE,IAAI,EAAE;gBACJ,IAAI;gBACJ,gBAAgB,EAAE,IAAI;aACvB;YACD,IAAI,EAAE;gBACJ,SAAS,EAAE,EAAE;aACd;SACF,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,cAAc,EAAE,OAAO,EAAE,CAC1C,CAAC;QACF,OAAO,GAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,OAA2C;QACnE,QAAQ,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACjC,KAAK,OAAO,CAAC,mBAAmB,CAAC,MAAM;gBACrC,IAAI,KAAK,IAAI,eAAe,IAAI,OAAO,EAAE,CAAC;oBACxC,MAAM,IAAI,SAAS,CAAC,qBAAqB,CAAC,iEAAiE,CAAC,CAAC;gBAC/G,CAAC;gBACD,OAAO,IAAI,CAAC,yBAAyB,CAAC,OAAO,CAAC,CAAC;YACjD,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO;gBACtC,IAAI,KAAK,IAAI,OAAO,IAAI,OAAO,EAAE,CAAC;oBAChC,MAAM,IAAI,SAAS,CAAC,qBAAqB,CACvC,qEAAqE,CACtE,CAAC;gBACJ,CAAC;gBACD,OAAO,IAAI,CAAC,0BAA0B,CAAC,OAAO,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,yBAAyB,CAAC,OAAoC;QAClE,QAAQ,IAAI,CAAC,mBAAmB,EAAE,CAAC;YACjC,KAAK,OAAO,CAAC,mBAAmB,CAAC,MAAM;gBACrC,OAAO,IAAI,CAAC,+BAA+B,CAAC,OAAO,CAAC,CAAC;YACvD,KAAK,OAAO,CAAC,mBAAmB,CAAC,OAAO;gBACtC,OAAO,IAAI,CAAC,gCAAgC,CAAC,OAAO,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAES,KAAK,CAAC,yBAAyB,CAAC,OAA6C;QACrF,MAAM,EAAE,OAAO,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC;QAC3C,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,wBAAwB,CAAC,OAAO,CAAC;YACzE,OAAO;YACP,aAAa;SACd,CAAC,CAAC;QACH,OAAO,mBAAmB,EAAE,UAAU,IAAI,IAAI,CAAC;IACjD,CAAC;IAES,KAAK,CAAC,0BAA0B,CAAC,OAA8C;QACvF,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;QACnC,4EAA4E;QAC5E,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACvB,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;YAChB,oDAAoD;YACpD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB,CAAC,OAAO,CAAC;YAClE,OAAO,EAAE,OAAO;YAChB,QAAQ,EAAE,EAAE,IAAI,EAAE,GAAG,EAAE;SACxB,CAAC,CAAC;QACH,OAAO,mBAAmB,EAAE,SAAS,IAAI,IAAI,CAAC;IAChD,CAAC;IAEO,KAAK,CAAC,gCAAgC,CAAC,OAAoC;QACjF,MAAM,KAAK,GAAG,IAAI,CAAC;QACnB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,iBAAiB;aAC5C,IAAI,CACH;YACE,gBAAgB,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,CAAC,cAAc,CAAC,GAAG,EAAE;SACxF,EACD;YACE,KAAK,EAAE,KAAK,GAAG,CAAC;YAChB,2DAA2D;YAC3D,+DAA+D;YAC/D,SAAS,EAAE,KAAK,GAAG,CAAC;YACpB,WAAW,EAAE,IAAI;SAClB,CACF;aACA,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;QAE1C,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC1D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;gBACtB,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QAED,OAAO;YACL,0BAA0B,EAAE,UAAU;YACtC,uBAAuB;SACxB,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,+BAA+B,CAAC,OAAoC;QAChF,MAAM,KAAK,GAAG,IAAI,CAAC;QACnB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,wBAAwB;aACnD,IAAI,CACH;YACE,KAAK,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,CAAC,cAAc,CAAC,UAAU,EAAE;SAC3F,EACD;YACE,KAAK,EAAE,KAAK,GAAG,CAAC;YAChB,2DAA2D;YAC3D,+DAA+D;YAC/D,SAAS,EAAE,KAAK,GAAG,CAAC;YACpB,WAAW,EAAE,IAAI;SAClB,CACF;aACA,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,KAAK,CAAC;QAE1C,MAAM,uBAAuB,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC1D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,KAAK,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;gBACtB,uBAAuB,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QAED,OAAO;YACL,0BAA0B,EAAE,UAAU;YACtC,uBAAuB;SACxB,CAAC;IACJ,CAAC;CACF;AAED,MAAM,CAAC,KAAK,UAAU,iCAAiC,CACrD,EAAkB,EAClB,OAA4B,EAC5B,WAAmD,EACnD,IAAkB;IAElB,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC5B,OAAO;IACT,CAAC;IAED,MAAM,EAAE,CAAC,wBAAwB,CAAC,SAAS,CACzC,WAAW,CAAC,GAAG,CAAC,CAAC,iBAAiB,EAAE,EAAE,CAAC,CAAC;QACtC,SAAS,EAAE;YACT,MAAM,EAAE,EAAE,OAAO,EAAE,iBAAiB,CAAC,OAAO,EAAE,aAAa,EAAE,iBAAiB,CAAC,aAAa,EAAE;YAC9F,MAAM,EAAE;gBACN,IAAI,EAAE;oBACJ,UAAU,EAAE,iBAAiB,CAAC,UAAU;oBACxC,aAAa,EAAE,iBAAiB,CAAC,aAAa;oBAC9C,KAAK,EAAE,IAAI;iBACZ;aACF;YACD,MAAM,EAAE,IAAI;SACb;KACF,CAAC,CAAC,EACH,EAAE,OAAO,EAAE,CACZ,CAAC;AACJ,CAAC"}

package/dist/storage/implementation/db.d.ts

@@ -48,5 +48,9 @@ export declare class PowerSyncMongo {
      * Only use in migrations and tests.
      */
     createBucketStateIndex(): Promise<void>;
+    /**
+     * Only use in migrations and tests.
+     */
+    createBucketStateIndex2(): Promise<void>;
 }
 export declare function createPowerSyncMongo(config: MongoStorageConfig, options?: lib_mongo.MongoConnectionOptions): PowerSyncMongo;

package/dist/storage/implementation/db.js

@@ -105,6 +105,16 @@ export class PowerSyncMongo {
             last_op: 1
         }, { name: 'bucket_updates', unique: true });
     }
+    /**
+     * Only use in migrations and tests.
+     */
+    async createBucketStateIndex2() {
+        // TODO: Implement a better mechanism to use migrations in tests
+        await this.bucket_state.createIndex({
+            '_id.g': 1,
+            'estimate_since_compact.count': -1
+        }, { name: 'dirty_count' });
+    }
 }
 export function createPowerSyncMongo(config, options) {
     return new PowerSyncMongo(lib_mongo.createMongoClient(config, {

package/dist/storage/implementation/db.js.map

@@ -1 +1 @@
-{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAwBrE,MAAM,OAAO,cAAc;IAChB,YAAY,CAAwC;IACpD,WAAW,CAAuC;IAClD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAqC;IAC/C,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAC9C,YAAY,CAAwC;IACpD,iBAAiB,CAA4C;IAE7D,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAEtB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;IACnE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAS,EAAE,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC;IACnF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gCAAgC;QACpC,6FAA6F;QAC7F,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aACnE,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBAChC,oEAAoE;gBACpE,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,iDAAiD;gBACjD,OAAO;YACT,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,EAAE;YAClD,MAAM,EAAE,IAAI;YACZ,sFAAsF;YACtF,mFAAmF;YACnF,iFAAiF;YACjF,4BAA4B;YAC5B,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,gBAAgB;YACjC,GAAG,EAAE,EAAE,CAAC,0BAA0B;SACnC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QAC1B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,CACzC,CAAC;IACJ,CAAC;CACF;AAED,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,OAA0C;IACzG,OAAO,IAAI,cAAc,CACvB,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAClC,gBAAgB,EAAE,iBAAiB;QACnC,GAAG,OAAO;KACX,CAAC,EACF,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAC9B,CAAC;AACJ,CAAC"}
+{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAwBrE,MAAM,OAAO,cAAc;IAChB,YAAY,CAAwC;IACpD,WAAW,CAAuC;IAClD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAqC;IAC/C,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAC9C,YAAY,CAAwC;IACpD,iBAAiB,CAA4C;IAE7D,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAEtB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;IACnE,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAS,EAAE,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC;IACnF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gCAAgC;QACpC,6FAA6F;QAC7F,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aACnE,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBAChC,oEAAoE;gBACpE,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,iDAAiD;gBACjD,OAAO;YACT,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,EAAE;YAClD,MAAM,EAAE,IAAI;YACZ,sFAAsF;YACtF,mFAAmF;YACnF,iFAAiF;YACjF,4BAA4B;YAC5B,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,gBAAgB;YACjC,GAAG,EAAE,EAAE,CAAC,0BAA0B;SACnC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QAC1B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,CACzC,CAAC;IACJ,CAAC;IACD;;OAEG;IACH,KAAK,CAAC,uBAAuB;QAC3B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,8BAA8B,EAAE,CAAC,CAAC;SACnC,EACD,EAAE,IAAI,EAAE,aAAa,EAAE,CACxB,CAAC;IACJ,CAAC;CACF;AAED,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,OAA0C;IACzG,OAAO,IAAI,cAAc,CACvB,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAClC,gBAAgB,EAAE,iBAAiB;QACnC,GAAG,OAAO;KACX,CAAC,EACF,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAC9B,CAAC;AACJ,CAAC"}

package/dist/storage/implementation/util.d.ts

@@ -14,7 +14,7 @@ export declare function generateSlotName(prefix: string, sync_rules_id: number):
  * However, that makes `has_more` detection very difficult, since the cursor is always closed
  * after the first batch. Instead, we do a workaround to only fetch a single batch below.
  *
- * For this to be effective, set batchSize = limit in the find command.
+ * For this to be effective, set batchSize = limit + 1 in the find command.
  */
 export declare function readSingleBatch<T>(cursor: mongo.AbstractCursor<T>): Promise<{
     data: T[];

package/dist/storage/implementation/util.js

@@ -33,7 +33,7 @@ export function generateSlotName(prefix, sync_rules_id) {
  * However, that makes `has_more` detection very difficult, since the cursor is always closed
  * after the first batch. Instead, we do a workaround to only fetch a single batch below.
  *
- * For this to be effective, set batchSize = limit in the find command.
+ * For this to be effective, set batchSize = limit + 1 in the find command.
  */
 export async function readSingleBatch(cursor) {
     try {

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-mongodb-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/index.d.ts",
-  "version": "0.12.7",
+  "version": "0.12.8",
   "main": "dist/index.js",
   "license": "FSL-1.1-ALv2",
   "type": "module",
@@ -22,20 +22,20 @@
     }
   },
   "dependencies": {
-    "bson": "^6.10.3",
+    "bson": "^6.10.4",
     "ix": "^5.0.0",
     "lru-cache": "^10.2.2",
     "ts-codec": "^1.3.0",
     "uuid": "^11.1.0",
-    "@powersync/lib-service-mongodb": "0.6.9",
-    "@powersync/lib-services-framework": "0.7.7",
-    "@powersync/service-core": "1.15.7",
+    "@powersync/lib-service-mongodb": "0.6.10",
+    "@powersync/lib-services-framework": "0.7.8",
+    "@powersync/service-core": "1.15.8",
     "@powersync/service-jsonbig": "0.17.11",
-    "@powersync/service-sync-rules": "0.29.4",
+    "@powersync/service-sync-rules": "0.29.5",
     "@powersync/service-types": "0.13.0"
   },
   "devDependencies": {
-    "@powersync/service-core-tests": "0.12.7"
+    "@powersync/service-core-tests": "0.12.8"
   },
   "scripts": {
     "build": "tsc -b",

package/src/migrations/db/migrations/1760433882550-bucket-state-index2.ts

@@ -0,0 +1,34 @@
+import { migrations } from '@powersync/service-core';
+import * as storage from '../../../storage/storage-index.js';
+import { MongoStorageConfig } from '../../../types/types.js';
+
+const INDEX_NAME = 'dirty_buckets';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    await db.createBucketStateIndex2();
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    if (await db.bucket_state.indexExists(INDEX_NAME)) {
+      await db.bucket_state.dropIndex(INDEX_NAME);
+    }
+  } finally {
+    await db.client.close();
+  }
+};

package/src/storage/implementation/MongoCompactor.ts

@@ -1,6 +1,13 @@
 import { mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb';
 import { logger, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
-import { addChecksums, InternalOpId, isPartialChecksum, storage, utils } from '@powersync/service-core';
+import {
+  addChecksums,
+  InternalOpId,
+  isPartialChecksum,
+  PopulateChecksumCacheResults,
+  storage,
+  utils
+} from '@powersync/service-core';
 
 import { PowerSyncMongo } from './db.js';
 import { BucketDataDocument, BucketDataKey, BucketStateDocument } from './models.js';
@@ -10,6 +17,7 @@ import { cacheKey } from './OperationBatch.js';
 interface CurrentBucketState {
   /** Bucket name */
   bucket: string;
+
   /**
    * Rows seen in the bucket, with the last op_id of each.
    */
@@ -96,67 +104,56 @@ export class MongoCompactor {
         // We can make this more efficient later on by iterating
         // through the buckets in a single query.
         // That makes batching more tricky, so we leave for later.
-        await this.compactInternal(bucket);
+        await this.compactSingleBucket(bucket);
       }
     } else {
-      await this.compactInternal(undefined);
+      await this.compactDirtyBuckets();
     }
   }
 
-  async compactInternal(bucket: string | undefined) {
-    const idLimitBytes = this.idLimitBytes;
+  private async compactDirtyBuckets() {
+    while (!this.signal?.aborted) {
+      // Process all buckets with 1 or more changes since last time
+      const buckets = await this.dirtyBucketBatch({ minBucketChanges: 1 });
+      if (buckets.length == 0) {
+        // All done
+        break;
+      }
+      for (let bucket of buckets) {
+        await this.compactSingleBucket(bucket);
+      }
+    }
+  }
 
-    let currentState: CurrentBucketState | null = null;
+  private async compactSingleBucket(bucket: string) {
+    const idLimitBytes = this.idLimitBytes;
 
-    let bucketLower: string | mongo.MinKey;
-    let bucketUpper: string | mongo.MaxKey;
+    let currentState: CurrentBucketState = {
+      bucket,
+      seen: new Map(),
+      trackingSize: 0,
+      lastNotPut: null,
+      opsSincePut: 0,
 
-    if (bucket == null) {
-      bucketLower = new mongo.MinKey();
-      bucketUpper = new mongo.MaxKey();
-    } else if (bucket.includes('[')) {
-      // Exact bucket name
-      bucketLower = bucket;
-      bucketUpper = bucket;
-    } else {
-      // Bucket definition name
-      bucketLower = `${bucket}[`;
-      bucketUpper = `${bucket}[\uFFFF`;
-    }
+      checksum: 0,
+      opCount: 0,
+      opBytes: 0
+    };
 
     // Constant lower bound
     const lowerBound: BucketDataKey = {
       g: this.group_id,
-      b: bucketLower as string,
+      b: bucket,
       o: new mongo.MinKey() as any
     };
 
     // Upper bound is adjusted for each batch
     let upperBound: BucketDataKey = {
       g: this.group_id,
-      b: bucketUpper as string,
+      b: bucket,
       o: new mongo.MaxKey() as any
     };
 
-    const doneWithBucket = async () => {
-      if (currentState == null) {
-        return;
-      }
-      // Free memory before clearing bucket
-      currentState.seen.clear();
-      if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
-        logger.info(
-          `Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
-        );
-        // Need flush() before clear()
-        await this.flush();
-        await this.clearBucket(currentState);
-      }
-
-      // Do this _after_ clearBucket so that we have accurate counts.
-      this.updateBucketChecksums(currentState);
-    };
-
     while (!this.signal?.aborted) {
       // Query one batch at a time, to avoid cursor timeouts
       const cursor = this.db.bucket_data.aggregate<BucketDataDocument & { size: number | bigint }>(
@@ -184,7 +181,11 @@ export class MongoCompactor {
             }
           }
         ],
-        { batchSize: this.moveBatchQueryLimit }
+        {
+          // batchSize is 1 more than limit to auto-close the cursor.
+          // See https://github.com/mongodb/node-mongodb-native/pull/4580
+          batchSize: this.moveBatchQueryLimit + 1
+        }
       );
       // We don't limit to a single batch here, since that often causes MongoDB to scan through more than it returns.
       // Instead, we load up to the limit.
@@ -199,22 +200,6 @@ export class MongoCompactor {
       upperBound = batch[batch.length - 1]._id;
 
       for (let doc of batch) {
-        if (currentState == null || doc._id.b != currentState.bucket) {
-          await doneWithBucket();
-
-          currentState = {
-            bucket: doc._id.b,
-            seen: new Map(),
-            trackingSize: 0,
-            lastNotPut: null,
-            opsSincePut: 0,
-
-            checksum: 0,
-            opCount: 0,
-            opBytes: 0
-          };
-        }
-
         if (doc._id.o > this.maxOpId) {
           continue;
         }
@@ -285,12 +270,22 @@ export class MongoCompactor {
         }
       }
 
-      if (currentState != null) {
-        logger.info(`Processed batch of length ${batch.length} current bucket: ${currentState.bucket}`);
-      }
+      logger.info(`Processed batch of length ${batch.length} current bucket: ${bucket}`);
+    }
+
+    // Free memory before clearing bucket
+    currentState.seen.clear();
+    if (currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
+      logger.info(
+        `Inserting CLEAR at ${this.group_id}:${bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
+      );
+      // Need flush() before clear()
+      await this.flush();
+      await this.clearBucket(currentState);
     }
 
-    await doneWithBucket();
+    // Do this _after_ clearBucket so that we have accurate counts.
+    this.updateBucketChecksums(currentState);
 
     // Need another flush after updateBucketChecksums()
     await this.flush();
@@ -478,50 +473,55 @@ export class MongoCompactor {
   /**
    * Subset of compact, only populating checksums where relevant.
    */
-  async populateChecksums() {
-    // This is updated after each batch
-    let lowerBound: BucketStateDocument['_id'] = {
-      g: this.group_id,
-      b: new mongo.MinKey() as any
-    };
-    // This is static
-    const upperBound: BucketStateDocument['_id'] = {
-      g: this.group_id,
-      b: new mongo.MaxKey() as any
-    };
+  async populateChecksums(options: { minBucketChanges: number }): Promise<PopulateChecksumCacheResults> {
+    let count = 0;
     while (!this.signal?.aborted) {
-      // By filtering buckets, we effectively make this "resumeable".
-      const filter: mongo.Filter<BucketStateDocument> = {
-        _id: {
-          $gt: lowerBound,
-          $lt: upperBound
-        },
-        compacted_state: { $exists: false }
-      };
+      const buckets = await this.dirtyBucketBatch(options);
+      if (buckets.length == 0) {
+        // All done
+        break;
+      }
+      const start = Date.now();
+      logger.info(`Calculating checksums for batch of ${buckets.length} buckets, starting at ${buckets[0]}`);
+
+      await this.updateChecksumsBatch(buckets);
+      logger.info(`Updated checksums for batch of ${buckets.length} buckets in ${Date.now() - start}ms`);
+      count += buckets.length;
+    }
+    return { buckets: count };
+  }
 
-      const bucketsWithoutChecksums = await this.db.bucket_state
-        .find(filter, {
+  /**
+   * Returns a batch of dirty buckets - buckets with most changes first.
+   *
+   * This cannot be used to iterate on its own - the client is expected to process these buckets and
+   * set estimate_since_compact.count: 0 when done, before fetching the next batch.
+   */
+  private async dirtyBucketBatch(options: { minBucketChanges: number }): Promise<string[]> {
+    if (options.minBucketChanges <= 0) {
+      throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+    }
+    // We make use of an index on {_id.g: 1, 'estimate_since_compact.count': -1}
+    const dirtyBuckets = await this.db.bucket_state
+      .find(
+        {
+          '_id.g': this.group_id,
+          'estimate_since_compact.count': { $gte: options.minBucketChanges }
+        },
+        {
           projection: {
             _id: 1
           },
           sort: {
-            _id: 1
+            'estimate_since_compact.count': -1
           },
           limit: 5_000,
           maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
-        })
-        .toArray();
-      if (bucketsWithoutChecksums.length == 0) {
-        // All done
-        break;
-      }
-
-      logger.info(`Calculating checksums for batch of ${bucketsWithoutChecksums.length} buckets`);
-
-      await this.updateChecksumsBatch(bucketsWithoutChecksums.map((b) => b._id.b));
+        }
+      )
+      .toArray();
 
-      lowerBound = bucketsWithoutChecksums[bucketsWithoutChecksums.length - 1]._id;
-    }
+    return dirtyBuckets.map((bucket) => bucket._id.b);
   }
 
   private async updateChecksumsBatch(buckets: string[]) {
@@ -555,6 +555,10 @@ export class MongoCompactor {
                 count: bucketChecksum.count,
                 checksum: BigInt(bucketChecksum.checksum),
                 bytes: null
+              },
+              estimate_since_compact: {
+                count: 0,
+                bytes: 0
               }
             }
           },

package/src/storage/implementation/MongoSyncBucketStorage.ts

@@ -16,6 +16,8 @@ import {
   InternalOpId,
   internalToExternalOpId,
   maxLsn,
+  PopulateChecksumCacheOptions,
+  PopulateChecksumCacheResults,
   ProtocolOpId,
   ReplicationCheckpoint,
   storage,
@@ -403,7 +405,9 @@ export class MongoSyncBucketStorage
         limit: batchLimit,
         // Increase batch size above the default 101, so that we can fill an entire batch in
         // one go.
-        batchSize: batchLimit,
+        // batchSize is 1 more than limit to auto-close the cursor.
+        // See https://github.com/mongodb/node-mongodb-native/pull/4580
+        batchSize: batchLimit + 1,
         // Raw mode is returns an array of Buffer instead of parsed documents.
         // We use it so that:
         // 1. We can calculate the document size accurately without serializing again.
@@ -663,7 +667,7 @@ export class MongoSyncBucketStorage
     }
   }
 
-  async populatePersistentChecksumCache(options: Required<Pick<CompactOptions, 'signal' | 'maxOpId'>>): Promise<void> {
+  async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
     logger.info(`Populating persistent checksum cache...`);
     const start = Date.now();
     // We do a minimal compact here.
@@ -674,9 +678,14 @@ export class MongoSyncBucketStorage
       memoryLimitMB: 0
     });
 
-    await compactor.populateChecksums();
+    const result = await compactor.populateChecksums({
+      // There are cases with millions of small buckets, in which case it can take very long to
+      // populate the checksums, with minimal benefit. We skip the small buckets here.
+      minBucketChanges: options.minBucketChanges ?? 10
+    });
     const duration = Date.now() - start;
     logger.info(`Populated persistent checksum cache in ${(duration / 1000).toFixed(1)}s`);
+    return result;
   }
 
   /**
@@ -905,7 +914,9 @@ export class MongoSyncBucketStorage
             '_id.b': 1
           },
           limit: limit + 1,
-          batchSize: limit + 1,
+          // batchSize is 1 more than limit to auto-close the cursor.
+          // See https://github.com/mongodb/node-mongodb-native/pull/4580
+          batchSize: limit + 2,
           singleBatch: true
         }
       )
@@ -935,7 +946,9 @@ export class MongoSyncBucketStorage
             lookup: 1
           },
           limit: limit + 1,
-          batchSize: limit + 1,
+          // batchSize is 1 more than limit to auto-close the cursor.
+          // See https://github.com/mongodb/node-mongodb-native/pull/4580
+          batchSize: limit + 2,
           singleBatch: true
         }
       )

package/src/storage/implementation/MongoTestStorageFactoryGenerator.ts

@@ -25,6 +25,7 @@ export const MongoTestStorageFactoryGenerator = (factoryOptions: MongoTestStorag
     // Full migrations are not currently run for tests, so we manually create the important ones
     await db.createCheckpointEventsCollection();
     await db.createBucketStateIndex();
+    await db.createBucketStateIndex2();
 
     return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }, factoryOptions.internalOptions);
   };

package/src/storage/implementation/MongoWriteCheckpointAPI.ts

@@ -111,7 +111,9 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
         },
         {
           limit: limit + 1,
-          batchSize: limit + 1,
+          // batchSize is 1 more than limit to auto-close the cursor.
+          // See https://github.com/mongodb/node-mongodb-native/pull/4580
+          batchSize: limit + 2,
           singleBatch: true
         }
       )
@@ -140,7 +142,9 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
         },
         {
           limit: limit + 1,
-          batchSize: limit + 1,
+          // batchSize is 1 more than limit to auto-close the cursor.
+          // See https://github.com/mongodb/node-mongodb-native/pull/4580
+          batchSize: limit + 2,
           singleBatch: true
         }
       )

package/src/storage/implementation/db.ts

@@ -141,6 +141,19 @@ export class PowerSyncMongo {
       { name: 'bucket_updates', unique: true }
     );
   }
+  /**
+   * Only use in migrations and tests.
+   */
+  async createBucketStateIndex2() {
+    // TODO: Implement a better mechanism to use migrations in tests
+    await this.bucket_state.createIndex(
+      {
+        '_id.g': 1,
+        'estimate_since_compact.count': -1
+      },
+      { name: 'dirty_count' }
+    );
+  }
 }
 
 export function createPowerSyncMongo(config: MongoStorageConfig, options?: lib_mongo.MongoConnectionOptions) {

package/src/storage/implementation/util.ts

@@ -41,7 +41,7 @@ export function generateSlotName(prefix: string, sync_rules_id: number) {
  * However, that makes `has_more` detection very difficult, since the cursor is always closed
  * after the first batch. Instead, we do a workaround to only fetch a single batch below.
  *
- * For this to be effective, set batchSize = limit in the find command.
+ * For this to be effective, set batchSize = limit + 1 in the find command.
  */
 export async function readSingleBatch<T>(cursor: mongo.AbstractCursor<T>): Promise<{ data: T[]; hasMore: boolean }> {
   try {

package/test/src/storage_compacting.test.ts

@@ -97,10 +97,25 @@ bucket_definitions:
       await populate(bucketStorage);
       const { checkpoint } = await bucketStorage.getCheckpoint();
 
-      await bucketStorage.populatePersistentChecksumCache({
+      // Default is to small small numbers - should be a no-op
+      const result0 = await bucketStorage.populatePersistentChecksumCache({
+        maxOpId: checkpoint
+      });
+      expect(result0.buckets).toEqual(0);
+
+      // This should cache the checksums for the two buckets
+      const result1 = await bucketStorage.populatePersistentChecksumCache({
+        maxOpId: checkpoint,
+        minBucketChanges: 1
+      });
+      expect(result1.buckets).toEqual(2);
+
+      // This should be a no-op, as the checksums are already cached
+      const result2 = await bucketStorage.populatePersistentChecksumCache({
         maxOpId: checkpoint,
-        signal: new AbortController().signal
+        minBucketChanges: 1
       });
+      expect(result2.buckets).toEqual(0);
 
       const checksumAfter = await bucketStorage.getChecksums(checkpoint, ['by_user2["u1"]', 'by_user2["u2"]']);
       expect(checksumAfter.get('by_user2["u1"]')).toEqual({