@powersync/service-module-mongodb-storage 0.0.0-dev-20260225160713 → 0.0.0-dev-20260511080634

package/dist/storage/implementation/db.js

@@ -1,6 +1,10 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
+import { DO_NOT_LOG } from '@powersync/lib-services-framework';
+import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js';
+import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
 export class PowerSyncMongo {
+    [DO_NOT_LOG] = true;
     current_data;
     bucket_data;
     bucket_parameters;
@@ -36,15 +40,90 @@ export class PowerSyncMongo {
         this.checkpoint_events = this.db.collection('checkpoint_events');
         this.connection_report_events = this.db.collection('connection_report_events');
     }
+    versioned(storageConfig) {
+        if (storageConfig.incrementalReprocessing) {
+            return new VersionedPowerSyncMongoV3(this, storageConfig);
+        }
+        return new VersionedPowerSyncMongoV1(this, storageConfig);
+    }
+    /**
+     * Not safe for user-provided prefix - only for hardcoded values.
+     */
+    async listBucketDataCollectionsV3(groupId) {
+        const prefix = groupId == null ? 'bucket_data_' : `bucket_data_${groupId}_`;
+        const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+        return collections
+            .filter((collection) => collection.name.startsWith(prefix))
+            .map((collection) => this.db.collection(collection.name));
+    }
+    /**
+     * Not safe for user-provided prefix - only for hardcoded values.
+     */
+    async collectionsByPrefix(prefix) {
+        const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
+        return collections
+            .filter((collection) => collection.name.startsWith(prefix))
+            .map((collection) => this.db.collection(collection.name));
+    }
+    /**
+     * List all parameter index collections across all replication streams.
+     *
+     * Primarily used to clear the db.
+     */
+    async listAllParameterIndexCollectionsV3() {
+        return this.collectionsByPrefix(`parameter_index_`);
+    }
+    /**
+     * List all parameter index collections across all replication streams.
+     *
+     * Primarily used to clear the db.
+     */
+    async listAllSourceRecordCollectionsV3() {
+        return this.collectionsByPrefix(`source_records_`);
+    }
+    async listAllBucketStateCollectionsV3() {
+        return this.collectionsByPrefix(`bucket_state_`);
+    }
+    sourceRecordsCollectionName(replicationStreamId, sourceTableId) {
+        return `source_records_${replicationStreamId}_${sourceTableId.toHexString()}`;
+    }
+    sourceTableCollectionName(replicationStreamId) {
+        return `source_table_${replicationStreamId}`;
+    }
+    async listSourceTableCollections(replicationStreamId) {
+        const filter = replicationStreamId == null
+            ? { name: new RegExp('^source_table_') }
+            : { name: this.sourceTableCollectionName(replicationStreamId) };
+        const prefix = replicationStreamId == null ? 'source_table_' : this.sourceTableCollectionName(replicationStreamId);
+        const collections = await this.db.listCollections(filter, { nameOnly: true }).toArray();
+        return collections
+            .filter((collection) => collection.name.startsWith(prefix))
+            .map((collection) => this.db.collection(collection.name));
+    }
     /**
      * Clear all collections.
      */
     async clear() {
         await this.current_data.deleteMany({});
+        for (const collection of await this.listAllSourceRecordCollectionsV3()) {
+            await collection.drop();
+        }
         await this.bucket_data.deleteMany({});
+        for (const collection of await this.listBucketDataCollectionsV3()) {
+            await collection.drop();
+        }
         await this.bucket_parameters.deleteMany({});
+        for (const collection of await this.listAllParameterIndexCollectionsV3()) {
+            await collection.drop();
+        }
+        for (const collection of await this.listAllBucketStateCollectionsV3()) {
+            await collection.drop();
+        }
         await this.op_id_sequence.deleteMany({});
         await this.sync_rules.deleteMany({});
+        for (const collection of await this.listSourceTableCollections()) {
+            await collection.drop();
+        }
         await this.source_tables.deleteMany({});
         await this.write_checkpoints.deleteMany({});
         await this.instance.deleteOne({});
@@ -61,7 +140,7 @@ export class PowerSyncMongo {
         await this.db.dropDatabase();
     }
     /**
-     * Call this after every checkpoint or sync rules status update. Rather call too often than too rarely.
+     * Call this after every checkpoint or replication stream status update. Rather call too often than too rarely.
      *
      * This is used in a similar way to the Postgres NOTIFY functionality.
      */

package/dist/storage/implementation/db.js.map

@@ -1 +1 @@
-{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAyBrE,MAAM,OAAO,cAAc;IAChB,YAAY,CAAwC;IACpD,WAAW,CAAuC;IAClD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAqC;IAC/C,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAC9C,YAAY,CAAwC;IACpD,iBAAiB,CAA4C;IAC7D,wBAAwB,CAA6C;IAErE,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAEtB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAsB,cAAc,CAAC,CAAC;QACvE,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QACjE,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;IACjF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAS,EAAE,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC;IACnF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gCAAgC;QACpC,6FAA6F;QAC7F,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aACnE,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBAChC,oEAAoE;gBACpE,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,iDAAiD;gBACjD,OAAO;YACT,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,EAAE;YAClD,MAAM,EAAE,IAAI;YACZ,sFAAsF;YACtF,mFAAmF;YACnF,iFAAiF;YACjF,4BAA4B;YAC5B,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,gBAAgB;YACjC,GAAG,EAAE,EAAE,CAAC,0BAA0B;SACnC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mCAAmC;QACvC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aAC1E,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QACD,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,0BAA0B,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QAC1B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,CACzC,CAAC;IACJ,CAAC;IACD;;OAEG;IACH,KAAK,CAAC,uBAAuB;QAC3B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,8BAA8B,EAAE,CAAC,CAAC;SACnC,EACD,EAAE,IAAI,EAAE,aAAa,EAAE,CACxB,CAAC;IACJ,CAAC;CACF;AAED,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,OAA0C;IACzG,OAAO,IAAI,cAAc,CACvB,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAClC,gBAAgB,EAAE,iBAAiB;QACnC,GAAG,OAAO;KACX,CAAC,EACF,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAC9B,CAAC;AACJ,CAAC"}
+{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAqB/D,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAE9E,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAS9E,MAAM,OAAO,cAAc;IACzB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAEX,YAAY,CAAwC;IACpD,WAAW,CAAyC;IACpD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAqC;IAC/C,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAC9C,YAAY,CAA0C;IACtD,iBAAiB,CAA4C;IAC7D,wBAAwB,CAA6C;IAErE,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAEtB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QACjE,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;IACjF,CAAC;IAED,SAAS,CAAC,aAA4B;QACpC,IAAI,aAAa,CAAC,uBAAuB,EAAE,CAAC;YAC1C,OAAO,IAAI,yBAAyB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,yBAAyB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAC5D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,2BAA2B,CAAC,OAAgB;QAChD,MAAM,MAAM,GAAG,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,OAAO,GAAG,CAAC;QAC5E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAEpH,OAAO,WAAW;aACf,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAuB,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACpF,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAC,MAAc;QAC9C,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAEpH,OAAO,WAAW;aACf,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAQ,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACrE,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kCAAkC;QACtC,OAAO,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,CAAC;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gCAAgC;QACpC,OAAO,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,+BAA+B;QACnC,OAAO,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IACnD,CAAC;IAED,2BAA2B,CAAC,mBAA2B,EAAE,aAA6B;QACpF,OAAO,kBAAkB,mBAAmB,IAAI,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC;IAChF,CAAC;IAED,yBAAyB,CAAC,mBAA2B;QACnD,OAAO,gBAAgB,mBAAmB,EAAE,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,0BAA0B,CAC9B,mBAA4B;QAE5B,MAAM,MAAM,GACV,mBAAmB,IAAI,IAAI;YACzB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,gBAAgB,CAAC,EAAE;YACxC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,EAAE,CAAC;QACpE,MAAM,MAAM,GAAG,mBAAmB,IAAI,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QACnH,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAExF,OAAO,WAAW;aACf,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAA4B,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACzF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,gCAAgC,EAAE,EAAE,CAAC;YACvE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,2BAA2B,EAAE,EAAE,CAAC;YAClE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,kCAAkC,EAAE,EAAE,CAAC;YACzE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,+BAA+B,EAAE,EAAE,CAAC;YACtE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,0BAA0B,EAAE,EAAE,CAAC;YACjE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAS,EAAE,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC,CAAC;IACnF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gCAAgC;QACpC,6FAA6F;QAC7F,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aACnE,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBAChC,oEAAoE;gBACpE,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,iDAAiD;gBACjD,OAAO;YACT,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,EAAE;YAClD,MAAM,EAAE,IAAI;YACZ,sFAAsF;YACtF,mFAAmF;YACnF,iFAAiF;YACjF,4BAA4B;YAC5B,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,gBAAgB;YACjC,GAAG,EAAE,EAAE,CAAC,0BAA0B;SACnC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mCAAmC;QACvC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aAC1E,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QACD,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,0BAA0B,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QAC1B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,CACzC,CAAC;IACJ,CAAC;IACD;;OAEG;IACH,KAAK,CAAC,uBAAuB;QAC3B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,8BAA8B,EAAE,CAAC,CAAC;SACnC,EACD,EAAE,IAAI,EAAE,aAAa,EAAE,CACxB,CAAC;IACJ,CAAC;CACF;AAOD,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,OAA0C;IACzG,OAAO,IAAI,cAAc,CACvB,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAClC,gBAAgB,EAAE,iBAAiB;QACnC,GAAG,OAAO;KACX,CAAC,EACF,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAC9B,CAAC;AACJ,CAAC"}

package/dist/storage/implementation/models.d.ts

@@ -1,7 +1,10 @@
 import { InternalOpId, SerializedSyncPlan, storage } from '@powersync/service-core';
 import { SqliteJsonValue } from '@powersync/service-sync-rules';
-import * as bson from 'bson';
 import { event_types } from '@powersync/service-types';
+import * as bson from 'bson';
+import { ParameterIndexId } from './BucketDefinitionMapping.js';
+import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js';
+import type { CurrentBucketV3, CurrentDataDocumentV3, RecordedLookupV3, SourceTableDocumentV3 } from './v3/models.js';
 /**
  * Replica id uniquely identifying a row on the source database.
  *
@@ -18,33 +21,38 @@ export interface SourceKey {
     /** source key */
     k: ReplicaId;
 }
+export interface SourceTableKey {
+    /** source table id */
+    t: bson.ObjectId;
+    /** source key */
+    k: ReplicaId;
+}
 export interface BucketDataKey {
-    /** group_id */
-    g: number;
     /** bucket name */
     b: string;
     /** op_id */
     o: bigint;
 }
-export interface CurrentDataDocument {
-    _id: SourceKey;
-    data: bson.Binary;
-    buckets: CurrentBucket[];
-    lookups: bson.Binary[];
-}
 export interface CurrentBucket {
     bucket: string;
     table: string;
     id: string;
 }
-export interface BucketParameterDocument {
+export interface BucketParameterDocumentBase<TKey> {
     _id: bigint;
-    key: SourceKey;
+    key: TKey;
     lookup: bson.Binary;
     bucket_parameters: Record<string, SqliteJsonValue>[];
 }
-export interface BucketDataDocument {
-    _id: BucketDataKey;
+export interface TaggedBucketParameterDocument extends BucketParameterDocumentBase<SourceKey | SourceTableKey> {
+    index: ParameterIndexId;
+}
+export declare function bucketParameterDocumentToTagged<TKey extends SourceKey | SourceTableKey>(document: BucketParameterDocumentBase<TKey>, index: ParameterIndexId): TaggedBucketParameterDocument;
+export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
+/**
+ * Common properties for storage V1, storage V3 and in-memory BucketDataDoc.
+ */
+export interface BucketDataProperties {
     op: OpType;
     source_table?: bson.ObjectId;
     source_key?: ReplicaId;
@@ -54,10 +62,21 @@ export interface BucketDataDocument {
     data: string | null;
     target_op?: bigint | null;
 }
-export type OpType = 'PUT' | 'REMOVE' | 'MOVE' | 'CLEAR';
+export interface BucketDataDocumentBase extends BucketDataProperties {
+    _id: {
+        b: string;
+    };
+}
+/**
+ * Internal-only tag used for v1 bucket_data rows before they are converted to the v1 on-disk shape.
+ */
+export declare const LEGACY_BUCKET_DATA_DEFINITION_ID = "0";
+/**
+ * Internal-only tag used for v1 bucket_parameters rows before they are converted to the v1 on-disk shape.
+ */
+export declare const LEGACY_BUCKET_PARAMETER_INDEX_ID = "0";
 export interface SourceTableDocument {
     _id: bson.ObjectId;
-    group_id: number;
     connection_id: number;
     relation_id: number | string | undefined;
     schema_name: string;
@@ -79,20 +98,21 @@ export interface SourceTableDocumentSnapshotStatus {
 /**
  * Record the state of each bucket.
  *
- * Right now, this is just used to track when buckets are updated, for efficient incremental sync.
- * In the future, this could be used to track operation counts, both for diagnostic purposes, and for
- * determining when a compact and/or defragment could be beneficial.
+ * The primary use case is to track when buckets are updated, for efficient incremental sync.
  *
- * Note: There is currently no migration to populate this collection from existing data - it is only
+ * The secondary use case is to track operation counts to determine whether or not a bucket should be compacted.
+ *
+ * Note: For storage V1, there is no migration to populate this collection from existing data - it is only
  * populated by new updates.
+ *
+ * For storage V3, these will always be present.
  */
-export interface BucketStateDocument {
+export interface BucketStateDocumentBase {
     _id: {
-        g: number;
         b: string;
     };
     /**
-     * Important: There is an unique index on {'_id.g': 1, last_op: 1}.
+     * Important: There is an unique index on last_op per logical stream.
      * That means the last_op must match an actual op in the bucket, and not the commit checkpoint.
      */
     last_op: bigint;
@@ -104,11 +124,11 @@ export interface BucketStateDocument {
         op_id: InternalOpId;
         count: number;
         checksum: bigint;
-        bytes: number | null;
+        bytes: number | bigint | null;
     };
     estimate_since_compact?: {
         count: number;
-        bytes: number;
+        bytes: number | bigint;
     };
 }
 export interface IdSequenceDocument {
@@ -177,6 +197,19 @@ export interface SyncRuleDocument {
     last_fatal_error_ts: Date | null;
     content: string;
     serialized_plan?: SerializedSyncPlan | null;
+    /**
+     * Required for V3+ storage.
+     */
+    rule_mapping?: {
+        /**
+         * Map of uniqueName -> id, unique per replication stream.
+         */
+        definitions: Record<string, string>;
+        /**
+         * Map of (lookupName, queryId) -> id, unique per replication stream.
+         */
+        parameter_indexes: Record<string, string>;
+    };
     lock?: {
         id: string;
         expires_at: Date;
@@ -191,8 +224,12 @@ export interface StorageConfig extends storage.StorageVersionConfig {
      * a Long before summing.
      */
     longChecksums: boolean;
+    /**
+     * Enables v3 MongoDB storage behavior used for incremental reprocessing.
+     */
+    incrementalReprocessing: boolean;
 }
-export declare function getMongoStorageConfig(storageVersion: number): StorageConfig | undefined;
+export declare function getMongoStorageConfig(storageVersion: number): StorageConfig;
 export interface CheckpointEventDocument {
     _id: bson.ObjectId;
 }
@@ -227,3 +264,7 @@ export interface InstanceDocument {
 }
 export interface ClientConnectionDocument extends event_types.ClientConnection {
 }
+export type CurrentDataDocumentId = CurrentDataDocument['_id'] | CurrentDataDocumentV3['_id'];
+export type CommonCurrentBucket = CurrentBucket | CurrentBucketV3;
+export type CommonCurrentLookup = bson.Binary | RecordedLookupV3;
+export type CommonSourceTableDocument = SourceTableDocumentV1 | SourceTableDocumentV3;

package/dist/storage/implementation/models.js

@@ -1,10 +1,30 @@
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import { storage } from '@powersync/service-core';
+export function bucketParameterDocumentToTagged(document, index) {
+    return {
+        ...document,
+        index
+    };
+}
+/**
+ * Internal-only tag used for v1 bucket_data rows before they are converted to the v1 on-disk shape.
+ */
+export const LEGACY_BUCKET_DATA_DEFINITION_ID = '0';
+/**
+ * Internal-only tag used for v1 bucket_parameters rows before they are converted to the v1 on-disk shape.
+ */
+export const LEGACY_BUCKET_PARAMETER_INDEX_ID = '0';
 const LONG_CHECKSUMS_STORAGE_VERSION = 2;
+const INCREMENTAL_REPROCESSING_STORAGE_VERSION = storage.STORAGE_VERSION_3;
 export function getMongoStorageConfig(storageVersion) {
     const baseConfig = storage.STORAGE_VERSION_CONFIG[storageVersion];
     if (baseConfig == null) {
-        return undefined;
+        throw new ServiceError(ErrorCode.PSYNC_S1005, `Unsupported storage version ${storageVersion}`);
     }
-    return { ...baseConfig, longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION };
+    return {
+        ...baseConfig,
+        longChecksums: storageVersion >= LONG_CHECKSUMS_STORAGE_VERSION,
+        incrementalReprocessing: storageVersion >= INCREMENTAL_REPROCESSING_STORAGE_VERSION
+    };
 }
 //# sourceMappingURL=models.js.map

package/dist/storage/implementation/models.js.map

@@ -1 +1 @@
-{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../src/storage/implementation/models.ts"],"names":[],"mappings":"AAAA,OAAO,EAAoC,OAAO,EAAE,MAAM,yBAAyB,CAAC;AA6NpF,MAAM,8BAA8B,GAAG,CAAC,CAAC;AAEzC,MAAM,UAAU,qBAAqB,CAAC,cAAsB;IAC1D,MAAM,UAAU,GAAG,OAAO,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;IAClE,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACvB,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,EAAE,GAAG,UAAU,EAAE,aAAa,EAAE,cAAc,IAAI,8BAA8B,EAAE,CAAC;AAC5F,CAAC"}
+{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../src/storage/implementation/models.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAoC,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAyDpF,MAAM,UAAU,+BAA+B,CAC7C,QAA2C,EAC3C,KAAuB;IAEvB,OAAO;QACL,GAAG,QAAQ;QACX,KAAK;KACN,CAAC;AACJ,CAAC;AAsBD;;GAEG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAC;AAEpD;;GAEG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAC;AA+KpD,MAAM,8BAA8B,GAAG,CAAC,CAAC;AACzC,MAAM,wCAAwC,GAAG,OAAO,CAAC,iBAAiB,CAAC;AAE3E,MAAM,UAAU,qBAAqB,CAAC,cAAsB;IAC1D,MAAM,UAAU,GAAG,OAAO,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;IAClE,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,+BAA+B,cAAc,EAAE,CAAC,CAAC;IACjG,CAAC;IAED,OAAO;QACL,GAAG,UAAU;QACb,aAAa,EAAE,cAAc,IAAI,8BAA8B;QAC/D,uBAAuB,EAAE,cAAc,IAAI,wCAAwC;KACpF,CAAC;AACJ,CAAC"}

package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts

@@ -0,0 +1,13 @@
+import { SourceTable } from '@powersync/service-core';
+import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
+import { PersistedBatch } from '../common/PersistedBatch.js';
+import { SourceRecordStore } from '../common/SourceRecordStore.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+export declare class MongoBucketBatchV1 extends MongoBucketBatch {
+    readonly db: VersionedPowerSyncMongoV1;
+    private readonly store;
+    constructor(options: MongoBucketBatchOptions);
+    protected createPersistedBatch(writtenSize: number): PersistedBatch;
+    protected get sourceRecordStore(): SourceRecordStore;
+    protected cleanupDroppedSourceTables(_tables: SourceTable[]): Promise<void>;
+}

package/dist/storage/implementation/v1/MongoBucketBatchV1.js

@@ -0,0 +1,22 @@
+import { MongoBucketBatch } from '../MongoBucketBatch.js';
+import { PersistedBatchV1 } from './PersistedBatchV1.js';
+import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
+export class MongoBucketBatchV1 extends MongoBucketBatch {
+    store;
+    constructor(options) {
+        super(options);
+        this.store = new SourceRecordStoreV1(this.db, this.group_id);
+    }
+    createPersistedBatch(writtenSize) {
+        return new PersistedBatchV1(this.db, this.group_id, this.mapping, writtenSize, {
+            logger: this.logger
+        });
+    }
+    get sourceRecordStore() {
+        return this.store;
+    }
+    async cleanupDroppedSourceTables(_tables) {
+        // No-op for V1: source records live in a shared collection.
+    }
+}
+//# sourceMappingURL=MongoBucketBatchV1.js.map

package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoBucketBatchV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoBucketBatchV1.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAA2B,MAAM,wBAAwB,CAAC;AAGnF,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAG/D,MAAM,OAAO,kBAAmB,SAAQ,gBAAgB;IAGrC,KAAK,CAAoB;IAE1C,YAAY,OAAgC;QAC1C,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,KAAK,GAAG,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC/D,CAAC;IAES,oBAAoB,CAAC,WAAmB;QAChD,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,WAAW,EAAE;YAC7E,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;IACL,CAAC;IAED,IAAc,iBAAiB;QAC7B,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;IAES,KAAK,CAAC,0BAA0B,CAAC,OAAsB;QAC/D,4DAA4D;IAC9D,CAAC;CACF"}

package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts

@@ -0,0 +1,12 @@
+import { BucketChecksum, FetchPartialBucketChecksum, InternalOpId, PartialChecksumMap } from '@powersync/service-core';
+import { FetchPartialBucketChecksumByBucket, MongoChecksums } from '../MongoChecksums.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+export declare class MongoChecksumsV1 extends MongoChecksums {
+    protected readonly db: VersionedPowerSyncMongoV1;
+    computePartialChecksumsDirectByBucket(batch: FetchPartialBucketChecksumByBucket[]): Promise<PartialChecksumMap>;
+    protected fetchPreStates(batch: FetchPartialBucketChecksum[]): Promise<Map<string, {
+        opId: InternalOpId;
+        checksum: BucketChecksum;
+    }>>;
+    protected computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap>;
+}

package/dist/storage/implementation/v1/MongoChecksumsV1.js

@@ -0,0 +1,56 @@
+import { bson } from '@powersync/service-core';
+import { MongoChecksums } from '../MongoChecksums.js';
+export class MongoChecksumsV1 extends MongoChecksums {
+    async computePartialChecksumsDirectByBucket(batch) {
+        return this.computePartialChecksumsForCollection(batch, this.db.bucketDataV1, (request) => ({
+            _id: {
+                $gt: {
+                    g: this.group_id,
+                    b: request.bucket,
+                    o: request.start ?? new bson.MinKey()
+                },
+                $lte: {
+                    g: this.group_id,
+                    b: request.bucket,
+                    o: request.end
+                }
+            }
+        }));
+    }
+    async fetchPreStates(batch) {
+        const preFilters = batch
+            .filter((request) => request.start == null)
+            .map((request) => ({
+            _id: {
+                g: this.group_id,
+                b: request.bucket
+            },
+            'compacted_state.op_id': { $exists: true, $lte: request.end }
+        }));
+        const preStates = new Map();
+        if (preFilters.length == 0) {
+            return preStates;
+        }
+        const states = await this.db.bucketStateV1
+            .find({
+            $or: preFilters
+        })
+            .toArray();
+        for (const state of states) {
+            const compactedState = state.compacted_state;
+            preStates.set(state._id.b, {
+                opId: compactedState.op_id,
+                checksum: {
+                    bucket: state._id.b,
+                    checksum: Number(compactedState.checksum),
+                    count: compactedState.count
+                }
+            });
+        }
+        return preStates;
+    }
+    async computePartialChecksumsInternal(batch) {
+        return this.computePartialChecksumsDirectByBucket(batch);
+    }
+}
+//# sourceMappingURL=MongoChecksumsV1.js.map

package/dist/storage/implementation/v1/MongoChecksumsV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoChecksumsV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoChecksumsV1.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,IAAI,EAKL,MAAM,yBAAyB,CAAC;AACjC,OAAO,EAAsC,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAG1F,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAGlD,KAAK,CAAC,qCAAqC,CACzC,KAA2C;QAE3C,OAAO,IAAI,CAAC,oCAAoC,CAAC,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YAC1F,GAAG,EAAE;gBACH,GAAG,EAAE;oBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;oBACjB,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE;iBACtC;gBACD,IAAI,EAAE;oBACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;oBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;oBACjB,CAAC,EAAE,OAAO,CAAC,GAAG;iBACf;aACF;SACF,CAAC,CAAC,CAAC;IACN,CAAC;IAES,KAAK,CAAC,cAAc,CAC5B,KAAmC;QAEnC,MAAM,UAAU,GAAG,KAAK;aACrB,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC;aAC1C,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;YACjB,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;aAClB;YACD,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE;SAC9D,CAAC,CAAC,CAAC;QAEN,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4D,CAAC;QACtF,IAAI,UAAU,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YAC3B,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa;aACvC,IAAI,CAAC;YACJ,GAAG,EAAE,UAAU;SAChB,CAAC;aACD,OAAO,EAAE,CAAC;QAEb,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,cAAc,GAAG,KAAK,CAAC,eAAgB,CAAC;YAC9C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE;gBACzB,IAAI,EAAE,cAAc,CAAC,KAAK;gBAC1B,QAAQ,EAAE;oBACR,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;oBACnB,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;oBACzC,KAAK,EAAE,cAAc,CAAC,KAAK;iBAC5B;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAES,KAAK,CAAC,+BAA+B,CAAC,KAAmC;QACjF,OAAO,IAAI,CAAC,qCAAqC,CAAC,KAAK,CAAC,CAAC;IAC3D,CAAC;CACF"}

package/dist/storage/implementation/v1/MongoCompactorV1.d.ts

@@ -0,0 +1,23 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { storage } from '@powersync/service-core';
+import { BucketDefinitionId } from '../BucketDefinitionMapping.js';
+import { SingleBucketStore } from '../common/SingleBucketStore.js';
+import { BucketStateDocumentBase } from '../models.js';
+import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js';
+import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+export declare class MongoCompactorV1 extends MongoCompactor {
+    protected readonly db: VersionedPowerSyncMongoV1;
+    protected readonly storage: MongoSyncBucketStorageV1;
+    dirtyBucketBatches(options: {
+        minBucketChanges: number;
+        minChangeRatio: number;
+    }): AsyncGenerator<DirtyBucket[]>;
+    dirtyBucketBatchForChecksums(options: {
+        minBucketChanges: number;
+    }): Promise<DirtyBucket[]>;
+    protected writeBucketStateUpdates(): Promise<void>;
+    protected computeChecksumsForBuckets(buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]): Promise<storage.PartialChecksumMap>;
+    protected bucketStateFilter(bucket: string, _definitionId: BucketDefinitionId | null): mongo.Filter<BucketStateDocumentBase>;
+    protected getBucketDataContext(bucket: string, _definitionId: BucketDefinitionId | null): Promise<SingleBucketStore | null>;
+}

package/dist/storage/implementation/v1/MongoCompactorV1.js

@@ -0,0 +1,52 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
+import { MongoCompactor } from '../MongoCompactor.js';
+import { SingleBucketStoreV1 } from './SingleBucketStoreV1.js';
+export class MongoCompactorV1 extends MongoCompactor {
+    async *dirtyBucketBatches(options) {
+        if (options.minBucketChanges <= 0) {
+            throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+        }
+        // Previously, we used an index on {_id.g: 1, estimate_since_compact.count: 1} to only scan buckets with changes.
+        // That works well if there are only a small number of dirty buckets, but it causes repeated rescans while data is
+        // still changing. We now iterate through all V1 bucket_state rows for the group and filter after projecting.
+        yield* this.dirtyBucketBatchesForCollection(this.db.bucketStateV1, { g: this.group_id, b: new mongo.MinKey() }, { g: this.group_id, b: new mongo.MaxKey() }, options, () => null);
+    }
+    async dirtyBucketBatchForChecksums(options) {
+        if (options.minBucketChanges <= 0) {
+            throw new ReplicationAssertionError('minBucketChanges must be >= 1');
+        }
+        // Unlike dirtyBucketBatches, this path is resumable after restart because populateChecksums resets
+        // estimate_since_compact as it progresses.
+        return this.dirtyBucketBatchForChecksumsForCollection(this.db.bucketStateV1, {
+            '_id.g': this.group_id,
+            'estimate_since_compact.count': { $gte: options.minBucketChanges }
+        }, () => null);
+    }
+    async writeBucketStateUpdates() {
+        await this.db.bucketStateV1.bulkWrite(this.bucketStateUpdates, { ordered: false });
+    }
+    async computeChecksumsForBuckets(buckets) {
+        return this.storage.checksums.computePartialChecksumsDirectByBucket(buckets.map(({ bucket }) => ({
+            bucket,
+            end: this.maxOpId
+        })));
+    }
+    bucketStateFilter(bucket, _definitionId) {
+        return {
+            _id: {
+                g: this.group_id,
+                b: bucket
+            }
+        };
+    }
+    async getBucketDataContext(bucket, _definitionId) {
+        return new SingleBucketStoreV1(this.db, {
+            replicationStreamId: this.group_id,
+            definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID,
+            bucket
+        });
+    }
+}
+//# sourceMappingURL=MongoCompactorV1.js.map

package/dist/storage/implementation/v1/MongoCompactorV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoCompactorV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AACvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAI9E,OAAO,EAA2B,gCAAgC,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAe,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAGnE,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAG/D,MAAM,OAAO,gBAAiB,SAAQ,cAAc;IAK3C,KAAK,CAAC,CAAC,kBAAkB,CAAC,OAGhC;QACC,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,iHAAiH;QACjH,kHAAkH;QAClH,6GAA6G;QAC7G,KAAK,CAAC,CAAC,IAAI,CAAC,+BAA+B,CACzC,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,EAAE,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAS,EAAE,EAClD,OAAO,EACP,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAEM,KAAK,CAAC,4BAA4B,CAAC,OAAqC;QAC7E,IAAI,OAAO,CAAC,gBAAgB,IAAI,CAAC,EAAE,CAAC;YAClC,MAAM,IAAI,yBAAyB,CAAC,+BAA+B,CAAC,CAAC;QACvE,CAAC;QACD,mGAAmG;QACnG,2CAA2C;QAC3C,OAAO,IAAI,CAAC,yCAAyC,CACnD,IAAI,CAAC,EAAE,CAAC,aAAa,EACrB;YACE,OAAO,EAAE,IAAI,CAAC,QAAQ;YACtB,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,gBAAgB,EAAE;SACnE,EACD,GAAG,EAAE,CAAC,IAAI,CACX,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,uBAAuB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,CACnC,IAAI,CAAC,kBAA0E,EAC/E,EAAE,OAAO,EAAE,KAAK,EAAE,CACnB,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,0BAA0B,CACxC,OAAuD;QAEvD,OAAO,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,qCAAqC,CACjE,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;YAC3B,MAAM;YACN,GAAG,EAAE,IAAI,CAAC,OAAO;SAClB,CAAC,CAAC,CACJ,CAAC;IACJ,CAAC;IAES,iBAAiB,CACzB,MAAc,EACd,aAAwC;QAExC,OAAO;YACL,GAAG,EAAE;gBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;gBAChB,CAAC,EAAE,MAAM;aACV;SACF,CAAC;IACJ,CAAC;IAES,KAAK,CAAC,oBAAoB,CAClC,MAAc,EACd,aAAwC;QAExC,OAAO,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,EAAE;YACtC,mBAAmB,EAAE,IAAI,CAAC,QAAQ;YAClC,YAAY,EAAE,gCAAgC;YAC9C,MAAM;SACP,CAAC,CAAC;IACL,CAAC;CACF"}

package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts

@@ -0,0 +1,9 @@
+import { mongo } from '@powersync/lib-service-mongodb';
+import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+export declare class MongoParameterCompactorV1 extends MongoParameterCompactor {
+    protected readonly db: VersionedPowerSyncMongoV1;
+    protected getCollections(): Promise<mongo.Collection<mongo.Document>[]>;
+    protected collectionFilter(): mongo.Document;
+    protected deleteFilter(doc: mongo.Document): mongo.Document;
+}

package/dist/storage/implementation/v1/MongoParameterCompactorV1.js

@@ -0,0 +1,20 @@
+import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
+export class MongoParameterCompactorV1 extends MongoParameterCompactor {
+    async getCollections() {
+        return [this.db.parameterIndexV1];
+    }
+    collectionFilter() {
+        return {
+            'key.g': this.group_id
+        };
+    }
+    deleteFilter(doc) {
+        return {
+            'key.g': doc.key.g,
+            lookup: doc.lookup,
+            _id: { $lte: doc._id },
+            key: doc.key
+        };
+    }
+}
+//# sourceMappingURL=MongoParameterCompactorV1.js.map

package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"MongoParameterCompactorV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/MongoParameterCompactorV1.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,uBAAuB,EAAE,MAAM,+BAA+B,CAAC;AAGxE,MAAM,OAAO,yBAA0B,SAAQ,uBAAuB;IAG1D,KAAK,CAAC,cAAc;QAC5B,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,gBAA+D,CAAC,CAAC;IACnF,CAAC;IAES,gBAAgB;QACxB,OAAO;YACL,OAAO,EAAE,IAAI,CAAC,QAAQ;SACvB,CAAC;IACJ,CAAC;IAES,YAAY,CAAC,GAAmB;QACxC,OAAO;YACL,OAAO,EAAE,GAAG,CAAC,GAAG,CAAC,CAAW;YAC5B,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,GAAG,EAAE,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE;YACtB,GAAG,EAAE,GAAG,CAAC,GAAG;SACb,CAAC;IACJ,CAAC;CACF"}

package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts

@@ -0,0 +1,41 @@
+import { CheckpointChanges, GetCheckpointChangesOptions, InternalOpId, storage, utils } from '@powersync/service-core';
+import { ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules';
+import * as bson from 'bson';
+import { MongoBucketStorage } from '../../MongoBucketStorage.js';
+import { MongoSyncBucketStorageCheckpoint, MongoSyncBucketStorageContext } from '../common/MongoSyncBucketStorageContext.js';
+import { CommonSourceTableDocument } from '../models.js';
+import { MongoBucketBatchOptions } from '../MongoBucketBatch.js';
+import { MongoChecksums } from '../MongoChecksums.js';
+import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
+import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
+import { MongoPersistedSyncRulesContent } from '../MongoPersistedSyncRulesContent.js';
+import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
+import { MongoChecksumsV1 } from './MongoChecksumsV1.js';
+import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
+export declare class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
+    readonly db: VersionedPowerSyncMongoV1;
+    readonly checksums: MongoChecksumsV1;
+    constructor(factory: MongoBucketStorage, group_id: number, sync_rules: MongoPersistedSyncRulesContent, slot_name: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions);
+    protected initializeVersionStorage(): Promise<void>;
+    protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch;
+    protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums;
+    createMongoCompactor(options: MongoCompactOptions): MongoCompactor;
+    protected createMongoParameterCompactor(checkpoint: InternalOpId, options: storage.CompactOptions): MongoParameterCompactor;
+    protected sourceTableBaseId(): Partial<CommonSourceTableDocument>;
+    protected augmentCreatedSourceTableDocument(_createDoc: CommonSourceTableDocument, _options: storage.ResolveTableOptions, _candidateSourceTable: storage.SourceTable): void;
+    protected initializeResolvedSourceRecords(_sourceTableId: bson.ObjectId): Promise<void>;
+    protected get versionContext(): MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>;
+    protected getParameterSetsImpl(checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[], limit: number): Promise<SqliteJsonRow[]>;
+    protected getBucketDataBatchImpl(checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
+    protected clearBucketData(signal?: AbortSignal): Promise<void>;
+    protected clearParameterIndexes(signal?: AbortSignal): Promise<void>;
+    protected clearSourceRecords(signal?: AbortSignal): Promise<void>;
+    protected clearBucketState(signal?: AbortSignal): Promise<void>;
+    protected clearSourceTables(signal?: AbortSignal): Promise<void>;
+    protected getDataBucketChangesImpl(options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>>;
+    protected getParameterBucketChangesImpl(options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;
+}
+export declare function getParameterSetsV1(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>, checkpoint: MongoSyncBucketStorageCheckpoint, lookups: ScopedParameterLookup[], limit: number): Promise<SqliteJsonRow[]>;
+export declare function getBucketDataBatchV1(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>, checkpoint: utils.InternalOpId, dataBuckets: storage.BucketDataRequest[], options?: storage.BucketDataBatchOptions): AsyncIterable<storage.SyncBucketDataChunk>;
+export declare function getDataBucketChangesV1(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>, options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>>;
+export declare function getParameterBucketChangesV1(ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV1>, options: GetCheckpointChangesOptions): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;