@powersync/service-module-mongodb-storage 0.15.3 → 0.16.0

package/dist/storage/implementation/MongoChecksums.d.ts

@@ -1,6 +1,19 @@
-import { BucketChecksumRequest, ChecksumMap, FetchPartialBucketChecksum, InternalOpId, PartialChecksumMap } from '@powersync/service-core';
-import { VersionedPowerSyncMongo } from './db.js';
-import { StorageConfig } from './models.js';
+import { BucketChecksum, BucketChecksumRequest, ChecksumMap, FetchPartialBucketChecksum, InternalOpId, PartialChecksumMap, PartialOrFullChecksum } from '@powersync/service-core';
+import type { VersionedPowerSyncMongo } from './db.js';
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { BucketDefinitionId, BucketDefinitionMapping } from './BucketDefinitionMapping.js';
+import { BucketDataDocumentBase, StorageConfig } from './models.js';
+export interface FetchPartialBucketChecksumV3 {
+    bucket: string;
+    definitionId: BucketDefinitionId;
+    start?: InternalOpId;
+    end: InternalOpId;
+}
+export interface FetchPartialBucketChecksumByBucket {
+    bucket: string;
+    start?: InternalOpId;
+    end: InternalOpId;
+}
 /**
  * Checksum calculation options, primarily for tests.
  */
@@ -14,20 +27,12 @@ export interface MongoChecksumOptions {
      */
     operationBatchLimit?: number;
     storageConfig: StorageConfig;
+    mapping?: BucketDefinitionMapping;
 }
-/**
- * Checksum query implementation.
- *
- * General implementation flow is:
- * 1. getChecksums() -> check cache for (partial) matches. If not found or partial match, query the remainder using computePartialChecksums().
- * 2. computePartialChecksums() -> query bucket_state for partial matches. Query the remainder using computePartialChecksumsDirect().
- * 3. computePartialChecksumsDirect() -> split into batches of 200 buckets at a time -> computePartialChecksumsInternal()
- * 4. computePartialChecksumsInternal() -> aggregate over 50_000 operations in bucket_data at a time
- */
-export declare class MongoChecksums {
-    private db;
-    private group_id;
-    private options;
+export declare abstract class MongoChecksums {
+    protected readonly db: VersionedPowerSyncMongo;
+    protected readonly group_id: number;
+    protected readonly options: MongoChecksumOptions;
     private _cache;
     private readonly storageConfig;
     constructor(db: VersionedPowerSyncMongo, group_id: number, options: MongoChecksumOptions);
@@ -71,5 +76,11 @@ export declare class MongoChecksums {
      *
      * `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this.
      */
-    private computePartialChecksumsInternal;
+    protected abstract computePartialChecksumsInternal(batch: FetchPartialBucketChecksum[]): Promise<PartialChecksumMap>;
+    protected abstract fetchPreStates(batch: FetchPartialBucketChecksum[]): Promise<Map<string, {
+        opId: InternalOpId;
+        checksum: BucketChecksum;
+    }>>;
+    protected computePartialChecksumsForCollection<TRequest extends FetchPartialBucketChecksumByBucket, TBucketDataDocument extends BucketDataDocumentBase>(batch: TRequest[], collection: lib_mongo.mongo.Collection<TBucketDataDocument>, createFilter: (request: TRequest) => any): Promise<PartialChecksumMap>;
 }
+export declare function emptyChecksumForRequest(request: Pick<FetchPartialBucketChecksum | FetchPartialBucketChecksumV3, 'bucket' | 'start'>): PartialOrFullChecksum;

package/dist/storage/implementation/MongoChecksums.js

@@ -1,16 +1,7 @@
+import { addPartialChecksums, ChecksumCache, isPartialChecksum } from '@powersync/service-core';
 import * as lib_mongo from '@powersync/lib-service-mongodb';
-import { addPartialChecksums, bson, ChecksumCache, isPartialChecksum } from '@powersync/service-core';
 const DEFAULT_BUCKET_BATCH_LIMIT = 200;
 const DEFAULT_OPERATION_BATCH_LIMIT = 50_000;
-/**
- * Checksum query implementation.
- *
- * General implementation flow is:
- * 1. getChecksums() -> check cache for (partial) matches. If not found or partial match, query the remainder using computePartialChecksums().
- * 2. computePartialChecksums() -> query bucket_state for partial matches. Query the remainder using computePartialChecksumsDirect().
- * 3. computePartialChecksumsDirect() -> split into batches of 200 buckets at a time -> computePartialChecksumsInternal()
- * 4. computePartialChecksumsInternal() -> aggregate over 50_000 operations in bucket_data at a time
- */
 export class MongoChecksums {
     db;
     group_id;
@@ -59,38 +50,7 @@ export class MongoChecksums {
         if (batch.length == 0) {
             return new Map();
         }
-        const preFilters = [];
-        for (let request of batch) {
-            if (request.start == null) {
-                preFilters.push({
-                    _id: {
-                        g: this.group_id,
-                        b: request.bucket
-                    },
-                    'compacted_state.op_id': { $exists: true, $lte: request.end }
-                });
-            }
-        }
-        const preStates = new Map();
-        if (preFilters.length > 0) {
-            // For un-cached bucket checksums, attempt to use the compacted state first.
-            const states = await this.db.bucket_state
-                .find({
-                $or: preFilters
-            })
-                .toArray();
-            for (let state of states) {
-                const compactedState = state.compacted_state;
-                preStates.set(state._id.b, {
-                    opId: compactedState.op_id,
-                    checksum: {
-                        bucket: state._id.b,
-                        checksum: Number(compactedState.checksum),
-                        count: compactedState.count
-                    }
-                });
-            }
-        }
+        const preStates = await this.fetchPreStates(batch);
         const mappedRequests = batch.map((request) => {
             let start = request.start;
             if (start == null) {
@@ -143,14 +103,7 @@ export class MongoChecksums {
         }
         return results;
     }
-    /**
-     * Query a batch of checksums.
-     *
-     * We limit the number of operations that the query aggregates in each sub-batch, to avoid potential query timeouts.
-     *
-     * `batch` must be limited to DEFAULT_BUCKET_BATCH_LIMIT buckets before calling this.
-     */
-    async computePartialChecksumsInternal(batch) {
+    async computePartialChecksumsForCollection(batch, collection, createFilter) {
         const batchLimit = this.options?.operationBatchLimit ?? DEFAULT_OPERATION_BATCH_LIMIT;
         // Map requests by bucket. We adjust this as we get partial results.
         let requests = new Map();
@@ -159,23 +112,7 @@ export class MongoChecksums {
         }
         const partialChecksums = new Map();
         while (requests.size > 0) {
-            const filters = [];
-            for (let request of requests.values()) {
-                filters.push({
-                    _id: {
-                        $gt: {
-                            g: this.group_id,
-                            b: request.bucket,
-                            o: request.start ?? new bson.MinKey()
-                        },
-                        $lte: {
-                            g: this.group_id,
-                            b: request.bucket,
-                            o: request.end
-                        }
-                    }
-                });
-            }
+            const filters = Array.from(requests.values(), createFilter);
             // Historically, checksum may be stored as 'int' or 'double'.
             // More recently, this should be a 'long'.
             // $toLong ensures that we always sum it as a long, avoiding inaccuracies in the calculations.
@@ -189,7 +126,7 @@ export class MongoChecksums {
             //    Returns: B[3-10], C[1-4]
             // 3. Query: C[5-end]
             //    Returns: C[5-10]
-            const aggregate = await this.db.bucket_data
+            const aggregate = await collection
                 .aggregate([
                 {
                     $match: {
@@ -239,10 +176,8 @@ export class MongoChecksums {
                     limitReached = true;
                     const req = requests.get(bucket);
                     requests.set(bucket, {
-                        bucket,
-                        source: req.source,
-                        start: doc.last_op,
-                        end: req.end
+                        ...req,
+                        start: doc.last_op
                     });
                 }
                 else {
@@ -276,6 +211,12 @@ export class MongoChecksums {
         }));
     }
 }
+export function emptyChecksumForRequest(request) {
+    if (request.start == null) {
+        return { bucket: request.bucket, count: 0, checksum: 0 };
+    }
+    return { bucket: request.bucket, partialCount: 0, partialChecksum: 0 };
+}
 /**
  * Convert output of the $group stage into a checksum.
  */

package/dist/storage/implementation/MongoChecksums.js.map

@@ -1 +1 @@
-{"version":3,"file":"MongoChecksums.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoChecksums.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EACL,mBAAmB,EACnB,IAAI,EAGJ,aAAa,EAIb,iBAAiB,EAIlB,MAAM,yBAAyB,CAAC;AAqBjC,MAAM,0BAA0B,GAAG,GAAG,CAAC;AACvC,MAAM,6BAA6B,GAAG,MAAM,CAAC;AAE7C;;;;;;;;GAQG;AACH,MAAM,OAAO,cAAc;IAKf;IACA;IACA;IANF,MAAM,CAA4B;IACzB,aAAa,CAAgB;IAE9C,YACU,EAA2B,EAC3B,QAAgB,EAChB,OAA6B;QAF7B,OAAE,GAAF,EAAE,CAAyB;QAC3B,aAAQ,GAAR,QAAQ,CAAQ;QAChB,YAAO,GAAP,OAAO,CAAsB;QAErC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACH,IAAY,KAAK;QACf,IAAI,CAAC,MAAM,KAAK,IAAI,aAAa,CAAC;YAChC,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE;gBACxB,OAAO,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;YAC7C,CAAC;SACF,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,UAAwB,EAAE,OAAgC;QAC3E,OAAO,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAED,UAAU;QACR,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,uBAAuB,CAAC,KAAmC;QACvE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,MAAM,UAAU,GAAU,EAAE,CAAC;QAC7B,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;gBAC1B,UAAU,CAAC,IAAI,CAAC;oBACd,GAAG,EAAE;wBACH,CAAC,EAAE,IAAI,CAAC,QAAQ;wBAChB,CAAC,EAAE,OAAO,CAAC,MAAM;qBAClB;oBACD,uBAAuB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,EAAE;iBAC9D,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAA4D,CAAC;QAEtF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,4EAA4E;YAC5E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY;iBACtC,IAAI,CAAC;gBACJ,GAAG,EAAE,UAAU;aAChB,CAAC;iBACD,OAAO,EAAE,CAAC;YACb,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;gBACzB,MAAM,cAAc,GAAG,KAAK,CAAC,eAAgB,CAAC;gBAC9C,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE;oBACzB,IAAI,EAAE,cAAc,CAAC,KAAK;oBAC1B,QAAQ,EAAE;wBACR,MAAM,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;wBACnB,QAAQ,EAAE,MAAM,CAAC,cAAc,CAAC,QAAQ,CAAC;wBACzC,KAAK,EAAE,cAAc,CAAC,KAAK;qBAC5B;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3C,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC/C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,OAAO;gBACL,GAAG,OAAO;gBACV,KAAK;aACN,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,cAAc,CAAC,CAAC;QAElF,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,oGAAoG;YACpG,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvC,kCAAkC;YAClC,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,IAAI,IAAI,EAAE,eAAe,IAAI,IAAI,CAAC,CAAC;YAEhG,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,6BAA6B,CAAC,KAAmC;QAC5E,sDAAsD;QACtD,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;QAEtF,IAAI,KAAK,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACrC,qEAAqE;YACrE,OAAO,MAAM,IAAI,CAAC,+BAA+B,CAAC,KAAK,CAAC,CAAC;QAC3D,CAAC;QACD,oCAAoC;QACpC,IAAI,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;QACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;YACxD,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,+BAA+B,CAAC,WAAW,CAAC,CAAC;YAC7E,KAAK,IAAI,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;;;OAMG;IACK,KAAK,CAAC,+BAA+B,CAAC,KAAmC;QAC/E,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,mBAAmB,IAAI,6BAA6B,CAAC;QAEtF,oEAAoE;QACpE,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAsC,CAAC;QAC7D,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAiC,CAAC;QAElE,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,OAAO,GAAU,EAAE,CAAC;YAC1B,KAAK,IAAI,OAAO,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE;wBACH,GAAG,EAAE;4BACH,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,OAAO,CAAC,MAAM;4BACjB,CAAC,EAAE,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE;yBACtC;wBACD,IAAI,EAAE;4BACJ,CAAC,EAAE,IAAI,CAAC,QAAQ;4BAChB,CAAC,EAAE,OAAO,CAAC,MAAM;4BACjB,CAAC,EAAE,OAAO,CAAC,GAAG;yBACf;qBACF;iBACF,CAAC,CAAC;YACL,CAAC;YAED,6DAA6D;YAC7D,0CAA0C;YAC1C,8FAA8F;YAC9F,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;YAE/F,6DAA6D;YAC7D,6FAA6F;YAC7F,+BAA+B;YAC/B,yCAAyC;YACzC,8BAA8B;YAC9B,+BAA+B;YAC/B,8BAA8B;YAC9B,qBAAqB;YACrB,sBAAsB;YACtB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,WAAW;iBACxC,SAAS,CACR;gBACE;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,OAAO;qBACb;iBACF;gBACD,mCAAmC;gBACnC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;gBACrB,EAAE,MAAM,EAAE,UAAU,EAAE;gBACtB;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,QAAQ;wBACb,cAAc,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE;wBACtC,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;wBAClB,YAAY,EAAE;4BACZ,IAAI,EAAE;gCACJ,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;6BACzC;yBACF;wBACD,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC5B;iBACF;gBACD,4DAA4D;gBAC5D,wEAAwE;gBACxE,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;aACtB,EACD,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,yBAAyB,EAAE,CAChG;iBACA,OAAO,EAAE;iBACT,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,MAAM,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YAEL,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,YAAY,GAAG,KAAK,CAAC;YACzB,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;gBACvB,MAAM,QAAQ,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;gBAE5C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC9C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAChF,CAAC;qBAAM,CAAC;oBACN,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACzC,CAAC;gBAED,UAAU,IAAI,GAAG,CAAC,KAAK,CAAC;gBACxB,IAAI,UAAU,IAAI,UAAU,EAAE,CAAC;oBAC7B,iDAAiD;oBACjD,4DAA4D;oBAC5D,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE;wBACnB,MAAM;wBACN,MAAM,EAAE,GAAI,CAAC,MAAM;wBACnB,KAAK,EAAE,GAAG,CAAC,OAAO;wBAClB,GAAG,EAAE,GAAI,CAAC,GAAG;qBACd,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,2BAA2B;oBAC3B,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;YACD,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,kCAAkC;YAClC,IAAI,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnD,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC5B,eAAe,GAAG;oBAChB,MAAM;oBACN,YAAY,EAAE,CAAC;oBACf,eAAe,EAAE,CAAC;iBACnB,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;gBAChE,eAAe,GAAG;oBAChB,MAAM;oBACN,KAAK,EAAE,eAAe,CAAC,YAAY;oBACnC,QAAQ,EAAE,eAAe,CAAC,eAAe;iBAC1C,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACnC,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,GAAkB;IAC/C,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,WAAW,CAAC,GAAG,UAAU,CAAC;IACtF,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;IAEvB,IAAI,GAAG,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,4CAA4C;YAC5C,MAAM;YACN,QAAQ,EAAE,eAAe;YACzB,KAAK,EAAE,GAAG,CAAC,KAAK;SACQ,CAAC;IAC7B,CAAC;SAAM,CAAC;QACN,OAAO;YACL,gDAAgD;YAChD,MAAM;YACN,YAAY,EAAE,GAAG,CAAC,KAAK;YACvB,eAAe;SACU,CAAC;IAC9B,CAAC;AACH,CAAC"}
+{"version":3,"file":"MongoChecksums.js","sourceRoot":"","sources":["../../../src/storage/implementation/MongoChecksums.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,mBAAmB,EAInB,aAAa,EAIb,iBAAiB,EAIlB,MAAM,yBAAyB,CAAC;AAGjC,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAmC5D,MAAM,0BAA0B,GAAG,GAAG,CAAC;AACvC,MAAM,6BAA6B,GAAG,MAAM,CAAC;AAE7C,MAAM,OAAgB,cAAc;IAKb;IACA;IACA;IANb,MAAM,CAA4B;IACzB,aAAa,CAAgB;IAE9C,YACqB,EAA2B,EAC3B,QAAgB,EAChB,OAA6B;QAF7B,OAAE,GAAF,EAAE,CAAyB;QAC3B,aAAQ,GAAR,QAAQ,CAAQ;QAChB,YAAO,GAAP,OAAO,CAAsB;QAEhD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAC7C,CAAC;IAED;;;;OAIG;IACH,IAAY,KAAK;QACf,IAAI,CAAC,MAAM,KAAK,IAAI,aAAa,CAAC;YAChC,cAAc,EAAE,CAAC,KAAK,EAAE,EAAE;gBACxB,OAAO,IAAI,CAAC,uBAAuB,CAAC,KAAK,CAAC,CAAC;YAC7C,CAAC;SACF,CAAC,CAAC;QACH,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,UAAwB,EAAE,OAAgC;QAC3E,OAAO,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;IACxD,CAAC;IAED,UAAU;QACR,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;IAED;;;;;;;;OAQG;IACK,KAAK,CAAC,uBAAuB,CAAC,KAAmC;QACvE,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,OAAO,IAAI,GAAG,EAAE,CAAC;QACnB,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;QAEnD,MAAM,cAAc,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YAC3C,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;YAC1B,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;gBAClB,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC/C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC;gBACxB,CAAC;YACH,CAAC;YACD,OAAO;gBACL,GAAG,OAAO;gBACV,KAAK;aACN,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,6BAA6B,CAAC,cAAc,CAAC,CAAC;QAElF,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,oGAAoG;YACpG,MAAM,QAAQ,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACvC,kCAAkC;YAClC,MAAM,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACrD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,IAAI,IAAI,EAAE,eAAe,IAAI,IAAI,CAAC,CAAC;YAEhG,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1B,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;IAED;;;;;;;;;OASG;IACI,KAAK,CAAC,6BAA6B,CAAC,KAAmC;QAC5E,sDAAsD;QACtD,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,EAAE,gBAAgB,IAAI,0BAA0B,CAAC;QAEtF,IAAI,KAAK,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC;YACrC,qEAAqE;YACrE,OAAO,MAAM,IAAI,CAAC,+BAA+B,CAAC,KAAK,CAAC,CAAC;QAC3D,CAAC;QACD,oCAAoC;QACpC,IAAI,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;QACvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,gBAAgB,EAAE,CAAC;YACxD,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,CAAC;YACzD,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,+BAA+B,CAAC,WAAW,CAAC,CAAC;YAC7E,KAAK,IAAI,CAAC,IAAI,YAAY,CAAC,MAAM,EAAE,EAAE,CAAC;gBACpC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAeS,KAAK,CAAC,oCAAoC,CAIlD,KAAiB,EACjB,UAA2D,EAC3D,YAAwC;QAExC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,EAAE,mBAAmB,IAAI,6BAA6B,CAAC;QAEtF,oEAAoE;QACpE,IAAI,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC3C,KAAK,IAAI,OAAO,IAAI,KAAK,EAAE,CAAC;YAC1B,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,GAAG,EAAiC,CAAC;QAElE,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC;YAE5D,6DAA6D;YAC7D,0CAA0C;YAC1C,8FAA8F;YAC9F,MAAM,YAAY,GAAG,IAAI,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,EAAE,CAAC;YAE/F,6DAA6D;YAC7D,6FAA6F;YAC7F,+BAA+B;YAC/B,yCAAyC;YACzC,8BAA8B;YAC9B,+BAA+B;YAC/B,8BAA8B;YAC9B,qBAAqB;YACrB,sBAAsB;YACtB,MAAM,SAAS,GAAG,MAAM,UAAU;iBAC/B,SAAS,CACR;gBACE;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,OAAO;qBACb;iBACF;gBACD,mCAAmC;gBACnC,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;gBACrB,EAAE,MAAM,EAAE,UAAU,EAAE;gBACtB;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE,QAAQ;wBACb,cAAc,EAAE,EAAE,IAAI,EAAE,YAAY,EAAE;wBACtC,KAAK,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE;wBAClB,YAAY,EAAE;4BACZ,IAAI,EAAE;gCACJ,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;6BACzC;yBACF;wBACD,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;qBAC5B;iBACF;gBACD,4DAA4D;gBAC5D,wEAAwE;gBACxE,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE;aACtB,EACD,EAAE,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,CAAC,yBAAyB,EAAE,CAChG;iBACA,OAAO,EAAE;iBACT,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,MAAM,SAAS,CAAC,aAAa,CAAC,CAAC,EAAE,yBAAyB,CAAC,CAAC;YAC9D,CAAC,CAAC,CAAC;YAEL,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,IAAI,YAAY,GAAG,KAAK,CAAC;YACzB,KAAK,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;gBACvB,MAAM,QAAQ,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC;gBAE5C,MAAM,QAAQ,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC9C,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;oBACrB,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;gBAChF,CAAC;qBAAM,CAAC;oBACN,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;gBACzC,CAAC;gBAED,UAAU,IAAI,GAAG,CAAC,KAAK,CAAC;gBACxB,IAAI,UAAU,IAAI,UAAU,EAAE,CAAC;oBAC7B,iDAAiD;oBACjD,4DAA4D;oBAC5D,YAAY,GAAG,IAAI,CAAC;oBACpB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjC,QAAQ,CAAC,GAAG,CAAC,MAAM,EAAE;wBACnB,GAAG,GAAI;wBACP,KAAK,EAAE,GAAG,CAAC,OAAO;qBACnB,CAAC,CAAC;gBACL,CAAC;qBAAM,CAAC;oBACN,2BAA2B;oBAC3B,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC;YACD,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,IAAI,GAAG,CACZ,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;YACpB,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;YAC9B,kCAAkC;YAClC,IAAI,eAAe,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACnD,IAAI,eAAe,IAAI,IAAI,EAAE,CAAC;gBAC5B,eAAe,GAAG;oBAChB,MAAM;oBACN,YAAY,EAAE,CAAC;oBACf,eAAe,EAAE,CAAC;iBACnB,CAAC;YACJ,CAAC;YACD,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,IAAI,iBAAiB,CAAC,eAAe,CAAC,EAAE,CAAC;gBAChE,eAAe,GAAG;oBAChB,MAAM;oBACN,KAAK,EAAE,eAAe,CAAC,YAAY;oBACnC,QAAQ,EAAE,eAAe,CAAC,eAAe;iBAC1C,CAAC;YACJ,CAAC;YAED,OAAO,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;QACnC,CAAC,CAAC,CACH,CAAC;IACJ,CAAC;CACF;AAED,MAAM,UAAU,uBAAuB,CACrC,OAA4F;IAE5F,IAAI,OAAO,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;QAC1B,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IAC3D,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,CAAC;AACzE,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,GAAkB;IAC/C,MAAM,eAAe,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,CAAC,GAAG,WAAW,CAAC,GAAG,UAAU,CAAC;IACtF,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC;IAEvB,IAAI,GAAG,CAAC,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO;YACL,4CAA4C;YAC5C,MAAM;YACN,QAAQ,EAAE,eAAe;YACzB,KAAK,EAAE,GAAG,CAAC,KAAK;SACQ,CAAC;IAC7B,CAAC;SAAM,CAAC;QACN,OAAO;YACL,gDAAgD;YAChD,MAAM;YACN,YAAY,EAAE,GAAG,CAAC,KAAK;YACvB,eAAe;SACU,CAAC;IAC9B,CAAC;AACH,CAAC"}

package/dist/storage/implementation/MongoCompactor.d.ts

@@ -1,80 +1,120 @@
-import { PopulateChecksumCacheResults, storage } from '@powersync/service-core';
-import { VersionedPowerSyncMongo } from './db.js';
-import { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js';
-/**
- * Additional options, primarily for testing.
- */
-export interface MongoCompactOptions extends storage.CompactOptions {
-}
-export declare class MongoCompactor {
-    private storage;
-    private db;
-    private updates;
-    private bucketStateUpdates;
-    private idLimitBytes;
-    private moveBatchLimit;
-    private moveBatchQueryLimit;
-    private clearBatchLimit;
-    private minBucketChanges;
-    private minChangeRatio;
-    private maxOpId;
-    private buckets;
-    private signal?;
-    private group_id;
-    constructor(storage: MongoSyncBucketStorage, db: VersionedPowerSyncMongo, options: MongoCompactOptions);
+import { mongo } from '@powersync/lib-service-mongodb';
+import { Logger } from '@powersync/lib-services-framework';
+import { InternalOpId, PopulateChecksumCacheResults, storage } from '@powersync/service-core';
+import { BucketDefinitionId } from './BucketDefinitionMapping.js';
+import { BucketDataDoc, BucketKey } from './common/BucketDataDoc.js';
+import { BucketDataDocumentGeneric, SingleBucketStore } from './common/SingleBucketStore.js';
+import type { VersionedPowerSyncMongo } from './db.js';
+import { BucketStateDocumentBase } from './models.js';
+import type { MongoSyncBucketStorage } from './MongoSyncBucketStorage.js';
+interface CurrentBucketState {
+    /** Bucket name */
+    bucket: string;
+    definitionId: BucketDefinitionId;
     /**
-     * Compact buckets by converting operations into MOVE and/or CLEAR operations.
-     *
-     * See /docs/compacting-operations.md for details.
+     * Rows seen in the bucket, with the last op_id of each.
      */
-    compact(): Promise<void>;
-    private compactDirtyBuckets;
+    seen: Map<string, InternalOpId>;
     /**
-     * Compaction for a single bucket, with retries on failure.
-     *
-     * This covers against occasional network or other database errors during a long compact job.
+     * Estimated memory usage of the seen Map.
      */
-    private compactSingleBucketRetried;
-    private compactSingleBucket;
+    trackingSize: number;
     /**
-     * Call when done with a bucket.
+     * Last (lowest) seen op_id that is not a PUT.
      */
-    private updateBucketChecksums;
-    private flush;
+    lastNotPut: InternalOpId | null;
     /**
-     * Perform a CLEAR compact for a bucket.
-     *
+     * Number of REMOVE/MOVE operations seen since lastNotPut.
+     */
+    opsSincePut: number;
+    /**
+     * Incrementally-updated checksum, up to maxOpId.
+     */
+    checksum: number;
+    /**
+     * Op count for the checksum.
+     */
+    opCount: number;
+    /**
+     * Byte size of ops covered by the checksum.
+     */
+    opBytes: number;
+}
+export interface MongoCompactOptions extends storage.CompactOptions {
+}
+export interface DirtyBucket {
+    bucket: string;
+    definitionId: BucketDefinitionId | null;
+    estimatedCount: number;
+    dirtyRatio?: number;
+}
+export declare abstract class MongoCompactor {
+    protected readonly storage: MongoSyncBucketStorage;
+    protected readonly db: VersionedPowerSyncMongo;
+    protected updates: mongo.AnyBulkWriteOperation<BucketDataDocumentGeneric>[];
+    protected bucketStateUpdates: mongo.AnyBulkWriteOperation<BucketStateDocumentBase>[];
+    protected readonly idLimitBytes: number;
+    protected readonly moveBatchLimit: number;
+    protected readonly moveBatchQueryLimit: number;
+    protected readonly clearBatchLimit: number;
+    protected readonly minBucketChanges: number;
+    protected readonly minChangeRatio: number;
+    protected readonly maxOpId: bigint;
+    protected readonly buckets: string[] | undefined;
+    protected readonly signal?: AbortSignal;
+    protected readonly group_id: number;
+    protected readonly logger: Logger;
+    constructor(storage: MongoSyncBucketStorage, db: VersionedPowerSyncMongo, options: MongoCompactOptions);
+    /**
+     * Compact buckets by converting operations into MOVE and/or CLEAR operations.
      *
-     * @param bucket bucket name
-     * @param op op_id of the last non-PUT operation, which will be converted to CLEAR.
+     * See /docs/compacting-operations.md for details.
      */
-    private clearBucket;
+    compact(): Promise<void>;
     /**
      * Subset of compact, only populating checksums where relevant.
      */
     populateChecksums(options: {
         minBucketChanges: number;
     }): Promise<PopulateChecksumCacheResults>;
+    protected dirtyBucketBatchesForCollection<TCollectionBucketState extends BucketStateDocumentBase>(collection: mongo.Collection<TCollectionBucketState>, lastId: TCollectionBucketState['_id'], maxId: TCollectionBucketState['_id'], options: {
+        minBucketChanges: number;
+        minChangeRatio: number;
+    }, getDefinitionId: (state: TCollectionBucketState) => BucketDefinitionId | null): AsyncGenerator<DirtyBucket[]>;
+    protected dirtyBucketBatchForChecksumsForCollection<TBucketState extends BucketStateDocumentBase>(collection: mongo.Collection<TBucketState>, filter: mongo.Filter<TBucketState>, getDefinitionId: (state: mongo.WithId<TBucketState>) => BucketDefinitionId | null): Promise<DirtyBucket[]>;
+    abstract dirtyBucketBatches(options: {
+        minBucketChanges: number;
+        minChangeRatio: number;
+    }): AsyncGenerator<DirtyBucket[]>;
+    abstract dirtyBucketBatchForChecksums(options: {
+        minBucketChanges: number;
+    }): Promise<DirtyBucket[]>;
+    protected compactDirtyBuckets(): Promise<void>;
     /**
-     * Return batches of dirty buckets.
-     *
-     * Can be used to iterate through all buckets.
+     * Compaction for a single bucket, with retries on failure.
      *
-     * minBucketChanges: minimum number of changes for a bucket to be included in the results.
-     * minChangeRatio: minimum ratio of changes to total ops for a bucket to be included in the results, number between 0 and 1.
+     * This covers against occasional network or other database errors during a long compact job.
      */
-    private dirtyBucketBatches;
+    protected compactSingleBucketRetried(bucket: string, definitionId?: BucketDefinitionId | null): Promise<void>;
+    protected compactSingleBucket(bucket: string, definitionId?: BucketDefinitionId | null): Promise<void>;
+    protected updateBucketChecksums(state: CurrentBucketState): void;
+    protected flush(col: SingleBucketStore): Promise<void>;
+    private flushBucketStateUpdates;
     /**
-     * Returns a batch of dirty buckets - buckets with most changes first.
-     *
-     * This cannot be used to iterate on its own - the client is expected to process these buckets and
-     * set estimate_since_compact.count: 0 when done, before fetching the next batch.
-     *
-     * Unlike dirtyBucketBatches, used for compacting, this is specifically designed to be resuamble after a restart,
-     * since it is used as the last step for initial replication.
+     * Perform a CLEAR compact for a bucket.
      *
-     * We currently don't get new data while doing populateChecksums, so we don't need to worry about buckets changing while processing.
+     * @param currentState tracks the last non-PUT op, which will be converted to CLEAR.
      */
-    private dirtyBucketBatchForChecksums;
-    private updateChecksumsBatch;
+    protected clearBucket(currentState: CurrentBucketState, col: SingleBucketStore): Promise<void>;
+    protected updateChecksumsBatch(buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]): Promise<void>;
+    protected formatBucketDataKey(doc: Pick<BucketDataDoc, 'bucketKey' | 'o'>): string;
+    protected abstract writeBucketStateUpdates(): Promise<void>;
+    protected abstract computeChecksumsForBuckets(buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]): Promise<storage.PartialChecksumMap>;
+    protected abstract bucketStateFilter(bucket: string, definitionId: BucketDefinitionId | null): mongo.Document;
+    protected abstract getBucketDataContext(bucket: string, definitionId: BucketDefinitionId | null): Promise<SingleBucketStore | null>;
+}
+export interface BucketDataCollectionContext<TBucketData extends mongo.Document> {
+    bucketKey: BucketKey;
+    collection: mongo.Collection<TBucketData>;
 }
+export {};