@powersync/service-module-mongodb-storage 0.15.4 → 0.16.0

package/dist/storage/implementation/v3/models.d.ts

@@ -0,0 +1,43 @@
+import { InternalOpId } from '@powersync/service-core';
+import * as bson from 'bson';
+import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js';
+import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
+import { BucketDataDocumentBase, BucketDataKey, BucketParameterDocumentBase, BucketStateDocumentBase, CurrentBucket, ReplicaId, SourceTableDocument, SourceTableKey, TaggedBucketParameterDocument } from '../models.js';
+export interface CurrentBucketV3 extends CurrentBucket {
+    def: BucketDefinitionId;
+}
+export interface RecordedLookupV3 {
+    i: ParameterIndexId;
+    l: bson.Binary;
+}
+export interface CurrentDataDocumentV3 {
+    _id: ReplicaId;
+    data: bson.Binary | null;
+    buckets: CurrentBucketV3[];
+    lookups: RecordedLookupV3[];
+    /**
+     * If set, this can be deleted, once there is a consistent checkpoint >= pending_delete.
+     *
+     * This must only be set if buckets = [], lookups = [].
+     */
+    pending_delete?: bigint;
+}
+export interface BucketParameterDocumentV3 extends BucketParameterDocumentBase<SourceTableKey> {
+}
+export type BucketDataKeyV3 = BucketDataKey;
+export interface BucketDataDocumentV3 extends BucketDataDocumentBase {
+    _id: BucketDataKeyV3;
+}
+export declare function serializeBucketDataV3(document: BucketDataDoc): BucketDataDocumentV3;
+export declare function loadBucketDataDocumentV3(context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>, doc: BucketDataDocumentV3): BucketDataDoc;
+export declare function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3;
+export interface SourceTableDocumentV3 extends SourceTableDocument {
+    bucket_data_source_ids: BucketDefinitionId[];
+    parameter_lookup_source_ids: ParameterIndexId[];
+    latest_pending_delete?: InternalOpId | undefined;
+}
+export interface BucketStateDocumentV3 extends BucketStateDocumentBase {
+    _id: BucketStateDocumentBase['_id'] & {
+        d: BucketDefinitionId;
+    };
+}

package/dist/storage/implementation/v3/models.js

@@ -0,0 +1,34 @@
+export function serializeBucketDataV3(document) {
+    const { bucketKey, o } = document;
+    return {
+        _id: {
+            b: bucketKey.bucket,
+            o: o
+        },
+        // List fields directly, so that we don't accidentally persist any unknown fields
+        op: document.op,
+        source_table: document.source_table,
+        source_key: document.source_key,
+        table: document.table,
+        row_id: document.row_id,
+        checksum: document.checksum,
+        data: document.data,
+        target_op: document.target_op
+    };
+}
+export function loadBucketDataDocumentV3(context, doc) {
+    const { _id, ...rest } = doc;
+    return {
+        bucketKey: {
+            ...context,
+            bucket: _id.b
+        },
+        o: _id.o,
+        ...rest
+    };
+}
+export function taggedBucketParameterDocumentToV3(document) {
+    const { index: _index, ...rest } = document;
+    return rest;
+}
+//# sourceMappingURL=models.js.map

package/dist/storage/implementation/v3/models.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/models.ts"],"names":[],"mappings":"AA8CA,MAAM,UAAU,qBAAqB,CAAC,QAAuB;IAC3D,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,QAAQ,CAAC;IAClC,OAAO;QACL,GAAG,EAAE;YACH,CAAC,EAAE,SAAS,CAAC,MAAM;YACnB,CAAC,EAAE,CAAC;SACL;QACD,iFAAiF;QACjF,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,OAAgE,EAChE,GAAyB;IAEzB,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC;IAC7B,OAAO;QACL,SAAS,EAAE;YACT,GAAG,OAAO;YACV,MAAM,EAAE,GAAG,CAAC,CAAC;SACd;QACD,CAAC,EAAE,GAAG,CAAC,CAAC;QACR,GAAG,IAAI;KACR,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAAC,QAAuC;IACvF,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;IAC5C,OAAO,IAAiC,CAAC;AAC3C,CAAC"}

package/dist/storage/storage-index.d.ts

@@ -1,14 +1,17 @@
 export * as test_utils from '../utils/test-utils.js';
 export * from '../utils/util.js';
+export * from './implementation/BucketDefinitionMapping.js';
+export * from './implementation/common/PersistedBatch.js';
+export * from './implementation/createMongoSyncBucketStorage.js';
 export * from './implementation/db.js';
 export * from './implementation/models.js';
-export * from './implementation/MongoBucketBatch.js';
 export * from './implementation/MongoIdSequence.js';
+export * from './implementation/MongoPersistedSyncRules.js';
 export * from './implementation/MongoPersistedSyncRulesContent.js';
 export * from './implementation/MongoStorageProvider.js';
-export * from './implementation/MongoSyncBucketStorage.js';
 export * from './implementation/MongoSyncRulesLock.js';
 export * from './implementation/OperationBatch.js';
-export * from './implementation/PersistedBatch.js';
+export * from './implementation/v1/models.js';
+export * from './implementation/v3/models.js';
 export * from './MongoBucketStorage.js';
 export * from './MongoReportStorage.js';

package/dist/storage/storage-index.js

@@ -1,15 +1,18 @@
 export * as test_utils from '../utils/test-utils.js';
 export * from '../utils/util.js';
+export * from './implementation/BucketDefinitionMapping.js';
+export * from './implementation/common/PersistedBatch.js';
+export * from './implementation/createMongoSyncBucketStorage.js';
 export * from './implementation/db.js';
 export * from './implementation/models.js';
-export * from './implementation/MongoBucketBatch.js';
 export * from './implementation/MongoIdSequence.js';
+export * from './implementation/MongoPersistedSyncRules.js';
 export * from './implementation/MongoPersistedSyncRulesContent.js';
 export * from './implementation/MongoStorageProvider.js';
-export * from './implementation/MongoSyncBucketStorage.js';
 export * from './implementation/MongoSyncRulesLock.js';
 export * from './implementation/OperationBatch.js';
-export * from './implementation/PersistedBatch.js';
+export * from './implementation/v1/models.js';
+export * from './implementation/v3/models.js';
 export * from './MongoBucketStorage.js';
 export * from './MongoReportStorage.js';
 //# sourceMappingURL=storage-index.js.map

package/dist/storage/storage-index.js.map

@@ -1 +1 @@
-{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,UAAU,MAAM,wBAAwB,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sCAAsC,CAAC;AACrD,cAAc,qCAAqC,CAAC;AACpD,cAAc,oDAAoD,CAAC;AACnE,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,wCAAwC,CAAC;AACvD,cAAc,oCAAoC,CAAC;AACnD,cAAc,oCAAoC,CAAC;AACnD,cAAc,yBAAyB,CAAC;AACxC,cAAc,yBAAyB,CAAC"}
+{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,UAAU,MAAM,wBAAwB,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,6CAA6C,CAAC;AAC5D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,kDAAkD,CAAC;AACjE,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qCAAqC,CAAC;AACpD,cAAc,6CAA6C,CAAC;AAC5D,cAAc,oDAAoD,CAAC;AACnE,cAAc,0CAA0C,CAAC;AACzD,cAAc,wCAAwC,CAAC;AACvD,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,yBAAyB,CAAC"}

package/dist/utils/util.d.ts

@@ -1,7 +1,8 @@
+import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 import { storage, utils } from '@powersync/service-core';
 import * as bson from 'bson';
-import { BucketDataDocument } from '../storage/implementation/models.js';
+import { BucketDataDoc } from '../storage/implementation/common/BucketDataDoc.js';
 export declare function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T>;
 export declare function generateSlotName(prefix: string, sync_rules_id: number): string;
 /**
@@ -19,12 +20,18 @@ export declare function readSingleBatch<T>(cursor: mongo.AbstractCursor<T>): Pro
     data: T[];
     hasMore: boolean;
 }>;
-export declare function mapOpEntry(row: BucketDataDocument): utils.OplogEntry;
+export declare function mapOpEntry(row: BucketDataDoc): utils.OplogEntry;
 export declare function replicaIdToSubkey(table: storage.SourceTableId, id: storage.ReplicaId): string;
 export declare function mongoTableId(table: storage.SourceTableId): bson.ObjectId;
 export declare function setSessionSnapshotTime(session: mongo.ClientSession, time: bson.Timestamp): void;
+export declare function retryOnMongoMaxTimeMSExpired<T>(operation: () => Promise<T>, options: {
+    signal?: AbortSignal;
+    abortMessage?: string;
+    retryDelayMs: number;
+    onRetry?: (retryCount: number) => void;
+}): Promise<T>;
 export declare const createPaginatedConnectionQuery: <T extends mongo.Document>(query: mongo.Filter<T>, collection: mongo.Collection<T>, limit: number, cursor?: string) => Promise<{
-    items: mongo.WithId<T>[];
+    items: lib_mongo.mongo.WithId<T>[];
     count: number;
     /** Setting the cursor to the connected at date of the last item in the list */
     cursor: any;

package/dist/utils/util.js

@@ -1,7 +1,9 @@
-import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
 import { storage, utils } from '@powersync/service-core';
 import * as bson from 'bson';
 import * as crypto from 'crypto';
+import * as timers from 'node:timers/promises';
 import * as uuid from 'uuid';
 export function idPrefixFilter(prefix, rest) {
     let filter = {
@@ -65,7 +67,7 @@ export async function readSingleBatch(cursor) {
 export function mapOpEntry(row) {
     if (row.op == 'PUT' || row.op == 'REMOVE') {
         return {
-            op_id: utils.internalToExternalOpId(row._id.o),
+            op_id: utils.internalToExternalOpId(row.o),
             op: row.op,
             object_type: row.table,
             object_id: row.row_id,
@@ -77,7 +79,7 @@ export function mapOpEntry(row) {
     else {
         // MOVE, CLEAR
         return {
-            op_id: utils.internalToExternalOpId(row._id.o),
+            op_id: utils.internalToExternalOpId(row.o),
             op: row.op,
             checksum: Number(row.checksum)
         };
@@ -120,6 +122,25 @@ export function setSessionSnapshotTime(session, time) {
         throw new ServiceAssertionError(`Session snapshotTime is already set`);
     }
 }
+export async function retryOnMongoMaxTimeMSExpired(operation, options) {
+    let retryCount = 0;
+    while (true) {
+        if (options.signal?.aborted) {
+            throw new ReplicationAbortedError(options.abortMessage ?? 'Aborted MongoDB operation', options.signal.reason);
+        }
+        try {
+            return await operation();
+        }
+        catch (e) {
+            if (!lib_mongo.isMongoServerError(e) || e.codeName !== 'MaxTimeMSExpired') {
+                throw e;
+            }
+            retryCount += 1;
+            options.onRetry?.(retryCount);
+            await timers.setTimeout(options.retryDelayMs);
+        }
+    }
+}
 export const createPaginatedConnectionQuery = async (query, collection, limit, cursor) => {
     const createQuery = (cursor) => {
         if (!cursor) {

package/dist/utils/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/utils/util.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAG7B,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA+B;IACtE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAA4B,EAAE,EAAqB;IACnF,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAA4B;IACvD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,wCAAwC,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAA4B;IACjD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;SAAM,CAAC;QACN,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,OAA4B,EAAE,IAAoB;IACvF,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;IACxE,CAAC;IACD,IAAK,OAAe,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QACzC,OAAe,CAAC,YAAY,GAAG,IAAI,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,8BAA8B,GAAG,KAAK,EACjD,KAAsB,EACtB,UAA+B,EAC/B,KAAa,EACb,MAAe,EACf,EAAE;IACF,MAAM,WAAW,GAAG,CAAC,MAAe,EAAE,EAAE;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY;YACrC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE;YAC1D,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO;YACL,GAAG,KAAK;YACR,YAAY;SACM,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE;QACtD,IAAI,EAAE;YACJ,6FAA6F;YAC7F,YAAY,EAAE,CAAC,CAAC;SACjB;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B;;SAEK;IACL,OAAO;QACL,KAAK;QACL,KAAK;QACL,+EAA+E;QAC/E,MAAM,EAAE,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;QACxF,IAAI,EAAE,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC;KACzB,CAAC;AACJ,CAAC,CAAC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/utils/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AACnG,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,MAAM,MAAM,sBAAsB,CAAC;AAC/C,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAG7B,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA+B;IACtE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAkB;IAC3C,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAA4B,EAAE,EAAqB;IACnF,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAA4B;IACvD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,wCAAwC,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAA4B;IACjD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;SAAM,CAAC;QACN,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,OAA4B,EAAE,IAAoB;IACvF,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;IACxE,CAAC;IACD,IAAK,OAAe,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QACzC,OAAe,CAAC,YAAY,GAAG,IAAI,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,SAA2B,EAC3B,OAKC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,uBAAuB,CAAC,OAAO,CAAC,YAAY,IAAI,2BAA2B,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChH,CAAC;QACD,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,kBAAkB,EAAE,CAAC;gBAC1E,MAAM,CAAC,CAAC;YACV,CAAC;YACD,UAAU,IAAI,CAAC,CAAC;YAChB,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;YAC9B,MAAM,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,8BAA8B,GAAG,KAAK,EACjD,KAAsB,EACtB,UAA+B,EAC/B,KAAa,EACb,MAAe,EACf,EAAE;IACF,MAAM,WAAW,GAAG,CAAC,MAAe,EAAE,EAAE;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY;YACrC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE;YAC1D,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO;YACL,GAAG,KAAK;YACR,YAAY;SACM,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE;QACtD,IAAI,EAAE;YACJ,6FAA6F;YAC7F,YAAY,EAAE,CAAC,CAAC;SACjB;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B;;SAEK;IACL,OAAO;QACL,KAAK;QACL,KAAK;QACL,+EAA+E;QAC/E,MAAM,EAAE,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;QACxF,IAAI,EAAE,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC;KACzB,CAAC;AACJ,CAAC,CAAC"}

package/package.json

@@ -2,7 +2,7 @@
   "name": "@powersync/service-module-mongodb-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
   "types": "dist/index.d.ts",
-  "version": "0.15.4",
+  "version": "0.16.0",
   "main": "dist/index.js",
   "license": "FSL-1.1-ALv2",
   "type": "module",
@@ -26,16 +26,16 @@
     "ix": "^5.0.0",
     "lru-cache": "^10.2.2",
     "ts-codec": "^1.3.0",
-    "uuid": "^11.1.0",
-    "@powersync/lib-service-mongodb": "0.6.24",
-    "@powersync/lib-services-framework": "0.9.3",
-    "@powersync/service-core": "1.20.5",
-    "@powersync/service-jsonbig": "0.17.12",
-    "@powersync/service-sync-rules": "0.35.0",
-    "@powersync/service-types": "0.15.1"
+    "uuid": "^14.0.0",
+    "@powersync/lib-service-mongodb": "0.6.25",
+    "@powersync/lib-services-framework": "0.9.4",
+    "@powersync/service-core": "1.21.0",
+    "@powersync/service-jsonbig": "0.17.13",
+    "@powersync/service-sync-rules": "0.36.0",
+    "@powersync/service-types": "0.15.2"
   },
   "devDependencies": {
-    "@powersync/service-core-tests": "0.15.4"
+    "@powersync/service-core-tests": "0.15.5"
   },
   "scripts": {
     "build": "tsc -b",

package/src/migrations/db/migrations/1688556755264-initial-sync-rules.ts

@@ -1,5 +1,5 @@
 export const up = async () => {
-  // No-op - we don't auto-create sync rules anymore
+  // No-op - we don't auto-create sync config anymore
 };
 
 export const down = async () => {};

package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts

@@ -6,21 +6,21 @@ import { MongoStorageConfig } from '../../../types/types.js';
 
 interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
   /**
-   * True if this is the active sync rules.
+   * True if this is the active replication stream.
    * requires `snapshot_done == true` and `replicating == true`.
    */
   active?: boolean;
 
   /**
-   * True if this sync rules should be used for replication.
+   * True if this replication stream should be used for replication.
    *
-   * During reprocessing, there is one sync rules with `replicating = true, active = true`,
+   * During reprocessing, there is one replication stream with `replicating = true, active = true`,
    * and one with `replicating = true, active = false, auto_activate = true`.
    */
   replicating?: boolean;
 
   /**
-   * True if the sync rules should set `active = true` when `snapshot_done` = true.
+   * True if the replication stream should set `active = true` when `snapshot_done` = true.
    */
   auto_activate?: boolean;
 }
@@ -35,7 +35,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => {
   try {
     // We keep the old flags for existing deployments still shutting down.
 
-    // 1. New sync rules: `active = false, snapshot_done = false, replicating = true, auto_activate = true`
+    // 1. New replication stream: `active = false, snapshot_done = false, replicating = true, auto_activate = true`
     await db.sync_rules.updateMany(
       {
         active: { $ne: true },
@@ -66,7 +66,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => {
     const remaining = await db.sync_rules.find({ state: null as any }).toArray();
     if (remaining.length > 0) {
       const slots = remaining.map((doc) => doc.slot_name).join(', ');
-      throw new ServiceAssertionError(`Invalid state for sync rules: ${slots}`);
+      throw new ServiceAssertionError(`Invalid state for replication stream: ${slots}`);
     }
   } finally {
     await db.client.close();

package/src/storage/MongoBucketStorage.ts

@@ -7,14 +7,16 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 
 import { generateSlotName } from '../utils/util.js';
+import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js';
+import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
+import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
 import { PowerSyncMongo } from './implementation/db.js';
 import { getMongoStorageConfig, SyncRuleDocument } from './implementation/models.js';
 import { MongoChecksumOptions } from './implementation/MongoChecksums.js';
 import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
-import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js';
 
 export interface MongoBucketStorageOptions {
-  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig' | 'mapping'>;
 }
 
 export class MongoBucketStorage extends storage.BucketStorageFactory {
@@ -53,7 +55,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       id = Number(id);
     }
     const storageConfig = (syncRules as MongoPersistedSyncRulesContent).getStorageConfig();
-    const storage = new MongoSyncBucketStorage(
+    const storage = createMongoSyncBucketStorage(
       this,
       id,
       syncRules as MongoPersistedSyncRulesContent,
@@ -100,7 +102,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
     const active = await this.getActiveSyncRulesContent();
 
     if (next != null && next.id == sync_rules_group_id) {
-      // We need to redo the "next" sync rules
+      // We need to redo the "next" replication stream
       await this.updateSyncRules(next.asUpdateOptions());
       // Pro-actively stop replicating
       await this.db.sync_rules.updateOne(
@@ -116,7 +118,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       );
       await this.db.notifyCheckpoint();
     } else if (next == null && active?.id == sync_rules_group_id) {
-      // Slot removed for "active" sync rules, while there is no "next" one.
+      // Slot removed for "active" replication stream, while there is no "next" one.
       await this.updateSyncRules(active.asUpdateOptions());
 
       // In this case we keep the old one as active for clients, so that that existing clients
@@ -136,7 +138,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       );
       await this.db.notifyCheckpoint();
     } else if (next != null && active?.id == sync_rules_group_id) {
-      // Already have next sync rules, but need to stop replicating the active one.
+      // Already have next replication stream, but need to stop replicating the active one.
 
       await this.db.sync_rules.updateOne(
         {
@@ -154,14 +156,14 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
   }
 
   async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent> {
-    const storageVersion = options.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
+    const storageVersion =
+      options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
     const storageConfig = getMongoStorageConfig(storageVersion);
-    await this.db.initializeStorageVersion(storageConfig);
 
     let rules: MongoPersistedSyncRulesContent | undefined = undefined;
 
     await this.session.withTransaction(async () => {
-      // Only have a single set of sync rules with PROCESSING.
+      // Only have a single replication stream with PROCESSING.
       await this.db.sync_rules.updateMany(
         {
           state: storage.SyncRuleState.PROCESSING
@@ -205,6 +207,10 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
         last_fatal_error_ts: null,
         last_keepalive_ts: null
       };
+      if (storageConfig.incrementalReprocessing) {
+        const parsed = options.config.parsed;
+        doc.rule_mapping = BucketDefinitionMapping.fromParsedSyncRules(parsed).serialize();
+      }
       await this.db.sync_rules.insertOne(doc);
       await this.db.notifyCheckpoint();
       rules = new MongoPersistedSyncRulesContent(this.db, doc);
@@ -296,63 +302,90 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       }
     };
 
-    const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' });
-    if (active_sync_rules == null) {
-      return {
-        operations_size_bytes: 0,
-        parameters_size_bytes: 0,
-        replication_size_bytes: 0
-      };
-    }
-    const operations_aggregate = await this.db.bucket_data
+    // For now, we get storage metrics over all v1 and v3 collections.
+    // In the future, we may split these metrics to report separately for active replication streams versus processing streams.
 
-      .aggregate([
-        {
-          $collStats: {
-            storageStats: {}
-          }
-        }
-      ])
-      .toArray()
-      .catch(ignoreNotExisting);
+    const aggregateStaticCollection = async <T extends mongo.Document>(collection: mongo.Collection<T>) => {
+      // We check whether the collection exists before getting the statistics. This avoids repeated
+      // errors in the MongoDB logs if the collection hasn't been created yet.
+      const exists =
+        (await this.db.db.listCollections({ name: collection.collectionName }, { nameOnly: true }).toArray()).length >
+        0;
+      if (!exists) {
+        return [{ storageStats: { size: 0 } }];
+      }
 
-    const parameters_aggregate = await this.db.bucket_parameters
-      .aggregate([
-        {
-          $collStats: {
-            storageStats: {}
+      return collection
+        .aggregate([
+          {
+            $collStats: {
+              storageStats: {}
+            }
           }
-        }
-      ])
-      .toArray()
-      .catch(ignoreNotExisting);
+        ])
+        .toArray()
+        .catch(ignoreNotExisting);
+    };
 
-    const v1_replication_aggregate = await this.db.current_data
-      .aggregate([
-        {
-          $collStats: {
-            storageStats: {}
-          }
-        }
-      ])
-      .toArray()
-      .catch(ignoreNotExisting);
+    const operations_aggregate = await aggregateStaticCollection(this.db.bucket_data);
+    const v3_operation_aggregates = await Promise.all(
+      (await this.db.listBucketDataCollectionsV3()).map((collection) =>
+        collection
+          .aggregate([
+            {
+              $collStats: {
+                storageStats: {}
+              }
+            }
+          ])
+          .toArray()
+          .catch(ignoreNotExisting)
+      )
+    );
 
-    const v3_replication_aggregate = await this.db.v3_current_data
-      .aggregate([
-        {
-          $collStats: {
-            storageStats: {}
-          }
-        }
-      ])
-      .toArray()
-      .catch(ignoreNotExisting);
+    const parameters_aggregate = await aggregateStaticCollection(this.db.bucket_parameters);
+
+    const v3_parameter_aggregates = await Promise.all(
+      (await this.db.listAllParameterIndexCollectionsV3()).map((collection) =>
+        collection
+          .aggregate([
+            {
+              $collStats: {
+                storageStats: {}
+              }
+            }
+          ])
+          .toArray()
+          .catch(ignoreNotExisting)
+      )
+    );
+
+    const v1_source_record_aggregate = await aggregateStaticCollection(this.db.current_data);
+
+    const source_record_aggregates = await Promise.all(
+      (await this.db.listAllSourceRecordCollectionsV3()).map((collection) =>
+        collection
+          .aggregate([
+            {
+              $collStats: {
+                storageStats: {}
+              }
+            }
+          ])
+          .toArray()
+          .catch(ignoreNotExisting)
+      )
+    );
     return {
-      operations_size_bytes: Number(operations_aggregate[0].storageStats.size),
-      parameters_size_bytes: Number(parameters_aggregate[0].storageStats.size),
+      operations_size_bytes:
+        Number(operations_aggregate[0].storageStats.size) +
+        v3_operation_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0),
+      parameters_size_bytes:
+        Number(parameters_aggregate[0].storageStats.size) +
+        v3_parameter_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0),
       replication_size_bytes:
-        Number(v1_replication_aggregate[0].storageStats.size) + Number(v3_replication_aggregate[0].storageStats.size)
+        Number(v1_source_record_aggregate[0]?.storageStats?.size ?? 0) +
+        source_record_aggregates.reduce((total, aggregate) => total + Number(aggregate[0]?.storageStats?.size ?? 0), 0)
     };
   }
 

package/src/storage/implementation/BucketDefinitionMapping.ts

@@ -0,0 +1,72 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import { BucketDataSource, ParameterIndexLookupCreator, SyncConfigWithErrors } from '@powersync/service-sync-rules';
+import { SyncRuleDocument } from './models.js';
+
+export type BucketDefinitionId = string;
+export type ParameterIndexId = string;
+
+export class BucketDefinitionMapping {
+  static fromSyncRules(doc: Pick<SyncRuleDocument, 'rule_mapping'>): BucketDefinitionMapping {
+    return new BucketDefinitionMapping(doc.rule_mapping?.definitions ?? {}, doc.rule_mapping?.parameter_indexes ?? {});
+  }
+
+  static fromParsedSyncRules(syncRules: SyncConfigWithErrors): BucketDefinitionMapping {
+    const definitionNames = syncRules.config.bucketDataSources.map((source) => source.uniqueName).sort();
+    const parameterKeys = syncRules.config.bucketParameterLookupSources
+      .map((source) => `${source.defaultLookupScope.lookupName}#${source.defaultLookupScope.queryId}`)
+      .sort();
+
+    const definitions: Record<string, BucketDefinitionId> = {};
+    const parameterLookups: Record<string, ParameterIndexId> = {};
+
+    for (const [index, uniqueName] of definitionNames.entries()) {
+      definitions[uniqueName] = (index + 1).toString(16);
+    }
+    for (const [index, key] of parameterKeys.entries()) {
+      parameterLookups[key] = (index + 1).toString(16);
+    }
+
+    return new BucketDefinitionMapping(definitions, parameterLookups);
+  }
+
+  constructor(
+    private definitions: Record<string, BucketDefinitionId> = {},
+    private parameterLookupMapping: Record<string, ParameterIndexId> = {}
+  ) {}
+
+  bucketSourceId(source: BucketDataSource): BucketDefinitionId {
+    const defId = this.definitions[source.uniqueName];
+    if (defId == null) {
+      throw new ServiceAssertionError(`No mapping found for bucket source ${source.uniqueName}`);
+    }
+    return defId;
+  }
+
+  allBucketDefinitionIds(): BucketDefinitionId[] {
+    return Object.values(this.definitions);
+  }
+
+  allParameterIndexIds(): ParameterIndexId[] {
+    return Object.values(this.parameterLookupMapping);
+  }
+
+  parameterLookupId(source: ParameterIndexLookupCreator): ParameterIndexId {
+    const key = this.parameterLookupKey(source.defaultLookupScope.lookupName, source.defaultLookupScope.queryId);
+    const defId = this.parameterLookupMapping[key];
+    if (defId == null) {
+      throw new ServiceAssertionError(`No mapping found for parameter lookup source ${key}`);
+    }
+    return defId;
+  }
+
+  private parameterLookupKey(lookupName: string, queryId: string) {
+    return `${lookupName}#${queryId}`;
+  }
+
+  serialize(): NonNullable<SyncRuleDocument['rule_mapping']> {
+    return {
+      definitions: { ...this.definitions },
+      parameter_indexes: { ...this.parameterLookupMapping }
+    };
+  }
+}