@powersync/service-module-mongodb-storage 0.18.3 → 0.20.0

package/dist/storage/implementation/v3/source-table-utils.js

@@ -45,9 +45,8 @@ class SourceTableReconciliationPlanner {
         this.context = context;
     }
     plan(candidateDocs) {
-        const { identity } = this.context;
         for (const doc of candidateDocs) {
-            if (matchingSourceTableIdentity(doc, identity)) {
+            if (this.isCompatible(doc)) {
                 this.retainDoc(doc);
             }
         }
@@ -55,11 +54,16 @@ class SourceTableReconciliationPlanner {
             tables: this.tables,
             narrowingUpdates: this.narrowingUpdates,
             newTableMemberships: this.newTableMemberships(),
-            // Non-retained overlapping docs represent renames / relation-id changes /
-            // replica-identity changes. Same-identity stale docs are retained in production.
-            dropDocs: candidateDocs.filter((doc) => !this.retainedDocIds.has(doc._id.toHexString()) && !matchingSourceTableIdentity(doc, identity))
+            // Compatible records remain available for reuse even when not needed by this config.
+            dropDocs: candidateDocs.filter((doc) => !this.isCompatible(doc))
         };
     }
+    isCompatible(doc) {
+        return this.compatibleTableFor(doc) != null;
+    }
+    compatibleTableFor(doc) {
+        return this.context.sourceCompatibleTables.find((table) => storage.sourceTableIdEquals(table.id, doc._id));
+    }
     retainDoc(doc) {
         const memberships = intersectMembershipIds(doc, this.context.desired);
         const coversDesiredMembership = hasMembershipIds(memberships);
@@ -116,22 +120,14 @@ class SourceTableReconciliationPlanner {
     }
     sourceTableFor(doc, memberships) {
         const { connectionTag, syncConfig, mapping, desired, storeCurrentData } = this.context;
-        const table = sourceTableFromDocument(doc, connectionTag, syncConfig, mapping, matchingSourcesFor(desired, memberships), memberships);
+        const built = sourceTableFromDocument(doc, connectionTag, syncConfig, mapping, matchingSourcesFor(desired, memberships), memberships);
+        // Use any metadata update returned by the reconciler.
+        const resolved = this.compatibleTableFor(doc);
+        const table = resolved == null ? built : built.withSourceMetadata(resolved.sourceMetadata);
         table.storeCurrentData = storeCurrentData;
         return table;
     }
 }
-function sameReplicaIdColumns(left, right) {
-    return (left != null &&
-        left.length == right.length &&
-        left.every((column, index) => column.name == right[index].name && column.type == right[index].type && column.type_oid == right[index].type_oid));
-}
-function matchingSourceTableIdentity(doc, identity) {
-    return (doc.schema_name == identity.schema &&
-        doc.table_name == identity.name &&
-        (identity.objectId == null || doc.relation_id == identity.objectId) &&
-        sameReplicaIdColumns(doc.replica_id_columns, identity.replicaIdColumns));
-}
 export function overlappingSourceTableFilter(connectionId, identity) {
     const clauses = [{ schema_name: identity.schema, table_name: identity.name }];
     if (identity.objectId != null) {
@@ -143,7 +139,7 @@ export function overlappingSourceTableFilter(connectionId, identity) {
     };
 }
 export function createNewSourceTable(id, memberships, context) {
-    const { connectionId, connectionTag, identity, syncConfig, mapping, desired, storeCurrentData } = context;
+    const { connectionId, connectionTag, identity, syncConfig, mapping, desired, storeCurrentData, newTableSourceMetadata } = context;
     const doc = {
         _id: id,
         connection_id: connectionId,
@@ -154,7 +150,9 @@ export function createNewSourceTable(id, memberships, context) {
         snapshot_done: false,
         snapshot_status: undefined,
         bucket_data_source_ids: memberships.bucketDataSourceIds,
-        parameter_lookup_source_ids: memberships.parameterLookupSourceIds
+        parameter_lookup_source_ids: memberships.parameterLookupSourceIds,
+        // Records created together share the same source metadata.
+        source_metadata: newTableSourceMetadata
     };
     const table = sourceTableFromDocument(doc, connectionTag, syncConfig, mapping, matchingSourcesFor(desired, memberships), memberships);
     table.storeCurrentData = storeCurrentData;
@@ -191,12 +189,13 @@ export function sourceTableFromDocument(doc, connectionTag, syncConfig, mapping,
             name: doc.table_name
         },
         objectId: doc.relation_id,
-        replicaIdColumns: doc.replica_id_columns.map((c) => ({ name: c.name, typeId: c.type_oid, type: c.type })),
+        replicaIdColumns: doc.replica_id_columns?.map((column) => ({ name: column.name, typeId: column.type_oid, type: column.type })) ?? [],
         snapshotComplete: doc.snapshot_done,
         bucketDataSources: resolvedMemberships.bucketDataSources,
         parameterLookupSources: resolvedMemberships.parameterLookupSources,
         bucketDataSourceIds: new Set(resolvedMembershipIds.bucketDataSourceIds),
-        parameterLookupSourceIds: new Set(resolvedMembershipIds.parameterLookupSourceIds)
+        parameterLookupSourceIds: new Set(resolvedMembershipIds.parameterLookupSourceIds),
+        sourceMetadata: doc.source_metadata ?? null
     });
     table.syncData = table.bucketDataSources.length > 0;
     table.syncParameters = table.parameterLookupSources.length > 0;

package/dist/storage/implementation/v3/source-table-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"source-table-utils.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/source-table-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAA6C,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAiE7F,MAAM,UAAU,4BAA4B,CAC1C,UAA8B,EAC9B,GAAmB,EACnB,OAAgC;IAEhC,MAAM,eAAe,GAAG,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAC3D,OAAO;QACL,gBAAgB,EAAE,IAAI,GAAG,CACvB,eAAe,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC,CACrG;QACD,yBAAyB,EAAE,IAAI,GAAG,CAChC,eAAe,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC,CAC7G;QACD,aAAa,EAAE,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC;KAClD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,6BAA6B,CAC3C,aAAsC,EACtC,OAAyC;IAEzC,OAAO,IAAI,gCAAgC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,eAAe,CAAC,IAAc,EAAE,KAAe;IACtD,OAAO,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AAC5F,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAqC,EACrC,WAAqC;IAErC,OAAO;QACL,iBAAiB,EAAE,WAAW,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QACjG,sBAAsB,EAAE,WAAW,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;KACrH,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,GAA0B,EAC1B,OAAqC;IAErC,OAAO;QACL,mBAAmB,EAAE,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChG,wBAAwB,EAAE,GAAG,CAAC,2BAA2B,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;KACpH,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,WAAqC;IAC7D,OAAO,WAAW,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,wBAAwB,CAAC,MAAM,GAAG,CAAC,CAAC;AACvG,CAAC;AAED,SAAS,iBAAiB,CAAC,GAA0B,EAAE,WAAqC;IAC1F,OAAO,CACL,eAAe,CAAC,GAAG,CAAC,sBAAsB,EAAE,WAAW,CAAC,mBAAmB,CAAC;QAC5E,eAAe,CAAC,GAAG,CAAC,2BAA2B,EAAE,WAAW,CAAC,wBAAwB,CAAC,CACvF,CAAC;AACJ,CAAC;AAED,MAAM,gCAAgC;IAQP;IAPZ,0BAA0B,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/C,+BAA+B,GAAG,IAAI,GAAG,EAAU,CAAC;IACpD,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,GAA0B,EAAE,CAAC;IACnC,gBAAgB,GAAkC,EAAE,CAAC;IAC9D,sBAAsB,GAAG,KAAK,CAAC;IAEvC,YAA6B,OAAyC;QAAzC,YAAO,GAAP,OAAO,CAAkC;IAAG,CAAC;IAE1E,IAAI,CAAC,aAAsC;QACzC,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,IAAI,2BAA2B,CAAC,GAAG,EAAE,QAAQ,CAAC,EAAE,CAAC;gBAC/C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAAE;YAC/C,0EAA0E;YAC1E,iFAAiF;YACjF,QAAQ,EAAE,aAAa,CAAC,MAAM,CAC5B,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,GAAG,EAAE,QAAQ,CAAC,CACxG;SACF,CAAC;IACJ,CAAC;IAEO,SAAS,CAAC,GAA0B;QAC1C,MAAM,WAAW,GAAG,sBAAsB,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtE,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;QAC9D,MAAM,oBAAoB,GACxB,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC;QAErG,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,WAAW,EAAE,uBAAuB,EAAE,oBAAoB,CAAC,CAAC;QAE1F,IAAI,uBAAuB,IAAI,oBAAoB,EAAE,CAAC;YACpD,IAAI,CAAC,sBAAsB,KAAK,oBAAoB,CAAC;YACrD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAEO,oBAAoB;QAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACjC,OAAO,OAAO,CAAC,gBAAgB,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,yBAAyB,CAAC,IAAI,GAAG,CAAC,CAAC;IACzF,CAAC;IAEO,cAAc,CAAC,GAA0B,EAAE,WAAqC;QACtF,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,oBAAoB,EAAE,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAC;QAC9G,IAAI,CAAC,WAAW,CACd,GAAG,EACH,yBAAyB,EACzB,IAAI,CAAC,+BAA+B,EACpC,WAAW,CAAC,wBAAwB,CACrC,CAAC;IACJ,CAAC;IAED,mFAAmF;IACnF,kFAAkF;IAClF,mFAAmF;IACnF,8EAA8E;IACtE,WAAW,CAAC,GAA0B,EAAE,IAAY,EAAE,OAAoB,EAAE,GAAa;QAC/F,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC/C,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,IAAI,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACpB,MAAM,IAAI,yBAAyB,CACjC,gBAAgB,GAAG,CAAC,GAAG,2BAA2B,IAAI,IAAI,EAAE,QAAQ,MAAM,IAAI,IAAI,EAAE,CACrF,CAAC;YACJ,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAEO,mBAAmB,CACzB,GAA0B,EAC1B,WAAqC,EACrC,uBAAgC,EAChC,oBAA6B;QAE7B,MAAM,YAAY,GAChB,CAAC,uBAAuB,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAElH,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;YACzB,EAAE,EAAE,GAAG,CAAC,GAAG;YACX,WAAW;SACZ,CAAC,CAAC;IACL,CAAC;IAEO,mBAAmB;QACzB,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACjC,MAAM,SAAS,GAA6B;YAC1C,mBAAmB,EAAE,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAC9D,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,EAAE,CAAC,CACjD;YACD,wBAAwB,EAAE,CAAC,GAAG,OAAO,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAC5E,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,EAAE,CAAC,CACtD;SACF,CAAC;QACF,IAAI,gBAAgB,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;YACtF,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,cAAc,CAAC,GAA0B,EAAE,WAAqC;QACtF,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACvF,MAAM,KAAK,GAAG,uBAAuB,CACnC,GAAG,EACH,aAAa,EACb,UAAU,EACV,OAAO,EACP,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,EACxC,WAAW,CACZ,CAAC;QACF,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QAC1C,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AAED,SAAS,oBAAoB,CAAC,IAAmC,EAAE,KAAwB;IACzF,OAAO,CACL,IAAI,IAAI,IAAI;QACZ,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM;QAC3B,IAAI,CAAC,KAAK,CACR,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAChB,MAAM,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,IAAI,MAAM,CAAC,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,CACnH,CACF,CAAC;AACJ,CAAC;AAED,SAAS,2BAA2B,CAAC,GAA0B,EAAE,QAA6B;IAC5F,OAAO,CACL,GAAG,CAAC,WAAW,IAAI,QAAQ,CAAC,MAAM;QAClC,GAAG,CAAC,UAAU,IAAI,QAAQ,CAAC,IAAI;QAC/B,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI,IAAI,GAAG,CAAC,WAAW,IAAI,QAAQ,CAAC,QAAQ,CAAC;QACnE,oBAAoB,CAAC,GAAG,CAAC,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB,CAAC,CACxE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,4BAA4B,CAC1C,YAAoB,EACpB,QAAmE;IAEnE,MAAM,OAAO,GAAG,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,IAAI,EAAE,CAA8B,CAAC;IAC3G,IAAI,QAAQ,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,OAAO;QACL,aAAa,EAAE,YAAY;QAC3B,GAAG,EAAE,OAAO;KACb,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,EAAiB,EACjB,WAAqC,EACrC,OAAyC;IAEzC,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,OAAO,CAAC;IAC1G,MAAM,GAAG,GAA0B;QACjC,GAAG,EAAE,EAAE;QACP,aAAa,EAAE,YAAY;QAC3B,WAAW,EAAE,QAAQ,CAAC,QAAQ;QAC9B,WAAW,EAAE,QAAQ,CAAC,MAAM;QAC5B,UAAU,EAAE,QAAQ,CAAC,IAAI;QACzB,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB;QAC7C,aAAa,EAAE,KAAK;QACpB,eAAe,EAAE,SAAS;QAC1B,sBAAsB,EAAE,WAAW,CAAC,mBAAmB;QACvD,2BAA2B,EAAE,WAAW,CAAC,wBAAwB;KAClE,CAAC;IACF,MAAM,KAAK,GAAG,uBAAuB,CACnC,GAAG,EACH,aAAa,EACb,UAAU,EACV,OAAO,EACP,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,EACxC,WAAW,CACZ,CAAC;IACF,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAE1C,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,MAA6B,EAAE,aAAsB;IACzF,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IACjF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,CAAC,SAAS,GAAG,KAAK,KAAK,YAAY,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,SAAS,kCAAkC,CACzC,GAA0B,EAC1B,UAA8B,EAC9B,OAAgC;IAEhC,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAChE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IAE1E,OAAO;QACL,iBAAiB,EAAE,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAChE,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CACxD;QACD,sBAAsB,EAAE,UAAU,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAChF,wBAAwB,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAChE;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,GAA0B,EAC1B,aAAqB,EACrB,UAA8B,EAC9B,OAAgC,EAChC,WAA6B,EAC7B,aAAwC;IAExC,MAAM,mBAAmB,GAAG,WAAW,IAAI,kCAAkC,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACxG,MAAM,qBAAqB,GAAG,aAAa,IAAI;QAC7C,mBAAmB,EAAE,mBAAmB,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC1G,wBAAwB,EAAE,mBAAmB,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAClF,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAClC;KACF,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC;QACpC,EAAE,EAAE,GAAG,CAAC,GAAG;QACX,GAAG,EAAE;YACH,aAAa;YACb,MAAM,EAAE,GAAG,CAAC,WAAW;YACvB,IAAI,EAAE,GAAG,CAAC,UAAU;SACrB;QACD,QAAQ,EAAE,GAAG,CAAC,WAAW;QACzB,gBAAgB,EAAE,GAAG,CAAC,kBAAmB,CAAC,GAAG,CAC3C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAA4B,CACvF;QACD,gBAAgB,EAAE,GAAG,CAAC,aAAa;QACnC,iBAAiB,EAAE,mBAAmB,CAAC,iBAAiB;QACxD,sBAAsB,EAAE,mBAAmB,CAAC,sBAAsB;QAClE,mBAAmB,EAAE,IAAI,GAAG,CAAC,qBAAqB,CAAC,mBAAmB,CAAC;QACvE,wBAAwB,EAAE,IAAI,GAAG,CAAC,qBAAqB,CAAC,wBAAwB,CAAC;KAClF,CAAC,CAAC;IACH,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;IACpD,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/D,KAAK,CAAC,SAAS,GAAG,UAAU,CAAC,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3D,KAAK,CAAC,cAAc;QAClB,GAAG,CAAC,eAAe,IAAI,IAAI;YACzB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC;gBACE,OAAO,EAAE,GAAG,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,IAAI,IAAI;gBACrD,mBAAmB,EAAE,GAAG,CAAC,eAAe,CAAC,qBAAqB;gBAC9D,eAAe,EAAE,GAAG,CAAC,eAAe,CAAC,gBAAgB;aACtD,CAAC;IACR,OAAO,KAAK,CAAC;AACf,CAAC"}
+{"version":3,"file":"source-table-utils.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/source-table-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAAwD,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAyExG,MAAM,UAAU,4BAA4B,CAC1C,UAA8B,EAC9B,GAAmB,EACnB,OAAgC;IAEhC,MAAM,eAAe,GAAG,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAC3D,OAAO;QACL,gBAAgB,EAAE,IAAI,GAAG,CACvB,eAAe,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC,CACrG;QACD,yBAAyB,EAAE,IAAI,GAAG,CAChC,eAAe,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC,CAC7G;QACD,aAAa,EAAE,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC;KAClD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,6BAA6B,CAC3C,aAAsC,EACtC,OAAyC;IAEzC,OAAO,IAAI,gCAAgC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,eAAe,CAAC,IAAc,EAAE,KAAe;IACtD,OAAO,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AAC5F,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAqC,EACrC,WAAqC;IAErC,OAAO;QACL,iBAAiB,EAAE,WAAW,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QACjG,sBAAsB,EAAE,WAAW,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;KACrH,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,GAA0B,EAC1B,OAAqC;IAErC,OAAO;QACL,mBAAmB,EAAE,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChG,wBAAwB,EAAE,GAAG,CAAC,2BAA2B,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;KACpH,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,WAAqC;IAC7D,OAAO,WAAW,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,CAAC,wBAAwB,CAAC,MAAM,GAAG,CAAC,CAAC;AACvG,CAAC;AAED,SAAS,iBAAiB,CAAC,GAA0B,EAAE,WAAqC;IAC1F,OAAO,CACL,eAAe,CAAC,GAAG,CAAC,sBAAsB,EAAE,WAAW,CAAC,mBAAmB,CAAC;QAC5E,eAAe,CAAC,GAAG,CAAC,2BAA2B,EAAE,WAAW,CAAC,wBAAwB,CAAC,CACvF,CAAC;AACJ,CAAC;AAED,MAAM,gCAAgC;IAQP;IAPZ,0BAA0B,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/C,+BAA+B,GAAG,IAAI,GAAG,EAAU,CAAC;IACpD,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,MAAM,GAA0B,EAAE,CAAC;IACnC,gBAAgB,GAAkC,EAAE,CAAC;IAC9D,sBAAsB,GAAG,KAAK,CAAC;IAEvC,YAA6B,OAAyC;QAAzC,YAAO,GAAP,OAAO,CAAkC;IAAG,CAAC;IAE1E,IAAI,CAAC,aAAsC;QACzC,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAAE;YAC/C,qFAAqF;YACrF,QAAQ,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;SACjE,CAAC;IACJ,CAAC;IAEO,YAAY,CAAC,GAA0B;QAC7C,OAAO,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;IAC9C,CAAC;IAEO,kBAAkB,CAAC,GAA0B;QACnD,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7G,CAAC;IAEO,SAAS,CAAC,GAA0B;QAC1C,MAAM,WAAW,GAAG,sBAAsB,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtE,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;QAC9D,MAAM,oBAAoB,GACxB,CAAC,IAAI,CAAC,oBAAoB,EAAE,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,aAAa,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC;QAErG,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,WAAW,EAAE,uBAAuB,EAAE,oBAAoB,CAAC,CAAC;QAE1F,IAAI,uBAAuB,IAAI,oBAAoB,EAAE,CAAC;YACpD,IAAI,CAAC,sBAAsB,KAAK,oBAAoB,CAAC;YACrD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAEO,oBAAoB;QAC1B,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACjC,OAAO,OAAO,CAAC,gBAAgB,CAAC,IAAI,GAAG,CAAC,IAAI,OAAO,CAAC,yBAAyB,CAAC,IAAI,GAAG,CAAC,CAAC;IACzF,CAAC;IAEO,cAAc,CAAC,GAA0B,EAAE,WAAqC;QACtF,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,oBAAoB,EAAE,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAC;QAC9G,IAAI,CAAC,WAAW,CACd,GAAG,EACH,yBAAyB,EACzB,IAAI,CAAC,+BAA+B,EACpC,WAAW,CAAC,wBAAwB,CACrC,CAAC;IACJ,CAAC;IAED,mFAAmF;IACnF,kFAAkF;IAClF,mFAAmF;IACnF,8EAA8E;IACtE,WAAW,CAAC,GAA0B,EAAE,IAAY,EAAE,OAAoB,EAAE,GAAa;QAC/F,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC/C,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,IAAI,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACpB,MAAM,IAAI,yBAAyB,CACjC,gBAAgB,GAAG,CAAC,GAAG,2BAA2B,IAAI,IAAI,EAAE,QAAQ,MAAM,IAAI,IAAI,EAAE,CACrF,CAAC;YACJ,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAEO,mBAAmB,CACzB,GAA0B,EAC1B,WAAqC,EACrC,uBAAgC,EAChC,oBAA6B;QAE7B,MAAM,YAAY,GAChB,CAAC,uBAAuB,IAAI,oBAAoB,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAElH,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;YACzB,EAAE,EAAE,GAAG,CAAC,GAAG;YACX,WAAW;SACZ,CAAC,CAAC;IACL,CAAC;IAEO,mBAAmB;QACzB,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACjC,MAAM,SAAS,GAA6B;YAC1C,mBAAmB,EAAE,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAC9D,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,EAAE,CAAC,CACjD;YACD,wBAAwB,EAAE,CAAC,GAAG,OAAO,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAC5E,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,EAAE,CAAC,CACtD;SACF,CAAC;QACF,IAAI,gBAAgB,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,EAAE,CAAC;YACtF,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,cAAc,CAAC,GAA0B,EAAE,WAAqC;QACtF,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACvF,MAAM,KAAK,GAAG,uBAAuB,CACnC,GAAG,EACH,aAAa,EACb,UAAU,EACV,OAAO,EACP,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,EACxC,WAAW,CACZ,CAAC;QACF,sDAAsD;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QAC3F,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QAC1C,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AAED,MAAM,UAAU,4BAA4B,CAC1C,YAAoB,EACpB,QAAmE;IAEnE,MAAM,OAAO,GAAG,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,IAAI,EAAE,CAA8B,CAAC;IAC3G,IAAI,QAAQ,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,OAAO;QACL,aAAa,EAAE,YAAY;QAC3B,GAAG,EAAE,OAAO;KACb,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,EAAiB,EACjB,WAAqC,EACrC,OAAyC;IAEzC,MAAM,EACJ,YAAY,EACZ,aAAa,EACb,QAAQ,EACR,UAAU,EACV,OAAO,EACP,OAAO,EACP,gBAAgB,EAChB,sBAAsB,EACvB,GAAG,OAAO,CAAC;IACZ,MAAM,GAAG,GAA0B;QACjC,GAAG,EAAE,EAAE;QACP,aAAa,EAAE,YAAY;QAC3B,WAAW,EAAE,QAAQ,CAAC,QAAQ;QAC9B,WAAW,EAAE,QAAQ,CAAC,MAAM;QAC5B,UAAU,EAAE,QAAQ,CAAC,IAAI;QACzB,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB;QAC7C,aAAa,EAAE,KAAK;QACpB,eAAe,EAAE,SAAS;QAC1B,sBAAsB,EAAE,WAAW,CAAC,mBAAmB;QACvD,2BAA2B,EAAE,WAAW,CAAC,wBAAwB;QACjE,2DAA2D;QAC3D,eAAe,EAAE,sBAAsB;KACxC,CAAC;IACF,MAAM,KAAK,GAAG,uBAAuB,CACnC,GAAG,EACH,aAAa,EACb,UAAU,EACV,OAAO,EACP,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,EACxC,WAAW,CACZ,CAAC;IACF,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAE1C,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,MAA6B,EAAE,aAAsB;IACzF,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO;IACT,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IACjF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,KAAK,CAAC,SAAS,GAAG,KAAK,KAAK,YAAY,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,SAAS,kCAAkC,CACzC,GAA0B,EAC1B,UAA8B,EAC9B,OAAgC;IAEhC,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAChE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IAE1E,OAAO;QACL,iBAAiB,EAAE,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAChE,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CACxD;QACD,sBAAsB,EAAE,UAAU,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAChF,wBAAwB,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAChE;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,GAA0B,EAC1B,aAAqB,EACrB,UAA8B,EAC9B,OAAgC,EAChC,WAA6B,EAC7B,aAAwC;IAExC,MAAM,mBAAmB,GAAG,WAAW,IAAI,kCAAkC,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACxG,MAAM,qBAAqB,GAAG,aAAa,IAAI;QAC7C,mBAAmB,EAAE,mBAAmB,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC1G,wBAAwB,EAAE,mBAAmB,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAClF,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAClC;KACF,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC;QACpC,EAAE,EAAE,GAAG,CAAC,GAAG;QACX,GAAG,EAAE;YACH,aAAa;YACb,MAAM,EAAE,GAAG,CAAC,WAAW;YACvB,IAAI,EAAE,GAAG,CAAC,UAAU;SACrB;QACD,QAAQ,EAAE,GAAG,CAAC,WAAW;QACzB,gBAAgB,EACd,GAAG,CAAC,kBAAkB,EAAE,GAAG,CACzB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAA4B,CAC3G,IAAI,EAAE;QACT,gBAAgB,EAAE,GAAG,CAAC,aAAa;QACnC,iBAAiB,EAAE,mBAAmB,CAAC,iBAAiB;QACxD,sBAAsB,EAAE,mBAAmB,CAAC,sBAAsB;QAClE,mBAAmB,EAAE,IAAI,GAAG,CAAC,qBAAqB,CAAC,mBAAmB,CAAC;QACvE,wBAAwB,EAAE,IAAI,GAAG,CAAC,qBAAqB,CAAC,wBAAwB,CAAC;QACjF,cAAc,EAAE,GAAG,CAAC,eAAe,IAAI,IAAI;KAC5C,CAAC,CAAC;IACH,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;IACpD,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/D,KAAK,CAAC,SAAS,GAAG,UAAU,CAAC,kBAAkB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3D,KAAK,CAAC,cAAc;QAClB,GAAG,CAAC,eAAe,IAAI,IAAI;YACzB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC;gBACE,OAAO,EAAE,GAAG,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,IAAI,IAAI;gBACrD,mBAAmB,EAAE,GAAG,CAAC,eAAe,CAAC,qBAAqB;gBAC9D,eAAe,EAAE,GAAG,CAAC,eAAe,CAAC,gBAAgB;aACtD,CAAC;IACR,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/types/types.d.ts

@@ -25,7 +25,42 @@ export declare const MongoStorageConfig: t.Intersection<t.Codec<{
      * This is an experimental option, and may be removed in a future release.
      */
     bulk_read_preference: t.OptionalCodec<t.Codec<"primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest", "primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest", string, t.CodecProps>>;
+    /**
+     * Throttle clear operations after sync config deploys, by pausing between batches.
+     * The rate produces a pause proportional to the previous batch duration.
+     *
+     * Increase this to reduce impact of clear operations on the storage cluster.
+     * Use 0 to clear as fast as possible.
+     *
+     * Defaults to 0.2.
+     */
+    clear_batch_throttle_rate: t.OptionalCodec<t.Codec<number, number, string, t.CodecProps>>;
+    object_storage: t.OptionalCodec<t.Codec<{
+        type: "s3";
+        bucket: string;
+        region?: string | undefined;
+        prefix?: string | undefined;
+        endpoint?: string | undefined;
+        force_path_style?: boolean | undefined;
+        access_key_id?: string | undefined;
+        secret_access_key?: string | undefined;
+        concurrency_limit?: number | undefined;
+        inline_threshold_bytes?: number | undefined;
+    }, {
+        type: "s3";
+        bucket: string;
+        region?: string | undefined;
+        prefix?: string | undefined;
+        endpoint?: string | undefined;
+        force_path_style?: boolean | undefined;
+        access_key_id?: string | undefined;
+        secret_access_key?: string | undefined;
+        concurrency_limit?: number | undefined;
+        inline_threshold_bytes?: number | undefined;
+    }, string, t.CodecProps>>;
 }>>;
 export type MongoStorageConfig = t.Encoded<typeof MongoStorageConfig>;
 export type MongoStorageConfigDecoded = t.Decoded<typeof MongoStorageConfig>;
+export declare const DEFAULT_CLEAR_BATCH_THROTTLE_RATE = 0.2;
+export declare function normalizeClearBatchThrottleRate(value: number | undefined): number;
 export declare function isMongoStorageConfig(config: service_types.configFile.GenericStorageConfig): config is MongoStorageConfig;

package/dist/types/types.js

@@ -1,5 +1,20 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
 import * as t from 'ts-codec';
+const S3ObjectStorageConfig = t.object({
+    type: t.literal('s3'),
+    bucket: t.string,
+    region: t.string.optional(),
+    prefix: t.string.optional(),
+    endpoint: t.string.optional(),
+    force_path_style: t.boolean.optional(),
+    access_key_id: t.string.optional(),
+    secret_access_key: t.string.optional(),
+    concurrency_limit: t.number.optional(),
+    // Chunks whose BSON-serialized size falls below this byte threshold
+    // stay inline in MongoDB instead of being offloaded to S3. Default 1024.
+    inline_threshold_bytes: t.number.optional()
+});
 export const MongoStorageReadPreference = t
     .literal('primary')
     .or(t.literal('primaryPreferred'))
@@ -14,8 +29,27 @@ export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(t.object({
      *
      * This is an experimental option, and may be removed in a future release.
      */
-    bulk_read_preference: MongoStorageReadPreference.optional()
+    bulk_read_preference: MongoStorageReadPreference.optional(),
+    /**
+     * Throttle clear operations after sync config deploys, by pausing between batches.
+     * The rate produces a pause proportional to the previous batch duration.
+     *
+     * Increase this to reduce impact of clear operations on the storage cluster.
+     * Use 0 to clear as fast as possible.
+     *
+     * Defaults to 0.2.
+     */
+    clear_batch_throttle_rate: t.number.optional(),
+    object_storage: S3ObjectStorageConfig.optional()
 }));
+export const DEFAULT_CLEAR_BATCH_THROTTLE_RATE = 0.2;
+export function normalizeClearBatchThrottleRate(value) {
+    const rate = value ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;
+    if (!Number.isFinite(rate) || rate < 0 || rate > 20) {
+        throw new ServiceError(ErrorCode.PSYNC_S3201, 'storage.clear_batch_throttle_rate must be a finite number between 0 and 20');
+    }
+    return rate;
+}
 export function isMongoStorageConfig(config) {
     return config.type == lib_mongo.MONGO_CONNECTION_TYPE;
 }

package/dist/types/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC;KACxC,OAAO,CAAC,SAAS,CAAC;KAClB,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;KACjC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;KAC1B,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;KACnC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;AAI5B,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAC7D,CAAC,CAAC,MAAM,CAAC;IACP;;;;;;OAMG;IACH,oBAAoB,EAAE,0BAA0B,CAAC,QAAQ,EAAE;CAC5D,CAAC,CACH,CAAC;AAKF,MAAM,UAAU,oBAAoB,CAClC,MAAqD;IAErD,OAAO,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,qBAAqB,CAAC;AACxD,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAE5E,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IACrB,MAAM,EAAE,CAAC,CAAC,MAAM;IAChB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC3B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IACtC,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAClC,iBAAiB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACtC,iBAAiB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACtC,oEAAoE;IACpE,yEAAyE;IACzE,sBAAsB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CAC5C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,CAAC;KACxC,OAAO,CAAC,SAAS,CAAC;KAClB,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;KACjC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;KAC1B,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;KACnC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;AAI5B,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAC7D,CAAC,CAAC,MAAM,CAAC;IACP;;;;;;OAMG;IACH,oBAAoB,EAAE,0BAA0B,CAAC,QAAQ,EAAE;IAC3D;;;;;;;;OAQG;IACH,yBAAyB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAE9C,cAAc,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CACjD,CAAC,CACH,CAAC;AAKF,MAAM,CAAC,MAAM,iCAAiC,GAAG,GAAG,CAAC;AAErD,MAAM,UAAU,+BAA+B,CAAC,KAAyB;IACvE,MAAM,IAAI,GAAG,KAAK,IAAI,iCAAiC,CAAC;IACxD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,EAAE,EAAE,CAAC;QACpD,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,4EAA4E,CAC7E,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,MAAqD;IAErD,OAAO,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,qBAAqB,CAAC;AACxD,CAAC"}

package/dist/utils/test-utils.js

@@ -18,7 +18,9 @@ export function mongoTestStorageFactoryGenerator(factoryOptions) {
             return new MongoBucketStorage(db, {
                 replicationStreamNamePrefix: 'test_',
                 checksumOptions: factoryOptions.checksumOptions,
-                supportsMultipleSyncConfigs: factoryOptions.supportsMultipleSyncConfigs
+                supportsMultipleSyncConfigs: factoryOptions.supportsMultipleSyncConfigs,
+                objectStorage: factoryOptions.objectStorage,
+                inlineThresholdBytes: factoryOptions.inlineThresholdBytes
             });
         },
         tableIdStrings: false

package/dist/utils/test-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAA6B,MAAM,kCAAkC,CAAC;AACjG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAOjE,MAAM,UAAU,gCAAgC,CAAC,cAAuC;IACtF,OAAO;QACL,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;YAEzE,8EAA8E;YAC9E,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBAC5F,MAAM,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;YAED,8EAA8E;YAC9E,MAAM,EAAE,CAAC,gCAAgC,EAAE,CAAC;YAE5C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;gBACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACnB,CAAC;YAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,EAAE;gBAChC,2BAA2B,EAAE,OAAO;gBACpC,eAAe,EAAE,cAAc,CAAC,eAAe;gBAC/C,2BAA2B,EAAE,cAAc,CAAC,2BAA2B;aACxE,CAAC,CAAC;QACL,CAAC;QACD,cAAc,EAAE,KAAK;KACtB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sCAAsC,CAAC,cAAuC;IAC5F,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;QAEzE,MAAM,EAAE,CAAC,mCAAmC,EAAE,CAAC;QAE/C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;YACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;QAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE;QACxC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC"}
+{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAA6B,MAAM,kCAAkC,CAAC;AACjG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAOjE,MAAM,UAAU,gCAAgC,CAAC,cAAuC;IACtF,OAAO;QACL,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;YAEzE,8EAA8E;YAC9E,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBAC5F,MAAM,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;YAED,8EAA8E;YAC9E,MAAM,EAAE,CAAC,gCAAgC,EAAE,CAAC;YAE5C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;gBACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACnB,CAAC;YAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,EAAE;gBAChC,2BAA2B,EAAE,OAAO;gBACpC,eAAe,EAAE,cAAc,CAAC,eAAe;gBAC/C,2BAA2B,EAAE,cAAc,CAAC,2BAA2B;gBACvE,aAAa,EAAE,cAAc,CAAC,aAAa;gBAC3C,oBAAoB,EAAE,cAAc,CAAC,oBAAoB;aAC1D,CAAC,CAAC;QACL,CAAC;QACD,cAAc,EAAE,KAAK;KACtB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sCAAsC,CAAC,cAAuC;IAC5F,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;QAEzE,MAAM,EAAE,CAAC,mCAAmC,EAAE,CAAC;QAE/C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;YACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;QAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE;QACxC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC"}

package/dist/utils/util.d.ts

@@ -1,9 +1,13 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
+import { Logger } from '@powersync/lib-services-framework';
 import { storage, utils } from '@powersync/service-core';
 import * as bson from 'bson';
 import { BucketDataDoc } from '../storage/implementation/common/BucketDataDoc.js';
-export declare function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T>;
+type ClearCollectionFilter<T extends mongo.Document> = mongo.Filter<T> & {
+    _id: mongo.FilterOperators<mongo.InferIdType<T>>;
+};
+export declare function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.FilterOperators<T>;
 export declare function generateReplicationStreamName(prefix: string, replicationStreamId: number): string;
 /**
  * Read a single batch of data from a cursor, then close it.
@@ -30,6 +34,8 @@ export declare function retryOnMongoMaxTimeMSExpired<T>(operation: () => Promise
     retryDelayMs: number;
     onRetry?: (retryCount: number) => void;
 }): Promise<T>;
+export declare function clearCollectionInIdRanges<T extends mongo.Document>(logger: Logger, label: string, collection: mongo.Collection<T>, filter: ClearCollectionFilter<T>, signal?: AbortSignal, throttleRate?: number): Promise<number>;
+export declare function clearCollectionInIdBatches<T extends mongo.Document>(logger: Logger, label: string, collection: mongo.Collection<T>, filter: mongo.Filter<T>, signal?: AbortSignal, throttleRate?: number): Promise<number>;
 export declare const createPaginatedConnectionQuery: <T extends mongo.Document>(query: mongo.Filter<T>, collection: mongo.Collection<T>, limit: number, cursor?: string) => Promise<{
     items: lib_mongo.mongo.WithId<T>[];
     count: number;
@@ -37,3 +43,4 @@ export declare const createPaginatedConnectionQuery: <T extends mongo.Document>(
     cursor: any;
     more: boolean;
 }>;
+export {};

package/dist/utils/util.js

@@ -5,6 +5,40 @@ import * as bson from 'bson';
 import * as crypto from 'crypto';
 import * as timers from 'node:timers/promises';
 import * as uuid from 'uuid';
+import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../types/types.js';
+/**
+ * Default and max batch size for clearing.
+ */
+const CLEAR_BATCH_SIZE = 10_000;
+/**
+ * Minimum batch size for clearing when a batch takes too long.
+ */
+const CLEAR_MIN_BATCH_SIZE = 100;
+/**
+ * We target batch duration to stay between the min and max. If the druation is outside this range,
+ * we increase or decrease the batch size.
+ *
+ * MONGO_CLEAR_OPERATION_TIMEOUT_MS controls the absolute max of individual find and delete operations,
+ * but we try to avoid hitting that.
+ */
+const CLEAR_BATCH_MIN_DURATION_MS = 500;
+const CLEAR_BATCH_MAX_DURATION_MS = 2000;
+function throwIfClearAborted(signal) {
+    if (signal?.aborted) {
+        throw new ReplicationAbortedError('Aborted clearing data', signal.reason);
+    }
+}
+async function waitWithSignal(delayMs, signal, abortMessage) {
+    try {
+        await timers.setTimeout(delayMs, undefined, { signal });
+    }
+    catch (error) {
+        if (signal?.aborted) {
+            throw new ReplicationAbortedError(abortMessage, signal.reason);
+        }
+        throw error;
+    }
+}
 export function idPrefixFilter(prefix, rest) {
     let filter = {
         $gte: {
@@ -133,15 +167,104 @@ export async function retryOnMongoMaxTimeMSExpired(operation, options) {
             return await operation();
         }
         catch (e) {
-            if (!lib_mongo.isMongoServerError(e) || e.codeName !== 'MaxTimeMSExpired') {
+            if (!lib_mongo.isMaxTimeMSExpiredError(e)) {
                 throw e;
             }
             retryCount += 1;
             options.onRetry?.(retryCount);
-            await timers.setTimeout(options.retryDelayMs);
+            await waitWithSignal(options.retryDelayMs, options.signal, options.abortMessage ?? 'Aborted MongoDB operation');
+        }
+    }
+}
+async function clearCollectionInBatches(logger, label, findAndDeleteBatch, signal, throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE) {
+    let batchSize = CLEAR_BATCH_SIZE;
+    let deletedCount = 0;
+    while (true) {
+        throwIfClearAborted(signal);
+        try {
+            const batchStartedAt = performance.now();
+            const foundBatchSize = await findAndDeleteBatch(batchSize);
+            const batchDurationMs = performance.now() - batchStartedAt;
+            deletedCount += foundBatchSize;
+            if (foundBatchSize > 0) {
+                logger.info(`Cleared batch of ${label} (${foundBatchSize} documents) in ${Math.round(batchDurationMs)}ms, continuing...`);
+            }
+            if (foundBatchSize < batchSize) {
+                return deletedCount;
+            }
+            if (batchDurationMs > CLEAR_BATCH_MAX_DURATION_MS) {
+                batchSize = Math.max(CLEAR_MIN_BATCH_SIZE, Math.floor(batchSize / 2));
+            }
+            else if (batchDurationMs < CLEAR_BATCH_MIN_DURATION_MS) {
+                batchSize = Math.min(CLEAR_BATCH_SIZE, batchSize * 2);
+            }
+            await waitWithSignal(batchDurationMs * throttleRate, signal, 'Aborted clearing data');
+        }
+        catch (error) {
+            if (!lib_mongo.isTimeoutError(error) || batchSize == CLEAR_MIN_BATCH_SIZE) {
+                throw error;
+            }
+            const nextBatchSize = Math.max(CLEAR_MIN_BATCH_SIZE, Math.floor(batchSize / 2));
+            logger.info(`Clearing batch of ${label} timed out with batch size ${batchSize}, retrying find and delete with ${nextBatchSize}...`);
+            batchSize = nextBatchSize;
+            // Pause for at least the equivalent of one query timeout before retrying, since the cause may be high database load.
+            const pauseDuration = Math.max(throttleRate, 1.0) * lib_mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS;
+            await waitWithSignal(pauseDuration, signal, 'Aborted clearing data');
         }
     }
 }
+export async function clearCollectionInIdRanges(logger, label, collection, filter, signal, throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE) {
+    return clearCollectionInBatches(logger, label, async (batchSize) => {
+        const queryOptions = {
+            maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
+            projection: { _id: 1 }
+        };
+        const [batchEnd] = await collection
+            .find(filter, queryOptions)
+            .sort({ _id: 1 })
+            .skip(batchSize)
+            .limit(1)
+            .toArray();
+        throwIfClearAborted(signal);
+        if (batchEnd == null) {
+            // We're on the last batch
+            const result = await collection.deleteMany(filter, {
+                maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
+            });
+            return result.deletedCount;
+        }
+        const idRange = {
+            ...filter._id,
+            $lt: batchEnd._id
+        };
+        await collection.deleteMany({
+            ...filter,
+            _id: idRange
+        }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS });
+        return batchSize;
+    }, signal, throttleRate);
+}
+export async function clearCollectionInIdBatches(logger, label, collection, filter, signal, throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE) {
+    return clearCollectionInBatches(logger, label, async (batchSize) => {
+        const documents = await collection
+            .find(filter, {
+            maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
+            projection: { _id: 1 }
+        })
+            .limit(batchSize)
+            .toArray();
+        throwIfClearAborted(signal);
+        if (documents.length === 0) {
+            return 0;
+        }
+        await collection.deleteMany({
+            _id: {
+                $in: documents.map((document) => document._id)
+            }
+        }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS });
+        return documents.length;
+    }, signal, throttleRate);
+}
 export const createPaginatedConnectionQuery = async (query, collection, limit, cursor) => {
     const createQuery = (cursor) => {
         if (!cursor) {

package/dist/utils/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/utils/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AACnG,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,MAAM,MAAM,sBAAsB,CAAC;AAC/C,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAG7B,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,MAAc,EAAE,mBAA2B;IACvF,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,mBAAmB,IAAI,WAAW,EAAE,CAAC;AAC1D,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA+B;IACtE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAkB;IAC3C,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAA4B,EAAE,EAAqB;IACnF,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAA4B;IACvD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,wCAAwC,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAA4B;IACjD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;SAAM,CAAC;QACN,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,OAA4B,EAAE,IAAoB;IACvF,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;IACxE,CAAC;IACD,IAAK,OAAe,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QACzC,OAAe,CAAC,YAAY,GAAG,IAAI,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,SAA2B,EAC3B,OAKC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,0EAA0E;IAC1E,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,uBAAuB,CAAC,OAAO,CAAC,YAAY,IAAI,2BAA2B,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChH,CAAC;QACD,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,kBAAkB,EAAE,CAAC;gBAC1E,MAAM,CAAC,CAAC;YACV,CAAC;YACD,UAAU,IAAI,CAAC,CAAC;YAChB,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;YAC9B,MAAM,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,8BAA8B,GAAG,KAAK,EACjD,KAAsB,EACtB,UAA+B,EAC/B,KAAa,EACb,MAAe,EACf,EAAE;IACF,MAAM,WAAW,GAAG,CAAC,MAAe,EAAE,EAAE;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY;YACrC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE;YAC1D,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO;YACL,GAAG,KAAK;YACR,YAAY;SACM,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE;QACtD,IAAI,EAAE;YACJ,6FAA6F;YAC7F,YAAY,EAAE,CAAC,CAAC;SACjB;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B;;SAEK;IACL,OAAO;QACL,KAAK;QACL,KAAK;QACL,+EAA+E;QAC/E,MAAM,EAAE,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;QACxF,IAAI,EAAE,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC;KACzB,CAAC;AACJ,CAAC,CAAC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/utils/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAU,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC3G,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,MAAM,MAAM,sBAAsB,CAAC;AAC/C,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,iCAAiC,EAAE,MAAM,mBAAmB,CAAC;AAEtE;;GAEG;AACH,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC;;GAEG;AACH,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,2BAA2B,GAAG,GAAG,CAAC;AACxC,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAMzC,SAAS,mBAAmB,CAAC,MAAoB;IAC/C,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,uBAAuB,CAAC,uBAAuB,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,OAAe,EAAE,MAA+B,EAAE,YAAoB;IAClG,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IAC1D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,uBAAuB,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,MAAc,EAAE,mBAA2B;IACvF,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,mBAAmB,IAAI,WAAW,EAAE,CAAC;AAC1D,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA+B;IACtE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAkB;IAC3C,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAA4B,EAAE,EAAqB;IACnF,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAA4B;IACvD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,wCAAwC,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAA4B;IACjD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;SAAM,CAAC;QACN,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,OAA4B,EAAE,IAAoB;IACvF,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;IACxE,CAAC;IACD,IAAK,OAAe,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QACzC,OAAe,CAAC,YAAY,GAAG,IAAI,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,SAA2B,EAC3B,OAKC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,0EAA0E;IAC1E,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,uBAAuB,CAAC,OAAO,CAAC,YAAY,IAAI,2BAA2B,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChH,CAAC;QACD,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1C,MAAM,CAAC,CAAC;YACV,CAAC;YACD,UAAU,IAAI,CAAC,CAAC;YAChB,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;YAC9B,MAAM,cAAc,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,YAAY,IAAI,2BAA2B,CAAC,CAAC;QAClH,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,MAAc,EACd,KAAa,EACb,kBAA0D,EAC1D,MAAoB,EACpB,YAAY,GAAG,iCAAiC;IAEhD,IAAI,SAAS,GAAG,gBAAgB,CAAC;IACjC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,OAAO,IAAI,EAAE,CAAC;QACZ,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YACzC,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,SAAS,CAAC,CAAC;YAC3D,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC;YAC3D,YAAY,IAAI,cAAc,CAAC;YAC/B,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,CAAC,IAAI,CACT,oBAAoB,KAAK,KAAK,cAAc,kBAAkB,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,mBAAmB,CAC7G,CAAC;YACJ,CAAC;YACD,IAAI,cAAc,GAAG,SAAS,EAAE,CAAC;gBAC/B,OAAO,YAAY,CAAC;YACtB,CAAC;YACD,IAAI,eAAe,GAAG,2BAA2B,EAAE,CAAC;gBAClD,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;YACxE,CAAC;iBAAM,IAAI,eAAe,GAAG,2BAA2B,EAAE,CAAC;gBACzD,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;YACxD,CAAC;YACD,MAAM,cAAc,CAAC,eAAe,GAAG,YAAY,EAAE,MAAM,EAAE,uBAAuB,CAAC,CAAC;QACxF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,SAAS,IAAI,oBAAoB,EAAE,CAAC;gBAC1E,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;YAChF,MAAM,CAAC,IAAI,CACT,qBAAqB,KAAK,8BAA8B,SAAS,mCAAmC,aAAa,KAAK,CACvH,CAAC;YACF,SAAS,GAAG,aAAa,CAAC;YAE1B,qHAAqH;YACrH,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,GAAG,SAAS,CAAC,gCAAgC,CAAC;YAC/F,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,uBAAuB,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,KAAa,EACb,UAA+B,EAC/B,MAAgC,EAChC,MAAoB,EACpB,YAAY,GAAG,iCAAiC;IAEhD,OAAO,wBAAwB,CAC7B,MAAM,EACN,KAAK,EACL,KAAK,EAAE,SAAS,EAAE,EAAE;QAClB,MAAM,YAAY,GAAyB;YACzC,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC;YACxD,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;SACvB,CAAC;QACF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,UAAU;aAChC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC;aAC1B,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;aAChB,IAAI,CAAC,SAAS,CAAC;aACf,KAAK,CAAC,CAAC,CAAC;aACR,OAAO,EAAE,CAAC;QACb,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAE5B,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACrB,0BAA0B;YAC1B,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,MAAM,EAAE;gBACjD,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC;aACzD,CAAC,CAAC;YACH,OAAO,MAAM,CAAC,YAAY,CAAC;QAC7B,CAAC;QAED,MAAM,OAAO,GAAgD;YAC3D,GAAG,MAAM,CAAC,GAAG;YACb,GAAG,EAAE,QAAQ,CAAC,GAAG;SAClB,CAAC;QACF,MAAM,UAAU,CAAC,UAAU,CACzB;YACE,GAAG,MAAM;YACT,GAAG,EAAE,OAAO;SACM,EACpB,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC,EAAE,CAC7D,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC,EACD,MAAM,EACN,YAAY,CACb,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,MAAc,EACd,KAAa,EACb,UAA+B,EAC/B,MAAuB,EACvB,MAAoB,EACpB,YAAY,GAAG,iCAAiC;IAEhD,OAAO,wBAAwB,CAC7B,MAAM,EACN,KAAK,EACL,KAAK,EAAE,SAAS,EAAE,EAAE;QAClB,MAAM,SAAS,GAAG,MAAM,UAAU;aAC/B,IAAI,CAAC,MAAM,EAAE;YACZ,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC;YACxD,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;SACvB,CAAC;aACD,KAAK,CAAC,SAAS,CAAC;aAChB,OAAO,EAAE,CAAC;QACb,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAE5B,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,UAAU,CAAC,UAAU,CACzB;YACE,GAAG,EAAE;gBACH,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;aAC/C;SACiB,EACpB,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC,EAAE,CAC7D,CAAC;QACF,OAAO,SAAS,CAAC,MAAM,CAAC;IAC1B,CAAC,EACD,MAAM,EACN,YAAY,CACb,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,8BAA8B,GAAG,KAAK,EACjD,KAAsB,EACtB,UAA+B,EAC/B,KAAa,EACb,MAAe,EACf,EAAE;IACF,MAAM,WAAW,GAAG,CAAC,MAAe,EAAE,EAAE;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY;YACrC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE;YAC1D,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO;YACL,GAAG,KAAK;YACR,YAAY;SACM,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE;QACtD,IAAI,EAAE;YACJ,6FAA6F;YAC7F,YAAY,EAAE,CAAC,CAAC;SACjB;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B;;SAEK;IACL,OAAO;QACL,KAAK;QACL,KAAK;QACL,+EAA+E;QAC/E,MAAM,EAAE,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;QACxF,IAAI,EAAE,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC;KACzB,CAAC;AACJ,CAAC,CAAC"}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@powersync/service-module-mongodb-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
-  "version": "0.18.3",
+  "version": "0.20.0",
   "license": "FSL-1.1-ALv2",
   "type": "module",
   "publishConfig": {
@@ -12,20 +12,23 @@
     "./types": "./dist/types/types.js"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.637.0",
+    "@smithy/core": "^3.24.6",
+    "async-mutex": "^0.5.0",
     "bson": "^6.10.4",
     "ix": "^5.0.0",
     "lru-cache": "^10.2.2",
     "ts-codec": "^1.3.0",
     "uuid": "^14.0.0",
-    "@powersync/lib-service-mongodb": "0.6.29",
-    "@powersync/lib-services-framework": "0.9.8",
-    "@powersync/service-core": "1.23.3",
+    "@powersync/lib-service-mongodb": "0.6.30",
+    "@powersync/lib-services-framework": "0.10.0",
+    "@powersync/service-core": "1.25.0",
     "@powersync/service-jsonbig": "0.17.13",
-    "@powersync/service-sync-rules": "0.39.0",
-    "@powersync/service-types": "0.16.1"
+    "@powersync/service-sync-rules": "0.40.0",
+    "@powersync/service-types": "0.17.0"
   },
   "devDependencies": {
-    "@powersync/service-core-tests": "0.17.2"
+    "@powersync/service-core-tests": "0.18.0"
   },
   "scripts": {
     "build": "tsc -b",

package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts

@@ -0,0 +1,55 @@
+import { migrations } from '@powersync/service-core';
+import * as storage from '../../../storage/storage-index.js';
+import { MongoStorageConfig } from '../../../types/types.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    await db.write_checkpoints.createIndex(
+      {
+        checkpoint_requested_at: 1
+      },
+      {
+        name: 'checkpoint_requested_at',
+        // Only client-requested checkpoints have this field; generated
+        // checkpoints leave it unset. This keeps the index limited to the
+        // documents the compact job's retention delete scans.
+        partialFilterExpression: { checkpoint_requested_at: { $exists: true } }
+      }
+    );
+    await db.custom_write_checkpoints.createIndex(
+      {
+        checkpoint_requested_at: 1
+      },
+      {
+        name: 'checkpoint_requested_at',
+        partialFilterExpression: { checkpoint_requested_at: { $exists: true } }
+      }
+    );
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    if (await db.write_checkpoints.indexExists('checkpoint_requested_at')) {
+      await db.write_checkpoints.dropIndex('checkpoint_requested_at');
+    }
+    if (await db.custom_write_checkpoints.indexExists('checkpoint_requested_at')) {
+      await db.custom_write_checkpoints.dropIndex('checkpoint_requested_at');
+    }
+  } finally {
+    await db.client.close();
+  }
+};

package/src/storage/MongoBucketStorage.ts

@@ -17,6 +17,7 @@ import {
   isCompatible
 } from '@powersync/service-core';
 import { ObjectId } from 'bson';
+import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../types/types.js';
 import { generateReplicationStreamName } from '../utils/util.js';
 import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
 import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
@@ -26,17 +27,21 @@ import { MongoChecksumOptions } from './implementation/MongoChecksums.js';
 import { MongoPersistedReplicationStream } from './implementation/MongoPersistedReplicationStream.js';
 import { syncRuleStateUpdatePipeline } from './implementation/SyncRuleStateUpdate.js';
 import { SyncRuleDocumentV1 } from './implementation/v1/models.js';
+import { ObjectStorage } from './implementation/v3/object-storage/ObjectStorage.js';
 import { VersionedPowerSyncMongoV3 } from './implementation/v3/VersionedPowerSyncMongoV3.js';
 import { ReplicationStreamDocumentV3, SyncConfigDefinition, SyncRuleConfigStateV3 } from './storage-index.js';
 
 export interface MongoBucketStorageOptions {
+  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
+  objectStorage?: ObjectStorage;
+  inlineThresholdBytes?: number;
   /**
    * Prefix for replication stream name and Postgres logical replication slot name.
    */
   replicationStreamNamePrefix: string;
   readPreference?: mongo.ReadPreference;
-  checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
   checksumCacheTtlMs?: number;
+  clearBatchThrottleRate?: number;
   /**
    * Reuse a compatible active replication stream by appending a new sync config.
    *
@@ -95,7 +100,10 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
         checksumOptions: this.options.checksumOptions,
         readPreference: this.options.readPreference,
         checksumCacheTtlMs: this.options.checksumCacheTtlMs,
-        storageConfig
+        storageConfig,
+        objectStorage: this.options.objectStorage,
+        inlineThresholdBytes: this.options.inlineThresholdBytes,
+        clearBatchThrottleRate: this.options.clearBatchThrottleRate ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE
       }
     );
     if (!options?.skipLifecycleHooks) {

package/src/storage/implementation/MongoBucketBatch.ts

@@ -32,6 +32,7 @@ import { MongoIdSequence } from './MongoIdSequence.js';
 import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
 import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js';
 import { OperationBatch, RecordOperation } from './OperationBatch.js';
+import { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
 
 // Currently, we can only have a single flush() at a time, since it locks the op_id sequence.
 // While the MongoDB transaction retry mechanism handles this okay, using an in-process Mutex
@@ -71,6 +72,9 @@ export interface MongoBucketBatchOptions {
 
   logger: Logger;
   tracer?: PerformanceTracer<'storage' | 'evaluate'>;
+
+  objectStorage?: ObjectStorage;
+  inlineThresholdBytes?: number;
 }
 
 export abstract class MongoBucketBatch