@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/dist/storage/implementation/v3/source-table-utils.js

@@ -0,0 +1,206 @@
+import { ReplicationAssertionError } from '@powersync/lib-services-framework';
+import { storage } from '@powersync/service-core';
+export function sourceTableDesiredResolution(syncConfig, ref, mapping, eventById) {
+    const matchingSources = syncConfig.getMatchingSources(ref);
+    return {
+        bucketSourceById: new Map(matchingSources.bucketDataSources.map((source) => [mapping.bucketSourceId(source), source])),
+        parameterLookupSourceById: new Map(matchingSources.parameterLookupSources.map((source) => [mapping.parameterLookupId(source), source])),
+        eventDefinitionById: new Map([...eventById].filter(([, event]) => event.tableTriggersEvent(ref)))
+    };
+}
+export function planSourceTableReconciliation(candidateDocs, context) {
+    return new SourceTableReconciliationPlanner(context).plan(candidateDocs);
+}
+function sameStringArray(left, right) {
+    return left.length == right.length && left.every((value, index) => value == right[index]);
+}
+function matchingSourcesFor(desired, memberships) {
+    return {
+        bucketDataSources: memberships.bucketDataSourceIds.map((id) => desired.bucketSourceById.get(id)),
+        parameterLookupSources: memberships.parameterLookupSourceIds.map((id) => desired.parameterLookupSourceById.get(id))
+    };
+}
+function intersectMembershipIds(doc, desired) {
+    return {
+        bucketDataSourceIds: doc.bucket_data_source_ids.filter((id) => desired.bucketSourceById.has(id)),
+        parameterLookupSourceIds: doc.parameter_lookup_source_ids.filter((id) => desired.parameterLookupSourceById.has(id)),
+        eventDefinitionIds: doc.event_definition_ids.filter((id) => desired.eventDefinitionById.has(id))
+    };
+}
+function hasMembershipIds(memberships) {
+    return (memberships.bucketDataSourceIds.length > 0 ||
+        memberships.parameterLookupSourceIds.length > 0 ||
+        memberships.eventDefinitionIds.length > 0);
+}
+function sameMembershipIds(doc, memberships) {
+    return (sameStringArray(doc.bucket_data_source_ids, memberships.bucketDataSourceIds) &&
+        sameStringArray(doc.parameter_lookup_source_ids, memberships.parameterLookupSourceIds) &&
+        sameStringArray(doc.event_definition_ids, memberships.eventDefinitionIds));
+}
+class SourceTableReconciliationPlanner {
+    context;
+    coveredBucketDataSourceIds = new Set();
+    coveredParameterLookupSourceIds = new Set();
+    coveredEventDefinitionIds = new Set();
+    tables = [];
+    narrowingUpdates = [];
+    constructor(context) {
+        this.context = context;
+    }
+    plan(candidateDocs) {
+        for (const doc of candidateDocs) {
+            if (this.isCompatible(doc)) {
+                this.retainDoc(doc);
+            }
+        }
+        return {
+            tables: this.tables,
+            narrowingUpdates: this.narrowingUpdates,
+            newTableMemberships: this.newTableMemberships(),
+            // Compatible records remain available for reuse even when not needed by this config.
+            dropDocs: candidateDocs.filter((doc) => !this.isCompatible(doc))
+        };
+    }
+    isCompatible(doc) {
+        return this.compatibleTableFor(doc) != null;
+    }
+    compatibleTableFor(doc) {
+        return this.context.sourceCompatibleTables.find((table) => storage.sourceTableIdEquals(table.id, doc._id));
+    }
+    retainDoc(doc) {
+        const memberships = intersectMembershipIds(doc, this.context.desired);
+        const coversDesiredMembership = hasMembershipIds(memberships);
+        this.recordCoverage(doc, memberships);
+        this.planNarrowingUpdate(doc, memberships, coversDesiredMembership);
+        if (coversDesiredMembership) {
+            this.tables.push(this.sourceTableFor(doc, memberships));
+        }
+    }
+    recordCoverage(doc, memberships) {
+        this.addCoverage(doc, 'bucket data source', this.coveredBucketDataSourceIds, memberships.bucketDataSourceIds);
+        this.addCoverage(doc, 'parameter lookup source', this.coveredParameterLookupSourceIds, memberships.parameterLookupSourceIds);
+        this.addCoverage(doc, 'event definition', this.coveredEventDefinitionIds, memberships.eventDefinitionIds);
+    }
+    // Membership sets must be pairwise disjoint across the docs of one physical table:
+    // each desired id is covered by exactly one doc, so each definition is evaluated through
+    // only one SourceTable for a physical row. Different docs may own different event ids.
+    // The algorithm maintains this (new docs only get uncovered ids, narrowing only removes ids) -
+    // overlap means the persisted state is corrupt.
+    addCoverage(doc, kind, covered, ids) {
+        const { schema, name } = this.context.identity;
+        for (const id of ids) {
+            if (covered.has(id)) {
+                throw new ReplicationAssertionError(`Source table ${doc._id} duplicates coverage of ${kind} ${id} for ${schema}.${name}`);
+            }
+            covered.add(id);
+        }
+    }
+    planNarrowingUpdate(doc, memberships, coversDesiredMembership) {
+        const shouldNarrow = coversDesiredMembership && !doc.snapshot_done && !sameMembershipIds(doc, memberships);
+        if (!shouldNarrow) {
+            return;
+        }
+        this.narrowingUpdates.push({
+            id: doc._id,
+            memberships
+        });
+    }
+    newTableMemberships() {
+        const { desired } = this.context;
+        const uncovered = {
+            bucketDataSourceIds: [...desired.bucketSourceById.keys()].filter((id) => !this.coveredBucketDataSourceIds.has(id)),
+            parameterLookupSourceIds: [...desired.parameterLookupSourceById.keys()].filter((id) => !this.coveredParameterLookupSourceIds.has(id)),
+            eventDefinitionIds: [...desired.eventDefinitionById.keys()].filter((id) => !this.coveredEventDefinitionIds.has(id))
+        };
+        if (hasMembershipIds(uncovered)) {
+            return uncovered;
+        }
+        return null;
+    }
+    sourceTableFor(doc, memberships) {
+        const { connectionTag, syncConfig, mapping, desired, storeCurrentData } = this.context;
+        const built = sourceTableFromDocument(doc, connectionTag, syncConfig, mapping, desired.eventDefinitionById, matchingSourcesFor(desired, memberships), memberships);
+        // Use any metadata update returned by the reconciler.
+        const resolved = this.compatibleTableFor(doc);
+        const table = resolved == null ? built : built.withSourceMetadata(resolved.sourceMetadata);
+        table.storeCurrentData = storeCurrentData;
+        return table;
+    }
+}
+export function overlappingSourceTableFilter(connectionId, identity) {
+    const clauses = [{ schema_name: identity.schema, table_name: identity.name }];
+    if (identity.objectId != null) {
+        clauses.push({ relation_id: identity.objectId });
+    }
+    return {
+        connection_id: connectionId,
+        $or: clauses
+    };
+}
+export function createNewSourceTable(id, memberships, context) {
+    const { connectionId, connectionTag, identity, syncConfig, mapping, desired, storeCurrentData, newTableSourceMetadata } = context;
+    const doc = {
+        _id: id,
+        connection_id: connectionId,
+        relation_id: identity.objectId,
+        schema_name: identity.schema,
+        table_name: identity.name,
+        replica_id_columns: identity.replicaIdColumns,
+        snapshot_done: false,
+        snapshot_status: undefined,
+        bucket_data_source_ids: memberships.bucketDataSourceIds,
+        parameter_lookup_source_ids: memberships.parameterLookupSourceIds,
+        event_definition_ids: memberships.eventDefinitionIds,
+        // Records created together share the same source metadata.
+        source_metadata: newTableSourceMetadata
+    };
+    const table = sourceTableFromDocument(doc, connectionTag, syncConfig, mapping, desired.eventDefinitionById, matchingSourcesFor(desired, memberships), memberships);
+    table.storeCurrentData = storeCurrentData;
+    return { doc, table };
+}
+function sourceTableMembershipsFromDocument(doc, syncConfig, mapping) {
+    const bucketDataSourceIds = new Set(doc.bucket_data_source_ids);
+    const parameterLookupSourceIds = new Set(doc.parameter_lookup_source_ids);
+    return {
+        bucketDataSources: syncConfig.bucketDataSources.filter((source) => bucketDataSourceIds.has(mapping.bucketSourceId(source))),
+        parameterLookupSources: syncConfig.bucketParameterLookupSources.filter((source) => parameterLookupSourceIds.has(mapping.parameterLookupId(source)))
+    };
+}
+export function sourceTableFromDocument(doc, connectionTag, syncConfig, mapping, eventById, memberships, membershipIds) {
+    const resolvedMemberships = memberships ?? sourceTableMembershipsFromDocument(doc, syncConfig, mapping);
+    const resolvedMembershipIds = membershipIds ?? {
+        bucketDataSourceIds: resolvedMemberships.bucketDataSources.map((source) => mapping.bucketSourceId(source)),
+        parameterLookupSourceIds: resolvedMemberships.parameterLookupSources.map((source) => mapping.parameterLookupId(source)),
+        eventDefinitionIds: doc.event_definition_ids.filter((id) => eventById.has(id))
+    };
+    const table = new storage.SourceTable({
+        id: doc._id,
+        ref: {
+            connectionTag,
+            schema: doc.schema_name,
+            name: doc.table_name
+        },
+        objectId: doc.relation_id,
+        replicaIdColumns: doc.replica_id_columns?.map((column) => ({ name: column.name, typeId: column.type_oid, type: column.type })) ?? [],
+        snapshotComplete: doc.snapshot_done,
+        bucketDataSources: resolvedMemberships.bucketDataSources,
+        parameterLookupSources: resolvedMemberships.parameterLookupSources,
+        bucketDataSourceIds: new Set(resolvedMembershipIds.bucketDataSourceIds),
+        parameterLookupSourceIds: new Set(resolvedMembershipIds.parameterLookupSourceIds),
+        eventDefinitionIds: new Set(resolvedMembershipIds.eventDefinitionIds),
+        sourceMetadata: doc.source_metadata ?? null
+    });
+    table.syncData = table.bucketDataSources.length > 0;
+    table.syncParameters = table.parameterLookupSources.length > 0;
+    table.syncEvent = table.eventDefinitionIds.size > 0;
+    table.snapshotStatus =
+        doc.snapshot_status == null
+            ? undefined
+            : {
+                lastKey: doc.snapshot_status.last_key?.buffer ?? null,
+                totalEstimatedCount: doc.snapshot_status.total_estimated_count,
+                replicatedCount: doc.snapshot_status.replicated_count
+            };
+    return table;
+}
+//# sourceMappingURL=source-table-utils.js.map

package/dist/storage/implementation/v3/source-table-utils.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"source-table-utils.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/source-table-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAC9E,OAAO,EAAwD,OAAO,EAAE,MAAM,yBAAyB,CAAC;AA4ExG,MAAM,UAAU,4BAA4B,CAC1C,UAA8B,EAC9B,GAAmB,EACnB,OAAgC,EAChC,SAAkE;IAElE,MAAM,eAAe,GAAG,UAAU,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;IAC3D,OAAO;QACL,gBAAgB,EAAE,IAAI,GAAG,CACvB,eAAe,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC,CACrG;QACD,yBAAyB,EAAE,IAAI,GAAG,CAChC,eAAe,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAU,CAAC,CAC7G;QACD,mBAAmB,EAAE,IAAI,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC,CAAC;KAClG,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,6BAA6B,CAC3C,aAAsC,EACtC,OAAyC;IAEzC,OAAO,IAAI,gCAAgC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,eAAe,CAAC,IAAc,EAAE,KAAe;IACtD,OAAO,IAAI,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;AAC5F,CAAC;AAED,SAAS,kBAAkB,CACzB,OAAqC,EACrC,WAAqC;IAErC,OAAO;QACL,iBAAiB,EAAE,WAAW,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;QACjG,sBAAsB,EAAE,WAAW,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC;KACrH,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAC7B,GAA0B,EAC1B,OAAqC;IAErC,OAAO;QACL,mBAAmB,EAAE,GAAG,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAChG,wBAAwB,EAAE,GAAG,CAAC,2BAA2B,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACnH,kBAAkB,EAAE,GAAG,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,CAAC,mBAAmB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;KACjG,CAAC;AACJ,CAAC;AAED,SAAS,gBAAgB,CAAC,WAAqC;IAC7D,OAAO,CACL,WAAW,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC;QAC1C,WAAW,CAAC,wBAAwB,CAAC,MAAM,GAAG,CAAC;QAC/C,WAAW,CAAC,kBAAkB,CAAC,MAAM,GAAG,CAAC,CAC1C,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,GAA0B,EAAE,WAAqC;IAC1F,OAAO,CACL,eAAe,CAAC,GAAG,CAAC,sBAAsB,EAAE,WAAW,CAAC,mBAAmB,CAAC;QAC5E,eAAe,CAAC,GAAG,CAAC,2BAA2B,EAAE,WAAW,CAAC,wBAAwB,CAAC;QACtF,eAAe,CAAC,GAAG,CAAC,oBAAoB,EAAE,WAAW,CAAC,kBAAkB,CAAC,CAC1E,CAAC;AACJ,CAAC;AAED,MAAM,gCAAgC;IAOP;IANZ,0BAA0B,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/C,+BAA+B,GAAG,IAAI,GAAG,EAAU,CAAC;IACpD,yBAAyB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC9C,MAAM,GAA0B,EAAE,CAAC;IACnC,gBAAgB,GAAkC,EAAE,CAAC;IAEtE,YAA6B,OAAyC;QAAzC,YAAO,GAAP,OAAO,CAAkC;IAAG,CAAC;IAE1E,IAAI,CAAC,aAAsC;QACzC,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,IAAI,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,OAAO;YACL,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,mBAAmB,EAAE,IAAI,CAAC,mBAAmB,EAAE;YAC/C,qFAAqF;YACrF,QAAQ,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC;SACjE,CAAC;IACJ,CAAC;IAEO,YAAY,CAAC,GAA0B;QAC7C,OAAO,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC;IAC9C,CAAC;IAEO,kBAAkB,CAAC,GAA0B;QACnD,OAAO,IAAI,CAAC,OAAO,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7G,CAAC;IAEO,SAAS,CAAC,GAA0B;QAC1C,MAAM,WAAW,GAAG,sBAAsB,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACtE,MAAM,uBAAuB,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAC;QAE9D,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QACtC,IAAI,CAAC,mBAAmB,CAAC,GAAG,EAAE,WAAW,EAAE,uBAAuB,CAAC,CAAC;QAEpE,IAAI,uBAAuB,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAEO,cAAc,CAAC,GAA0B,EAAE,WAAqC;QACtF,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,oBAAoB,EAAE,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,mBAAmB,CAAC,CAAC;QAC9G,IAAI,CAAC,WAAW,CACd,GAAG,EACH,yBAAyB,EACzB,IAAI,CAAC,+BAA+B,EACpC,WAAW,CAAC,wBAAwB,CACrC,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,kBAAkB,EAAE,IAAI,CAAC,yBAAyB,EAAE,WAAW,CAAC,kBAAkB,CAAC,CAAC;IAC5G,CAAC;IAED,mFAAmF;IACnF,yFAAyF;IACzF,uFAAuF;IACvF,+FAA+F;IAC/F,gDAAgD;IACxC,WAAW,CAAC,GAA0B,EAAE,IAAY,EAAE,OAAoB,EAAE,GAAa;QAC/F,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC;QAC/C,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;YACrB,IAAI,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBACpB,MAAM,IAAI,yBAAyB,CACjC,gBAAgB,GAAG,CAAC,GAAG,2BAA2B,IAAI,IAAI,EAAE,QAAQ,MAAM,IAAI,IAAI,EAAE,CACrF,CAAC;YACJ,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;IAEO,mBAAmB,CACzB,GAA0B,EAC1B,WAAqC,EACrC,uBAAgC;QAEhC,MAAM,YAAY,GAAG,uBAAuB,IAAI,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;QAE3G,IAAI,CAAC,YAAY,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QAED,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;YACzB,EAAE,EAAE,GAAG,CAAC,GAAG;YACX,WAAW;SACZ,CAAC,CAAC;IACL,CAAC;IAEO,mBAAmB;QACzB,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACjC,MAAM,SAAS,GAA6B;YAC1C,mBAAmB,EAAE,CAAC,GAAG,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAC9D,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,EAAE,CAAC,CACjD;YACD,wBAAwB,EAAE,CAAC,GAAG,OAAO,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAC5E,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,GAAG,CAAC,EAAE,CAAC,CACtD;YACD,kBAAkB,EAAE,CAAC,GAAG,OAAO,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAChE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,EAAE,CAAC,CAChD;SACF,CAAC;QACF,IAAI,gBAAgB,CAAC,SAAS,CAAC,EAAE,CAAC;YAChC,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,cAAc,CAAC,GAA0B,EAAE,WAAqC;QACtF,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;QACvF,MAAM,KAAK,GAAG,uBAAuB,CACnC,GAAG,EACH,aAAa,EACb,UAAU,EACV,OAAO,EACP,OAAO,CAAC,mBAAmB,EAC3B,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,EACxC,WAAW,CACZ,CAAC;QACF,sDAAsD;QACtD,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAC;QAC9C,MAAM,KAAK,GAAG,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,kBAAkB,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;QAC3F,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QAC1C,OAAO,KAAK,CAAC;IACf,CAAC;CACF;AAED,MAAM,UAAU,4BAA4B,CAC1C,YAAoB,EACpB,QAAmE;IAEnE,MAAM,OAAO,GAAG,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,IAAI,EAAE,CAA8B,CAAC;IAC3G,IAAI,QAAQ,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,QAAQ,CAAC,QAAQ,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,OAAO;QACL,aAAa,EAAE,YAAY;QAC3B,GAAG,EAAE,OAAO;KACb,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,EAAiB,EACjB,WAAqC,EACrC,OAAyC;IAEzC,MAAM,EACJ,YAAY,EACZ,aAAa,EACb,QAAQ,EACR,UAAU,EACV,OAAO,EACP,OAAO,EACP,gBAAgB,EAChB,sBAAsB,EACvB,GAAG,OAAO,CAAC;IACZ,MAAM,GAAG,GAA0B;QACjC,GAAG,EAAE,EAAE;QACP,aAAa,EAAE,YAAY;QAC3B,WAAW,EAAE,QAAQ,CAAC,QAAQ;QAC9B,WAAW,EAAE,QAAQ,CAAC,MAAM;QAC5B,UAAU,EAAE,QAAQ,CAAC,IAAI;QACzB,kBAAkB,EAAE,QAAQ,CAAC,gBAAgB;QAC7C,aAAa,EAAE,KAAK;QACpB,eAAe,EAAE,SAAS;QAC1B,sBAAsB,EAAE,WAAW,CAAC,mBAAmB;QACvD,2BAA2B,EAAE,WAAW,CAAC,wBAAwB;QACjE,oBAAoB,EAAE,WAAW,CAAC,kBAAkB;QACpD,2DAA2D;QAC3D,eAAe,EAAE,sBAAsB;KACxC,CAAC;IACF,MAAM,KAAK,GAAG,uBAAuB,CACnC,GAAG,EACH,aAAa,EACb,UAAU,EACV,OAAO,EACP,OAAO,CAAC,mBAAmB,EAC3B,kBAAkB,CAAC,OAAO,EAAE,WAAW,CAAC,EACxC,WAAW,CACZ,CAAC;IACF,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAE1C,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;AACxB,CAAC;AAED,SAAS,kCAAkC,CACzC,GAA0B,EAC1B,UAA8B,EAC9B,OAAgC;IAEhC,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAChE,MAAM,wBAAwB,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;IAE1E,OAAO;QACL,iBAAiB,EAAE,UAAU,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAChE,mBAAmB,CAAC,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CACxD;QACD,sBAAsB,EAAE,UAAU,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAChF,wBAAwB,CAAC,GAAG,CAAC,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC,CAChE;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,GAA0B,EAC1B,aAAqB,EACrB,UAA8B,EAC9B,OAAgC,EAChC,SAAkE,EAClE,WAA6B,EAC7B,aAAwC;IAExC,MAAM,mBAAmB,GAAG,WAAW,IAAI,kCAAkC,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IACxG,MAAM,qBAAqB,GAAG,aAAa,IAAI;QAC7C,mBAAmB,EAAE,mBAAmB,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;QAC1G,wBAAwB,EAAE,mBAAmB,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAClF,OAAO,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAClC;QACD,kBAAkB,EAAE,GAAG,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;KAC/E,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,OAAO,CAAC,WAAW,CAAC;QACpC,EAAE,EAAE,GAAG,CAAC,GAAG;QACX,GAAG,EAAE;YACH,aAAa;YACb,MAAM,EAAE,GAAG,CAAC,WAAW;YACvB,IAAI,EAAE,GAAG,CAAC,UAAU;SACrB;QACD,QAAQ,EAAE,GAAG,CAAC,WAAW;QACzB,gBAAgB,EACd,GAAG,CAAC,kBAAkB,EAAE,GAAG,CACzB,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAA4B,CAC3G,IAAI,EAAE;QACT,gBAAgB,EAAE,GAAG,CAAC,aAAa;QACnC,iBAAiB,EAAE,mBAAmB,CAAC,iBAAiB;QACxD,sBAAsB,EAAE,mBAAmB,CAAC,sBAAsB;QAClE,mBAAmB,EAAE,IAAI,GAAG,CAAC,qBAAqB,CAAC,mBAAmB,CAAC;QACvE,wBAAwB,EAAE,IAAI,GAAG,CAAC,qBAAqB,CAAC,wBAAwB,CAAC;QACjF,kBAAkB,EAAE,IAAI,GAAG,CAAC,qBAAqB,CAAC,kBAAkB,CAAC;QACrE,cAAc,EAAE,GAAG,CAAC,eAAe,IAAI,IAAI;KAC5C,CAAC,CAAC;IACH,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC;IACpD,KAAK,CAAC,cAAc,GAAG,KAAK,CAAC,sBAAsB,CAAC,MAAM,GAAG,CAAC,CAAC;IAC/D,KAAK,CAAC,SAAS,GAAG,KAAK,CAAC,kBAAmB,CAAC,IAAI,GAAG,CAAC,CAAC;IACrD,KAAK,CAAC,cAAc;QAClB,GAAG,CAAC,eAAe,IAAI,IAAI;YACzB,CAAC,CAAC,SAAS;YACX,CAAC,CAAC;gBACE,OAAO,EAAE,GAAG,CAAC,eAAe,CAAC,QAAQ,EAAE,MAAM,IAAI,IAAI;gBACrD,mBAAmB,EAAE,GAAG,CAAC,eAAe,CAAC,qBAAqB;gBAC9D,eAAe,EAAE,GAAG,CAAC,eAAe,CAAC,gBAAgB;aACtD,CAAC;IACR,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/storage/storage-index.d.ts

@@ -1,17 +1,18 @@
 export * as test_utils from '../utils/test-utils.js';
 export * from '../utils/util.js';
-export * from './implementation/BucketDefinitionMapping.js';
 export * from './implementation/common/PersistedBatch.js';
 export * from './implementation/createMongoSyncBucketStorage.js';
 export * from './implementation/db.js';
 export * from './implementation/models.js';
 export * from './implementation/MongoIdSequence.js';
-export * from './implementation/MongoPersistedSyncRules.js';
-export * from './implementation/MongoPersistedSyncRulesContent.js';
+export * from './implementation/MongoParsedSyncConfigSet.js';
+export * from './implementation/MongoPersistedReplicationStream.js';
 export * from './implementation/MongoStorageProvider.js';
 export * from './implementation/MongoSyncRulesLock.js';
 export * from './implementation/OperationBatch.js';
 export * from './implementation/v1/models.js';
-export * from './implementation/v3/models.js';
+export { loadBucketDataDocument, serializeBucketData } from './implementation/v3/bucket-format.js';
+export { BucketDataDocumentV3, BucketOperation, ObjectStorageUsageDocument, RecordedLookupV3, ReplicationStreamDocumentV3, SyncConfigDefinition, SyncRuleConfigStateV3, taggedBucketParameterDocumentToTagged } from './implementation/v3/models.js';
+export * from './implementation/v3/object-storage/ObjectStorageUsage.js';
 export * from './MongoBucketStorage.js';
 export * from './MongoReportStorage.js';

package/dist/storage/storage-index.js

@@ -1,18 +1,19 @@
 export * as test_utils from '../utils/test-utils.js';
 export * from '../utils/util.js';
-export * from './implementation/BucketDefinitionMapping.js';
 export * from './implementation/common/PersistedBatch.js';
 export * from './implementation/createMongoSyncBucketStorage.js';
 export * from './implementation/db.js';
 export * from './implementation/models.js';
 export * from './implementation/MongoIdSequence.js';
-export * from './implementation/MongoPersistedSyncRules.js';
-export * from './implementation/MongoPersistedSyncRulesContent.js';
+export * from './implementation/MongoParsedSyncConfigSet.js';
+export * from './implementation/MongoPersistedReplicationStream.js';
 export * from './implementation/MongoStorageProvider.js';
 export * from './implementation/MongoSyncRulesLock.js';
 export * from './implementation/OperationBatch.js';
 export * from './implementation/v1/models.js';
-export * from './implementation/v3/models.js';
+export { loadBucketDataDocument, serializeBucketData } from './implementation/v3/bucket-format.js';
+export { taggedBucketParameterDocumentToTagged } from './implementation/v3/models.js';
+export * from './implementation/v3/object-storage/ObjectStorageUsage.js';
 export * from './MongoBucketStorage.js';
 export * from './MongoReportStorage.js';
 //# sourceMappingURL=storage-index.js.map

package/dist/storage/storage-index.js.map

@@ -1 +1 @@
-{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,UAAU,MAAM,wBAAwB,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,6CAA6C,CAAC;AAC5D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,kDAAkD,CAAC;AACjE,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qCAAqC,CAAC;AACpD,cAAc,6CAA6C,CAAC;AAC5D,cAAc,oDAAoD,CAAC;AACnE,cAAc,0CAA0C,CAAC;AACzD,cAAc,wCAAwC,CAAC;AACvD,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,yBAAyB,CAAC"}
+{"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,UAAU,MAAM,wBAAwB,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,2CAA2C,CAAC;AAC1D,cAAc,kDAAkD,CAAC;AACjE,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qCAAqC,CAAC;AACpD,cAAc,8CAA8C,CAAC;AAC7D,cAAc,qDAAqD,CAAC;AACpE,cAAc,0CAA0C,CAAC;AACzD,cAAc,wCAAwC,CAAC;AACvD,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,OAAO,EAAE,sBAAsB,EAAE,mBAAmB,EAAE,MAAM,sCAAsC,CAAC;AACnG,OAAO,EAQL,qCAAqC,EACtC,MAAM,+BAA+B,CAAC;AACvC,cAAc,0DAA0D,CAAC;AACzE,cAAc,yBAAyB,CAAC;AACxC,cAAc,yBAAyB,CAAC"}

package/dist/types/types.d.ts

@@ -1,20 +1,88 @@
 import * as service_types from '@powersync/service-types';
 import * as t from 'ts-codec';
+/**
+ * AWS defaults mode, as used by the AWS SDK.
+ *
+ * This is the baseline for the object storage request timeouts, so that they do not each need to be
+ * configured individually.
+ *
+ * The SDK's `legacy` and `auto` modes are not selectable here: `legacy` defines no timeouts at all,
+ * and `auto` makes the timeouts depend on where the process happens to be running. Both are still
+ * handled when they come from the AWS environment, and are then treated as `standard`.
+ */
+export declare const S3DefaultsMode: t.Codec<"standard" | "in-region" | "cross-region" | "mobile", "standard" | "in-region" | "cross-region" | "mobile">;
+export type S3DefaultsMode = t.Encoded<typeof S3DefaultsMode>;
+export declare const MongoStorageReadPreference: t.Codec<"primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest", "primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest">;
+export type MongoStorageReadPreference = t.Encoded<typeof MongoStorageReadPreference>;
 export declare const MongoStorageConfig: t.Intersection<t.Codec<{
-    uri: string;
+    type: string;
+    max_pool_size?: number | undefined;
+    default_storage_version?: number | undefined;
+} & {
     type: "mongodb";
+    uri: string;
     database?: string | undefined;
     username?: string | undefined;
     password?: string | undefined;
     reject_ip_ranges?: string[] | undefined;
 }, {
-    uri: string;
+    type: string;
+    max_pool_size?: number | undefined;
+    default_storage_version?: number | undefined;
+} & {
     type: "mongodb";
+    uri: string;
     database?: string | undefined;
     username?: string | undefined;
     password?: string | undefined;
     reject_ip_ranges?: string[] | undefined;
-}, string, t.CodecProps>, t.ObjectCodec<{}>>;
+}, string, t.CodecProps>, t.ObjectCodec<{
+    /**
+     * Read preference for bulk checksum and bucket data reads.
+     *
+     * If unset, MongoDB driver defaults are used for backwards compatibility.
+     *
+     * This is an experimental option, and may be removed in a future release.
+     */
+    bulk_read_preference: t.OptionalCodec<t.Codec<"primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest", "primary" | "primaryPreferred" | "secondary" | "secondaryPreferred" | "nearest", string, t.CodecProps>>;
+    /**
+     * Throttle clear operations after sync config deploys, by pausing between batches.
+     * The rate produces a pause proportional to the previous batch duration.
+     *
+     * Increase this to reduce impact of clear operations on the storage cluster.
+     * Use 0 to clear as fast as possible.
+     *
+     * Defaults to 0.2.
+     */
+    clear_batch_throttle_rate: t.OptionalCodec<t.Codec<number, number, string, t.CodecProps>>;
+    object_storage: t.OptionalCodec<t.Codec<{
+        type: "s3";
+        bucket: string;
+        region?: string | undefined;
+        prefix?: string | undefined;
+        endpoint?: string | undefined;
+        force_path_style?: boolean | undefined;
+        access_key_id?: string | undefined;
+        secret_access_key?: string | undefined;
+        concurrency_limit?: number | undefined;
+        defaults_mode?: "standard" | "in-region" | "cross-region" | "mobile" | undefined;
+        inline_threshold_bytes?: number | undefined;
+    }, {
+        type: "s3";
+        bucket: string;
+        region?: string | undefined;
+        prefix?: string | undefined;
+        endpoint?: string | undefined;
+        force_path_style?: boolean | undefined;
+        access_key_id?: string | undefined;
+        secret_access_key?: string | undefined;
+        concurrency_limit?: number | undefined;
+        defaults_mode?: "standard" | "in-region" | "cross-region" | "mobile" | undefined;
+        inline_threshold_bytes?: number | undefined;
+    }, string, t.CodecProps>>;
+}>>;
 export type MongoStorageConfig = t.Encoded<typeof MongoStorageConfig>;
 export type MongoStorageConfigDecoded = t.Decoded<typeof MongoStorageConfig>;
+export declare const DEFAULT_CLEAR_BATCH_THROTTLE_RATE = 0.2;
+export declare function normalizeClearBatchThrottleRate(value: number | undefined): number;
 export declare function isMongoStorageConfig(config: service_types.configFile.GenericStorageConfig): config is MongoStorageConfig;

package/dist/types/types.js

@@ -1,8 +1,64 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
+import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
+import * as service_types from '@powersync/service-types';
 import * as t from 'ts-codec';
-export const MongoStorageConfig = lib_mongo.BaseMongoConfig.and(t.object({
-// Add any mongo specific storage settings here in future
+/**
+ * AWS defaults mode, as used by the AWS SDK.
+ *
+ * This is the baseline for the object storage request timeouts, so that they do not each need to be
+ * configured individually.
+ *
+ * The SDK's `legacy` and `auto` modes are not selectable here: `legacy` defines no timeouts at all,
+ * and `auto` makes the timeouts depend on where the process happens to be running. Both are still
+ * handled when they come from the AWS environment, and are then treated as `standard`.
+ */
+export const S3DefaultsMode = service_types.enumLiteral('standard', 'in-region', 'cross-region', 'mobile');
+const S3ObjectStorageConfig = t.object({
+    type: t.literal('s3'),
+    bucket: t.string,
+    region: t.string.optional(),
+    prefix: t.string.optional(),
+    endpoint: t.string.optional(),
+    force_path_style: t.boolean.optional(),
+    access_key_id: t.string.optional(),
+    secret_access_key: t.string.optional(),
+    concurrency_limit: t.number.optional(),
+    // Defaults to the AWS_DEFAULTS_MODE environment variable.
+    defaults_mode: S3DefaultsMode.optional(),
+    // Chunks whose BSON-serialized size falls below this byte threshold
+    // stay inline in MongoDB instead of being offloaded to S3. Default 1024.
+    inline_threshold_bytes: t.number.optional()
+});
+export const MongoStorageReadPreference = service_types.enumLiteral('primary', 'primaryPreferred', 'secondary', 'secondaryPreferred', 'nearest');
+export const MongoStorageConfig = service_types.configFile.BaseStorageConfig.and(lib_mongo.BaseMongoConfig).and(t.object({
+    /**
+     * Read preference for bulk checksum and bucket data reads.
+     *
+     * If unset, MongoDB driver defaults are used for backwards compatibility.
+     *
+     * This is an experimental option, and may be removed in a future release.
+     */
+    bulk_read_preference: MongoStorageReadPreference.optional(),
+    /**
+     * Throttle clear operations after sync config deploys, by pausing between batches.
+     * The rate produces a pause proportional to the previous batch duration.
+     *
+     * Increase this to reduce impact of clear operations on the storage cluster.
+     * Use 0 to clear as fast as possible.
+     *
+     * Defaults to 0.2.
+     */
+    clear_batch_throttle_rate: t.number.optional(),
+    object_storage: S3ObjectStorageConfig.optional()
 }));
+export const DEFAULT_CLEAR_BATCH_THROTTLE_RATE = 0.2;
+export function normalizeClearBatchThrottleRate(value) {
+    const rate = value ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;
+    if (!Number.isFinite(rate) || rate < 0 || rate > 20) {
+        throw new ServiceError(ErrorCode.PSYNC_S3201, 'storage.clear_batch_throttle_rate must be a finite number between 0 and 20');
+    }
+    return rate;
+}
 export function isMongoStorageConfig(config) {
     return config.type == lib_mongo.MONGO_CONNECTION_TYPE;
 }

package/dist/types/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAC,eAAe,CAAC,GAAG,CAC7D,CAAC,CAAC,MAAM,CAAC;AACP,yDAAyD;CAC1D,CAAC,CACH,CAAC;AAKF,MAAM,UAAU,oBAAoB,CAClC,MAAqD;IAErD,OAAO,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,qBAAqB,CAAC;AACxD,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,aAAa,CAAC,WAAW,CAAC,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;AAI3G,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IACrB,MAAM,EAAE,CAAC,CAAC,MAAM;IAChB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC3B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IACtC,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAClC,iBAAiB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACtC,iBAAiB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACtC,0DAA0D;IAC1D,aAAa,EAAE,cAAc,CAAC,QAAQ,EAAE;IACxC,oEAAoE;IACpE,yEAAyE;IACzE,sBAAsB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CAC5C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,aAAa,CAAC,WAAW,CACjE,SAAS,EACT,kBAAkB,EAClB,WAAW,EACX,oBAAoB,EACpB,SAAS,CACV,CAAC;AAIF,MAAM,CAAC,MAAM,kBAAkB,GAAG,aAAa,CAAC,UAAU,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,GAAG,CAC7G,CAAC,CAAC,MAAM,CAAC;IACP;;;;;;OAMG;IACH,oBAAoB,EAAE,0BAA0B,CAAC,QAAQ,EAAE;IAC3D;;;;;;;;OAQG;IACH,yBAAyB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAE9C,cAAc,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CACjD,CAAC,CACH,CAAC;AAKF,MAAM,CAAC,MAAM,iCAAiC,GAAG,GAAG,CAAC;AAErD,MAAM,UAAU,+BAA+B,CAAC,KAAyB;IACvE,MAAM,IAAI,GAAG,KAAK,IAAI,iCAAiC,CAAC;IACxD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,EAAE,EAAE,CAAC;QACpD,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,4EAA4E,CAC7E,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,MAAqD;IAErD,OAAO,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,qBAAqB,CAAC;AACxD,CAAC"}

package/dist/utils/test-utils.d.ts

@@ -5,8 +5,7 @@ import { PowerSyncMongo } from '../storage/implementation/db.js';
 export type MongoTestStorageOptions = {
     url: string;
     isCI: boolean;
-    internalOptions?: MongoBucketStorageOptions;
-};
+} & Omit<MongoBucketStorageOptions, 'replicationStreamNamePrefix'>;
 export declare function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions): {
     factory: (options?: TestStorageOptions) => Promise<MongoBucketStorage>;
     tableIdStrings: boolean;

package/dist/utils/test-utils.js

@@ -15,7 +15,13 @@ export function mongoTestStorageFactoryGenerator(factoryOptions) {
             if (!options?.doNotClear) {
                 await db.clear();
             }
-            return new MongoBucketStorage(db, { slot_name_prefix: 'test_' }, factoryOptions.internalOptions);
+            return new MongoBucketStorage(db, {
+                replicationStreamNamePrefix: 'test_',
+                checksumOptions: factoryOptions.checksumOptions,
+                supportsMultipleSyncConfigs: factoryOptions.supportsMultipleSyncConfigs,
+                objectStorage: factoryOptions.objectStorage,
+                inlineThresholdBytes: factoryOptions.inlineThresholdBytes
+            });
         },
         tableIdStrings: false
     };

package/dist/utils/test-utils.js.map

@@ -1 +1 @@
-{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAA6B,MAAM,kCAAkC,CAAC;AACjG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAQjE,MAAM,UAAU,gCAAgC,CAAC,cAAuC;IACtF,OAAO;QACL,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;YAEzE,8EAA8E;YAC9E,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBAC5F,MAAM,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;YAED,8EAA8E;YAC9E,MAAM,EAAE,CAAC,gCAAgC,EAAE,CAAC;YAE5C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;gBACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACnB,CAAC;YAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,EAAE,EAAE,gBAAgB,EAAE,OAAO,EAAE,EAAE,cAAc,CAAC,eAAe,CAAC,CAAC;QACnG,CAAC;QACD,cAAc,EAAE,KAAK;KACtB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sCAAsC,CAAC,cAAuC;IAC5F,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;QAEzE,MAAM,EAAE,CAAC,mCAAmC,EAAE,CAAC;QAE/C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;YACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;QAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE;QACxC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC"}
+{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAA6B,MAAM,kCAAkC,CAAC;AACjG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAOjE,MAAM,UAAU,gCAAgC,CAAC,cAAuC;IACtF,OAAO;QACL,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;YAEzE,8EAA8E;YAC9E,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBAC5F,MAAM,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;YAED,8EAA8E;YAC9E,MAAM,EAAE,CAAC,gCAAgC,EAAE,CAAC;YAE5C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;gBACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACnB,CAAC;YAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,EAAE;gBAChC,2BAA2B,EAAE,OAAO;gBACpC,eAAe,EAAE,cAAc,CAAC,eAAe;gBAC/C,2BAA2B,EAAE,cAAc,CAAC,2BAA2B;gBACvE,aAAa,EAAE,cAAc,CAAC,aAAa;gBAC3C,oBAAoB,EAAE,cAAc,CAAC,oBAAoB;aAC1D,CAAC,CAAC;QACL,CAAC;QACD,cAAc,EAAE,KAAK;KACtB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sCAAsC,CAAC,cAAuC;IAC5F,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;QAEzE,MAAM,EAAE,CAAC,mCAAmC,EAAE,CAAC;QAE/C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;YACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;QAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE;QACxC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC"}

package/dist/utils/util.d.ts

@@ -1,10 +1,14 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
+import { Logger } from '@powersync/lib-services-framework';
 import { storage, utils } from '@powersync/service-core';
 import * as bson from 'bson';
 import { BucketDataDoc } from '../storage/implementation/common/BucketDataDoc.js';
-export declare function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T>;
-export declare function generateSlotName(prefix: string, sync_rules_id: number): string;
+type ClearCollectionFilter<T extends mongo.Document> = mongo.Filter<T> & {
+    _id: mongo.FilterOperators<mongo.InferIdType<T>>;
+};
+export declare function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.FilterOperators<T>;
+export declare function generateReplicationStreamName(prefix: string, replicationStreamId: number): string;
 /**
  * Read a single batch of data from a cursor, then close it.
  *
@@ -30,6 +34,8 @@ export declare function retryOnMongoMaxTimeMSExpired<T>(operation: () => Promise
     retryDelayMs: number;
     onRetry?: (retryCount: number) => void;
 }): Promise<T>;
+export declare function clearCollectionInIdRanges<T extends mongo.Document>(logger: Logger, label: string, collection: mongo.Collection<T>, filter: ClearCollectionFilter<T>, signal?: AbortSignal, throttleRate?: number): Promise<number>;
+export declare function clearCollectionInIdBatches<T extends mongo.Document>(logger: Logger, label: string, collection: mongo.Collection<T>, filter: mongo.Filter<T>, signal?: AbortSignal, throttleRate?: number): Promise<number>;
 export declare const createPaginatedConnectionQuery: <T extends mongo.Document>(query: mongo.Filter<T>, collection: mongo.Collection<T>, limit: number, cursor?: string) => Promise<{
     items: lib_mongo.mongo.WithId<T>[];
     count: number;
@@ -37,3 +43,4 @@ export declare const createPaginatedConnectionQuery: <T extends mongo.Document>(
     cursor: any;
     more: boolean;
 }>;
+export {};