@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/dist/utils/util.js

@@ -5,6 +5,40 @@ import * as bson from 'bson';
 import * as crypto from 'crypto';
 import * as timers from 'node:timers/promises';
 import * as uuid from 'uuid';
+import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../types/types.js';
+/**
+ * Default and max batch size for clearing.
+ */
+const CLEAR_BATCH_SIZE = 10_000;
+/**
+ * Minimum batch size for clearing when a batch takes too long.
+ */
+const CLEAR_MIN_BATCH_SIZE = 100;
+/**
+ * We target batch duration to stay between the min and max. If the druation is outside this range,
+ * we increase or decrease the batch size.
+ *
+ * MONGO_CLEAR_OPERATION_TIMEOUT_MS controls the absolute max of individual find and delete operations,
+ * but we try to avoid hitting that.
+ */
+const CLEAR_BATCH_MIN_DURATION_MS = 500;
+const CLEAR_BATCH_MAX_DURATION_MS = 2000;
+function throwIfClearAborted(signal) {
+    if (signal?.aborted) {
+        throw new ReplicationAbortedError('Aborted clearing data', signal.reason);
+    }
+}
+async function waitWithSignal(delayMs, signal, abortMessage) {
+    try {
+        await timers.setTimeout(delayMs, undefined, { signal });
+    }
+    catch (error) {
+        if (signal?.aborted) {
+            throw new ReplicationAbortedError(abortMessage, signal.reason);
+        }
+        throw error;
+    }
+}
 export function idPrefixFilter(prefix, rest) {
     let filter = {
         $gte: {
@@ -20,9 +54,9 @@ export function idPrefixFilter(prefix, rest) {
     }
     return filter;
 }
-export function generateSlotName(prefix, sync_rules_id) {
+export function generateReplicationStreamName(prefix, replicationStreamId) {
     const slot_suffix = crypto.randomBytes(2).toString('hex');
-    return `${prefix}${sync_rules_id}_${slot_suffix}`;
+    return `${prefix}${replicationStreamId}_${slot_suffix}`;
 }
 /**
  * Read a single batch of data from a cursor, then close it.
@@ -72,7 +106,9 @@ export function mapOpEntry(row) {
             object_type: row.table,
             object_id: row.row_id,
             checksum: Number(row.checksum),
-            subkey: replicaIdToSubkey(row.source_table, row.source_key),
+            // Storage V3 always persists this during replication. The fallback serves V1 data
+            // written before subkey persistence was introduced.
+            subkey: row.subkey ?? replicaIdToSubkey(row.source_table, row.source_key),
             data: row.data
         };
     }
@@ -124,6 +160,7 @@ export function setSessionSnapshotTime(session, time) {
 }
 export async function retryOnMongoMaxTimeMSExpired(operation, options) {
     let retryCount = 0;
+    // Retry indefinitely on MaxTimeMSExpired errors with exponential backoff.
     while (true) {
         if (options.signal?.aborted) {
             throw new ReplicationAbortedError(options.abortMessage ?? 'Aborted MongoDB operation', options.signal.reason);
@@ -132,15 +169,104 @@ export async function retryOnMongoMaxTimeMSExpired(operation, options) {
             return await operation();
         }
         catch (e) {
-            if (!lib_mongo.isMongoServerError(e) || e.codeName !== 'MaxTimeMSExpired') {
+            if (!lib_mongo.isMaxTimeMSExpiredError(e)) {
                 throw e;
             }
             retryCount += 1;
             options.onRetry?.(retryCount);
-            await timers.setTimeout(options.retryDelayMs);
+            await waitWithSignal(options.retryDelayMs, options.signal, options.abortMessage ?? 'Aborted MongoDB operation');
+        }
+    }
+}
+async function clearCollectionInBatches(logger, label, findAndDeleteBatch, signal, throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE) {
+    let batchSize = CLEAR_BATCH_SIZE;
+    let deletedCount = 0;
+    while (true) {
+        throwIfClearAborted(signal);
+        try {
+            const batchStartedAt = performance.now();
+            const foundBatchSize = await findAndDeleteBatch(batchSize);
+            const batchDurationMs = performance.now() - batchStartedAt;
+            deletedCount += foundBatchSize;
+            if (foundBatchSize > 0) {
+                logger.info(`Cleared batch of ${label} (${foundBatchSize} documents) in ${Math.round(batchDurationMs)}ms, continuing...`);
+            }
+            if (foundBatchSize < batchSize) {
+                return deletedCount;
+            }
+            if (batchDurationMs > CLEAR_BATCH_MAX_DURATION_MS) {
+                batchSize = Math.max(CLEAR_MIN_BATCH_SIZE, Math.floor(batchSize / 2));
+            }
+            else if (batchDurationMs < CLEAR_BATCH_MIN_DURATION_MS) {
+                batchSize = Math.min(CLEAR_BATCH_SIZE, batchSize * 2);
+            }
+            await waitWithSignal(batchDurationMs * throttleRate, signal, 'Aborted clearing data');
+        }
+        catch (error) {
+            if (!lib_mongo.isTimeoutError(error) || batchSize == CLEAR_MIN_BATCH_SIZE) {
+                throw error;
+            }
+            const nextBatchSize = Math.max(CLEAR_MIN_BATCH_SIZE, Math.floor(batchSize / 2));
+            logger.info(`Clearing batch of ${label} timed out with batch size ${batchSize}, retrying find and delete with ${nextBatchSize}...`);
+            batchSize = nextBatchSize;
+            // Pause for at least the equivalent of one query timeout before retrying, since the cause may be high database load.
+            const pauseDuration = Math.max(throttleRate, 1.0) * lib_mongo.MONGO_CLEAR_OPERATION_TIMEOUT_MS;
+            await waitWithSignal(pauseDuration, signal, 'Aborted clearing data');
         }
     }
 }
+export async function clearCollectionInIdRanges(logger, label, collection, filter, signal, throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE) {
+    return clearCollectionInBatches(logger, label, async (batchSize) => {
+        const queryOptions = {
+            maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
+            projection: { _id: 1 }
+        };
+        const [batchEnd] = await collection
+            .find(filter, queryOptions)
+            .sort({ _id: 1 })
+            .skip(batchSize)
+            .limit(1)
+            .toArray();
+        throwIfClearAborted(signal);
+        if (batchEnd == null) {
+            // We're on the last batch
+            const result = await collection.deleteMany(filter, {
+                maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
+            });
+            return result.deletedCount;
+        }
+        const idRange = {
+            ...filter._id,
+            $lt: batchEnd._id
+        };
+        await collection.deleteMany({
+            ...filter,
+            _id: idRange
+        }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS });
+        return batchSize;
+    }, signal, throttleRate);
+}
+export async function clearCollectionInIdBatches(logger, label, collection, filter, signal, throttleRate = DEFAULT_CLEAR_BATCH_THROTTLE_RATE) {
+    return clearCollectionInBatches(logger, label, async (batchSize) => {
+        const documents = await collection
+            .find(filter, {
+            maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
+            projection: { _id: 1 }
+        })
+            .limit(batchSize)
+            .toArray();
+        throwIfClearAborted(signal);
+        if (documents.length === 0) {
+            return 0;
+        }
+        await collection.deleteMany({
+            _id: {
+                $in: documents.map((document) => document._id)
+            }
+        }, { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS });
+        return documents.length;
+    }, signal, throttleRate);
+}
 export const createPaginatedConnectionQuery = async (query, collection, limit, cursor) => {
     const createQuery = (cursor) => {
         if (!cursor) {

package/dist/utils/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/utils/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AACnG,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,MAAM,MAAM,sBAAsB,CAAC;AAC/C,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAG7B,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA+B;IACtE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAkB;IAC3C,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAA4B,EAAE,EAAqB;IACnF,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAA4B;IACvD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,wCAAwC,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAA4B;IACjD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;SAAM,CAAC;QACN,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,OAA4B,EAAE,IAAoB;IACvF,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;IACxE,CAAC;IACD,IAAK,OAAe,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QACzC,OAAe,CAAC,YAAY,GAAG,IAAI,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,SAA2B,EAC3B,OAKC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,uBAAuB,CAAC,OAAO,CAAC,YAAY,IAAI,2BAA2B,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChH,CAAC;QACD,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,kBAAkB,EAAE,CAAC;gBAC1E,MAAM,CAAC,CAAC;YACV,CAAC;YACD,UAAU,IAAI,CAAC,CAAC;YAChB,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;YAC9B,MAAM,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,8BAA8B,GAAG,KAAK,EACjD,KAAsB,EACtB,UAA+B,EAC/B,KAAa,EACb,MAAe,EACf,EAAE;IACF,MAAM,WAAW,GAAG,CAAC,MAAe,EAAE,EAAE;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY;YACrC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE;YAC1D,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO;YACL,GAAG,KAAK;YACR,YAAY;SACM,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE;QACtD,IAAI,EAAE;YACJ,6FAA6F;YAC7F,YAAY,EAAE,CAAC,CAAC;SACjB;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B;;SAEK;IACL,OAAO;QACL,KAAK;QACL,KAAK;QACL,+EAA+E;QAC/E,MAAM,EAAE,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;QACxF,IAAI,EAAE,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC;KACzB,CAAC;AACJ,CAAC,CAAC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/utils/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAU,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC3G,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,MAAM,MAAM,sBAAsB,CAAC;AAC/C,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,iCAAiC,EAAE,MAAM,mBAAmB,CAAC;AAEtE;;GAEG;AACH,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEhC;;GAEG;AACH,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAEjC;;;;;;GAMG;AACH,MAAM,2BAA2B,GAAG,GAAG,CAAC;AACxC,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAMzC,SAAS,mBAAmB,CAAC,MAAoB;IAC/C,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,uBAAuB,CAAC,uBAAuB,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5E,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,OAAe,EAAE,MAA+B,EAAE,YAAoB;IAClG,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,SAAS,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;IAC1D,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,uBAAuB,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;QACjE,CAAC;QACD,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,MAAc,EAAE,mBAA2B;IACvF,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,mBAAmB,IAAI,WAAW,EAAE,CAAC;AAC1D,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA+B;IACtE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAkB;IAC3C,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,kFAAkF;YAClF,oDAAoD;YACpD,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC3E,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAA4B,EAAE,EAAqB;IACnF,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAA4B;IACvD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,wCAAwC,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAA4B;IACjD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;SAAM,CAAC;QACN,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,OAA4B,EAAE,IAAoB;IACvF,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;IACxE,CAAC;IACD,IAAK,OAAe,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QACzC,OAAe,CAAC,YAAY,GAAG,IAAI,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,SAA2B,EAC3B,OAKC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,0EAA0E;IAC1E,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,uBAAuB,CAAC,OAAO,CAAC,YAAY,IAAI,2BAA2B,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChH,CAAC;QACD,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,uBAAuB,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC1C,MAAM,CAAC,CAAC;YACV,CAAC;YACD,UAAU,IAAI,CAAC,CAAC;YAChB,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;YAC9B,MAAM,cAAc,CAAC,OAAO,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,YAAY,IAAI,2BAA2B,CAAC,CAAC;QAClH,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,MAAc,EACd,KAAa,EACb,kBAA0D,EAC1D,MAAoB,EACpB,YAAY,GAAG,iCAAiC;IAEhD,IAAI,SAAS,GAAG,gBAAgB,CAAC;IACjC,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,OAAO,IAAI,EAAE,CAAC;QACZ,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,CAAC;YACH,MAAM,cAAc,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YACzC,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,SAAS,CAAC,CAAC;YAC3D,MAAM,eAAe,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,cAAc,CAAC;YAC3D,YAAY,IAAI,cAAc,CAAC;YAC/B,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;gBACvB,MAAM,CAAC,IAAI,CACT,oBAAoB,KAAK,KAAK,cAAc,kBAAkB,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,mBAAmB,CAC7G,CAAC;YACJ,CAAC;YACD,IAAI,cAAc,GAAG,SAAS,EAAE,CAAC;gBAC/B,OAAO,YAAY,CAAC;YACtB,CAAC;YACD,IAAI,eAAe,GAAG,2BAA2B,EAAE,CAAC;gBAClD,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;YACxE,CAAC;iBAAM,IAAI,eAAe,GAAG,2BAA2B,EAAE,CAAC;gBACzD,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;YACxD,CAAC;YACD,MAAM,cAAc,CAAC,eAAe,GAAG,YAAY,EAAE,MAAM,EAAE,uBAAuB,CAAC,CAAC;QACxF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,KAAK,CAAC,IAAI,SAAS,IAAI,oBAAoB,EAAE,CAAC;gBAC1E,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;YAChF,MAAM,CAAC,IAAI,CACT,qBAAqB,KAAK,8BAA8B,SAAS,mCAAmC,aAAa,KAAK,CACvH,CAAC;YACF,SAAS,GAAG,aAAa,CAAC;YAE1B,qHAAqH;YACrH,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,GAAG,SAAS,CAAC,gCAAgC,CAAC;YAC/F,MAAM,cAAc,CAAC,aAAa,EAAE,MAAM,EAAE,uBAAuB,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,MAAc,EACd,KAAa,EACb,UAA+B,EAC/B,MAAgC,EAChC,MAAoB,EACpB,YAAY,GAAG,iCAAiC;IAEhD,OAAO,wBAAwB,CAC7B,MAAM,EACN,KAAK,EACL,KAAK,EAAE,SAAS,EAAE,EAAE;QAClB,MAAM,YAAY,GAAyB;YACzC,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC;YACxD,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;SACvB,CAAC;QACF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,UAAU;aAChC,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC;aAC1B,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;aAChB,IAAI,CAAC,SAAS,CAAC;aACf,KAAK,CAAC,CAAC,CAAC;aACR,OAAO,EAAE,CAAC;QACb,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAE5B,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACrB,0BAA0B;YAC1B,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,MAAM,EAAE;gBACjD,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC;aACzD,CAAC,CAAC;YACH,OAAO,MAAM,CAAC,YAAY,CAAC;QAC7B,CAAC;QAED,MAAM,OAAO,GAAgD;YAC3D,GAAG,MAAM,CAAC,GAAG;YACb,GAAG,EAAE,QAAQ,CAAC,GAAG;SAClB,CAAC;QACF,MAAM,UAAU,CAAC,UAAU,CACzB;YACE,GAAG,MAAM;YACT,GAAG,EAAE,OAAO;SACM,EACpB,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC,EAAE,CAC7D,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC,EACD,MAAM,EACN,YAAY,CACb,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,MAAc,EACd,KAAa,EACb,UAA+B,EAC/B,MAAuB,EACvB,MAAoB,EACpB,YAAY,GAAG,iCAAiC;IAEhD,OAAO,wBAAwB,CAC7B,MAAM,EACN,KAAK,EACL,KAAK,EAAE,SAAS,EAAE,EAAE;QAClB,MAAM,SAAS,GAAG,MAAM,UAAU;aAC/B,IAAI,CAAC,MAAM,EAAE;YACZ,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC;YACxD,UAAU,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE;SACvB,CAAC;aACD,KAAK,CAAC,SAAS,CAAC;aAChB,OAAO,EAAE,CAAC;QACb,mBAAmB,CAAC,MAAM,CAAC,CAAC;QAE5B,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,UAAU,CAAC,UAAU,CACzB;YACE,GAAG,EAAE;gBACH,GAAG,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC;aAC/C;SACiB,EACpB,EAAE,SAAS,EAAE,SAAS,CAAC,EAAE,CAAC,gCAAgC,EAAE,CAC7D,CAAC;QACF,OAAO,SAAS,CAAC,MAAM,CAAC;IAC1B,CAAC,EACD,MAAM,EACN,YAAY,CACb,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,8BAA8B,GAAG,KAAK,EACjD,KAAsB,EACtB,UAA+B,EAC/B,KAAa,EACb,MAAe,EACf,EAAE;IACF,MAAM,WAAW,GAAG,CAAC,MAAe,EAAE,EAAE;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY;YACrC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE;YAC1D,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO;YACL,GAAG,KAAK;YACR,YAAY;SACM,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE;QACtD,IAAI,EAAE;YACJ,6FAA6F;YAC7F,YAAY,EAAE,CAAC,CAAC;SACjB;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B;;SAEK;IACL,OAAO;QACL,KAAK;QACL,KAAK;QACL,+EAA+E;QAC/E,MAAM,EAAE,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;QACxF,IAAI,EAAE,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC;KACzB,CAAC;AACJ,CAAC,CAAC"}

package/package.json

@@ -1,41 +1,35 @@
 {
   "name": "@powersync/service-module-mongodb-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
-  "types": "dist/index.d.ts",
-  "version": "0.0.0-dev-20260602073133",
-  "main": "dist/index.js",
+  "version": "0.0.0-dev-20260909133214",
   "license": "FSL-1.1-ALv2",
   "type": "module",
   "publishConfig": {
     "access": "public"
   },
   "exports": {
-    ".": {
-      "import": "./dist/index.js",
-      "require": "./dist/index.js",
-      "default": "./dist/index.js"
-    },
-    "./types": {
-      "import": "./dist/types/types.js",
-      "require": "./dist/types/types.js",
-      "default": "./dist/types/types.js"
-    }
+    ".": "./dist/index.js",
+    "./types": "./dist/types/types.js"
   },
   "dependencies": {
+    "@aws-sdk/client-s3": "^3.1116.0",
+    "@smithy/core": "^3.33.3",
+    "@smithy/node-http-handler": "^4.11.3",
+    "async-mutex": "^0.5.0",
     "bson": "^6.10.4",
     "ix": "^5.0.0",
     "lru-cache": "^10.2.2",
     "ts-codec": "^1.3.0",
     "uuid": "^14.0.0",
-    "@powersync/lib-service-mongodb": "0.0.0-dev-20260602073133",
-    "@powersync/lib-services-framework": "0.0.0-dev-20260602073133",
-    "@powersync/service-core": "0.0.0-dev-20260602073133",
+    "@powersync/lib-service-mongodb": "0.0.0-dev-20260909133214",
+    "@powersync/lib-services-framework": "0.0.0-dev-20260909133214",
+    "@powersync/service-core": "0.0.0-dev-20260909133214",
     "@powersync/service-jsonbig": "0.17.13",
-    "@powersync/service-sync-rules": "0.0.0-dev-20260602073133",
-    "@powersync/service-types": "0.15.2"
+    "@powersync/service-sync-rules": "0.0.0-dev-20260909133214",
+    "@powersync/service-types": "0.18.0"
   },
   "devDependencies": {
-    "@powersync/service-core-tests": "0.0.0-dev-20260602073133"
+    "@powersync/service-core-tests": "0.0.0-dev-20260909133214"
   },
   "scripts": {
     "build": "tsc -b",

package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts

@@ -0,0 +1,55 @@
+import { migrations } from '@powersync/service-core';
+import * as storage from '../../../storage/storage-index.js';
+import { MongoStorageConfig } from '../../../types/types.js';
+
+export const up: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    await db.write_checkpoints.createIndex(
+      {
+        checkpoint_requested_at: 1
+      },
+      {
+        name: 'checkpoint_requested_at',
+        // Only client-requested checkpoints have this field; generated
+        // checkpoints leave it unset. This keeps the index limited to the
+        // documents the compact job's retention delete scans.
+        partialFilterExpression: { checkpoint_requested_at: { $exists: true } }
+      }
+    );
+    await db.custom_write_checkpoints.createIndex(
+      {
+        checkpoint_requested_at: 1
+      },
+      {
+        name: 'checkpoint_requested_at',
+        partialFilterExpression: { checkpoint_requested_at: { $exists: true } }
+      }
+    );
+  } finally {
+    await db.client.close();
+  }
+};
+
+export const down: migrations.PowerSyncMigrationFunction = async (context) => {
+  const {
+    service_context: { configuration }
+  } = context;
+
+  const db = storage.createPowerSyncMongo(configuration.storage as MongoStorageConfig);
+
+  try {
+    if (await db.write_checkpoints.indexExists('checkpoint_requested_at')) {
+      await db.write_checkpoints.dropIndex('checkpoint_requested_at');
+    }
+    if (await db.custom_write_checkpoints.indexExists('checkpoint_requested_at')) {
+      await db.custom_write_checkpoints.dropIndex('checkpoint_requested_at');
+    }
+  } finally {
+    await db.client.close();
+  }
+};