@powersync/service-module-mongodb-storage 0.23.2 → 0.24.1

package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts

@@ -1,3 +1,4 @@
+import { mongo } from '@powersync/lib-service-mongodb';
 import { Logger } from '@powersync/lib-services-framework';
 import * as bson from 'bson';
 import { MongoWriteBatch } from '../../MongoWriteBatch.js';
@@ -5,6 +6,12 @@ import { VersionedPowerSyncMongoV3 } from '../VersionedPowerSyncMongoV3.js';
 import { BucketDataObjectStorage } from './BucketDataObjectStorage.js';
 import { ObjectStorage, ObjectStorageOperationOptions } from './ObjectStorage.js';
 export declare const OBJECT_STORAGE_UPLOAD_LEASE_MS: number;
+/**
+ * Leave time for the bounded publication transaction and clock skew before
+ * cleanup can select a marker. Publication must finish within this margin after
+ * its eligibility check; checked marker deletion alone cannot protect an object
+ * from a cleanup worker that already selected an expired marker.
+ */
 export declare const OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS: number;
 export declare const OBJECT_STORAGE_REFERENCE_GRACE_MS: number;
 export interface PreparedObjectStorageUpload {
@@ -12,6 +19,10 @@ export interface PreparedObjectStorageUpload {
     path: string;
     deleteAfter: Date;
 }
+export interface ObjectStorageUploadPayload {
+    upload: PreparedObjectStorageUpload;
+    payload: Uint8Array;
+}
 /**
  * Coordinates the MongoDB outbox used to make object-storage changes recoverable.
  *
@@ -35,10 +46,24 @@ export declare class ObjectStorageLifecycle {
      * Indicate that files will be uploaded to S3.
      *
      * These markers are expected to be removed using publishUploads(). If that does not happen within
-     * the grace period, the files will be deleted.
+     * the grace period, the files will be deleted. Registration must be majority
+     * acknowledged before any PUT starts: losing a marker on failover would lose
+     * the only cleanup obligation for an unpublished object. A failed/ambiguous
+     * insert must therefore prevent uploads, even if some markers were inserted.
      */
-    prepareUploads(paths: string[], now?: Date): Promise<PreparedObjectStorageUpload[]>;
+    prepareUploads(paths: string[], now?: Date, session?: mongo.ClientSession): Promise<PreparedObjectStorageUpload[]>;
     canPublish(upload: PreparedObjectStorageUpload, now?: Date): boolean;
+    /**
+     * Upload a durably registered generation, joining every started PUT on failure.
+     * The shared adapter limits concurrency; its queue must observe the same signal
+     * as an active request so no queued PUT can start after the protection deadline.
+     *
+     * Failed/ambiguous PUTs retain their original markers. S3's bounded request
+     * timeouts are far shorter than the one-hour orphan grace, including when a
+     * response is lost or the process dies. Cancellation does not prove an object
+     * is absent, so never delete an object or its marker on upload failure.
+     */
+    storeUploads(uploads: ObjectStorageUploadPayload[], options?: ObjectStorageOperationOptions): Promise<void>;
     /**
      * Remove markers for files that have finished uploading, and are now referenced from other documents.
      *

package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js

@@ -1,7 +1,14 @@
 import * as bson from 'bson';
 import { createHash, randomUUID } from 'node:crypto';
+import { REPLICATION_WRITE_OPTIONS } from '../../MongoReplicationOptions.js';
 import { BucketDataObjectStorage } from './BucketDataObjectStorage.js';
 export const OBJECT_STORAGE_UPLOAD_LEASE_MS = 60 * 60 * 1000;
+/**
+ * Leave time for the bounded publication transaction and clock skew before
+ * cleanup can select a marker. Publication must finish within this margin after
+ * its eligibility check; checked marker deletion alone cannot protect an object
+ * from a cleanup worker that already selected an expired marker.
+ */
 export const OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS = 2 * 60 * 1000;
 export const OBJECT_STORAGE_REFERENCE_GRACE_MS = 15 * 60 * 1000;
 /**
@@ -50,23 +57,71 @@ export class ObjectStorageLifecycle {
      * Indicate that files will be uploaded to S3.
      *
      * These markers are expected to be removed using publishUploads(). If that does not happen within
-     * the grace period, the files will be deleted.
+     * the grace period, the files will be deleted. Registration must be majority
+     * acknowledged before any PUT starts: losing a marker on failover would lose
+     * the only cleanup obligation for an unpublished object. A failed/ambiguous
+     * insert must therefore prevent uploads, even if some markers were inserted.
      */
-    async prepareUploads(paths, now = new Date()) {
+    async prepareUploads(paths, now = new Date(), session) {
         const deleteAfter = new Date(now.getTime() + OBJECT_STORAGE_UPLOAD_LEASE_MS);
         const uploads = paths.map((path) => ({ markerId: new bson.ObjectId(), path, deleteAfter }));
-        if (uploads.length) {
+        if (uploads.length > 0) {
             await this.db.pendingObjectStorageDeletes(this.replicationStreamId).insertMany(uploads.map((upload) => ({
                 _id: upload.markerId,
                 path: upload.path,
                 delete_after: upload.deleteAfter
-            })));
+            })), { ...REPLICATION_WRITE_OPTIONS, session });
         }
         return uploads;
     }
     canPublish(upload, now = new Date()) {
         return now.getTime() < upload.deleteAfter.getTime() - OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS;
     }
+    /**
+     * Upload a durably registered generation, joining every started PUT on failure.
+     * The shared adapter limits concurrency; its queue must observe the same signal
+     * as an active request so no queued PUT can start after the protection deadline.
+     *
+     * Failed/ambiguous PUTs retain their original markers. S3's bounded request
+     * timeouts are far shorter than the one-hour orphan grace, including when a
+     * response is lost or the process dies. Cancellation does not prove an object
+     * is absent, so never delete an object or its marker on upload failure.
+     */
+    async storeUploads(uploads, options) {
+        if (uploads.length === 0) {
+            return;
+        }
+        const deadline = Math.min(...uploads.map(({ upload }) => upload.deleteAfter.getTime())) -
+            OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS;
+        const remaining = deadline - Date.now();
+        if (remaining <= 0) {
+            throw new Error('Object storage upload lease expired before PUT');
+        }
+        const expiration = new AbortController();
+        const timer = setTimeout(() => expiration.abort(new Error('Object storage upload lease expired during PUT')), remaining);
+        timer.unref();
+        const signal = options?.signal ? AbortSignal.any([options.signal, expiration.signal]) : expiration.signal;
+        let results;
+        try {
+            results = await Promise.allSettled(uploads.map(async ({ upload, payload }) => {
+                signal.throwIfAborted();
+                // The timer handles time in the adapter's queue; this check also catches
+                // wall-clock changes before dispatch. Publication checks again later.
+                if (!this.canPublish(upload)) {
+                    throw new Error(`Object storage upload lease expired for ${upload.path}`);
+                }
+                await this.bucketData.storeSerialized(upload.path, payload, { signal });
+            }));
+        }
+        finally {
+            clearTimeout(timer);
+        }
+        const failure = results.find((result) => result.status === 'rejected');
+        if (failure?.status === 'rejected') {
+            throw failure.reason;
+        }
+        signal.throwIfAborted();
+    }
     /**
      * Remove markers for files that have finished uploading, and are now referenced from other documents.
      *
@@ -82,8 +137,9 @@ export class ObjectStorageLifecycle {
             return;
         }
         writes.deleteMany(this.db.pendingObjectStorageDeletes(this.replicationStreamId), { _id: { $in: uploads.map((upload) => upload.markerId) } }, (deletedCount) => {
-            if (deletedCount !== uploads.length)
+            if (deletedCount !== uploads.length) {
                 throw new Error('Missing object storage publication markers');
+            }
         });
     }
     /**

package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map

@@ -1 +1 @@
-{"version":3,"file":"ObjectStorageLifecycle.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAGvE,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC7D,MAAM,CAAC,MAAM,2CAA2C,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AACzE,MAAM,CAAC,MAAM,iCAAiC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAChE;;;;GAIG;AACH,MAAM,iCAAiC,GAAG,GAAG,CAAC;AAQ9C;;;;;;;;;GASG;AACH,MAAM,OAAO,sBAAsB;IAId;IACA;IACA;IALV,UAAU,CAA0B;IAE7C,YACmB,EAA6B,EAC7B,mBAA2B,EAC3B,aAA4B;QAF5B,OAAE,GAAF,EAAE,CAA2B;QAC7B,wBAAmB,GAAnB,mBAAmB,CAAQ;QAC3B,kBAAa,GAAb,aAAa,CAAe;QAE7C,IAAI,CAAC,UAAU,GAAG,IAAI,uBAAuB,CAAC,aAAa,CAAC,CAAC;IAC/D,CAAC;IAED,YAAY;QACV,OAAO,eAAe,IAAI,CAAC,mBAAmB,GAAG,CAAC;IACpD,CAAC;IAED,gBAAgB,CAAC,YAAoB;QACnC,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,GAAG,YAAY,GAAG,CAAC;IAClD,CAAC;IAED,YAAY,CAAC,YAAoB,EAAE,MAAc,EAAE,KAAa,EAAE,KAAa;QAC7E,wEAAwE;QACxE,oEAAoE;QACpE,MAAM,aAAa,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,aAAa,GACjB,MAAM,CAAC,UAAU,CAAC,aAAa,EAAE,MAAM,CAAC,IAAI,iCAAiC;YAC3E,CAAC,CAAC,aAAa;YACf,CAAC,CAAC,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QAC1E,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,GAAG,aAAa,IAAI,KAAK,IAAI,KAAK,IAAI,UAAU,EAAE,OAAO,CAAC;IACzG,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAAC,KAAe,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE;QACpD,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,8BAA8B,CAAC,CAAC;QAC7E,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;QAC5F,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,UAAU,CAC5E,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACvB,GAAG,EAAE,MAAM,CAAC,QAAQ;gBACpB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,YAAY,EAAE,MAAM,CAAC,WAAW;aACjC,CAAC,CAAC,CACJ,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,UAAU,CAAC,MAAmC,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE;QAC9D,OAAO,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,2CAA2C,CAAC;IACpG,CAAC;IAED;;;;OAIG;IACH,cAAc,CAAC,OAAsC,EAAE,MAAuB;QAC5E,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,gDAAgD,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YACjF,CAAC;QACH,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;QACT,CAAC;QAED,MAAM,CAAC,UAAU,CACf,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAC7D,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,EAC1D,CAAC,YAAY,EAAE,EAAE;YACf,IAAI,YAAY,KAAK,OAAO,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QACrG,CAAC,CACF,CAAC;IACJ,CAAC;IAED;;;;;;;;;;OAUG;IACH,MAAM,CAAC,KAAuB,EAAE,MAAuB,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE;QACvE,MAAM,OAAO,GAAkC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1E,GAAG,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE;YACxB,IAAI;YACJ,YAAY,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,iCAAiC,CAAC;SAC1E,CAAC,CAAC,CAAC;QACJ,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,OAAuC;QACxE,OAAO,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAAuC;QACnE,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACpE,IAAI,KAAK,GAAkC,EAAE,CAAC;QAE9C,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;YAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO;YACT,CAAC;YACD,MAAM,QAAQ,GAAG,KAAK,CAAC;YACvB,KAAK,GAAG,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAC1B,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EACrC,EAAE,MAAM,EAAE,CACX,CAAC;gBACF,MAAM,OAAO,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;YACnF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;oBACpB,kFAAkF;oBAClF,uFAAuF;oBACvF,4DAA4D;oBAC5D,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,2FAA2F,EAAE,KAAK,CAAC,CAAC;YAClH,CAAC;QACH,CAAC,CAAC;QAEF,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;YAClC,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACnB,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACzB,MAAM,WAAW,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;QACD,MAAM,WAAW,EAAE,CAAC;IACtB,CAAC;CACF"}
+{"version":3,"file":"ObjectStorageLifecycle.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,yBAAyB,EAAE,MAAM,kCAAkC,CAAC;AAI7E,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAGvE,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC7D;;;;;GAKG;AACH,MAAM,CAAC,MAAM,2CAA2C,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AACzE,MAAM,CAAC,MAAM,iCAAiC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAChE;;;;GAIG;AACH,MAAM,iCAAiC,GAAG,GAAG,CAAC;AAa9C;;;;;;;;;GASG;AACH,MAAM,OAAO,sBAAsB;IAId;IACA;IACA;IALV,UAAU,CAA0B;IAE7C,YACmB,EAA6B,EAC7B,mBAA2B,EAC3B,aAA4B;QAF5B,OAAE,GAAF,EAAE,CAA2B;QAC7B,wBAAmB,GAAnB,mBAAmB,CAAQ;QAC3B,kBAAa,GAAb,aAAa,CAAe;QAE7C,IAAI,CAAC,UAAU,GAAG,IAAI,uBAAuB,CAAC,aAAa,CAAC,CAAC;IAC/D,CAAC;IAED,YAAY;QACV,OAAO,eAAe,IAAI,CAAC,mBAAmB,GAAG,CAAC;IACpD,CAAC;IAED,gBAAgB,CAAC,YAAoB;QACnC,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,GAAG,YAAY,GAAG,CAAC;IAClD,CAAC;IAED,YAAY,CAAC,YAAoB,EAAE,MAAc,EAAE,KAAa,EAAE,KAAa;QAC7E,wEAAwE;QACxE,oEAAoE;QACpE,MAAM,aAAa,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,aAAa,GACjB,MAAM,CAAC,UAAU,CAAC,aAAa,EAAE,MAAM,CAAC,IAAI,iCAAiC;YAC3E,CAAC,CAAC,aAAa;YACf,CAAC,CAAC,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QAC1E,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,GAAG,aAAa,IAAI,KAAK,IAAI,KAAK,IAAI,UAAU,EAAE,OAAO,CAAC;IACzG,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,CAAC,cAAc,CAClB,KAAe,EACf,GAAG,GAAG,IAAI,IAAI,EAAE,EAChB,OAA6B;QAE7B,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,8BAA8B,CAAC,CAAC;QAC7E,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;QAC5F,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,UAAU,CAC5E,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACvB,GAAG,EAAE,MAAM,CAAC,QAAQ;gBACpB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,YAAY,EAAE,MAAM,CAAC,WAAW;aACjC,CAAC,CAAC,EACH,EAAE,GAAG,yBAAyB,EAAE,OAAO,EAAE,CAC1C,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,UAAU,CAAC,MAAmC,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE;QAC9D,OAAO,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,2CAA2C,CAAC;IACpG,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,CAAC,YAAY,CAAC,OAAqC,EAAE,OAAuC;QAC/F,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;QACT,CAAC;QACD,MAAM,QAAQ,GACZ,IAAI,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC;YACtE,2CAA2C,CAAC;QAC9C,MAAM,SAAS,GAAG,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACxC,IAAI,SAAS,IAAI,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QACD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CACtB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC,EACnF,SAAS,CACV,CAAC;QACF,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,MAAM,CAAC;QAC1G,IAAI,OAAwC,CAAC;QAC7C,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAChC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE;gBACxC,MAAM,CAAC,cAAc,EAAE,CAAC;gBACxB,yEAAyE;gBACzE,sEAAsE;gBACtE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC7B,MAAM,IAAI,KAAK,CAAC,2CAA2C,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;gBAC5E,CAAC;gBACD,MAAM,IAAI,CAAC,UAAU,CAAC,eAAe,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;YAC1E,CAAC,CAAC,CACH,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACtB,CAAC;QACD,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;QACvE,IAAI,OAAO,EAAE,MAAM,KAAK,UAAU,EAAE,CAAC;YACnC,MAAM,OAAO,CAAC,MAAM,CAAC;QACvB,CAAC;QACD,MAAM,CAAC,cAAc,EAAE,CAAC;IAC1B,CAAC;IAED;;;;OAIG;IACH,cAAc,CAAC,OAAsC,EAAE,MAAuB;QAC5E,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,gDAAgD,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YACjF,CAAC;QACH,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;QACT,CAAC;QAED,MAAM,CAAC,UAAU,CACf,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAC7D,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,EAC1D,CAAC,YAAY,EAAE,EAAE;YACf,IAAI,YAAY,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;gBACpC,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAChE,CAAC;QACH,CAAC,CACF,CAAC;IACJ,CAAC;IAED;;;;;;;;;;OAUG;IACH,MAAM,CAAC,KAAuB,EAAE,MAAuB,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE;QACvE,MAAM,OAAO,GAAkC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1E,GAAG,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE;YACxB,IAAI;YACJ,YAAY,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,iCAAiC,CAAC;SAC1E,CAAC,CAAC,CAAC;QACJ,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,IAAI,CAAC,mBAAmB,CAAC,EAAE,OAAO,CAAC,CAAC;QAC5F,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,OAAuC;QACxE,OAAO,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAAuC;QACnE,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACpE,IAAI,KAAK,GAAkC,EAAE,CAAC;QAE9C,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;YAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO;YACT,CAAC;YACD,MAAM,QAAQ,GAAG,KAAK,CAAC;YACvB,KAAK,GAAG,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAC1B,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EACrC,EAAE,MAAM,EAAE,CACX,CAAC;gBACF,MAAM,OAAO,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;YACnF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;oBACpB,kFAAkF;oBAClF,uFAAuF;oBACvF,4DAA4D;oBAC5D,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,2FAA2F,EAAE,KAAK,CAAC,CAAC;YAClH,CAAC;QACH,CAAC,CAAC;QAEF,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;YAClC,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACnB,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACzB,MAAM,WAAW,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;QACD,MAAM,WAAW,EAAE,CAAC;IACtB,CAAC;CACF"}

package/package.json

@@ -1,8 +1,8 @@
 {
   "name": "@powersync/service-module-mongodb-storage",
   "repository": "https://github.com/powersync-ja/powersync-service",
-  "version": "0.23.2",
-  "license": "FSL-1.1-ALv2",
+  "version": "0.24.1",
+  "license": "Apache-2.0",
   "type": "module",
   "publishConfig": {
     "access": "public"
@@ -17,19 +17,18 @@
     "@smithy/node-http-handler": "^4.11.3",
     "async-mutex": "^0.5.0",
     "bson": "^6.10.4",
-    "ix": "^5.0.0",
     "lru-cache": "^10.2.2",
     "ts-codec": "^1.3.0",
     "uuid": "^14.0.0",
-    "@powersync/lib-service-mongodb": "0.6.34",
-    "@powersync/lib-services-framework": "0.10.4",
-    "@powersync/service-core": "1.27.2",
-    "@powersync/service-jsonbig": "0.17.13",
-    "@powersync/service-sync-rules": "0.44.0",
-    "@powersync/service-types": "0.20.0"
+    "@powersync/lib-service-mongodb": "0.6.36",
+    "@powersync/lib-services-framework": "0.10.5",
+    "@powersync/service-core": "1.28.1",
+    "@powersync/service-jsonbig": "0.17.14",
+    "@powersync/service-sync-rules": "0.45.0",
+    "@powersync/service-types": "0.21.0"
   },
   "devDependencies": {
-    "@powersync/service-core-tests": "0.18.4"
+    "@powersync/service-core-tests": "0.18.5"
   },
   "scripts": {
     "build": "tsc -b",

package/src/storage/MongoBucketStorage.ts

@@ -652,6 +652,10 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
    *
    * Use in the same transaction that the replacement stream is created.
    */
+  async stopProcessingSyncConfigs(): Promise<void> {
+    await this.withTransaction((session) => this.stopExistingProcessingWork(session));
+  }
+
   private async stopExistingProcessingWork(session: mongo.ClientSession) {
     await this.db.sync_rules.updateMany(
       {
@@ -824,89 +828,45 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
   }
 
   async getStorageMetrics(): Promise<storage.StorageMetrics> {
-    const ignoreNotExisting = (e: unknown) => {
-      if (lib_mongo.isMongoNamespaceNotFoundError(e)) {
-        // Collection doesn't exist - return 0
-        return [{ storageStats: { size: 0 } }];
-      } else {
-        return Promise.reject(e);
-      }
-    };
-
-    const aggregateStaticCollection = async <T extends mongo.Document>(collection: mongo.Collection<T>) => {
-      // We check whether the collection exists before getting the statistics. This avoids repeated
-      // errors in the MongoDB logs if the collection hasn't been created yet.
-      const exists =
-        (await this.db.db.listCollections({ name: collection.collectionName }, { nameOnly: true }).toArray()).length >
-        0;
-      if (!exists) {
-        return [{ storageStats: { size: 0 } }];
-      }
-
-      return collection
-        .aggregate([
-          {
-            $collStats: {
-              storageStats: {}
-            }
+    // We check which collections exist before getting statistics to avoid repeated errors in
+    // MongoDB logs for collections that have not been created yet.
+    const collections = await this.db.db
+      .listCollections(
+        {
+          name: {
+            $regex: '^(bucket_data|bucket_parameters|current_data)$|^(bucket_data_|parameter_index_|source_records_)'
           }
-        ])
-        .toArray()
-        .catch(ignoreNotExisting);
-    };
-
-    const operations_aggregate = await aggregateStaticCollection(this.db.bucket_data);
-    const v3OperationCollections = await this.db.listBucketDataCollectionsV3();
-    const v3_operation_aggregates = await Promise.all(
-      v3OperationCollections.map((collection) =>
-        collection
-          .aggregate([
-            {
-              $collStats: {
-                storageStats: {}
-              }
-            }
-          ])
-          .toArray()
-          .catch(ignoreNotExisting)
-      )
-    );
-
-    const parameters_aggregate = await aggregateStaticCollection(this.db.bucket_parameters);
-
-    const v3ParameterCollections = await this.db.listAllParameterIndexCollectionsV3();
-    const v3_parameter_aggregates = await Promise.all(
-      v3ParameterCollections.map((collection) =>
-        collection
-          .aggregate([
-            {
-              $collStats: {
-                storageStats: {}
-              }
-            }
-          ])
-          .toArray()
-          .catch(ignoreNotExisting)
-      )
-    );
-
-    const v1_source_record_aggregate = await aggregateStaticCollection(this.db.current_data);
-
-    const v3SourceRecordCollections = await this.db.listAllSourceRecordCollectionsV3();
-    const source_record_aggregates = await Promise.all(
-      v3SourceRecordCollections.map((collection) =>
-        collection
-          .aggregate([
-            {
-              $collStats: {
-                storageStats: {}
-              }
-            }
-          ])
-          .toArray()
-          .catch(ignoreNotExisting)
+        },
+        { nameOnly: true }
       )
-    );
+      .toArray();
+    const sizesByCollection = new Map<string, number>();
+    const statsPipeline: mongo.Document[] = [
+      { $collStats: { storageStats: {} } },
+      { $project: { ns: 1, size: '$storageStats.size' } }
+    ];
+    // MongoDB has a limit of 1000 stages per aggregation pipeline. Additionally, we want
+    // to avoid too much work per aggregation, so we limit the number of collections per batch.
+    const COLLECTIONS_PER_BATCH = 100;
+    // Query batches sequentially so metrics collection does not fill the connection pool on standby pods.
+    // If a collection disappears during aggregation, let the next metrics collection retry.
+    for (let offset = 0; offset < collections.length; offset += COLLECTIONS_PER_BATCH) {
+      const batch = collections.slice(offset, offset + COLLECTIONS_PER_BATCH);
+      const pipeline = [
+        ...statsPipeline,
+        ...batch.slice(1).map((collection) => ({
+          $unionWith: { coll: collection.name, pipeline: statsPipeline }
+        }))
+      ];
+      const stats = await this.db.db
+        .collection(batch[0].name)
+        .aggregate<{ ns: string; size: number | bigint }>(pipeline)
+        .toArray();
+      for (const stat of stats) {
+        const name = stat.ns.slice(this.db.db.databaseName.length + 1);
+        sizesByCollection.set(name, (sizesByCollection.get(name) ?? 0) + Number(stat.size));
+      }
+    }
 
     const v3StorageConfig = getMongoStorageConfig(storage.STORAGE_VERSION_3) as StorageConfig & {
       incrementalReprocessing: true;
@@ -918,29 +878,8 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       .find({ storage_version: { $gte: storage.STORAGE_VERSION_3 } }, { projection: { _id: 1, sync_configs: 1 } })
       .toArray()) as unknown as Pick<ReplicationStreamDocumentV3, '_id' | 'sync_configs'>[];
 
-    const collectionSizes = new Map<string, number>();
-    const addCollectionSize = (
-      collection: { collectionName: string },
-      aggregate: { storageStats?: { size?: number | bigint } }[],
-      prefix: string,
-      sizes: Map<string, number>
-    ) => {
-      const match = collection.collectionName.match(new RegExp(`^${prefix}(\\d+)_(.+)$`));
-      if (match == null) {
-        return;
-      }
-      sizes.set(`${prefix}${match[1]}:${match[2]}`, Number(aggregate[0]?.storageStats?.size ?? 0));
-    };
-
-    v3OperationCollections.forEach((collection, index) =>
-      addCollectionSize(collection, v3_operation_aggregates[index], 'bucket_data_', collectionSizes)
-    );
-    v3ParameterCollections.forEach((collection, index) =>
-      addCollectionSize(collection, v3_parameter_aggregates[index], 'parameter_index_', collectionSizes)
-    );
-    v3SourceRecordCollections.forEach((collection, index) =>
-      addCollectionSize(collection, source_record_aggregates[index], 'source_records_', collectionSizes)
-    );
+    const sumCollectionSizesByPrefix = (prefix: string) =>
+      [...sizesByCollection].reduce((total, [name, size]) => total + (name.startsWith(prefix) ? size : 0), 0);
 
     const syncConfigIds = v3StreamDocs.flatMap((stream) => (stream.sync_configs ?? []).map((config) => config._id));
     const syncConfigDefinitions =
@@ -957,15 +896,13 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       number,
       { _id: ObjectId; bucket_data_source_ids: string[]; parameter_lookup_source_ids: string[] }[]
     >();
-    await Promise.all(
-      v3StreamDocs.map(async (stream) => {
-        const sourceTables = await v3Db
-          .sourceTables(stream._id)
-          .find({}, { projection: { _id: 1, bucket_data_source_ids: 1, parameter_lookup_source_ids: 1 } })
-          .toArray();
-        sourceTablesByStream.set(stream._id, sourceTables);
-      })
-    );
+    for (const stream of v3StreamDocs) {
+      const sourceTables = await v3Db
+        .sourceTables(stream._id)
+        .find({}, { projection: { _id: 1, bucket_data_source_ids: 1, parameter_lookup_source_ids: 1 } })
+        .toArray();
+      sourceTablesByStream.set(stream._id, sourceTables);
+    }
 
     const objectStorageSizeByDefinition = new Map<string, number>();
     for (const usage of objectStorageDefinitionUsage) {
@@ -976,7 +913,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
     }
 
     const sumCollectionSizes = (prefix: string, streamId: number, ids: ReadonlySet<string>) =>
-      [...ids].reduce((total, id) => total + (collectionSizes.get(`${prefix}${streamId}:${id}`) ?? 0), 0);
+      [...ids].reduce((total, id) => total + (sizesByCollection.get(`${prefix}${streamId}_${id}`) ?? 0), 0);
 
     const syncConfigMetrics: storage.StorageSyncConfigMetrics[] = [];
     for (const stream of v3StreamDocs) {
@@ -999,7 +936,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
           ) {
             return total;
           }
-          return total + (collectionSizes.get(`source_records_${stream._id}:${sourceTable._id.toHexString()}`) ?? 0);
+          return total + (sizesByCollection.get(`source_records_${stream._id}_${sourceTable._id.toHexString()}`) ?? 0);
         }, 0);
 
         syncConfigMetrics.push({
@@ -1022,15 +959,11 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
       0n
     );
     return {
-      operations_size_bytes:
-        Number(operations_aggregate[0].storageStats.size) +
-        v3_operation_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0),
+      operations_size_bytes: (sizesByCollection.get('bucket_data') ?? 0) + sumCollectionSizesByPrefix('bucket_data_'),
       parameters_size_bytes:
-        Number(parameters_aggregate[0].storageStats.size) +
-        v3_parameter_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0),
+        (sizesByCollection.get('bucket_parameters') ?? 0) + sumCollectionSizesByPrefix('parameter_index_'),
       replication_size_bytes:
-        Number(v1_source_record_aggregate[0]?.storageStats?.size ?? 0) +
-        source_record_aggregates.reduce((total, aggregate) => total + Number(aggregate[0]?.storageStats?.size ?? 0), 0),
+        (sizesByCollection.get('current_data') ?? 0) + sumCollectionSizesByPrefix('source_records_'),
       object_storage_size_bytes: Number(totalObjectStorageSize),
       sync_config_metrics: syncConfigMetrics
     };