@powersync/service-module-mongodb-storage 0.19.0 → 0.21.0

package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts

@@ -3,6 +3,8 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { ServiceAssertionError } from '@powersync/lib-services-framework';
 import {
   CheckpointChanges,
+  CompactInitialReplicationOptions,
+  CompactInitialReplicationResults,
   GetCheckpointChangesOptions,
   InternalOpId,
   internalToExternalOpId,
@@ -18,12 +20,20 @@ import * as bson from 'bson';
 import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
 import { MongoBucketStorage } from '../../MongoBucketStorage.js';
 import { BucketDataDoc } from '../common/BucketDataDoc.js';
-import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js';
+import {
+  MongoGetCheckpointChangesOptions,
+  MongoSyncBucketStorageCheckpoint
+} from '../common/MongoSyncBucketStorageCheckpoint.js';
 import { MongoChecksums } from '../MongoChecksums.js';
 import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
 import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
 import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js';
-import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
+import {
+  MongoCheckpointState,
+  MongoSyncBucketStorage,
+  MongoSyncBucketStorageOptions
+} from '../MongoSyncBucketStorage.js';
+import { MongoCheckpointAPIOptions } from '../MongoWriteCheckpointAPI.js';
 import { loadBucketDataDocument, maxOpId } from './bucket-format.js';
 import {
   BucketDataDocumentV3,
@@ -36,10 +46,13 @@ import {
 import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js';
 import { MongoChecksumsV3 } from './MongoChecksumsV3.js';
 import { MongoCompactorV3 } from './MongoCompactorV3.js';
+import { MongoParameterCompactorV3 } from './MongoParameterCompactorV3.js';
 import { MongoStoppedSyncConfigCleanup } from './MongoStoppedSyncConfigCleanup.js';
+import { MongoWriteCheckpointAPIV3 } from './MongoWriteCheckpointAPIV3.js';
 import { hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js';
 import { ObjectStorage } from './object-storage/ObjectStorage.js';
 import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
+import { ObjectStorageUsage } from './object-storage/ObjectStorageUsage.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
 
 export interface MongoSyncBucketStorageContextV3 {
@@ -188,21 +201,39 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     });
   }
 
+  protected override createWriteCheckpointAPI(options: MongoCheckpointAPIOptions): MongoWriteCheckpointAPIV3 {
+    return new MongoWriteCheckpointAPIV3({
+      ...options,
+      db: this.db,
+      syncConfigMapping: () => this.singleSyncConfigMapping()
+    });
+  }
+
   createMongoCompactor(options: MongoCompactOptions): MongoCompactor {
     return new MongoCompactorV3(this, this.db, options);
   }
 
+  override async compactInitialReplication(
+    options: CompactInitialReplicationOptions
+  ): Promise<CompactInitialReplicationResults> {
+    this.logger.info(`Compacting chunks after initial replication...`);
+    const start = Date.now();
+    const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined;
+    const compactedBuckets = await this.createMongoCompactor({
+      ...options,
+      maxOpId,
+      compactChunksOnly: true,
+      logger: this.logger
+    }).compact();
+    this.logger.info(`Compacted chunks after initial replication in ${(Date.now() - start) / 1000}s`);
+    return { buckets: compactedBuckets };
+  }
+
   protected createMongoParameterCompactor(
     checkpoint: InternalOpId,
     options: storage.CompactOptions
   ): MongoParameterCompactor {
-    return new MongoParameterCompactor(this.db, this.replicationStreamId, checkpoint, options, () =>
-      this.db
-        .listParameterIndexCollections(this.replicationStreamId)
-        .then((collections) =>
-          collections.map((c) => c.collection as unknown as lib_mongo.mongo.Collection<lib_mongo.mongo.Document>)
-        )
-    );
+    return new MongoParameterCompactorV3(this.db, this.replicationStreamId, checkpoint, options);
   }
 
   protected async fetchPersistedOpHead(): Promise<InternalOpId | null> {
@@ -214,13 +245,14 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
   }
 
   protected async createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
+    const batchOptions = this.writerBatchOptions(options);
     const doc = await this.syncRulesCollection.findOne(
       { _id: this.replicationStreamId },
       { projection: { resume_lsn: 1 } }
     );
 
     return new MongoBucketBatchV3({
-      ...this.writerBatchOptions(options),
+      ...batchOptions,
       // The stream-level replication position - per-config checkpoint LSNs are consistency
       // markers and do not affect where replication resumes.
       resumeFromLsn: doc?.resume_lsn ?? null,
@@ -228,16 +260,15 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     });
   }
 
-  protected async fetchCheckpointState(
-    session: mongo.ClientSession
-  ): Promise<{ checkpoint: bigint; lsn: string | null } | null> {
+  protected async fetchCheckpointState(session: mongo.ClientSession): Promise<MongoCheckpointState | null> {
     const doc = await this.syncRulesCollection.findOne(
       this.syncConfigMatch({
         state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
       }),
       {
         session,
-        projection: this.syncConfigProjection()
+        // The invalidation fence must be read in the same snapshot as the checkpoint.
+        projection: this.syncConfigProjection({ 'parameter_compaction.checkpoint_changes_invalid_before': 1 })
       }
     );
     // Checkpoints are served from the single active config. A PROCESSING config in the same
@@ -258,7 +289,10 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     }
     return {
       checkpoint: syncConfig.last_checkpoint ?? 0n,
-      lsn: syncConfig.last_checkpoint_lsn ?? null
+      lsn: syncConfig.last_checkpoint_lsn ?? null,
+      // Stream-level state: shared by all sync configs. Defaults to 0n for streams that have
+      // never been compacted.
+      parameterChangesInvalidBefore: doc?.parameter_compaction?.checkpoint_changes_invalid_before ?? 0n
     };
   }
 
@@ -311,7 +345,8 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
         },
         $unset: {
           resume_lsn: 1,
-          last_persisted_op: 1
+          last_persisted_op: 1,
+          parameter_compaction: 1
         }
       },
       {
@@ -376,6 +411,8 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
     for (const collection of await this.db.listBucketDataCollections(this.replicationStreamId)) {
       await collection.drop();
     }
+    const usage = new ObjectStorageUsage(this.db, this.replicationStreamId);
+    await this.db.client.withSession((session) => session.withTransaction(() => usage.removeStream(session)));
     if (this.objectStorage) {
       const lifecycle = new ObjectStorageLifecycle(this.db, this.replicationStreamId, this.objectStorage);
       await lifecycle.deletePrefix(lifecycle.streamPrefix(), { signal });
@@ -384,7 +421,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
       .pendingObjectStorageDeletes(this.replicationStreamId)
       .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
       .catch((error) => {
-        if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
+        if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
           return;
         }
         throw error;
@@ -408,7 +445,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
       .bucketState(this.replicationStreamId)
       .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
       .catch((error) => {
-        if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
+        if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
           return;
         }
         throw error;
@@ -420,11 +457,23 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
       .sourceTables(this.replicationStreamId)
       .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
       .catch((error) => {
+        if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
+          return;
+        }
+        throw error;
+      });
+  }
+
+  protected override async clearCustomCheckpointRequests(signal?: AbortSignal): Promise<void> {
+    for (const collection of await this.db.listCustomCheckpointRequestCollections(this.replicationStreamId)) {
+      signal?.throwIfAborted();
+      await collection.drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }).catch((error) => {
         if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
           return;
         }
         throw error;
       });
+    }
   }
 
   async cleanupStoppedSyncConfigs(
@@ -449,7 +498,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
   }
 
   protected getParameterBucketChangesImpl(
-    options: GetCheckpointChangesOptions
+    options: MongoGetCheckpointChangesOptions
   ): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
     return getParameterBucketChangesV3(this.versionContext, options);
   }
@@ -647,7 +696,7 @@ export async function* getBucketDataBatchV3(
     // Hydrate any operations from object storage.
     // In the future we can do this in a more pipelined fashion, but for now we hydrate
     // the entire batch at once.
-    await hydrateBucketDataDocuments(docs, ctx.objectStorage, { signal: options?.signal });
+    await hydrateBucketDataDocuments(docs, ctx.objectStorage, { signal: options?.signal, tracer: options?.tracer });
 
     let currentChunkSizeBytes = 0;
     let currentChunk: utils.SyncBucketData | null = null;
@@ -784,9 +833,17 @@ export async function getDataBucketChangesV3(
   };
 }
 
+/**
+ * Query the parameter entries changed between the two checkpoints, to determine which parameter
+ * lookups need to be re-evaluated.
+ *
+ * This runs at the next checkpoint's snapshot, so it still sees entries that parameter compaction
+ * deleted after that snapshot. Compaction that deleted entries before the snapshot is covered by
+ * the invalidation fence, checked before we get here.
+ */
 export async function getParameterBucketChangesV3(
   ctx: MongoSyncBucketStorageContextV3,
-  options: GetCheckpointChangesOptions
+  options: MongoGetCheckpointChangesOptions
 ): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
   const limit = 1000;
   const indexIds = ctx.mapping.allParameterIndexIds();
@@ -816,28 +873,41 @@ export async function getParameterBucketChangesV3(
     }
   ];
   const [firstCollection, ...remainingCollections] = collections;
-  const parameterUpdates = await firstCollection.collection
-    .aggregate<{ lookup: bson.Binary; indexId: string }>(
-      [
-        ...pipelineForCollection(firstCollection.indexId),
-        ...remainingCollections.map((collection) => {
-          return {
-            $unionWith: {
-              coll: collection.collection.collectionName,
-              pipeline: pipelineForCollection(collection.indexId)
-            }
-          };
-        }),
+  const parameterUpdates = await ctx.db.client.withSession({ snapshot: true }, async (session) => {
+    setSessionSnapshotTime(session, options.nextCheckpoint.snapshotTime);
+    return await firstCollection.collection
+      .aggregate<{ lookup: bson.Binary; indexId: string }>(
+        [
+          ...pipelineForCollection(firstCollection.indexId),
+          ...remainingCollections.map((collection) => {
+            return {
+              $unionWith: {
+                coll: collection.collection.collectionName,
+                pipeline: pipelineForCollection(collection.indexId)
+              }
+            };
+          }),
+          {
+            $limit: limit + 1
+          }
+        ],
         {
-          $limit: limit + 1
+          session,
+          readConcern: 'snapshot',
+          batchSize: limit + 2,
+          maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
         }
-      ],
-      {
-        batchSize: limit + 2,
-        maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
-      }
-    )
-    .toArray();
+      )
+      .toArray()
+      .catch((e) => {
+        // Includes the case where the checkpoint snapshot has expired. Degrading to
+        // invalidateParameterBuckets would be safe in itself - it reads nothing - but it gains
+        // nothing: the caller responds to that by re-evaluating the parameter queries at this
+        // same snapshot, which fails too. The checkpoint has to be refetched instead, which is
+        // what the existing sync retry behavior does.
+        throw lib_mongo.mapQueryError(e, 'while querying parameter changes');
+      });
+  });
 
   const invalidateParameterUpdates = parameterUpdates.length > limit;
 

package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts

@@ -0,0 +1,111 @@
+import { ServiceAssertionError } from '@powersync/lib-services-framework';
+import {
+  CustomWriteCheckpointFilters,
+  GetCheckpointChangesOptions,
+  SingleSyncConfigBucketDefinitionMapping,
+  storage
+} from '@powersync/service-core';
+import { EventDefinitionId } from '@powersync/service-sync-rules';
+import { MongoCheckpointAPIOptions, MongoWriteCheckpointAPI } from '../MongoWriteCheckpointAPI.js';
+import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
+
+export type MongoCheckpointAPIV3Options = Omit<MongoCheckpointAPIOptions, 'db'> & {
+  db: VersionedPowerSyncMongoV3;
+  syncConfigMapping: () => SingleSyncConfigBucketDefinitionMapping;
+};
+
+export class MongoWriteCheckpointAPIV3 extends MongoWriteCheckpointAPI {
+  declare db: VersionedPowerSyncMongoV3;
+
+  // Supplied via the constructor or the setWriteCheckpointMode setter.
+  private _customWriteCheckpointEventName: string | undefined;
+  private readonly syncConfigMapping: () => SingleSyncConfigBucketDefinitionMapping;
+
+  constructor(options: MongoCheckpointAPIV3Options) {
+    super(options);
+    this.syncConfigMapping = options.syncConfigMapping;
+    // Enforce the CUSTOM-requires-event-name invariant on construction too, not only setWriteCheckpointMode.
+    this._customWriteCheckpointEventName = this.requireCustomEventName(options.writeCheckpointMode);
+  }
+
+  protected resolveEventId(): EventDefinitionId {
+    const eventName = this._customWriteCheckpointEventName;
+    if (!eventName) {
+      throw new ServiceAssertionError(`No eventName has been supplied via setWriteCheckpointMode.`);
+    }
+    return this.syncConfigMapping().eventDefinitionIdByName(eventName);
+  }
+
+  protected override async lastCustomWriteCheckpoint(filters: CustomWriteCheckpointFilters): Promise<bigint | null> {
+    const { user_id } = filters;
+    const eventId = this.resolveEventId();
+
+    const lastWriteCheckpoint = await this.db
+      .customCheckpointRequests({
+        eventId,
+        replicationStreamId: this.replicationStreamId
+      })
+      .findOne({
+        user_id
+      });
+    return lastWriteCheckpoint?.checkpoint ?? null;
+  }
+
+  override setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
+    this._customWriteCheckpointEventName = this.requireCustomEventName(config);
+    super.setWriteCheckpointMode(config);
+  }
+
+  /**
+   * Validates and returns the event name for the given mode config. Shared by the constructor and
+   * {@link setWriteCheckpointMode} so both paths enforce that CUSTOM mode always names its checkpoint event.
+   */
+  private requireCustomEventName(config: storage.WriteCheckpointModeConfig): string | undefined {
+    if (config.mode != storage.WriteCheckpointMode.CUSTOM) {
+      return undefined;
+    }
+    if (!config.eventName) {
+      throw new ServiceAssertionError(`V3 incremental reprocessing requires an eventName to be supplied.`);
+    }
+    return config.eventName;
+  }
+
+  protected override async getCustomWriteCheckpointChanges(
+    options: GetCheckpointChangesOptions
+  ): Promise<{ invalidateWriteCheckpoints: boolean; updatedWriteCheckpoints: Map<string, bigint> }> {
+    const eventId = this.resolveEventId();
+
+    const limit = 1000;
+    const changes = await this.db
+      .customCheckpointRequests({
+        replicationStreamId: this.replicationStreamId,
+        eventId
+      })
+      .find(
+        {
+          op_id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint }
+        },
+        {
+          limit: limit + 1,
+          // batchSize is 1 more than limit to auto-close the cursor.
+          // See https://github.com/mongodb/node-mongodb-native/pull/4580
+          batchSize: limit + 2,
+          singleBatch: true
+        }
+      )
+      .toArray();
+    const invalidate = changes.length > limit;
+
+    const updatedWriteCheckpoints = new Map<string, bigint>();
+    if (!invalidate) {
+      for (let c of changes) {
+        updatedWriteCheckpoints.set(c.user_id, c.checkpoint);
+      }
+    }
+
+    return {
+      invalidateWriteCheckpoints: invalidate,
+      updatedWriteCheckpoints
+    };
+  }
+}

package/src/storage/implementation/v3/PersistedBatchV3.ts

@@ -15,6 +15,7 @@ import {
 import { SourceRecordLookupState } from '../common/SourceRecordStore.js';
 import { serializeBucketData } from './bucket-format.js';
 import { chunkBucketData } from './chunking.js';
+import { DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS } from './compaction-constants.js';
 import {
   BucketDataDocumentV3,
   BucketStateDocumentV3,
@@ -24,12 +25,14 @@ import {
   taggedBucketParameterDocumentToTagged
 } from './models.js';
 import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
+import { ObjectStorageUsage } from './object-storage/ObjectStorageUsage.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
 
 export class PersistedBatchV3 extends PersistedBatch {
   currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation<CurrentDataDocumentV3> }[] = [];
   sourceTablePendingDeletes = new Map<string, InternalOpId>();
   protected readonly objectStorageLifecycle?: ObjectStorageLifecycle;
+  protected readonly objectStorageUsage?: ObjectStorageUsage;
 
   declare protected readonly db: VersionedPowerSyncMongoV3;
 
@@ -43,6 +46,7 @@ export class PersistedBatchV3 extends PersistedBatch {
     super(db, group_id, mapping, writtenSize, options);
     if (this.objectStorage) {
       this.objectStorageLifecycle = new ObjectStorageLifecycle(this.db, this.group_id, this.objectStorage);
+      this.objectStorageUsage = new ObjectStorageUsage(this.db, this.group_id, this.objectStorageUsageWriterId);
     }
   }
 
@@ -225,15 +229,17 @@ export class PersistedBatchV3 extends PersistedBatch {
       operationsByDefinition.set(document.bucketKey.definitionId, existing);
     }
 
-    let uploadCount = 0;
     const plans = Array.from(operationsByDefinition, ([definitionId, documents]) => {
       const operationsByBucket = Map.groupBy(documents, (document) => document.bucketKey.bucket);
       const lifecycle = this.objectStorageLifecycle;
-      const createInserts: (() => Promise<mongo.AnyBulkWriteOperation<BucketDataDocumentV3>>)[] = [];
+      type BucketDataInsert = { insertOne: { document: BucketDataDocumentV3 } };
+      const createInserts: (() => Promise<BucketDataInsert>)[] = [];
 
       for (const [bucket, ops] of operationsByBucket) {
+        this.resetBucketPersistedBytes(definitionId, bucket);
         for (const chunk of chunkBucketData(ops)) {
           const serialized = serializeBucketData(bucket, chunk);
+          this.incrementBucketPersistedChunk(definitionId, bucket, serialized.size);
           if (lifecycle == null || serialized.size <= this.inlineThresholdBytes) {
             createInserts.push(async () => ({
               insertOne: {
@@ -243,13 +249,12 @@ export class PersistedBatchV3 extends PersistedBatch {
             continue;
           }
 
-          uploadCount += 1;
           createInserts.push(async () => {
             const minOp = chunk[0].o;
             const maxOp = chunk[chunk.length - 1].o;
             const { ops: bucketOps, ...metadata } = serialized;
             const path = lifecycle.allocatePath(definitionId, bucket, minOp, maxOp);
-            const { fileSize } = await lifecycle.bucketData.store(path, bucketOps!);
+            const { fileSize } = await lifecycle.bucketData.store(path, bucketOps!, { signal: this.signal });
             return {
               insertOne: {
                 document: {
@@ -281,7 +286,17 @@ export class PersistedBatchV3 extends PersistedBatch {
     // separate limiter here.
     const writes = await createAllInserts();
 
+    const usageDeltas = this.objectStorageUsage == null ? undefined : new Map<BucketDefinitionId, bigint>();
     for (const { definitionId, inserts } of writes) {
+      if (usageDeltas != null) {
+        const delta = inserts.reduce(
+          (sum, operation) => sum + ObjectStorageUsage.bytes(operation.insertOne.document),
+          0n
+        );
+        if (delta !== 0n) {
+          usageDeltas.set(definitionId, delta);
+        }
+      }
       if (inserts.length > 0) {
         await this.db.bucketData(this.group_id, definitionId).bulkWrite(inserts, {
           session,
@@ -289,6 +304,9 @@ export class PersistedBatchV3 extends PersistedBatch {
         });
       }
     }
+    if (usageDeltas != null) {
+      await this.objectStorageUsage!.applyDeltas(usageDeltas, session);
+    }
   }
 
   protected async flushBucketParameters(session: mongo.ClientSession) {
@@ -379,15 +397,42 @@ export class PersistedBatchV3 extends PersistedBatch {
               b: state.bucket
             }
           },
-          update: {
-            $set: {
-              last_op: state.lastOp
-            },
-            $inc: {
-              'estimate_since_compact.count': state.incrementCount,
-              'estimate_since_compact.bytes': state.incrementBytes
+          // A pipeline update makes initialisation and scheduling one atomic
+          // writer operation. In particular, a later write cannot move an
+          // already-due compact check into the future.
+          update: [
+            {
+              $set: {
+                last_op: state.lastOp,
+                bucket_stats: {
+                  count: { $add: [{ $ifNull: ['$bucket_stats.count', 0] }, state.incrementCount] },
+                  bytes: { $add: [{ $ifNull: ['$bucket_stats.bytes', 0n] }, BigInt(state.incrementBytes)] },
+                  chunks: { $add: [{ $ifNull: ['$bucket_stats.chunks', 0] }, state.incrementChunks] }
+                },
+                first_uncompacted_write: { $ifNull: ['$first_uncompacted_write', '$$NOW'] },
+                next_compact_check: {
+                  $let: {
+                    vars: {
+                      requested: {
+                        $dateAdd: {
+                          startDate: '$$NOW',
+                          unit: 'millisecond',
+                          amount: DEFAULT_MIN_COMPACT_CHUNK_INTERVAL_MS
+                        }
+                      }
+                    },
+                    in: {
+                      $cond: [
+                        { $lt: [{ $ifNull: ['$next_compact_check', '$$requested'] }, '$$requested'] },
+                        '$next_compact_check',
+                        '$$requested'
+                      ]
+                    }
+                  }
+                }
+              }
             }
-          },
+          ],
           upsert: true
         }
       } satisfies mongo.AnyBulkWriteOperation<BucketStateDocumentV3>;

package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts

@@ -1,5 +1,5 @@
 import { mongo } from '@powersync/lib-service-mongodb';
-import { BucketDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
+import { BucketDefinitionId, EventDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
 import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
 import { CommonSourceTableDocument } from '../models.js';
 import {
@@ -7,11 +7,19 @@ import {
   BucketParameterDocumentV3,
   BucketStateDocumentV3,
   CurrentDataDocumentV3,
+  CustomCheckpointRequestDocumentV3,
   ObjectStorageDeletionMarker,
+  ObjectStorageUsageDocument,
   SourceTableDocumentV3,
   SyncConfigDefinition
 } from './models.js';
 
+export const OBJECT_STORAGE_USAGE_COLLECTION = 'object_storage_usage';
+export type CustomCheckpointRequestCollectionSpecifier = {
+  replicationStreamId: number;
+  eventId: EventDefinitionId;
+};
+
 export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
   constructor(
     upstream: ConstructorParameters<typeof BaseVersionedPowerSyncMongo>[0],
@@ -74,6 +82,42 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
     return this.db.collection<ObjectStorageDeletionMarker>(`pending_object_storage_deletes_${replicationStreamId}`);
   }
 
+  get objectStorageUsage(): mongo.Collection<ObjectStorageUsageDocument> {
+    return this.db.collection<ObjectStorageUsageDocument>(OBJECT_STORAGE_USAGE_COLLECTION);
+  }
+
+  customCheckpointRequests(options: CustomCheckpointRequestCollectionSpecifier) {
+    const collectionName = this.customCheckpointRequestCollectionName(options);
+    return this.db.collection<CustomCheckpointRequestDocumentV3>(collectionName);
+  }
+
+  async initializeCustomCheckpointRequestsCollection(options: CustomCheckpointRequestCollectionSpecifier) {
+    // createIndexes also creates the collection on first use. Callers invoke
+    // this lazily before the transaction which writes the first checkpoint.
+    await this.customCheckpointRequests(options).createIndexes([
+      {
+        key: { user_id: 1 },
+        name: 'user_unique',
+        unique: true
+      },
+      {
+        // Find checkpoint changes over an operation-id range.
+        key: { op_id: 1 },
+        name: 'op_id'
+      },
+      {
+        // Limit retention scans to temporary client-requested checkpoints.
+        key: { checkpoint_requested_at: 1 },
+        name: 'checkpoint_requested_at',
+        partialFilterExpression: { checkpoint_requested_at: { $exists: true } }
+      }
+    ]);
+  }
+
+  listCustomCheckpointRequestCollections(replicationStreamId: number) {
+    return this.upstream.listCustomCheckpointRequestCollections(replicationStreamId);
+  }
+
   listBucketDataCollections(replicationStreamId: number) {
     return this.listCollectionsByPrefix(`bucket_data_${replicationStreamId}_`);
   }
@@ -117,11 +161,18 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
       },
       { name: 'bucket_updates', unique: true }
     );
+    // V3 workers claim only buckets that have reached their scheduled check
+    // time, keeping scheduled scans proportional to pending work.
     await bucketState.createIndex(
+      { next_compact_check: 1 },
       {
-        'estimate_since_compact.count': -1
-      },
-      { name: 'dirty_count' }
+        name: 'next_compact_check',
+        partialFilterExpression: { next_compact_check: { $exists: true } }
+      }
     );
   }
+
+  customCheckpointRequestCollectionName(options: CustomCheckpointRequestCollectionSpecifier) {
+    return this.upstream.customCheckpointRequestCollectionName(options);
+  }
 }

package/src/storage/implementation/v3/bucket-format.ts

@@ -32,6 +32,7 @@ export function serializeBucketData(
       op: op.op,
       source_table: op.source_table,
       source_key: op.source_key,
+      subkey: op.subkey,
       table: op.table,
       row_id: op.row_id,
       checksum: op.checksum,
@@ -78,6 +79,7 @@ export function* loadBucketDataDocument(
       op: op.op,
       source_table: op.source_table,
       source_key: op.source_key,
+      subkey: op.subkey,
       table: op.table,
       row_id: op.row_id,
       checksum: op.checksum,