@powersync/service-module-mongodb-storage 0.20.0 → 0.21.0

package/src/storage/implementation/db.ts

@@ -24,8 +24,9 @@ import {
   CurrentDataDocument
 } from './v1/models.js';
 import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js';
-import { BucketDataDocumentV3 } from './v3/models.js';
-import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
+import { BucketDataDocumentV3, CustomCheckpointRequestDocumentV3 } from './v3/models.js';
+import type { CustomCheckpointRequestCollectionSpecifier } from './v3/VersionedPowerSyncMongoV3.js';
+import { OBJECT_STORAGE_USAGE_COLLECTION, VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
 
 export interface PowerSyncMongoOptions {
   /**
@@ -137,6 +138,20 @@ export class PowerSyncMongo {
     return `source_table_${replicationStreamId}`;
   }
 
+  customCheckpointRequestCollectionName(options: CustomCheckpointRequestCollectionSpecifier) {
+    return `custom_checkpoint_requests_${options.replicationStreamId}_${options.eventId}`;
+  }
+
+  async listCustomCheckpointRequestCollections(
+    replicationStreamId?: number
+  ): Promise<mongo.Collection<CustomCheckpointRequestDocumentV3>[]> {
+    const prefix =
+      replicationStreamId == null
+        ? 'custom_checkpoint_requests_'
+        : `custom_checkpoint_requests_${replicationStreamId}_`;
+    return this.collectionsByPrefix(prefix);
+  }
+
   async listSourceTableCollections(
     replicationStreamId?: number
   ): Promise<mongo.Collection<CommonSourceTableDocument>[]> {
@@ -178,6 +193,10 @@ export class PowerSyncMongo {
     await this.locks.deleteMany({});
     await this.bucket_state.deleteMany({});
     await this.custom_write_checkpoints.deleteMany({});
+    await this.db.collection(OBJECT_STORAGE_USAGE_COLLECTION).deleteMany({});
+    for (const collection of await this.listCustomCheckpointRequestCollections()) {
+      await collection.drop();
+    }
   }
 
   /**

package/src/storage/implementation/models.ts

@@ -74,6 +74,12 @@ export interface BucketDataProperties {
   op: OpType;
   source_table?: bson.ObjectId;
   source_key?: ReplicaId;
+  /**
+   * Stable identifier for the source record in the client protocol.
+   *
+   * This is derived from (source_table, source_key), so persistence is optional but preferred.
+   */
+  subkey?: string;
   table?: string;
   row_id?: string;
   checksum: bigint;
@@ -215,6 +221,34 @@ export interface SyncRuleDocumentBase {
   } | null;
 
   storage_version?: number;
+
+  /** Optional operator-supplied label identifying the sync config version. */
+  version_label?: string | null;
+
+  /**
+   * Incremental parameter compaction state for the replication stream.
+   *
+   * Operation ids are allocated across all parameter indexes of a stream, so this is
+   * stream-level state, shared by all sync configs of the stream.
+   */
+  parameter_compaction?: {
+    /**
+     * The exclusive operation-id boundary through which every parameter index in this stream has
+     * been compacted.
+     */
+    compacted_before: InternalOpId;
+
+    /**
+     * Parameter entries below this boundary may no longer be available for checkpoint change
+     * detection, since compaction may have deleted them.
+     *
+     * This is advanced before the first delete of a compaction pass, while
+     * {@link compacted_before} is only advanced after every delete of the pass completed.
+     * A checkpoint transition starting below this boundary must conservatively invalidate all
+     * parameter buckets, since the individual changes may no longer be available.
+     */
+    checkpoint_changes_invalid_before?: InternalOpId;
+  };
 }
 
 export interface SyncRuleCheckpointFields {

package/src/storage/implementation/v1/MongoBucketBatchV1.ts

@@ -1,3 +1,4 @@
+import { mongo } from '@powersync/lib-service-mongodb';
 import { ReplicationAssertionError } from '@powersync/lib-services-framework';
 import { ColumnDescriptor, InternalOpId, SourceTable, storage } from '@powersync/service-core';
 import { HydratedSyncConfig } from '@powersync/service-sync-rules';
@@ -50,6 +51,36 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
     // No-op for V1: source records live in a shared collection.
   }
 
+  protected override async batchCreateCustomWriteCheckpoints(
+    session: mongo.ClientSession,
+    opId: InternalOpId
+  ): Promise<void> {
+    await this.db.custom_write_checkpoints.bulkWrite(
+      this.write_checkpoint_batch.map((checkpointOptions) => {
+        const set: Record<string, unknown> = {
+          checkpoint: checkpointOptions.checkpoint,
+          sync_rules_id: checkpointOptions.sync_rules_id,
+          op_id: opId
+        };
+        if (checkpointOptions.checkpoint_requested_at != null) {
+          set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
+        }
+
+        return {
+          updateOne: {
+            filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
+            update: {
+              $set: set,
+              ...(checkpointOptions.checkpoint_requested_at == null ? { $unset: { checkpoint_requested_at: 1 } } : {})
+            },
+            upsert: true
+          }
+        };
+      }),
+      { session }
+    );
+  }
+
   async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
     const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
     const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;

package/src/storage/implementation/v1/MongoCompactorV1.ts

@@ -1,10 +1,18 @@
-import { mongo } from '@powersync/lib-service-mongodb';
-import { ReplicationAssertionError } from '@powersync/lib-services-framework';
-import { addChecksums, storage, utils } from '@powersync/service-core';
+import { mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
+import {
+  addChecksums,
+  CompactInitialReplicationResults,
+  InternalOpId,
+  isPartialChecksum,
+  storage,
+  SyncRuleState,
+  utils
+} from '@powersync/service-core';
 import { BucketDefinitionId } from '@powersync/service-sync-rules';
 import { BucketDataDoc } from '../common/BucketDataDoc.js';
 import { BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
-import { CurrentBucketState, DirtyBucket, MongoCompactor } from '../MongoCompactor.js';
+import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
 import { cacheKey } from '../OperationBatch.js';
 import { BucketDataDocumentV1, BucketStateDocumentV1 } from './models.js';
 import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js';
@@ -13,12 +21,79 @@ import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
 
 type CompactClearProperties = 'op' | 'checksum' | 'target_op';
 
+const DEFAULT_MIN_BUCKET_CHANGES = 10;
+const DEFAULT_MIN_CHANGE_RATIO = 0.1;
+const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000;
+
+interface CurrentBucketState {
+  bucket: string;
+  definitionId: BucketDefinitionId;
+  seen: Map<string, InternalOpId>;
+  trackingSize: number;
+  lastNotPut: InternalOpId | null;
+  opsSincePut: number;
+  checksum: number;
+  opCount: number;
+  opBytes: number;
+}
+
+interface DirtyBucket {
+  bucket: string;
+  definitionId: BucketDefinitionId | null;
+  estimatedCount: number;
+  dirtyRatio?: number;
+}
+
 export class MongoCompactorV1 extends MongoCompactor {
   // Override types to the more specific ones
   declare protected readonly db: VersionedPowerSyncMongoV1;
   declare protected readonly storage: MongoSyncBucketStorageV1;
 
   private updates: mongo.AnyBulkWriteOperation<BucketDataDocumentV1>[] = [];
+  private bucketStateUpdates: mongo.AnyBulkWriteOperation<BucketStateDocumentBase>[] = [];
+  private readonly minBucketChanges: number;
+  private readonly minChangeRatio: number;
+  private readonly maxOpId: bigint;
+
+  constructor(
+    bucketStorage: MongoSyncBucketStorageV1,
+    db: VersionedPowerSyncMongoV1,
+    private options: MongoCompactOptions
+  ) {
+    super(bucketStorage, db, options);
+    this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
+    this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO;
+    this.maxOpId = options.maxOpId ?? 0n;
+  }
+
+  /**
+   * Compact buckets by converting operations into MOVE and/or CLEAR operations.
+   *
+   * See /docs/storage/compacting-operations.md for details.
+   */
+  override async compact(): Promise<number> {
+    if (this.options?.incrementalOnly) {
+      // Not supported for V1
+      this.logger.info('Incremental compacting is not supported on MongoDB storage V1/V2');
+      return 0;
+    } else if (this.storage.replicationStream.state != SyncRuleState.ACTIVE) {
+      this.logger.info(`Skipping compacting of replication stream in ${this.storage.replicationStream.state} state.`);
+      return 0;
+    }
+    await this.deleteOldCheckpointRequests();
+
+    if (this.buckets) {
+      for (const bucket of this.buckets) {
+        // We can make this more efficient later on by iterating through the buckets in a single query.
+        // That makes batching more tricky, so we leave for later.
+        await this.compactSingleBucketRetried(bucket);
+      }
+    } else {
+      await this.compactDirtyBuckets();
+    }
+
+    return this.compactedBucketCount;
+  }
 
   public async *dirtyBucketBatches(options: {
     minBucketChanges: number;
@@ -34,8 +109,7 @@ export class MongoCompactorV1 extends MongoCompactor {
       this.db.bucketStateV1,
       { g: this.group_id, b: new mongo.MinKey() as any },
       { g: this.group_id, b: new mongo.MaxKey() as any },
-      options,
-      () => null
+      options
     );
   }
 
@@ -45,14 +119,225 @@ export class MongoCompactorV1 extends MongoCompactor {
     }
     // Unlike dirtyBucketBatches, this path is resumable after restart because populateChecksums resets
     // estimate_since_compact as it progresses.
-    return this.dirtyBucketBatchForChecksumsForCollection(
-      this.db.bucketStateV1,
-      {
-        '_id.g': this.group_id,
-        'estimate_since_compact.count': { $gte: options.minBucketChanges }
-      },
-      () => null
-    );
+    return this.dirtyBucketBatchForChecksumsForCollection({
+      '_id.g': this.group_id,
+      'estimate_since_compact.count': { $gte: options.minBucketChanges }
+    });
+  }
+
+  private async *dirtyBucketBatchesForCollection<TBucketState extends BucketStateDocumentBase>(
+    collection: mongo.Collection<TBucketState>,
+    lastId: TBucketState['_id'],
+    maxId: TBucketState['_id'],
+    options: { minBucketChanges: number; minChangeRatio: number }
+  ): AsyncGenerator<DirtyBucket[]> {
+    // Paginate through the bucket state collection using cursor-based scanning.
+    while (true) {
+      // To avoid timeouts from too many buckets not meeting the minimum-change
+      // criteria, scan a fixed batch and only return matching buckets.
+      const [result] = await collection
+        .aggregate<{ buckets: TBucketState[]; cursor: Pick<TBucketState, '_id'>[] }>(
+          [
+            { $match: { _id: { $gt: lastId, $lt: maxId } } },
+            { $sort: { _id: 1 } },
+            // Scan a fixed number of docs each query so sparse matches don't block progress.
+            { $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE },
+            {
+              $facet: {
+                buckets: [
+                  { $match: { 'estimate_since_compact.count': { $gte: options.minBucketChanges } } },
+                  { $project: { _id: 1, estimate_since_compact: 1, compacted_state: 1 } }
+                ],
+                cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }]
+              }
+            }
+          ],
+          { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS }
+        )
+        .toArray();
+
+      const cursor = result?.cursor?.[0];
+      if (cursor == null) {
+        break;
+      }
+      lastId = cursor._id;
+
+      const dirtyBuckets = (result?.buckets ?? []).map((bucketState) => {
+        // BigInt precision is not needed here since this is only an estimate.
+        const updatedCount = bucketState.estimate_since_compact?.count ?? 0;
+        const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount;
+        const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0);
+        const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes;
+        return {
+          bucket: bucketState._id.b,
+          definitionId: null,
+          estimatedCount: totalCount,
+          dirtyRatio: Math.max(
+            totalCount > 0 ? updatedCount / totalCount : 0,
+            totalBytes > 0 ? updatedBytes / totalBytes : 0
+          )
+        };
+      });
+
+      yield dirtyBuckets.filter(
+        (bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio
+      );
+    }
+  }
+
+  private async dirtyBucketBatchForChecksumsForCollection(
+    filter: mongo.Filter<BucketStateDocumentV1>
+  ): Promise<DirtyBucket[]> {
+    const dirtyBuckets = await this.db.bucketStateV1
+      .find(filter, {
+        projection: { _id: 1, estimate_since_compact: 1, compacted_state: 1 },
+        sort: { 'estimate_since_compact.count': -1 },
+        limit: 200,
+        maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
+      })
+      .toArray();
+
+    return dirtyBuckets.map((bucket) => ({
+      bucket: bucket._id.b,
+      definitionId: null,
+      estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0)
+    }));
+  }
+
+  private async compactSingleBucketRetried(bucket: string, _definitionId: BucketDefinitionId | null = null) {
+    await this.retryCompaction(bucket, () => this.compactSingleBucket(bucket));
+    this.compactedBucketCount++;
+  }
+
+  private async compactDirtyBuckets() {
+    for await (const buckets of this.dirtyBucketBatches({
+      minBucketChanges: this.minBucketChanges,
+      minChangeRatio: this.minChangeRatio
+    })) {
+      this.signal?.throwIfAborted();
+      for (const { bucket, definitionId } of buckets) {
+        await this.compactSingleBucketRetried(bucket, definitionId);
+      }
+    }
+  }
+
+  /**
+   * Subset of compact, only populating checksums where relevant.
+   */
+  async populateChecksums(options: { minBucketChanges: number }): Promise<CompactInitialReplicationResults> {
+    let count = 0;
+    // Paginate through dirty buckets in batches until no more buckets meet the criteria.
+    while (true) {
+      this.signal?.throwIfAborted();
+      const buckets = await this.dirtyBucketBatchForChecksums(options);
+      if (buckets.length == 0) {
+        break;
+      }
+      this.signal?.throwIfAborted();
+
+      const start = Date.now();
+      // Filter batch by estimated bucket size, to reduce possibility of timeouts.
+      const checkBuckets: typeof buckets = [];
+      let totalCountEstimate = 0;
+      for (const bucket of buckets) {
+        checkBuckets.push(bucket);
+        totalCountEstimate += bucket.estimatedCount;
+        if (totalCountEstimate > 50_000) {
+          break;
+        }
+      }
+      this.logger.info(
+        `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}`
+      );
+      await this.updateChecksumsBatch(checkBuckets);
+      this.logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`);
+      count += checkBuckets.length;
+    }
+    return { buckets: count };
+  }
+
+  private collectBucketStateUpdate(
+    state: CurrentBucketState,
+    compactedOpId: bigint
+  ): mongo.AnyBulkWriteOperation<BucketStateDocumentBase> {
+    if (state.opCount < 0) {
+      throw new ServiceAssertionError(
+        `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}`
+      );
+    }
+    return {
+      updateOne: {
+        filter: this.bucketStateFilter(state.bucket),
+        update: {
+          $set: {
+            compacted_state: {
+              op_id: compactedOpId,
+              count: state.opCount,
+              checksum: BigInt(state.checksum),
+              bytes: state.opBytes
+            },
+            estimate_since_compact: {
+              // There could have been a whole bunch of new operations added to the bucket while compacting,
+              // which we don't currently cater for. We could potentially query for that, but that adds overhead.
+              count: 0,
+              bytes: 0
+            }
+          }
+        } satisfies mongo.UpdateFilter<BucketStateDocumentBase>,
+        // We generally expect this to have been created before.
+        // We don't create new ones here, to avoid issues with the unique index on bucket_updates.
+        upsert: false
+      }
+    };
+  }
+
+  private updateBucketChecksums(state: CurrentBucketState, compactedOpId: bigint) {
+    this.bucketStateUpdates.push(this.collectBucketStateUpdate(state, compactedOpId));
+  }
+
+  private async flushBucketStateUpdates() {
+    if (this.bucketStateUpdates.length > 0) {
+      this.logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`);
+      await this.writeBucketStateUpdates();
+      this.bucketStateUpdates = [];
+    }
+  }
+
+  private async updateChecksumsBatch(buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]) {
+    const checksums = await this.computeChecksumsForBuckets(buckets);
+    const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId]));
+
+    for (const bucketChecksum of checksums.values()) {
+      if (isPartialChecksum(bucketChecksum)) {
+        // Should never happen since we don't specify `start`.
+        throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`);
+      }
+
+      this.bucketStateUpdates.push({
+        updateOne: {
+          filter: this.bucketStateFilter(bucketChecksum.bucket),
+          update: {
+            $set: {
+              compacted_state: {
+                op_id: this.maxOpId,
+                count: bucketChecksum.count,
+                checksum: BigInt(bucketChecksum.checksum),
+                bytes: null
+              },
+              estimate_since_compact: {
+                count: 0,
+                bytes: 0
+              }
+            }
+          } satisfies mongo.UpdateFilter<BucketStateDocumentBase>,
+          // We don't create new ones here - it gets tricky to get the last_op right with the unique index on
+          // bucket_updates.
+          upsert: false
+        }
+      });
+    }
+
+    await this.flushBucketStateUpdates();
   }
 
   protected async writeBucketStateUpdates(): Promise<void> {
@@ -74,10 +359,7 @@ export class MongoCompactorV1 extends MongoCompactor {
     );
   }
 
-  protected bucketStateFilter(
-    bucket: string,
-    _definitionId: BucketDefinitionId | null
-  ): mongo.Filter<BucketStateDocumentBase> {
+  private bucketStateFilter(bucket: string): mongo.Filter<BucketStateDocumentBase> {
     return {
       _id: {
         g: this.group_id,
@@ -95,8 +377,9 @@ export class MongoCompactorV1 extends MongoCompactor {
   }
 
   protected async compactSingleBucket(bucket: string) {
-    // Do not carry queued writes from a failed attempt into the rescan
+    // Do not carry queued writes from a failed attempt into the rescan.
     this.updates = [];
+    this.bucketStateUpdates = [];
 
     const idLimitBytes = this.idLimitBytes;
     const bucketContext = this.getBucketDataContext(bucket);
@@ -200,6 +483,7 @@ export class MongoCompactorV1 extends MongoCompactor {
                   $unset: {
                     source_table: 1,
                     source_key: 1,
+                    subkey: 1,
                     table: 1,
                     row_id: 1,
                     data: 1

package/src/storage/implementation/v1/MongoParameterCompactorV1.ts

@@ -1,4 +1,5 @@
 import { mongo } from '@powersync/lib-service-mongodb';
+import { bson, InternalOpId } from '@powersync/service-core';
 import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
 import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
 
@@ -9,18 +10,33 @@ export class MongoParameterCompactorV1 extends MongoParameterCompactor {
     return [this.db.parameterIndexV1 as unknown as mongo.Collection<mongo.Document>];
   }
 
-  protected collectionFilter(): mongo.Document {
-    return {
-      'key.g': this.group_id
-    };
+  // The shared V1 collection is scanned using only its default `_id` index. Filter the stream in
+  // code so compaction does not require an index on `key.g`.
+  protected override shouldCompactDocument(doc: { key: mongo.Document }): boolean {
+    return doc.key.g === this.replicationStreamId;
   }
 
-  protected deleteFilter(doc: mongo.Document): mongo.Document {
+  /**
+   * Uses the legacy `{ 'key.g': 1, lookup: 1, _id: 1 }` index to narrow the stream, lookup and
+   * operation-id range. `key` is not part of that index at all, so it is a residual predicate applied
+   * to every document the range scan returns.
+   *
+   * That scan may therefore have to filter through many keys for the same lookup, but the cost is
+   * amortized: a single scan covers up to 1000 keys, and identities seen again in a later batch skip
+   * the scan entirely - they are deleted by `_id`.
+   *
+   * The V3 storage format uses an index more suitable for this.
+   */
+  protected leadingHistoryDeleteFilter(
+    lookup: bson.Binary,
+    keys: mongo.Document[],
+    before: InternalOpId
+  ): mongo.Document {
     return {
-      'key.g': doc.key.g as number,
-      lookup: doc.lookup,
-      _id: { $lte: doc._id },
-      key: doc.key
+      'key.g': this.replicationStreamId,
+      lookup,
+      key: { $in: keys },
+      _id: { $lt: before }
     };
   }
 }