@powersync/service-module-mongodb-storage 0.19.0 → 0.21.0

package/src/storage/implementation/v1/MongoCompactorV1.ts

@@ -1,10 +1,18 @@
-import { mongo } from '@powersync/lib-service-mongodb';
-import { ReplicationAssertionError } from '@powersync/lib-services-framework';
-import { addChecksums, storage, utils } from '@powersync/service-core';
+import { mongo, MONGO_OPERATION_TIMEOUT_MS } from '@powersync/lib-service-mongodb';
+import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
+import {
+  addChecksums,
+  CompactInitialReplicationResults,
+  InternalOpId,
+  isPartialChecksum,
+  storage,
+  SyncRuleState,
+  utils
+} from '@powersync/service-core';
 import { BucketDefinitionId } from '@powersync/service-sync-rules';
 import { BucketDataDoc } from '../common/BucketDataDoc.js';
 import { BucketStateDocumentBase, LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
-import { CurrentBucketState, DirtyBucket, MongoCompactor } from '../MongoCompactor.js';
+import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
 import { cacheKey } from '../OperationBatch.js';
 import { BucketDataDocumentV1, BucketStateDocumentV1 } from './models.js';
 import type { MongoSyncBucketStorageV1 } from './MongoSyncBucketStorageV1.js';
@@ -13,12 +21,79 @@ import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
 
 type CompactClearProperties = 'op' | 'checksum' | 'target_op';
 
+const DEFAULT_MIN_BUCKET_CHANGES = 10;
+const DEFAULT_MIN_CHANGE_RATIO = 0.1;
+const DIRTY_BUCKET_SCAN_BATCH_SIZE = 2_000;
+
+interface CurrentBucketState {
+  bucket: string;
+  definitionId: BucketDefinitionId;
+  seen: Map<string, InternalOpId>;
+  trackingSize: number;
+  lastNotPut: InternalOpId | null;
+  opsSincePut: number;
+  checksum: number;
+  opCount: number;
+  opBytes: number;
+}
+
+interface DirtyBucket {
+  bucket: string;
+  definitionId: BucketDefinitionId | null;
+  estimatedCount: number;
+  dirtyRatio?: number;
+}
+
 export class MongoCompactorV1 extends MongoCompactor {
   // Override types to the more specific ones
   declare protected readonly db: VersionedPowerSyncMongoV1;
   declare protected readonly storage: MongoSyncBucketStorageV1;
 
   private updates: mongo.AnyBulkWriteOperation<BucketDataDocumentV1>[] = [];
+  private bucketStateUpdates: mongo.AnyBulkWriteOperation<BucketStateDocumentBase>[] = [];
+  private readonly minBucketChanges: number;
+  private readonly minChangeRatio: number;
+  private readonly maxOpId: bigint;
+
+  constructor(
+    bucketStorage: MongoSyncBucketStorageV1,
+    db: VersionedPowerSyncMongoV1,
+    private options: MongoCompactOptions
+  ) {
+    super(bucketStorage, db, options);
+    this.minBucketChanges = options.minBucketChanges ?? DEFAULT_MIN_BUCKET_CHANGES;
+    this.minChangeRatio = options.minChangeRatio ?? DEFAULT_MIN_CHANGE_RATIO;
+    this.maxOpId = options.maxOpId ?? 0n;
+  }
+
+  /**
+   * Compact buckets by converting operations into MOVE and/or CLEAR operations.
+   *
+   * See /docs/storage/compacting-operations.md for details.
+   */
+  override async compact(): Promise<number> {
+    if (this.options?.incrementalOnly) {
+      // Not supported for V1
+      this.logger.info('Incremental compacting is not supported on MongoDB storage V1/V2');
+      return 0;
+    } else if (this.storage.replicationStream.state != SyncRuleState.ACTIVE) {
+      this.logger.info(`Skipping compacting of replication stream in ${this.storage.replicationStream.state} state.`);
+      return 0;
+    }
+    await this.deleteOldCheckpointRequests();
+
+    if (this.buckets) {
+      for (const bucket of this.buckets) {
+        // We can make this more efficient later on by iterating through the buckets in a single query.
+        // That makes batching more tricky, so we leave for later.
+        await this.compactSingleBucketRetried(bucket);
+      }
+    } else {
+      await this.compactDirtyBuckets();
+    }
+
+    return this.compactedBucketCount;
+  }
 
   public async *dirtyBucketBatches(options: {
     minBucketChanges: number;
@@ -34,8 +109,7 @@ export class MongoCompactorV1 extends MongoCompactor {
       this.db.bucketStateV1,
       { g: this.group_id, b: new mongo.MinKey() as any },
       { g: this.group_id, b: new mongo.MaxKey() as any },
-      options,
-      () => null
+      options
     );
   }
 
@@ -45,14 +119,225 @@ export class MongoCompactorV1 extends MongoCompactor {
     }
     // Unlike dirtyBucketBatches, this path is resumable after restart because populateChecksums resets
     // estimate_since_compact as it progresses.
-    return this.dirtyBucketBatchForChecksumsForCollection(
-      this.db.bucketStateV1,
-      {
-        '_id.g': this.group_id,
-        'estimate_since_compact.count': { $gte: options.minBucketChanges }
-      },
-      () => null
-    );
+    return this.dirtyBucketBatchForChecksumsForCollection({
+      '_id.g': this.group_id,
+      'estimate_since_compact.count': { $gte: options.minBucketChanges }
+    });
+  }
+
+  private async *dirtyBucketBatchesForCollection<TBucketState extends BucketStateDocumentBase>(
+    collection: mongo.Collection<TBucketState>,
+    lastId: TBucketState['_id'],
+    maxId: TBucketState['_id'],
+    options: { minBucketChanges: number; minChangeRatio: number }
+  ): AsyncGenerator<DirtyBucket[]> {
+    // Paginate through the bucket state collection using cursor-based scanning.
+    while (true) {
+      // To avoid timeouts from too many buckets not meeting the minimum-change
+      // criteria, scan a fixed batch and only return matching buckets.
+      const [result] = await collection
+        .aggregate<{ buckets: TBucketState[]; cursor: Pick<TBucketState, '_id'>[] }>(
+          [
+            { $match: { _id: { $gt: lastId, $lt: maxId } } },
+            { $sort: { _id: 1 } },
+            // Scan a fixed number of docs each query so sparse matches don't block progress.
+            { $limit: DIRTY_BUCKET_SCAN_BATCH_SIZE },
+            {
+              $facet: {
+                buckets: [
+                  { $match: { 'estimate_since_compact.count': { $gte: options.minBucketChanges } } },
+                  { $project: { _id: 1, estimate_since_compact: 1, compacted_state: 1 } }
+                ],
+                cursor: [{ $sort: { _id: -1 } }, { $limit: 1 }, { $project: { _id: 1 } }]
+              }
+            }
+          ],
+          { maxTimeMS: MONGO_OPERATION_TIMEOUT_MS }
+        )
+        .toArray();
+
+      const cursor = result?.cursor?.[0];
+      if (cursor == null) {
+        break;
+      }
+      lastId = cursor._id;
+
+      const dirtyBuckets = (result?.buckets ?? []).map((bucketState) => {
+        // BigInt precision is not needed here since this is only an estimate.
+        const updatedCount = bucketState.estimate_since_compact?.count ?? 0;
+        const totalCount = (bucketState.compacted_state?.count ?? 0) + updatedCount;
+        const updatedBytes = Number(bucketState.estimate_since_compact?.bytes ?? 0);
+        const totalBytes = Number(bucketState.compacted_state?.bytes ?? 0) + updatedBytes;
+        return {
+          bucket: bucketState._id.b,
+          definitionId: null,
+          estimatedCount: totalCount,
+          dirtyRatio: Math.max(
+            totalCount > 0 ? updatedCount / totalCount : 0,
+            totalBytes > 0 ? updatedBytes / totalBytes : 0
+          )
+        };
+      });
+
+      yield dirtyBuckets.filter(
+        (bucket) => bucket.estimatedCount >= options.minBucketChanges && bucket.dirtyRatio >= options.minChangeRatio
+      );
+    }
+  }
+
+  private async dirtyBucketBatchForChecksumsForCollection(
+    filter: mongo.Filter<BucketStateDocumentV1>
+  ): Promise<DirtyBucket[]> {
+    const dirtyBuckets = await this.db.bucketStateV1
+      .find(filter, {
+        projection: { _id: 1, estimate_since_compact: 1, compacted_state: 1 },
+        sort: { 'estimate_since_compact.count': -1 },
+        limit: 200,
+        maxTimeMS: MONGO_OPERATION_TIMEOUT_MS
+      })
+      .toArray();
+
+    return dirtyBuckets.map((bucket) => ({
+      bucket: bucket._id.b,
+      definitionId: null,
+      estimatedCount: Number(bucket.estimate_since_compact!.count) + Number(bucket.compacted_state?.count ?? 0)
+    }));
+  }
+
+  private async compactSingleBucketRetried(bucket: string, _definitionId: BucketDefinitionId | null = null) {
+    await this.retryCompaction(bucket, () => this.compactSingleBucket(bucket));
+    this.compactedBucketCount++;
+  }
+
+  private async compactDirtyBuckets() {
+    for await (const buckets of this.dirtyBucketBatches({
+      minBucketChanges: this.minBucketChanges,
+      minChangeRatio: this.minChangeRatio
+    })) {
+      this.signal?.throwIfAborted();
+      for (const { bucket, definitionId } of buckets) {
+        await this.compactSingleBucketRetried(bucket, definitionId);
+      }
+    }
+  }
+
+  /**
+   * Subset of compact, only populating checksums where relevant.
+   */
+  async populateChecksums(options: { minBucketChanges: number }): Promise<CompactInitialReplicationResults> {
+    let count = 0;
+    // Paginate through dirty buckets in batches until no more buckets meet the criteria.
+    while (true) {
+      this.signal?.throwIfAborted();
+      const buckets = await this.dirtyBucketBatchForChecksums(options);
+      if (buckets.length == 0) {
+        break;
+      }
+      this.signal?.throwIfAborted();
+
+      const start = Date.now();
+      // Filter batch by estimated bucket size, to reduce possibility of timeouts.
+      const checkBuckets: typeof buckets = [];
+      let totalCountEstimate = 0;
+      for (const bucket of buckets) {
+        checkBuckets.push(bucket);
+        totalCountEstimate += bucket.estimatedCount;
+        if (totalCountEstimate > 50_000) {
+          break;
+        }
+      }
+      this.logger.info(
+        `Calculating checksums for batch of ${buckets.length} buckets, estimated count of ${totalCountEstimate}`
+      );
+      await this.updateChecksumsBatch(checkBuckets);
+      this.logger.info(`Updated checksums for batch of ${checkBuckets.length} buckets in ${Date.now() - start}ms`);
+      count += checkBuckets.length;
+    }
+    return { buckets: count };
+  }
+
+  private collectBucketStateUpdate(
+    state: CurrentBucketState,
+    compactedOpId: bigint
+  ): mongo.AnyBulkWriteOperation<BucketStateDocumentBase> {
+    if (state.opCount < 0) {
+      throw new ServiceAssertionError(
+        `Invalid opCount: ${state.opCount} checksum ${state.checksum} opsSincePut: ${state.opsSincePut} maxOpId: ${this.maxOpId}`
+      );
+    }
+    return {
+      updateOne: {
+        filter: this.bucketStateFilter(state.bucket),
+        update: {
+          $set: {
+            compacted_state: {
+              op_id: compactedOpId,
+              count: state.opCount,
+              checksum: BigInt(state.checksum),
+              bytes: state.opBytes
+            },
+            estimate_since_compact: {
+              // There could have been a whole bunch of new operations added to the bucket while compacting,
+              // which we don't currently cater for. We could potentially query for that, but that adds overhead.
+              count: 0,
+              bytes: 0
+            }
+          }
+        } satisfies mongo.UpdateFilter<BucketStateDocumentBase>,
+        // We generally expect this to have been created before.
+        // We don't create new ones here, to avoid issues with the unique index on bucket_updates.
+        upsert: false
+      }
+    };
+  }
+
+  private updateBucketChecksums(state: CurrentBucketState, compactedOpId: bigint) {
+    this.bucketStateUpdates.push(this.collectBucketStateUpdate(state, compactedOpId));
+  }
+
+  private async flushBucketStateUpdates() {
+    if (this.bucketStateUpdates.length > 0) {
+      this.logger.info(`Updating ${this.bucketStateUpdates.length} bucket states`);
+      await this.writeBucketStateUpdates();
+      this.bucketStateUpdates = [];
+    }
+  }
+
+  private async updateChecksumsBatch(buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]) {
+    const checksums = await this.computeChecksumsForBuckets(buckets);
+    const definitionIdByBucket = new Map(buckets.map((bucket) => [bucket.bucket, bucket.definitionId]));
+
+    for (const bucketChecksum of checksums.values()) {
+      if (isPartialChecksum(bucketChecksum)) {
+        // Should never happen since we don't specify `start`.
+        throw new ServiceAssertionError(`Full checksum expected, got ${JSON.stringify(bucketChecksum)}`);
+      }
+
+      this.bucketStateUpdates.push({
+        updateOne: {
+          filter: this.bucketStateFilter(bucketChecksum.bucket),
+          update: {
+            $set: {
+              compacted_state: {
+                op_id: this.maxOpId,
+                count: bucketChecksum.count,
+                checksum: BigInt(bucketChecksum.checksum),
+                bytes: null
+              },
+              estimate_since_compact: {
+                count: 0,
+                bytes: 0
+              }
+            }
+          } satisfies mongo.UpdateFilter<BucketStateDocumentBase>,
+          // We don't create new ones here - it gets tricky to get the last_op right with the unique index on
+          // bucket_updates.
+          upsert: false
+        }
+      });
+    }
+
+    await this.flushBucketStateUpdates();
   }
 
   protected async writeBucketStateUpdates(): Promise<void> {
@@ -74,10 +359,7 @@ export class MongoCompactorV1 extends MongoCompactor {
     );
   }
 
-  protected bucketStateFilter(
-    bucket: string,
-    _definitionId: BucketDefinitionId | null
-  ): mongo.Filter<BucketStateDocumentBase> {
+  private bucketStateFilter(bucket: string): mongo.Filter<BucketStateDocumentBase> {
     return {
       _id: {
         g: this.group_id,
@@ -95,8 +377,9 @@ export class MongoCompactorV1 extends MongoCompactor {
   }
 
   protected async compactSingleBucket(bucket: string) {
-    // Do not carry queued writes from a failed attempt into the rescan
+    // Do not carry queued writes from a failed attempt into the rescan.
     this.updates = [];
+    this.bucketStateUpdates = [];
 
     const idLimitBytes = this.idLimitBytes;
     const bucketContext = this.getBucketDataContext(bucket);
@@ -200,6 +483,7 @@ export class MongoCompactorV1 extends MongoCompactor {
                   $unset: {
                     source_table: 1,
                     source_key: 1,
+                    subkey: 1,
                     table: 1,
                     row_id: 1,
                     data: 1

package/src/storage/implementation/v1/MongoParameterCompactorV1.ts

@@ -1,4 +1,5 @@
 import { mongo } from '@powersync/lib-service-mongodb';
+import { bson, InternalOpId } from '@powersync/service-core';
 import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
 import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
 
@@ -9,18 +10,33 @@ export class MongoParameterCompactorV1 extends MongoParameterCompactor {
     return [this.db.parameterIndexV1 as unknown as mongo.Collection<mongo.Document>];
   }
 
-  protected collectionFilter(): mongo.Document {
-    return {
-      'key.g': this.group_id
-    };
+  // The shared V1 collection is scanned using only its default `_id` index. Filter the stream in
+  // code so compaction does not require an index on `key.g`.
+  protected override shouldCompactDocument(doc: { key: mongo.Document }): boolean {
+    return doc.key.g === this.replicationStreamId;
   }
 
-  protected deleteFilter(doc: mongo.Document): mongo.Document {
+  /**
+   * Uses the legacy `{ 'key.g': 1, lookup: 1, _id: 1 }` index to narrow the stream, lookup and
+   * operation-id range. `key` is not part of that index at all, so it is a residual predicate applied
+   * to every document the range scan returns.
+   *
+   * That scan may therefore have to filter through many keys for the same lookup, but the cost is
+   * amortized: a single scan covers up to 1000 keys, and identities seen again in a later batch skip
+   * the scan entirely - they are deleted by `_id`.
+   *
+   * The V3 storage format uses an index more suitable for this.
+   */
+  protected leadingHistoryDeleteFilter(
+    lookup: bson.Binary,
+    keys: mongo.Document[],
+    before: InternalOpId
+  ): mongo.Document {
     return {
-      'key.g': doc.key.g as number,
-      lookup: doc.lookup,
-      _id: { $lte: doc._id },
-      key: doc.key
+      'key.g': this.replicationStreamId,
+      lookup,
+      key: { $in: keys },
+      _id: { $lt: before }
     };
   }
 }

package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts

@@ -3,6 +3,8 @@ import { mongo } from '@powersync/lib-service-mongodb';
 import { ServiceAssertionError } from '@powersync/lib-services-framework';
 import {
   CheckpointChanges,
+  CompactInitialReplicationOptions,
+  CompactInitialReplicationResults,
   deserializeParameterLookup,
   GetCheckpointChangesOptions,
   InternalOpId,
@@ -26,13 +28,20 @@ import {
   setSessionSnapshotTime
 } from '../../../utils/util.js';
 import { MongoBucketStorage } from '../../MongoBucketStorage.js';
-import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js';
+import {
+  MongoGetCheckpointChangesOptions,
+  MongoSyncBucketStorageCheckpoint
+} from '../common/MongoSyncBucketStorageCheckpoint.js';
 import { SourceKey } from '../models.js';
 import { MongoChecksums } from '../MongoChecksums.js';
-import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
+import { MongoCompactOptions } from '../MongoCompactor.js';
 import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
 import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js';
-import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
+import {
+  MongoCheckpointState,
+  MongoSyncBucketStorage,
+  MongoSyncBucketStorageOptions
+} from '../MongoSyncBucketStorage.js';
 import {
   BucketDataDocumentV1,
   BucketDataKeyV1,
@@ -100,14 +109,20 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
     });
   }
 
-  protected async fetchCheckpointState(
-    session: mongo.ClientSession
-  ): Promise<{ checkpoint: bigint; lsn: string | null } | null> {
+  protected async fetchCheckpointState(session: mongo.ClientSession): Promise<MongoCheckpointState | null> {
     const doc = (await this.db.sync_rules.findOne(
       { _id: this.replicationStreamId },
       {
         session,
-        projection: { _id: 1, state: 1, last_checkpoint: 1, last_checkpoint_lsn: 1, snapshot_done: 1 }
+        projection: {
+          _id: 1,
+          state: 1,
+          last_checkpoint: 1,
+          last_checkpoint_lsn: 1,
+          snapshot_done: 1,
+          // Must be read in the same snapshot as the checkpoint.
+          'parameter_compaction.checkpoint_changes_invalid_before': 1
+        }
       }
     )) as SyncRuleDocumentV1;
     if (!doc?.snapshot_done || ![storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED].includes(doc.state)) {
@@ -115,7 +130,9 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
     }
     return {
       checkpoint: doc.last_checkpoint ?? 0n,
-      lsn: doc.last_checkpoint_lsn ?? null
+      lsn: doc.last_checkpoint_lsn ?? null,
+      // Defaults to 0n for streams that have never been compacted.
+      parameterChangesInvalidBefore: doc.parameter_compaction?.checkpoint_changes_invalid_before ?? 0n
     };
   }
 
@@ -171,7 +188,8 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
           no_checkpoint_before: null
         },
         $unset: {
-          snapshot_lsn: 1
+          snapshot_lsn: 1,
+          parameter_compaction: 1
         }
       },
       { maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
@@ -186,10 +204,31 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
     });
   }
 
-  createMongoCompactor(options: MongoCompactOptions): MongoCompactor {
+  createMongoCompactor(options: MongoCompactOptions): MongoCompactorV1 {
     return new MongoCompactorV1(this, this.db, options);
   }
 
+  async compactInitialReplication(
+    options: CompactInitialReplicationOptions
+  ): Promise<CompactInitialReplicationResults> {
+    this.logger.info(`Compacting after initial replication...`);
+    const start = Date.now();
+    const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined;
+    const compactor = this.createMongoCompactor({
+      ...options,
+      maxOpId,
+      memoryLimitMB: 0,
+      logger: this.logger
+    });
+
+    const result = await compactor.populateChecksums({
+      minBucketChanges: options.minBucketChanges ?? 10
+    });
+    const duration = Date.now() - start;
+    this.logger.info(`Compacted after initial replication in ${(duration / 1000).toFixed(1)}s`);
+    return result;
+  }
+
   protected createMongoParameterCompactor(
     checkpoint: InternalOpId,
     options: storage.CompactOptions
@@ -303,7 +342,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
   }
 
   protected getParameterBucketChangesImpl(
-    options: GetCheckpointChangesOptions
+    options: MongoGetCheckpointChangesOptions
   ): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
     return getParameterBucketChangesV1(this.versionContext, options);
   }
@@ -548,27 +587,48 @@ export async function getDataBucketChangesV1(
   };
 }
 
+/**
+ * Query the parameter entries changed between the two checkpoints, to determine which parameter
+ * lookups need to be re-evaluated.
+ *
+ * This runs at the next checkpoint's snapshot, so it still sees entries that parameter compaction
+ * deleted after that snapshot. Compaction that deleted entries before the snapshot is covered by
+ * the invalidation fence, checked before we get here.
+ */
 export async function getParameterBucketChangesV1(
   ctx: MongoSyncBucketStorageContextV1,
-  options: GetCheckpointChangesOptions
+  options: MongoGetCheckpointChangesOptions
 ): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
   const limit = 1000;
-  const parameterUpdates = await ctx.db.parameterIndexV1
-    .find(
-      {
-        _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint },
-        'key.g': ctx.replicationStreamId
-      },
-      {
-        projection: {
-          lookup: 1
+  const parameterUpdates = await ctx.db.client.withSession({ snapshot: true }, async (session) => {
+    setSessionSnapshotTime(session, options.nextCheckpoint.snapshotTime);
+    return await ctx.db.parameterIndexV1
+      .find(
+        {
+          _id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint },
+          'key.g': ctx.replicationStreamId
         },
-        limit: limit + 1,
-        batchSize: limit + 2,
-        singleBatch: true
-      }
-    )
-    .toArray();
+        {
+          session,
+          readConcern: 'snapshot',
+          projection: {
+            lookup: 1
+          },
+          limit: limit + 1,
+          batchSize: limit + 2,
+          singleBatch: true
+        }
+      )
+      .toArray()
+      .catch((e) => {
+        // Includes the case where the checkpoint snapshot has expired. Degrading to
+        // invalidateParameterBuckets would be safe in itself - it reads nothing - but it gains
+        // nothing: the caller responds to that by re-evaluating the parameter queries at this
+        // same snapshot, which fails too. The checkpoint has to be refetched instead, which is
+        // what the existing sync retry behavior does.
+        throw lib_mongo.mapQueryError(e, 'while querying parameter changes');
+      });
+  });
   const invalidateParameterUpdates = parameterUpdates.length > limit;
 
   return {

package/src/storage/implementation/v1/models.ts

@@ -48,6 +48,7 @@ export function serializeBucketDataV1(document: BucketDataDoc): BucketDataDocume
     op: document.op,
     source_table: document.source_table,
     source_key: document.source_key,
+    subkey: document.subkey,
     table: document.table,
     row_id: document.row_id,
     checksum: document.checksum,