@powersync/service-module-mongodb-storage 0.17.0 → 0.18.1

package/src/storage/implementation/v3/PersistedBatchV3.ts

@@ -4,66 +4,76 @@ import { InternalOpId, storage } from '@powersync/service-core';
 import { BucketDataSource, BucketDefinitionId } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { mongoTableId } from '../../../utils/util.js';
+import { BucketDataDoc } from '../common/BucketDataDoc.js';
 import {
   BucketStateUpdate,
   PersistedBatch,
   SaveParameterDataOptions,
   UpsertCurrentDataOptions
 } from '../common/PersistedBatch.js';
-import { SourceTableKey } from '../models.js';
+import { SourceRecordLookupState } from '../common/SourceRecordStore.js';
+import { serializeBucketData } from './bucket-format.js';
+import { chunkBucketData } from './chunking.js';
 import {
-  BucketParameterDocumentV3,
+  BucketDataDocumentV3,
   BucketStateDocumentV3,
   CurrentDataDocumentV3,
-  serializeBucketDataV3,
+  serializeParameterLookup,
   SourceTableDocumentV3,
-  taggedBucketParameterDocumentToV3
+  taggedBucketParameterDocumentToTagged
 } from './models.js';
-import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
 
 export class PersistedBatchV3 extends PersistedBatch {
-  declare protected readonly db: VersionedPowerSyncMongoV3;
-
   currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation<CurrentDataDocumentV3> }[] = [];
   sourceTablePendingDeletes = new Map<string, InternalOpId>();
 
-  protected checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId {
+  declare protected readonly db: VersionedPowerSyncMongoV3;
+
+  // Abstract override from PersistedBatch (V3-specific error message)
+
+  protected override checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId {
     if (definitionId == null) {
-      // This is required for V3 storage.
       throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled');
     }
     return definitionId;
   }
 
+  // Concrete implementations from PersistedBatchShared
+
   protected getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId {
     return this.mapping.bucketSourceId(bucketSource);
   }
 
   saveParameterData(data: SaveParameterDataOptions) {
     const { sourceTable, sourceKey, evaluated } = data;
-    const remaining_lookups = new Map<string, SaveParameterDataOptions['existing_lookups'][number]>();
+    const remaining_lookups = new Map<string, SourceRecordLookupState>();
 
     for (let lookup of data.existing_lookups) {
       if (lookup.indexId == null) {
-        throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled');
+        throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
+      }
+      if (sourceTable.parameterLookupSourceIds != null && !sourceTable.parameterLookupSourceIds.has(lookup.indexId)) {
+        // The parameter index is not active anymore.
+        // We don't cleanup these references upfront, but do need to ignore them after the definition is removed.
+        continue;
       }
       remaining_lookups.set(`${lookup.indexId}.${lookup.lookup.toString('base64')}`, lookup);
     }
 
     for (let result of evaluated) {
       const sourceDefinitionId = this.mapping.parameterLookupId(result.lookup.source);
-      const binLookup = serializeParameterLookupV3(result.lookup);
+      const binLookup = serializeParameterLookup(result.lookup);
       remaining_lookups.delete(`${sourceDefinitionId}.${binLookup.toString('base64')}`);
 
       const op_id = data.op_seq.next();
       this.debugLastOpId = op_id;
-      const values: BucketParameterDocumentV3 = {
+      const values = {
         _id: op_id,
         key: {
           t: mongoTableId(sourceTable.id),
           k: sourceKey
-        } satisfies SourceTableKey,
+        },
         lookup: binLookup,
         bucket_parameters: result.bucketParameters
       };
@@ -80,14 +90,14 @@ export class PersistedBatchV3 extends PersistedBatch {
       this.debugLastOpId = op_id;
       const indexId = lookup.indexId;
       if (indexId == null) {
-        throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled');
+        throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
       }
-      const values: BucketParameterDocumentV3 = {
+      const values = {
         _id: op_id,
         key: {
           t: mongoTableId(sourceTable.id),
           k: sourceKey
-        } satisfies SourceTableKey,
+        },
         lookup: lookup.lookup,
         bucket_parameters: []
       };
@@ -125,8 +135,8 @@ export class PersistedBatchV3 extends PersistedBatch {
           update: {
             $set: {
               data: null,
-              buckets: [] as CurrentDataDocumentV3['buckets'],
-              lookups: [] as CurrentDataDocumentV3['lookups'],
+              buckets: [],
+              lookups: [],
               pending_delete: checkpointGreaterThan
             }
           },
@@ -146,7 +156,7 @@ export class PersistedBatchV3 extends PersistedBatch {
   upsertCurrentData(values: UpsertCurrentDataOptions) {
     const buckets = values.buckets.map((bucket) => {
       if (bucket.definitionId == null) {
-        throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled');
+        throw new ReplicationAssertionError('Expected bucket when incrementalReprocessing is enabled');
       }
       return {
         def: bucket.definitionId,
@@ -157,7 +167,7 @@ export class PersistedBatchV3 extends PersistedBatch {
     });
     const lookups = values.lookups.map((lookup) => {
       if (lookup.indexId == null) {
-        throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled');
+        throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
       }
       return {
         i: lookup.indexId,
@@ -189,8 +199,10 @@ export class PersistedBatchV3 extends PersistedBatch {
     return this.currentData.length;
   }
 
+  // Flush methods
+
   protected async flushBucketData(session: mongo.ClientSession) {
-    const operationsByDefinition = new Map<BucketDefinitionId, typeof this.bucketData>();
+    const operationsByDefinition = new Map<BucketDefinitionId, BucketDataDoc[]>();
     for (const document of this.bucketData) {
       const existing = operationsByDefinition.get(document.bucketKey.definitionId) ?? [];
       existing.push(document);
@@ -198,17 +210,31 @@ export class PersistedBatchV3 extends PersistedBatch {
     }
 
     for (const [definitionId, documents] of operationsByDefinition.entries()) {
-      await this.db.bucketDataV3(this.group_id, definitionId).bulkWrite(
-        documents.map((document) => ({
-          insertOne: {
-            document: serializeBucketDataV3(document)
-          }
-        })),
-        {
+      const operationsByBucket = new Map<string, BucketDataDoc[]>();
+      for (const document of documents) {
+        const existing = operationsByBucket.get(document.bucketKey.bucket) ?? [];
+        existing.push(document);
+        operationsByBucket.set(document.bucketKey.bucket, existing);
+      }
+
+      const inserts: mongo.AnyBulkWriteOperation<BucketDataDocumentV3>[] = [];
+      for (const [bucket, ops] of operationsByBucket.entries()) {
+        const chunks = chunkBucketData(ops);
+        for (const chunk of chunks) {
+          inserts.push({
+            insertOne: {
+              document: serializeBucketData(bucket, chunk)
+            }
+          });
+        }
+      }
+
+      if (inserts.length > 0) {
+        await this.db.bucketData(this.group_id, definitionId).bulkWrite(inserts, {
           session,
           ordered: false
-        }
-      );
+        });
+      }
     }
   }
 
@@ -221,10 +247,10 @@ export class PersistedBatchV3 extends PersistedBatch {
     }
 
     for (const [indexId, documents] of operationsByIndex.entries()) {
-      await this.db.parameterIndexV3(this.group_id, indexId).bulkWrite(
+      await this.db.parameterIndex(this.group_id, indexId).bulkWrite(
         documents.map((document) => ({
           insertOne: {
-            document: taggedBucketParameterDocumentToV3(document)
+            document: taggedBucketParameterDocumentToTagged(document)
           }
         })),
         {
@@ -260,12 +286,12 @@ export class PersistedBatchV3 extends PersistedBatch {
     });
 
     if (sourceTableUpdates.length > 0) {
-      await this.db.sourceTablesV3(this.group_id).bulkWrite(sourceTableUpdates, { session, ordered: false });
+      await this.db.sourceTables(this.group_id).bulkWrite(sourceTableUpdates, { session, ordered: false });
     }
 
     for (const operations of operationsBySourceTable.values()) {
       const sourceTableId = operations[0]!.sourceTableId;
-      await this.db.sourceRecordsV3(this.group_id, sourceTableId).bulkWrite(
+      await this.db.sourceRecords(this.group_id, sourceTableId).bulkWrite(
         operations.map((entry) => entry.operation),
         {
           session,
@@ -276,7 +302,7 @@ export class PersistedBatchV3 extends PersistedBatch {
   }
 
   protected async flushBucketStates(session: mongo.ClientSession) {
-    await this.db.bucketStateV3(this.group_id).bulkWrite(this.getBucketStateUpdates(), {
+    await this.db.bucketState(this.group_id).bulkWrite(this.getBucketStateUpdates(), {
       session,
       ordered: false
     });

package/src/storage/implementation/v3/SingleBucketStoreV3.ts

@@ -6,9 +6,38 @@ import {
   BucketDataDocumentGenericId,
   SingleBucketStore
 } from '../common/SingleBucketStore.js';
-import { BucketDataProperties } from '../models.js';
+import { BucketDataKey, BucketDataProperties } from '../models.js';
+import { loadBucketDataDocument, serializeBucketData } from './bucket-format.js';
+import { BucketDataDocumentV3 } from './models.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
-import { BucketDataDocumentV3, BucketDataKeyV3, loadBucketDataDocumentV3, serializeBucketDataV3 } from './models.js';
+
+function extractPartialDocumentFields(doc: unknown): { _id: { o: bigint }; [key: string]: unknown } {
+  if (typeof doc !== 'object' || doc === null) {
+    throw new Error('Invalid partial document: expected object');
+  }
+  const d = doc as Record<string, unknown>;
+  const id = d._id;
+  if (typeof id !== 'object' || id === null || !('o' in id)) {
+    throw new Error('Invalid partial document: missing _id.o');
+  }
+  return d as { _id: { o: bigint }; [key: string]: unknown };
+}
+
+// MongoDB's MinKey/MaxKey are special sentinel values that don't match the bigint type
+// for _id.o in BucketDataDocumentGenericId, so we need an explicit cast.
+function minKeyForBucket(bucket: string): BucketDataDocumentGenericId {
+  return {
+    b: bucket,
+    o: new mongo.MinKey()
+  } as unknown as BucketDataDocumentGenericId;
+}
+
+function maxKeyForBucket(bucket: string): BucketDataDocumentGenericId {
+  return {
+    b: bucket,
+    o: new mongo.MaxKey()
+  } as unknown as BucketDataDocumentGenericId;
+}
 
 export class SingleBucketStoreV3 implements SingleBucketStore {
   public readonly collection: mongo.Collection<BucketDataDocumentGeneric>;
@@ -17,52 +46,63 @@ export class SingleBucketStoreV3 implements SingleBucketStore {
     private db: VersionedPowerSyncMongoV3,
     public readonly key: BucketKey
   ) {
-    this.collection = db.bucketDataV3(
+    // Cast from the version-specific collection type to the generic interface
+    // used across storage versions.
+    this.collection = db.bucketData(
       key.replicationStreamId,
       key.definitionId
     ) as unknown as mongo.Collection<BucketDataDocumentGeneric>;
   }
 
   docId(o: InternalOpId): BucketDataDocumentGenericId {
-    // `satisfies BucketDataKeyV3` checks that we use the correct type for V3 storage
+    // `satisfies BucketDataKey` checks that we use the correct type for V3 storage
     // `as BucketDataDocumentGenericId` does a cast to get the interface virtual type
     return {
       b: this.key.bucket,
       o
-    } satisfies BucketDataKeyV3 as BucketDataDocumentGenericId;
+    } satisfies BucketDataKey as BucketDataDocumentGenericId;
   }
 
   get minId(): BucketDataDocumentGenericId {
-    return {
-      b: this.key.bucket,
-      o: new mongo.MinKey()
-    } as any; // No way to properly type this
+    return minKeyForBucket(this.key.bucket);
   }
 
   get maxId(): BucketDataDocumentGenericId {
-    return {
-      b: this.key.bucket,
-      o: new mongo.MaxKey()
-    } as any; // No way to properly type this
+    return maxKeyForBucket(this.key.bucket);
   }
 
   toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentGeneric {
-    return serializeBucketDataV3({ bucketKey: this.key, ...source }) as BucketDataDocumentGeneric;
+    return serializeBucketData(this.key.bucket, [
+      { bucketKey: this.key, ...source }
+    ]) as unknown as BucketDataDocumentGeneric;
   }
 
   fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc {
-    return loadBucketDataDocumentV3(this.key, doc as BucketDataDocumentV3);
+    const generator = loadBucketDataDocument(this.key, doc as unknown as BucketDataDocumentV3);
+    const first = generator.next();
+    if (first.done) {
+      throw new Error('Empty ops array in BucketDataDocumentV3');
+    }
+    return first.value;
   }
 
   fromPartialPersistedDocument<T extends keyof BucketDataProperties>(
     doc: Pick<BucketDataDocumentGeneric, '_id' | T>
   ): Pick<BucketDataDoc, 'bucketKey' | 'o' | T> {
-    const document = doc as Pick<BucketDataDocumentV3, '_id' | T>;
-    const { _id, ...rest } = document;
+    const document = doc as unknown as Pick<BucketDataDocumentV3, '_id' | 'ops'>;
+    const op = document.ops?.[0];
+    if (op == null) {
+      const fields = extractPartialDocumentFields(doc);
+      const { _id, ...rest } = fields;
+      return {
+        bucketKey: this.key,
+        o: _id.o,
+        ...rest
+      } as Pick<BucketDataDoc, 'bucketKey' | 'o' | T>;
+    }
     return {
       bucketKey: this.key,
-      o: _id.o,
-      ...rest
+      ...op
     } as Pick<BucketDataDoc, 'bucketKey' | 'o' | T>;
   }
 }

package/src/storage/implementation/v3/SourceRecordStoreV3.ts

@@ -1,21 +1,19 @@
 import * as lib_mongo from '@powersync/lib-service-mongodb';
 import { mongo } from '@powersync/lib-service-mongodb';
 import { Logger } from '@powersync/lib-services-framework';
-import { storage } from '@powersync/service-core';
+import { BucketDefinitionMapping, storage } from '@powersync/service-core';
 import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
 import * as bson from 'bson';
 import { retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js';
-import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js';
 import { cacheKey } from '../OperationBatch.js';
 import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from '../common/SourceRecordStore.js';
-import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js';
 import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
-import { CurrentDataDocumentV3, SourceTableDocumentV3 } from './models.js';
+import { CurrentDataDocumentV3, serializeParameterLookup, SourceTableDocumentV3 } from './models.js';
 
 export class SourceRecordStoreV3 implements SourceRecordStore {
   constructor(
     private readonly db: VersionedPowerSyncMongoV3,
-    private readonly groupId: number,
+    private readonly replicationStreamId: number,
     private readonly mapping: BucketDefinitionMapping
   ) {}
 
@@ -31,7 +29,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
   mapParameterLookups(paramEvaluated: EvaluatedParameters[]): LoadedSourceRecord['lookups'] {
     return paramEvaluated.map((entry) => ({
       indexId: this.mapping.parameterLookupId(entry.lookup.source),
-      lookup: serializeParameterLookupV3(entry.lookup)
+      lookup: serializeParameterLookup(entry.lookup)
     }));
   }
 
@@ -67,7 +65,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
         _id: { $in: replicaIds as any[] }
       } as unknown as mongo.Filter<CurrentDataDocumentV3>;
       const sizeCursor: mongo.AggregationCursor<CurrentDataDocumentV3 & { size: number }> = this.db
-        .sourceRecordsV3(this.groupId, sourceTableId)
+        .sourceRecords(this.replicationStreamId, sourceTableId)
         .aggregate(
           [
             {
@@ -100,7 +98,9 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
       const filter = {
         _id: { $in: replicaIds as any[] }
       } as unknown as mongo.Filter<CurrentDataDocumentV3>;
-      const cursor = this.db.sourceRecordsV3(this.groupId, sourceTableId).find(filter, { session, projection });
+      const cursor = this.db
+        .sourceRecords(this.replicationStreamId, sourceTableId)
+        .find(filter, { session, projection });
       for await (const doc of cursor.stream()) {
         const loaded = this.createLoadedDocument(
           sourceTableId,
@@ -120,7 +120,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
     sourceTableId: bson.ObjectId,
     limit: number
   ): Promise<LoadedSourceRecord[]> {
-    const cursor = this.db.sourceRecordsV3(this.groupId, sourceTableId).find(
+    const cursor = this.db.sourceRecords(this.replicationStreamId, sourceTableId).find(
       {
         pending_delete: { $exists: false }
       },
@@ -145,7 +145,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
     // ones have dirty deletes in source_tables.
 
     const dirtySourceTables = await this.db
-      .sourceTablesV3(this.groupId)
+      .sourceTables(this.replicationStreamId)
       .find(
         {
           latest_pending_delete: { $exists: true }
@@ -159,7 +159,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
     let deletedCount = 0;
     const sourceTableUpdates: mongo.AnyBulkWriteOperation<SourceTableDocumentV3>[] = [];
     for (const sourceTable of dirtySourceTables) {
-      const collection = this.db.sourceRecordsV3(this.groupId, sourceTable._id);
+      const collection = this.db.sourceRecords(this.replicationStreamId, sourceTable._id);
       const result = await this.deletePendingDeletes(collection, sourceTable._id, lastCheckpoint, logger);
       deletedCount += result.deletedCount;
 
@@ -182,7 +182,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
     }
 
     if (sourceTableUpdates.length > 0) {
-      await this.db.sourceTablesV3(this.groupId).bulkWrite(sourceTableUpdates, { ordered: false });
+      await this.db.sourceTables(this.replicationStreamId).bulkWrite(sourceTableUpdates, { ordered: false });
     }
     if (deletedCount > 0) {
       logger.info(`Cleaned up ${deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}`);

package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts

@@ -12,53 +12,80 @@ import {
 } from './models.js';
 
 export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
+  constructor(
+    upstream: ConstructorParameters<typeof BaseVersionedPowerSyncMongo>[0],
+    storageConfig: ConstructorParameters<typeof BaseVersionedPowerSyncMongo>[1]
+  ) {
+    super(upstream, storageConfig);
+  }
+
   get syncConfigDefinitions() {
     return this.db.collection<SyncConfigDefinition>('sync_config');
   }
 
-  sourceRecordsV3(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection<CurrentDataDocumentV3> {
+  sourceRecords(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection<CurrentDataDocumentV3> {
     const collectionName = this.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
     return this.db.collection<CurrentDataDocumentV3>(collectionName);
   }
 
-  async listSourceRecordCollectionsV3(replicationStreamId: number): Promise<mongo.Collection<CurrentDataDocumentV3>[]> {
+  async listSourceRecordCollections(replicationStreamId: number): Promise<mongo.Collection<CurrentDataDocumentV3>[]> {
     return this.listCollectionsByPrefix<CurrentDataDocumentV3>(`source_records_${replicationStreamId}_`);
   }
 
-  async initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
-    await this.sourceRecordsV3(replicationStreamId, sourceTableId).createIndex(
+  async initializeSourceRecordsCollection(
+    replicationStreamId: number,
+    sourceTableId: mongo.ObjectId,
+    session?: mongo.ClientSession
+  ) {
+    await this.sourceRecords(replicationStreamId, sourceTableId).createIndex(
       {
         pending_delete: 1
       },
       {
         partialFilterExpression: { pending_delete: { $exists: true } },
-        name: 'pending_delete'
+        name: 'pending_delete',
+        session
       }
     );
   }
 
   commonSourceTables(replicationStreamId: number): mongo.Collection<CommonSourceTableDocument> {
-    return this.sourceTablesV3(replicationStreamId) as mongo.Collection<CommonSourceTableDocument>;
+    return this.sourceTables(replicationStreamId) as any as mongo.Collection<CommonSourceTableDocument>;
   }
 
-  bucketStateV3(replicationStreamId: number): mongo.Collection<BucketStateDocumentV3> {
+  bucketState(replicationStreamId: number): mongo.Collection<BucketStateDocumentV3> {
     return this.db.collection(`bucket_state_${replicationStreamId}`);
   }
 
-  parameterIndexV3(
-    replicationStreamId: number,
-    indexId: ParameterIndexId
-  ): mongo.Collection<BucketParameterDocumentV3> {
+  parameterIndex(replicationStreamId: number, indexId: ParameterIndexId): mongo.Collection<BucketParameterDocumentV3> {
     return this.db.collection(`parameter_index_${replicationStreamId}_${indexId}`);
   }
 
-  sourceTablesV3(replicationStreamId: number): mongo.Collection<SourceTableDocumentV3> {
+  sourceTables(replicationStreamId: number): mongo.Collection<SourceTableDocumentV3> {
     return this.db.collection<SourceTableDocumentV3>(this.sourceTableCollectionName(replicationStreamId));
   }
 
+  bucketData(replicationStreamId: number, definitionId: BucketDefinitionId): mongo.Collection<BucketDataDocumentV3> {
+    return this.db.collection<BucketDataDocumentV3>(`bucket_data_${replicationStreamId}_${definitionId}`);
+  }
+
+  listBucketDataCollections(replicationStreamId: number) {
+    return this.listCollectionsByPrefix(`bucket_data_${replicationStreamId}_`);
+  }
+
+  async listParameterIndexCollections(replicationStreamId: number) {
+    const prefix = `parameter_index_${replicationStreamId}_`;
+    const collections = await this.listCollectionsByPrefix(prefix);
+
+    return collections.map((collection) => ({
+      collection,
+      indexId: collection.collectionName.slice(prefix.length)
+    }));
+  }
+
   async initializeStreamStorage(replicationStreamId: number) {
-    const sourceTables = this.sourceTablesV3(replicationStreamId);
-    const bucketState = this.bucketStateV3(replicationStreamId);
+    const sourceTables = this.sourceTables(replicationStreamId);
+    const bucketState = this.bucketState(replicationStreamId);
     await sourceTables.createIndex(
       {
         connection_id: 1,
@@ -92,26 +119,4 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
       { name: 'dirty_count' }
     );
   }
-
-  bucketDataV3(replicationStreamId: number, definitionId: BucketDefinitionId) {
-    return this.db.collection<BucketDataDocumentV3>(`bucket_data_${replicationStreamId}_${definitionId}`);
-  }
-
-  listBucketDataCollectionsV3(replicationStreamId: number) {
-    return this.upstream.listBucketDataCollectionsV3(replicationStreamId);
-  }
-
-  async listParameterIndexCollectionsV3(
-    replicationStreamId: number
-  ): Promise<{ collection: mongo.Collection<BucketParameterDocumentV3>; indexId: ParameterIndexId }[]> {
-    const prefix = `parameter_index_${replicationStreamId}_`;
-    const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
-
-    return collections
-      .filter((collection) => collection.name.startsWith(prefix))
-      .map((collection) => ({
-        collection: this.db.collection<BucketParameterDocumentV3>(collection.name),
-        indexId: collection.name.slice(prefix.length)
-      }));
-  }
 }

package/src/storage/implementation/v3/bucket-format.ts

@@ -0,0 +1,70 @@
+import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
+import { BucketDataDocumentV3, BucketOperation } from './models.js';
+
+export function serializeBucketData(bucket: string, operations: BucketDataDoc[]): BucketDataDocumentV3 {
+  const minOp = operations[0].o;
+  const maxOp = operations[operations.length - 1].o;
+
+  let totalChecksum = 0n;
+  let totalSize = 0;
+  let maxTargetOp: bigint | null = null;
+
+  const ops: BucketOperation[] = operations.map((op) => {
+    totalChecksum += op.checksum;
+    totalSize += op.data?.length ?? 0;
+
+    if (op.target_op != null && (maxTargetOp == null || op.target_op > maxTargetOp)) {
+      maxTargetOp = op.target_op;
+    }
+
+    return {
+      o: op.o,
+      op: op.op,
+      source_table: op.source_table,
+      source_key: op.source_key,
+      table: op.table,
+      row_id: op.row_id,
+      checksum: op.checksum,
+      data: op.data
+    };
+  });
+
+  return {
+    _id: {
+      b: bucket,
+      o: maxOp
+    },
+    min_op: minOp,
+    checksum: totalChecksum,
+    count: operations.length,
+    size: totalSize,
+    target_op: maxTargetOp,
+    ops
+  };
+}
+
+export function* loadBucketDataDocument(
+  context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>,
+  doc: BucketDataDocumentV3
+): Generator<BucketDataDoc> {
+  const { _id, ops } = doc;
+  const bucketKey = {
+    ...context,
+    bucket: _id.b
+  };
+
+  for (const op of ops) {
+    yield {
+      bucketKey,
+      o: op.o,
+      op: op.op,
+      source_table: op.source_table,
+      source_key: op.source_key,
+      table: op.table,
+      row_id: op.row_id,
+      checksum: op.checksum,
+      data: op.data,
+      target_op: doc.target_op ?? null
+    };
+  }
+}

package/src/storage/implementation/v3/chunking.ts

@@ -0,0 +1,38 @@
+import { BucketDataDoc } from '../common/BucketDataDoc.js';
+
+export const DEFAULT_MAX_DOC_SIZE_BYTES = 1024 * 1024; // 1MB
+
+/**
+ * Split an array of bucket data operations into chunks where each chunk's
+ * total data size stays close to `maxDocSizeBytes`.
+ *
+ * The threshold is a target, not a hard ceiling — a single operation whose
+ * `data` field exceeds the limit is placed in its own chunk and is NOT split.
+ */
+export function chunkBucketData(
+  operations: BucketDataDoc[],
+  maxDocSizeBytes = DEFAULT_MAX_DOC_SIZE_BYTES
+): BucketDataDoc[][] {
+  const chunks: BucketDataDoc[][] = [];
+  let currentChunk: BucketDataDoc[] = [];
+  let currentSize = 0;
+
+  for (const op of operations) {
+    const opSize = op.data?.length ?? 0;
+
+    if (currentSize + opSize > maxDocSizeBytes && currentChunk.length > 0) {
+      chunks.push(currentChunk);
+      currentChunk = [];
+      currentSize = 0;
+    }
+
+    currentChunk.push(op);
+    currentSize += opSize;
+  }
+
+  if (currentChunk.length > 0) {
+    chunks.push(currentChunk);
+  }
+
+  return chunks;
+}