@powersync/service-module-mongodb-storage 0.16.0 → 0.18.0

package/dist/storage/implementation/MongoBucketBatch.js

@@ -1,7 +1,7 @@
 import { mongo } from '@powersync/lib-service-mongodb';
 import * as bson from 'bson';
 import { BaseObserver, container, ErrorCode, errors, ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework';
-import { deserializeBson, isCompleteRow, PerformanceTracer, SaveOperationTag, storage, SyncRuleState, utils } from '@powersync/service-core';
+import { deserializeBson, isCompleteRow, PerformanceTracer, SaveOperationTag, storage, utils } from '@powersync/service-core';
 import * as timers from 'node:timers/promises';
 import { mongoTableId } from '../../utils/util.js';
 import { MAX_ROW_SIZE } from './MongoBucketBatchShared.js';
@@ -15,38 +15,40 @@ import { OperationBatch, RecordOperation } from './OperationBatch.js';
 // In the future, we can investigate allowing multiple replication streams operating independently.
 const replicationMutex = new utils.Mutex();
 export class MongoBucketBatch extends BaseObserver {
+    options;
     logger;
     client;
     db;
     session;
     sync_rules;
-    group_id;
-    slot_name;
+    replicationStreamId;
+    replicationStreamName;
+    /**
+     * Source-level setting for whether raw row data should be stored in current_data.
+     *
+     * Some sources always send complete rows (MongoDB, MySQL with binlog_row_image=full),
+     * in which case this is false for the whole batch. For sources where it depends on the
+     * table (Postgres REPLICA IDENTITY), this is true and the decision is refined per-table
+     * via SourceTable.storeCurrentData. The effective per-record value is the conjunction of
+     * the two.
+     */
     storeCurrentData;
     skipExistingRows;
     mapping;
     batch = null;
     write_checkpoint_batch = [];
     markRecordUnavailable;
+    hooks;
     clearedError = false;
     tracer;
-    /**
-     * Last LSN received associated with a checkpoint.
-     *
-     * This could be either:
-     * 1. A commit LSN.
-     * 2. A keepalive message LSN.
-     */
-    last_checkpoint_lsn = null;
-    persisted_op = null;
     /**
      * Last written op, if any. This may not reflect a consistent checkpoint.
      */
     last_flushed_op = null;
     /**
-     * lastCheckpointLsn is the last consistent commit.
+     * LSN to resume replication from.
      *
-     * While that is generally a "safe" point to resume from, there are cases where we may want to resume from a different point:
+     * This is typically the last commit LSN, but there are cases where it differs:
      * 1. After an initial snapshot, we don't have a consistent commit yet, but need to resume from the snapshot LSN.
      * 2. If "no_checkpoint_before_lsn" is set far in advance, it may take a while to reach that point. We
      *    may want to resume at incremental points before that.
@@ -54,35 +56,31 @@ export class MongoBucketBatch extends BaseObserver {
      * This is set when creating the batch, but may not be updated afterwards.
      */
     resumeFromLsn = null;
-    needsActivation = true;
     constructor(options) {
         super();
         this.logger = options.logger;
+        this.options = options;
         this.client = options.db.client;
         this.db = options.db;
-        this.group_id = options.groupId;
-        this.last_checkpoint_lsn = options.lastCheckpointLsn;
+        this.replicationStreamId = options.replicationStreamId;
         this.resumeFromLsn = options.resumeFromLsn;
         this.session = this.client.startSession();
-        this.slot_name = options.slotName;
-        this.sync_rules = options.syncRules;
+        this.replicationStreamName = options.replicationStreamName;
+        this.sync_rules = options.parsedSyncConfig.hydratedSyncConfig;
         this.storeCurrentData = options.storeCurrentData;
-        this.mapping = options.mapping;
+        this.mapping = options.parsedSyncConfig.mapping;
         this.skipExistingRows = options.skipExistingRows;
         this.markRecordUnavailable = options.markRecordUnavailable;
+        this.hooks = options.hooks;
         this.batch = new OperationBatch();
-        this.persisted_op = options.keepaliveOp ?? null;
         this.tracer = options.tracer ?? new PerformanceTracer('MongoDB storage');
     }
     addCustomWriteCheckpoint(checkpoint) {
         this.write_checkpoint_batch.push({
             ...checkpoint,
-            sync_rules_id: this.group_id
+            sync_rules_id: this.replicationStreamId
         });
     }
-    get lastCheckpointLsn() {
-        return this.last_checkpoint_lsn;
-    }
     async flush(options) {
         let result = null;
         // One flush may be split over multiple transactions.
@@ -100,6 +98,7 @@ export class MongoBucketBatch extends BaseObserver {
         let last_op = null;
         let resumeBatch = null;
         using _ = this.tracer.span('storage', 'flush');
+        await this.hooks?.beforeBatchFlush?.(this);
         await this.withReplicationTransaction(`Flushing ${batch?.length ?? 0} ops`, async (session, opSeq) => {
             if (batch != null) {
                 resumeBatch = await this.replicateBatch(session, batch, opSeq, options);
@@ -116,15 +115,19 @@ export class MongoBucketBatch extends BaseObserver {
         if (last_op == null) {
             throw new ReplicationAssertionError('Unexpected last_op == null');
         }
-        this.persisted_op = last_op;
+        this.recordPersistedOp(last_op);
         this.last_flushed_op = last_op;
+        await this.hooks?.afterBatchFlush?.(this);
         return { flushed_op: last_op };
     }
     async replicateBatch(session, batch, op_seq, options) {
         let sizes = undefined;
         using _ = this.tracer.span('storage', 'replicate_batch');
-        if (this.storeCurrentData && !this.skipExistingRows) {
-            // We skip this step if we don't store current_data, since the sizes will
+        // Only look up current_data sizes if the batch stores current_data and at least one
+        // table in it does too (per-table can disable it, e.g. Postgres REPLICA IDENTITY FULL).
+        const anyTableStoresCurrentData = this.storeCurrentData && batch.batch.some((r) => r.record.sourceTable.storeCurrentData);
+        if (anyTableStoresCurrentData && !this.skipExistingRows) {
+            // We skip this step if no tables store current_data, since the sizes will
             // always be small in that case.
             // With skipExistingRows, we don't load the full documents into memory,
             // so we can also skip the size lookup step.
@@ -135,7 +138,11 @@ export class MongoBucketBatch extends BaseObserver {
             // (automatically limited to 48MB(?) per batch by MongoDB). The issue is that it changes
             // the order of processing, which then becomes really tricky to manage.
             // This now takes 2+ queries, but doesn't have any issues with order of operations.
-            const sizeLookups = batch.batch.map((r) => ({
+            // Within this branch this.storeCurrentData is true, so the per-table flag is the
+            // effective value - only look up sizes for tables that actually store current_data.
+            const sizeLookups = batch.batch
+                .filter((r) => r.record.sourceTable.storeCurrentData)
+                .map((r) => ({
                 sourceTableId: mongoTableId(r.record.sourceTable.id),
                 replicaId: r.beforeId
             }));
@@ -217,6 +224,9 @@ export class MongoBucketBatch extends BaseObserver {
         const afterId = operation.afterId;
         let after = record.after;
         const sourceTable = record.sourceTable;
+        // Effective per-record flag: store current_data only if both the batch (source-level,
+        // e.g. Postgres) and the table (e.g. non-FULL replica identity) require it.
+        const storeCurrentData = this.storeCurrentData && sourceTable.storeCurrentData;
         let existing_buckets = [];
         let new_buckets = [];
         let existing_lookups = [];
@@ -242,7 +252,7 @@ export class MongoBucketBatch extends BaseObserver {
                 // Not an error if we re-apply a transaction
                 existing_buckets = [];
                 existing_lookups = [];
-                if (!isCompleteRow(this.storeCurrentData, after)) {
+                if (!isCompleteRow(storeCurrentData, after)) {
                     if (this.markRecordUnavailable != null) {
                         // This will trigger a "resnapshot" of the record.
                         // This is not relevant if storeCurrentData is false, since we'll get the full row
@@ -258,7 +268,7 @@ export class MongoBucketBatch extends BaseObserver {
             else {
                 existing_buckets = result.buckets;
                 existing_lookups = result.lookups;
-                if (this.storeCurrentData && result.data != null) {
+                if (storeCurrentData && result.data != null) {
                     const data = deserializeBson(result.data.buffer);
                     after = storage.mergeToast(after, data);
                 }
@@ -270,7 +280,9 @@ export class MongoBucketBatch extends BaseObserver {
                 // Not an error if we re-apply a transaction
                 existing_buckets = [];
                 existing_lookups = [];
-                // Log to help with debugging if there was a consistency issue
+                // Log to help with debugging if there was a consistency issue.
+                // Gate on the batch-level flag: FULL tables (per-record flag false) still get a
+                // current_data entry, so a missing record on DELETE is meaningful for them too.
                 if (this.storeCurrentData && this.markRecordUnavailable == null) {
                     this.logger.warn(`Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`);
                 }
@@ -281,7 +293,7 @@ export class MongoBucketBatch extends BaseObserver {
             }
         }
         let afterData = null;
-        if (afterId != null && !this.storeCurrentData) {
+        if (afterId != null && !storeCurrentData) {
             afterData = null;
         }
         else if (afterId != null) {
@@ -302,7 +314,7 @@ export class MongoBucketBatch extends BaseObserver {
                 container.reporter.captureMessage(`Data too big on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${e.message}`, {
                     level: errors.ErrorSeverity.WARNING,
                     metadata: {
-                        replication_slot: this.slot_name,
+                        replication_slot: this.replicationStreamName,
                         table: record.sourceTable.qualifiedName
                     }
                 });
@@ -341,18 +353,20 @@ export class MongoBucketBatch extends BaseObserver {
         // However, it will be valid by the end of the transaction.
         //
         // In this case, we don't save the op, but we do save the current data.
-        if (afterId && after && utils.isCompleteRow(this.storeCurrentData, after)) {
+        if (afterId && after && utils.isCompleteRow(storeCurrentData, after)) {
             // Insert or update
             if (sourceTable.syncData) {
-                const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({
+                const { results, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({
                     record: after,
-                    sourceTable
+                    sourceTable: sourceTable.ref,
+                    bucketDataSources: sourceTable.bucketDataSources
                 });
+                const evaluated = results;
                 for (let error of syncErrors) {
                     container.reporter.captureMessage(`Failed to evaluate data query on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.error}`, {
                         level: errors.ErrorSeverity.WARNING,
                         metadata: {
-                            replication_slot: this.slot_name,
+                            replication_slot: this.replicationStreamName,
                             table: record.sourceTable.qualifiedName
                         }
                     });
@@ -370,12 +384,12 @@ export class MongoBucketBatch extends BaseObserver {
             }
             if (sourceTable.syncParameters) {
                 // Parameters
-                const { results: paramEvaluated, errors: paramErrors } = this.sync_rules.evaluateParameterRowWithErrors(sourceTable, after);
+                const { results: paramEvaluated, errors: paramErrors } = this.sync_rules.evaluateParameterRowWithErrors(sourceTable.ref, after, { parameterLookupSources: sourceTable.parameterLookupSources });
                 for (let error of paramErrors) {
                     container.reporter.captureMessage(`Failed to evaluate parameter query on ${record.sourceTable.qualifiedName}.${record.after?.id}: ${error.error}`, {
                         level: errors.ErrorSeverity.WARNING,
                         metadata: {
-                            replication_slot: this.slot_name,
+                            replication_slot: this.replicationStreamName,
                             table: record.sourceTable.qualifiedName
                         }
                     });
@@ -481,17 +495,43 @@ export class MongoBucketBatch extends BaseObserver {
                 session
             });
             await this.db.sync_rules.updateOne({
-                _id: this.group_id
+                _id: this.replicationStreamId
             }, {
                 $set: {
                     last_keepalive_ts: new Date()
                 }
             }, { session });
+            // Allow subclasses to persist additional flush-time state in the same transaction
+            // (e.g. v3 advances the stream-level last_persisted_op).
+            await this.onReplicationTransactionFlush(session, opSeq.last());
             // We don't notify checkpoint here - we don't make any checkpoint updates directly
         });
     }
+    /**
+     * Hook called inside the replication flush transaction, after ops have been persisted.
+     *
+     * The base implementation does nothing; v3 storage overrides this to `$max` the stream-level
+     * `last_persisted_op` durably within the same transaction.
+     */
+    async onReplicationTransactionFlush(_session, _lastOp) {
+        // No-op by default (v1 behaviour unchanged).
+    }
+    /**
+     * Called after a replication transaction has successfully committed, with the last persisted op id.
+     *
+     * v1 storage tracks this in memory to fold into the next checkpoint. v3 storage does not need it:
+     * the stream-level `last_persisted_op` is already advanced durably by
+     * {@link onReplicationTransactionFlush} within the same transaction, and checkpoints read it
+     * from the document.
+     *
+     * Keep calls to this adjacent to {@link withReplicationTransaction} usage - both must observe
+     * every path that persists ops.
+     */
+    recordPersistedOp(_lastOp) {
+        // No-op by default.
+    }
     async [Symbol.asyncDispose]() {
-        if (this.batch != null || this.write_checkpoint_batch.length > 0) {
+        if (this.batch?.hasData() || this.write_checkpoint_batch.length > 0) {
             // We don't error here, since:
             // 1. In error states, this is expected (we can't distinguish between disposing after success or error).
             // 2. SuppressedError is messy to deal with.
@@ -503,221 +543,25 @@ export class MongoBucketBatch extends BaseObserver {
     async dispose() {
         await this[Symbol.asyncDispose]();
     }
-    lastWaitingLogThottled = 0;
-    async commit(lsn, options) {
-        const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
-        await this.flush(options);
-        const now = new Date();
-        // Mark relevant write checkpoints as "processed".
-        // This makes it easier to identify write checkpoints that are "valid" in order.
-        await this.db.write_checkpoints.updateMany({
-            processed_at_lsn: null,
-            'lsns.1': { $lte: lsn }
-        }, {
-            $set: {
-                processed_at_lsn: lsn
-            }
-        }, {
-            session: this.session
-        });
-        const can_checkpoint = {
-            $and: [
-                { $eq: ['$snapshot_done', true] },
-                {
-                    $or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
-                },
-                {
-                    $or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
-                }
-            ]
-        };
-        const new_keepalive_op = {
-            $cond: [
-                can_checkpoint,
-                { $literal: null },
-                {
-                    $toString: {
-                        $max: [{ $toLong: '$keepalive_op' }, { $literal: this.persisted_op }, 0n]
-                    }
-                }
-            ]
-        };
-        const new_last_checkpoint = {
-            $cond: [
-                can_checkpoint,
-                {
-                    $max: ['$last_checkpoint', { $literal: this.persisted_op }, { $toLong: '$keepalive_op' }, 0n]
-                },
-                '$last_checkpoint'
-            ]
-        };
-        // For this query, we need to handle multiple cases, depending on the state:
-        // 1. Normal commit - advance last_checkpoint to this.persisted_op.
-        // 2. Commit delayed by no_checkpoint_before due to snapshot. In this case we only advance keepalive_op.
-        // 3. Commit with no new data - here may may set last_checkpoint = keepalive_op, if a delayed commit is relevant.
-        // We want to do as much as possible in a single atomic database operation, which makes this somewhat complex.
-        let preUpdateDocument = await this.db.sync_rules.findOneAndUpdate({ _id: this.group_id }, [
-            {
-                $set: {
-                    _can_checkpoint: can_checkpoint,
-                    _not_empty: createEmptyCheckpoints
-                        ? true
-                        : {
-                            $or: [
-                                { $literal: createEmptyCheckpoints },
-                                { $ne: ['$keepalive_op', new_keepalive_op] },
-                                { $ne: ['$last_checkpoint', new_last_checkpoint] }
-                            ]
-                        }
-                }
-            },
-            {
-                $set: {
-                    last_checkpoint_lsn: {
-                        $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
-                    },
-                    last_checkpoint_ts: {
-                        $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
-                    },
-                    last_keepalive_ts: { $literal: now },
-                    last_fatal_error: { $literal: null },
-                    last_fatal_error_ts: { $literal: null },
-                    keepalive_op: new_keepalive_op,
-                    last_checkpoint: new_last_checkpoint,
-                    // Unset snapshot_lsn on checkpoint
-                    snapshot_lsn: {
-                        $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
-                    }
-                }
-            },
-            {
-                $unset: ['_can_checkpoint', '_not_empty']
-            }
-        ], {
-            session: this.session,
-            // We return the before document, so that we can check the previous state to determine if a checkpoint was actually created or if we were blocked by snapshot/no_checkpoint_before.
-            returnDocument: 'before',
-            projection: {
-                snapshot_done: 1,
-                last_checkpoint_lsn: 1,
-                no_checkpoint_before: 1,
-                keepalive_op: 1,
-                last_checkpoint: 1
-            }
-        });
-        if (preUpdateDocument == null) {
-            throw new ReplicationAssertionError('Failed to update checkpoint - no matching sync_rules document for _id: ' + this.group_id);
-        }
-        // This re-implements the same logic as in the pipeline, to determine what was actually updated.
-        // Unfortunately we cannot return these from the pipeline directly, so we need to re-implement the logic.
-        const canCheckpoint = preUpdateDocument.snapshot_done === true &&
-            (preUpdateDocument.last_checkpoint_lsn == null || preUpdateDocument.last_checkpoint_lsn <= lsn) &&
-            (preUpdateDocument.no_checkpoint_before == null || preUpdateDocument.no_checkpoint_before <= lsn);
-        const keepaliveOp = preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op);
-        const maxKeepalive = [keepaliveOp ?? 0n, this.persisted_op ?? 0n, 0n].reduce((a, b) => (a > b ? a : b));
-        const newKeepaliveOp = canCheckpoint ? null : maxKeepalive.toString();
-        const newLastCheckpoint = canCheckpoint
-            ? [preUpdateDocument.last_checkpoint ?? 0n, this.persisted_op ?? 0n, keepaliveOp ?? 0n, 0n].reduce((a, b) => a > b ? a : b)
-            : preUpdateDocument.last_checkpoint;
-        const notEmpty = createEmptyCheckpoints ||
-            preUpdateDocument.keepalive_op !== newKeepaliveOp ||
-            preUpdateDocument.last_checkpoint !== newLastCheckpoint;
-        const checkpointCreated = canCheckpoint && notEmpty;
-        const checkpointBlocked = !canCheckpoint;
-        if (checkpointBlocked) {
-            // Failed on snapshot_done or no_checkpoint_before.
-            if (Date.now() - this.lastWaitingLogThottled > 5_000) {
-                this.logger.info(`Waiting before creating checkpoint, currently at ${lsn} / ${preUpdateDocument.keepalive_op}. Current state: ${JSON.stringify({
-                    snapshot_done: preUpdateDocument.snapshot_done,
-                    last_checkpoint_lsn: preUpdateDocument.last_checkpoint_lsn,
-                    no_checkpoint_before: preUpdateDocument.no_checkpoint_before
-                })}`);
-                this.lastWaitingLogThottled = Date.now();
-            }
-        }
-        else {
-            if (checkpointCreated) {
-                this.logger.debug(`Created checkpoint at ${lsn} / ${newLastCheckpoint}`);
-            }
-            await this.autoActivate(lsn);
-            await this.db.notifyCheckpoint();
-            this.persisted_op = null;
-            this.last_checkpoint_lsn = lsn;
-            if (newLastCheckpoint != null) {
-                await this.sourceRecordStore.postCommitCleanup(newLastCheckpoint, this.logger);
-            }
-        }
-        return { checkpointBlocked, checkpointCreated };
-    }
-    /**
-     * Switch from processing -> active if relevant.
-     *
-     * Called on new commits.
-     */
-    async autoActivate(lsn) {
-        if (!this.needsActivation) {
-            return;
-        }
-        // Activate the batch, so it can start processing.
-        // This is done automatically when the first save() is called.
-        const session = this.session;
-        let activated = false;
-        await session.withTransaction(async () => {
-            const doc = await this.db.sync_rules.findOne({ _id: this.group_id }, { session });
-            if (doc && doc.state == SyncRuleState.PROCESSING && doc.snapshot_done && doc.last_checkpoint != null) {
-                await this.db.sync_rules.updateOne({
-                    _id: this.group_id
-                }, {
-                    $set: {
-                        state: storage.SyncRuleState.ACTIVE
-                    }
-                }, { session });
-                await this.db.sync_rules.updateMany({
-                    _id: { $ne: this.group_id },
-                    state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
-                }, {
-                    $set: {
-                        state: storage.SyncRuleState.STOP
-                    }
-                }, { session });
-                activated = true;
-            }
-            else if (doc?.state != SyncRuleState.PROCESSING) {
-                this.needsActivation = false;
-            }
-        });
-        if (activated) {
-            this.logger.info(`Activated new replication stream at ${lsn}`);
-            await this.db.notifyCheckpoint();
-            this.needsActivation = false;
-        }
-    }
-    async keepalive(lsn) {
-        return await this.commit(lsn, { createEmptyCheckpoints: true });
-    }
-    async setResumeLsn(lsn) {
-        const update = {
-            snapshot_lsn: lsn
-        };
-        await this.db.sync_rules.updateOne({
-            _id: this.group_id
-        }, {
-            $set: update
-        }, { session: this.session });
-    }
     async save(record) {
         const { after, before, sourceTable, tag } = record;
-        for (const event of this.getTableEvents(sourceTable)) {
-            this.iterateListeners((cb) => cb.replicationEvent?.({
-                batch: this,
-                table: sourceTable,
-                data: {
-                    op: tag,
-                    after: after && utils.isCompleteRow(this.storeCurrentData, after) ? after : undefined,
-                    before: before && utils.isCompleteRow(this.storeCurrentData, before) ? before : undefined
-                },
-                event
-            }));
+        const storeCurrentData = this.storeCurrentData && sourceTable.storeCurrentData;
+        // syncEvent is the per-table designation from resolveTables. With v3 storage, multiple
+        // SourceTables can exist for the same ref, with a row change saved once per table -
+        // only the designated event carrier may fire events, so each event fires once per row.
+        if (sourceTable.syncEvent) {
+            for (const event of this.getTableEvents(sourceTable)) {
+                this.iterateListeners((cb) => cb.replicationEvent?.({
+                    batch: this,
+                    table: sourceTable,
+                    data: {
+                        op: tag,
+                        after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined,
+                        before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined
+                    },
+                    event
+                }));
+            }
         }
         /**
          * Return if the table is just an event table
@@ -744,7 +588,7 @@ export class MongoBucketBatch extends BaseObserver {
         const result = await this.flush();
         await this.withTransaction(async () => {
             for (let table of sourceTables) {
-                await this.db.commonSourceTables(this.group_id).deleteOne({ _id: mongoTableId(table.id) });
+                await this.db.commonSourceTables(this.replicationStreamId).deleteOne({ _id: mongoTableId(table.id) });
             }
         });
         await this.cleanupDroppedSourceTables(sourceTables);
@@ -757,7 +601,7 @@ export class MongoBucketBatch extends BaseObserver {
             last_op = await this.truncateSingle(table);
         }
         if (last_op) {
-            this.persisted_op = last_op;
+            this.recordPersistedOp(last_op);
             return {
                 flushed_op: last_op
             };
@@ -814,7 +658,7 @@ export class MongoBucketBatch extends BaseObserver {
         };
         copy.snapshotStatus = snapshotStatus;
         await this.withTransaction(async () => {
-            await this.db.commonSourceTables(this.group_id).updateOne({ _id: mongoTableId(table.id) }, {
+            await this.db.commonSourceTables(this.replicationStreamId).updateOne({ _id: mongoTableId(table.id) }, {
                 $set: {
                     snapshot_status: {
                         last_key: snapshotStatus.lastKey == null ? null : new bson.Binary(snapshotStatus.lastKey),
@@ -826,66 +670,13 @@ export class MongoBucketBatch extends BaseObserver {
         });
         return copy;
     }
-    async markAllSnapshotDone(no_checkpoint_before_lsn) {
-        await this.db.sync_rules.updateOne({
-            _id: this.group_id
-        }, {
-            $set: {
-                snapshot_done: true,
-                last_keepalive_ts: new Date()
-            },
-            $max: {
-                no_checkpoint_before: no_checkpoint_before_lsn
-            }
-        }, { session: this.session });
-    }
-    async markTableSnapshotRequired(table) {
-        await this.db.sync_rules.updateOne({
-            _id: this.group_id
-        }, {
-            $set: {
-                snapshot_done: false
-            }
-        }, { session: this.session });
-    }
-    async markTableSnapshotDone(tables, no_checkpoint_before_lsn) {
-        const session = this.session;
-        const ids = tables.map((table) => mongoTableId(table.id));
-        await this.withTransaction(async () => {
-            await this.db.commonSourceTables(this.group_id).updateMany({ _id: { $in: ids } }, {
-                $set: {
-                    snapshot_done: true
-                },
-                $unset: {
-                    snapshot_status: 1
-                }
-            }, { session });
-            if (no_checkpoint_before_lsn != null) {
-                await this.db.sync_rules.updateOne({
-                    _id: this.group_id
-                }, {
-                    $set: {
-                        last_keepalive_ts: new Date()
-                    },
-                    $max: {
-                        no_checkpoint_before: no_checkpoint_before_lsn
-                    }
-                }, { session: this.session });
-            }
-        });
-        return tables.map((table) => {
-            const copy = table.clone();
-            copy.snapshotComplete = true;
-            return copy;
-        });
-    }
     async clearError() {
         // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
         if (this.clearedError) {
             return;
         }
         await this.db.sync_rules.updateOne({
-            _id: this.group_id
+            _id: this.replicationStreamId
         }, {
             $set: {
                 last_fatal_error: null,
@@ -898,7 +689,7 @@ export class MongoBucketBatch extends BaseObserver {
      * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
      */
     getTableEvents(table) {
-        return this.sync_rules.eventDescriptors.filter((evt) => [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table)));
+        return this.sync_rules.eventDescriptors.filter((evt) => [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref)));
     }
 }
 //# sourceMappingURL=MongoBucketBatch.js.map