@powersync/service-module-postgres-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

package/src/storage/batch/PostgresBucketBatch.ts

@@ -19,6 +19,7 @@ import {
   utils
 } from '@powersync/service-core';
 import * as sync_rules from '@powersync/service-sync-rules';
+import { orNull } from '@powersync/service-types';
 import * as timers from 'timers/promises';
 import * as t from 'ts-codec';
 import * as uuid from 'uuid';
@@ -38,8 +39,8 @@ export interface PostgresBucketBatchOptions {
   logger: Logger;
   db: lib_postgres.DatabaseClient;
   sync_rules: sync_rules.HydratedSyncConfig;
-  group_id: number;
-  slot_name: string;
+  replicationStreamId: number;
+  replicationStreamName: string;
   last_checkpoint_lsn: string | null;
   store_current_data: boolean;
   keep_alive_op?: InternalOpId | null;
@@ -60,15 +61,14 @@ export interface PostgresBucketBatchOptions {
  * via the Postgres NOTIFY protocol.
  */
 const StatefulCheckpoint = models.ActiveCheckpoint.and(t.object({ state: t.Enum(storage.SyncRuleState) }));
-type StatefulCheckpointDecoded = t.Decoded<typeof StatefulCheckpoint>;
 
 const CheckpointWithStatus = StatefulCheckpoint.and(
   t.object({
     snapshot_done: t.boolean,
-    no_checkpoint_before: t.string.or(t.Null),
+    no_checkpoint_before: orNull(t.string),
     can_checkpoint: t.boolean,
-    keepalive_op: bigint.or(t.Null),
-    new_last_checkpoint: bigint.or(t.Null),
+    keepalive_op: orNull(bigint),
+    new_last_checkpoint: orNull(bigint),
     created_checkpoint: t.boolean
   })
 );
@@ -112,7 +112,7 @@ export class PostgresBucketBatch
     super();
     this.logger = options.logger;
     this.db = options.db;
-    this.group_id = options.group_id;
+    this.group_id = options.replicationStreamId;
     this.last_checkpoint_lsn = options.last_checkpoint_lsn;
     this.resumeFromLsn = options.resumeFromLsn;
     this.skipExistingRows = options.skip_existing_rows;
@@ -129,10 +129,6 @@ export class PostgresBucketBatch
     }
   }
 
-  get lastCheckpointLsn() {
-    return this.last_checkpoint_lsn;
-  }
-
   async [Symbol.asyncDispose]() {
     if (this.batch != null || this.write_checkpoint_batch.length > 0) {
       // We don't error here, since:
@@ -148,19 +144,22 @@ export class PostgresBucketBatch
   }
 
   async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
-    const syncRules = options.syncRules ?? this.sync_rules;
+    const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
+    const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
     const { connection_id, source } = options;
-    const { schema, name: table, objectId, replicaIdColumns, connectionTag } = source;
+    const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
 
     const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
       name: column.name,
       type: column.type,
       type_oid: typeof column.typeId !== 'undefined' ? Number(column.typeId) : column.typeId
     }));
+
     return this.db.transaction(async (db) => {
-      let sourceTableRow: SourceTableDecoded | null;
+      // Find records that overlap by name or relation id.
+      let candidateRows: SourceTableDecoded[];
       if (objectId != null) {
-        sourceTableRow = await db.sql`
+        candidateRows = await db.sql`
           SELECT
             *
           FROM
@@ -168,15 +167,18 @@ export class PostgresBucketBatch
           WHERE
             group_id = ${{ type: 'int4', value: this.group_id }}
             AND connection_id = ${{ type: 'int4', value: connection_id }}
-            AND relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
-            AND schema_name = ${{ type: 'varchar', value: schema }}
-            AND table_name = ${{ type: 'varchar', value: table }}
-            AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
+            AND (
+              relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
+              OR (
+                schema_name = ${{ type: 'varchar', value: schema }}
+                AND table_name = ${{ type: 'varchar', value: table }}
+              )
+            )
         `
           .decoded(models.SourceTable)
-          .first();
+          .rows();
       } else {
-        sourceTableRow = await db.sql`
+        candidateRows = await db.sql`
           SELECT
             *
           FROM
@@ -186,15 +188,42 @@ export class PostgresBucketBatch
             AND connection_id = ${{ type: 'int4', value: connection_id }}
             AND schema_name = ${{ type: 'varchar', value: schema }}
             AND table_name = ${{ type: 'varchar', value: table }}
-            AND replica_id_columns = ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
         `
           .decoded(models.SourceTable)
-          .first();
+          .rows();
+      }
+
+      const candidateTables = candidateRows.map((row) => {
+        const table = this.sourceTableFromRow(row, connectionTag, syncRules);
+        table.storeCurrentData = sendsCompleteRows !== true;
+        return table;
+      });
+      const candidates = candidateTables.map((table) => table.clone());
+      const resolution = await reconcile({ source, candidates });
+      storage.validateSourceTableCandidateResolution(candidates, resolution);
+
+      for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
+        await db.sql`
+          UPDATE source_tables
+          SET
+            source_metadata = ${{ type: 'jsonb', value: sourceMetadata ?? null }}
+          WHERE
+            id = ${{ type: 'varchar', value: id.toString() }}
+        `.execute();
       }
 
-      if (sourceTableRow == null) {
+      const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
+      const compatibleTables = materializedResolution.compatibleTables;
+
+      // Keep one record, preferring one that has already been snapshotted.
+      const reuse = compatibleTables.find((candidate) => candidate.snapshotComplete) ?? compatibleTables[0] ?? null;
+
+      let sourceTable: storage.SourceTable;
+      if (reuse != null) {
+        sourceTable = reuse;
+      } else {
         const id = options.idGenerator ? postgresTableId(options.idGenerator()) : uuid.v4();
-        sourceTableRow = await db.sql`
+        const insertedRow = await db.sql`
           INSERT INTO
             source_tables (
               id,
@@ -203,7 +232,8 @@ export class PostgresBucketBatch
               relation_id,
               schema_name,
               table_name,
-              replica_id_columns
+              replica_id_columns,
+              source_metadata
             )
           VALUES
             (
@@ -213,94 +243,27 @@ export class PostgresBucketBatch
               ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }},
               ${{ type: 'varchar', value: schema }},
               ${{ type: 'varchar', value: table }},
-              ${{ type: 'jsonb', value: normalizedReplicaIdColumns }}
+              ${{ type: 'jsonb', value: normalizedReplicaIdColumns }},
+              ${{ type: 'jsonb', value: resolution.newTableValues.sourceMetadata ?? null }}
             )
           RETURNING
             *
         `
           .decoded(models.SourceTable)
           .first();
+        sourceTable = this.sourceTableFromRow(insertedRow!, connectionTag, syncRules);
+        sourceTable.storeCurrentData = sendsCompleteRows !== true;
       }
 
-      const sourceTable = new storage.SourceTable({
-        id: sourceTableRow!.id,
-        ref: source,
-        objectId,
-        replicaIdColumns,
-        snapshotComplete: sourceTableRow!.snapshot_done ?? true,
-        ...syncRules.getMatchingSources(source)
-      });
-      if (!sourceTable.snapshotComplete) {
-        sourceTable.snapshotStatus = {
-          totalEstimatedCount: Number(sourceTableRow!.snapshot_total_estimated_count ?? -1n),
-          replicatedCount: Number(sourceTableRow!.snapshot_replicated_count ?? 0n),
-          lastKey: sourceTableRow!.snapshot_last_key
-        };
-      }
-      sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
-      sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
-      sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
-
-      let truncatedTables: SourceTableDecoded[] = [];
-      if (objectId != null) {
-        truncatedTables = await db.sql`
-          SELECT
-            *
-          FROM
-            source_tables
-          WHERE
-            group_id = ${{ type: 'int4', value: this.group_id }}
-            AND connection_id = ${{ type: 'int4', value: connection_id }}
-            AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
-            AND (
-              relation_id = ${{ type: 'jsonb', value: { object_id: objectId } satisfies StoredRelationId }}
-              OR (
-                schema_name = ${{ type: 'varchar', value: schema }}
-                AND table_name = ${{ type: 'varchar', value: table }}
-              )
-            )
-        `
-          .decoded(models.SourceTable)
-          .rows();
-      } else {
-        truncatedTables = await db.sql`
-          SELECT
-            *
-          FROM
-            source_tables
-          WHERE
-            group_id = ${{ type: 'int4', value: this.group_id }}
-            AND connection_id = ${{ type: 'int4', value: connection_id }}
-            AND id != ${{ type: 'varchar', value: sourceTableRow!.id }}
-            AND (
-              schema_name = ${{ type: 'varchar', value: schema }}
-              AND table_name = ${{ type: 'varchar', value: table }}
-            )
-        `
-          .decoded(models.SourceTable)
-          .rows();
-      }
+      const dropTables = [
+        ...materializedResolution.incompatibleTables,
+        // PostgreSQL storage only keeps one SourceTable per physical table.
+        ...compatibleTables.filter((candidate) => !storage.sourceTableIdEquals(candidate.id, sourceTable.id))
+      ];
 
       return {
         tables: [sourceTable],
-        dropTables: truncatedTables.map((doc) => {
-          const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
-          const dropTable = new storage.SourceTable({
-            id: doc.id,
-            ref,
-            objectId: doc.relation_id?.object_id ?? 0,
-            replicaIdColumns:
-              doc.replica_id_columns?.map(
-                (c) => ({ name: c.name, typeId: c.typeId, type: c.type }) satisfies ColumnDescriptor
-              ) ?? [],
-            snapshotComplete: doc.snapshot_done ?? true,
-            ...syncRules.getMatchingSources(ref)
-          });
-          dropTable.syncEvent = syncRules.tableTriggersEvent(ref);
-          dropTable.syncData = dropTable.bucketDataSources.length > 0;
-          dropTable.syncParameters = dropTable.parameterLookupSources.length > 0;
-          return dropTable;
-        })
+        dropTables
       };
     });
   }
@@ -337,9 +300,10 @@ export class PostgresBucketBatch
       objectId: row.relation_id?.object_id,
       replicaIdColumns:
         row.replica_id_columns?.map(
-          (c) => ({ name: c.name, typeId: c.typeId, type: c.type }) satisfies ColumnDescriptor
+          (column) => ({ name: column.name, typeId: column.type_oid, type: column.type }) satisfies ColumnDescriptor
         ) ?? [],
       snapshotComplete: row.snapshot_done ?? true,
+      sourceMetadata: row.source_metadata,
       ...syncRules.getMatchingSources(ref)
     });
 
@@ -359,19 +323,25 @@ export class PostgresBucketBatch
   async save(record: storage.SaveOptions): Promise<storage.FlushedResult | null> {
     // TODO maybe share with abstract class
     const { after, before, sourceTable, tag } = record;
-    for (const event of this.getTableEvents(sourceTable)) {
-      this.iterateListeners((cb) =>
-        cb.replicationEvent?.({
-          batch: this,
-          table: sourceTable,
-          data: {
-            op: tag,
-            after: after && utils.isCompleteRow(this.options.store_current_data, after) ? after : undefined,
-            before: before && utils.isCompleteRow(this.options.store_current_data, before) ? before : undefined
-          },
-          event
-        })
-      );
+    const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
+    // PostgreSQL storage resolves each physical table to one SourceTable. syncEvent records whether
+    // that table matches any event definition; each matching definition is evaluated once here and
+    // may produce multiple payloads from its matching queries.
+    if (sourceTable.syncEvent) {
+      for (const event of this.getTableEvents(sourceTable)) {
+        this.iterateListeners((cb) =>
+          cb.replicationEvent?.({
+            batch: this,
+            table: sourceTable,
+            data: {
+              op: tag,
+              after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined,
+              before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined
+            },
+            event
+          })
+        );
+      }
     }
     /**
      * Return if the table is just an event table
@@ -420,7 +390,9 @@ export class PostgresBucketBatch
     let processedCount = 0;
     while (lastBatchCount == BATCH_LIMIT) {
       lastBatchCount = 0;
+      let clearedError = false;
       await this.withReplicationTransaction(async (db) => {
+        clearedError = false;
         const persistedBatch = new PostgresPersistedBatch({
           group_id: this.group_id,
           storageConfig: this.storageConfig,
@@ -460,10 +432,14 @@ export class PostgresBucketBatch
           }
         }
         const { flushedAny } = await persistedBatch.flush(db);
-        if (flushedAny) {
-          await this.clearError(db);
-        }
+        clearedError = flushedAny && !this.clearedError;
       });
+      if (clearedError) {
+        // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
+        // Cleared outside the replication transaction - see flushInner.
+        await this.clearError();
+        this.clearedError = true;
+      }
     }
     if (processedCount == 0) {
       // The op sequence should not have progressed
@@ -516,13 +492,25 @@ export class PostgresBucketBatch
     await this.hooks?.beforeBatchFlush?.(this);
 
     let resumeBatch: OperationBatch | null = null;
+    let clearedError = false;
 
     const lastOp = await this.withReplicationTransaction(async (db) => {
-      resumeBatch = await this.replicateBatch(db, batch);
+      clearedError = false;
+      const result = await this.replicateBatch(db, batch);
+      resumeBatch = result.resumeBatch;
+      clearedError ||= result.clearedError;
 
       return this.getLastOpIdSequence(db);
     });
 
+    if (clearedError) {
+      // Clear the error outside the replication transaction (plain autocommit update,
+      // like the keepalive), to avoid serialization conflicts on the sync_rules row
+      // when multiple writers flush concurrently.
+      await this.clearError();
+      this.clearedError = true;
+    }
+
     // null if done, set if we need another flush
     this.batch = resumeBatch;
 
@@ -545,136 +533,144 @@ export class PostgresBucketBatch
 
     const persisted_op = this.persisted_op ?? null;
 
-    const result = await this.db.sql`
-      WITH
-        selected AS (
-          SELECT
-            id,
-            state,
-            last_checkpoint,
-            last_checkpoint_lsn,
-            snapshot_done,
-            no_checkpoint_before,
-            keepalive_op,
-            (
-              snapshot_done = TRUE
-              AND (
-                last_checkpoint_lsn IS NULL
-                OR last_checkpoint_lsn <= ${{ type: 'varchar', value: lsn }}
-              )
+    // Transaction failures restart replication from the last durable checkpoint.
+    const result = await this.db.transaction(async (db) => {
+      const checkpoint = await db.sql`
+        WITH
+          selected AS (
+            SELECT
+              id,
+              state,
+              last_checkpoint,
+              last_checkpoint_lsn,
+              snapshot_done,
+              no_checkpoint_before,
+              keepalive_op,
+              (
+                snapshot_done = TRUE
+                AND (
+                  last_checkpoint_lsn IS NULL
+                  OR last_checkpoint_lsn <= ${{ type: 'varchar', value: lsn }}
+                )
+                AND (
+                  no_checkpoint_before IS NULL
+                  OR no_checkpoint_before <= ${{ type: 'varchar', value: lsn }}
+                )
+              ) AS can_checkpoint
+            FROM
+              sync_rules
+            WHERE
+              id = ${{ type: 'int4', value: this.group_id }}
+            FOR UPDATE
+          ),
+          computed AS (
+            SELECT
+              selected.*,
+              CASE
+                WHEN selected.can_checkpoint THEN GREATEST(
+                  selected.last_checkpoint,
+                  ${{ type: 'int8', value: persisted_op }},
+                  selected.keepalive_op,
+                  0
+                )
+                ELSE selected.last_checkpoint
+              END AS new_last_checkpoint,
+              CASE
+                WHEN selected.can_checkpoint THEN NULL
+                ELSE GREATEST(
+                  selected.keepalive_op,
+                  ${{ type: 'int8', value: persisted_op }},
+                  0
+                )
+              END AS new_keepalive_op
+            FROM
+              selected
+          ),
+          updated AS (
+            UPDATE sync_rules AS sr
+            SET
+              last_checkpoint_lsn = CASE
+                WHEN computed.can_checkpoint THEN ${{ type: 'varchar', value: lsn }}
+                ELSE sr.last_checkpoint_lsn
+              END,
+              last_checkpoint_ts = CASE
+                WHEN computed.can_checkpoint THEN ${{ type: 1184, value: now }}
+                ELSE sr.last_checkpoint_ts
+              END,
+              last_keepalive_ts = ${{ type: 1184, value: now }},
+              last_fatal_error = CASE
+                WHEN computed.can_checkpoint THEN NULL
+                ELSE sr.last_fatal_error
+              END,
+              keepalive_op = computed.new_keepalive_op,
+              last_checkpoint = computed.new_last_checkpoint,
+              snapshot_lsn = CASE
+                WHEN computed.can_checkpoint THEN NULL
+                ELSE sr.snapshot_lsn
+              END
+            FROM
+              computed
+            WHERE
+              sr.id = computed.id
               AND (
-                no_checkpoint_before IS NULL
-                OR no_checkpoint_before <= ${{ type: 'varchar', value: lsn }}
+                sr.keepalive_op IS DISTINCT FROM computed.new_keepalive_op
+                OR sr.last_checkpoint IS DISTINCT FROM computed.new_last_checkpoint
+                OR ${{ type: 'bool', value: createEmptyCheckpoints }}
               )
-            ) AS can_checkpoint
-          FROM
-            sync_rules
-          WHERE
-            id = ${{ type: 'int4', value: this.group_id }}
-          FOR UPDATE
-        ),
-        computed AS (
-          SELECT
-            selected.*,
-            CASE
-              WHEN selected.can_checkpoint THEN GREATEST(
-                selected.last_checkpoint,
-                ${{ type: 'int8', value: persisted_op }},
-                selected.keepalive_op,
-                0
-              )
-              ELSE selected.last_checkpoint
-            END AS new_last_checkpoint,
-            CASE
-              WHEN selected.can_checkpoint THEN NULL
-              ELSE GREATEST(
-                selected.keepalive_op,
-                ${{ type: 'int8', value: persisted_op }},
-                0
-              )
-            END AS new_keepalive_op
-          FROM
-            selected
-        ),
-        updated AS (
-          UPDATE sync_rules AS sr
-          SET
-            last_checkpoint_lsn = CASE
-              WHEN computed.can_checkpoint THEN ${{ type: 'varchar', value: lsn }}
-              ELSE sr.last_checkpoint_lsn
-            END,
-            last_checkpoint_ts = CASE
-              WHEN computed.can_checkpoint THEN ${{ type: 1184, value: now }}
-              ELSE sr.last_checkpoint_ts
-            END,
-            last_keepalive_ts = ${{ type: 1184, value: now }},
-            last_fatal_error = CASE
-              WHEN computed.can_checkpoint THEN NULL
-              ELSE sr.last_fatal_error
-            END,
-            keepalive_op = computed.new_keepalive_op,
-            last_checkpoint = computed.new_last_checkpoint,
-            snapshot_lsn = CASE
-              WHEN computed.can_checkpoint THEN NULL
-              ELSE sr.snapshot_lsn
-            END
-          FROM
-            computed
-          WHERE
-            sr.id = computed.id
-            AND (
-              sr.keepalive_op IS DISTINCT FROM computed.new_keepalive_op
-              OR sr.last_checkpoint IS DISTINCT FROM computed.new_last_checkpoint
-              OR ${{ type: 'bool', value: createEmptyCheckpoints }}
-            )
-          RETURNING
-            sr.id,
-            sr.state,
-            sr.last_checkpoint,
-            sr.last_checkpoint_lsn,
-            sr.snapshot_done,
-            sr.no_checkpoint_before,
-            computed.can_checkpoint,
-            computed.keepalive_op,
-            computed.new_last_checkpoint
-        )
-      SELECT
-        id,
-        state,
-        last_checkpoint,
-        last_checkpoint_lsn,
-        snapshot_done,
-        no_checkpoint_before,
-        can_checkpoint,
-        keepalive_op,
-        new_last_checkpoint,
-        TRUE AS created_checkpoint
-      FROM
-        updated
-      UNION ALL
-      SELECT
-        id,
-        state,
-        new_last_checkpoint AS last_checkpoint,
-        last_checkpoint_lsn,
-        snapshot_done,
-        no_checkpoint_before,
-        can_checkpoint,
-        keepalive_op,
-        new_last_checkpoint,
-        FALSE AS created_checkpoint
-      FROM
-        computed
-      WHERE
-        NOT EXISTS (
-          SELECT
-            1
-          FROM
-            updated
-        )
-    `
-      .decoded(CheckpointWithStatus)
-      .first();
+            RETURNING
+              sr.id,
+              sr.state,
+              sr.last_checkpoint,
+              sr.last_checkpoint_lsn,
+              sr.snapshot_done,
+              sr.no_checkpoint_before,
+              computed.can_checkpoint,
+              computed.keepalive_op,
+              computed.new_last_checkpoint
+          )
+        SELECT
+          id,
+          state,
+          last_checkpoint,
+          last_checkpoint_lsn,
+          snapshot_done,
+          no_checkpoint_before,
+          can_checkpoint,
+          keepalive_op,
+          new_last_checkpoint,
+          TRUE AS created_checkpoint
+        FROM
+          updated
+        UNION ALL
+        SELECT
+          id,
+          state,
+          new_last_checkpoint AS last_checkpoint,
+          last_checkpoint_lsn,
+          snapshot_done,
+          no_checkpoint_before,
+          can_checkpoint,
+          keepalive_op,
+          new_last_checkpoint,
+          FALSE AS created_checkpoint
+        FROM
+          computed
+        WHERE
+          NOT EXISTS (
+            SELECT
+              1
+            FROM
+              updated
+          )
+      `
+        .decoded(CheckpointWithStatus)
+        .first();
+
+      if (checkpoint?.can_checkpoint && checkpoint.state == storage.SyncRuleState.ACTIVE) {
+        await notifySyncRulesUpdate(db, checkpoint);
+      }
+      return checkpoint;
+    });
 
     if (result == null) {
       throw new ReplicationAssertionError('Failed to update sync_rules during checkpoint');
@@ -706,10 +702,8 @@ export class PostgresBucketBatch
         });
       }
     }
-    await this.autoActivate(lsn);
-    await notifySyncRulesUpdate(this.db, {
+    await this.autoActivate(lsn, {
       id: result.id,
-      state: result.state,
       last_checkpoint: result.last_checkpoint,
       last_checkpoint_lsn: result.last_checkpoint_lsn
     });
@@ -758,23 +752,22 @@ export class PostgresBucketBatch
 
   async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void> {
     await this.db.transaction(async (db) => {
-      const snapshotRequiredCount = await db.sql`
+      const blockingTables = await db.sql`
         SELECT
-          COUNT(*) AS count
+          schema_name,
+          table_name
         FROM
           source_tables
         WHERE
           group_id = ${{ type: 'int4', value: this.group_id }}
           AND snapshot_done = FALSE
       `
-        .decoded(t.object({ count: bigint }))
-        .first();
-      if ((snapshotRequiredCount?.count ?? 0n) > 0n) {
+        .decoded(t.object({ schema_name: t.string, table_name: t.string }))
+        .rows();
+      if (blockingTables.length > 0) {
         if (options?.throwOnConflict ?? true) {
           throw new ReplicationAssertionError(
-            `Cannot mark snapshot done while ${snapshotRequiredCount?.count} source table${
-              snapshotRequiredCount?.count == 1n ? '' : 's'
-            } still require snapshotting`
+            `Cannot mark snapshot done while source tables still require snapshotting. Tables: ${blockingTables.map((t) => `${t.schema_name}.${t.table_name}`).join(', ')}`
           );
         } else {
           return;
@@ -890,10 +883,17 @@ export class PostgresBucketBatch
     });
   }
 
-  protected async replicateBatch(db: lib_postgres.WrappedConnection, batch: OperationBatch) {
+  protected async replicateBatch(
+    db: lib_postgres.WrappedConnection,
+    batch: OperationBatch
+  ): Promise<{ resumeBatch: OperationBatch | null; clearedError: boolean }> {
     let sizes: Map<string, number> | undefined = undefined;
-    if (this.options.store_current_data && !this.options.skip_existing_rows) {
-      // We skip this step if we don't store current_data, since the sizes will
+    // Check if any table in this batch needs to store current_data
+    const anyTableStoresCurrentData =
+      this.options.store_current_data && batch.batch.some((r) => r.record.sourceTable.storeCurrentData);
+
+    if (anyTableStoresCurrentData && !this.options.skip_existing_rows) {
+      // We skip this step if no tables store current_data, since the sizes will
       // always be small in that case.
 
       // With skipExistingRows, we don't load the full documents into memory,
@@ -901,15 +901,19 @@ export class PostgresBucketBatch
 
       // Find sizes of current_data documents, to assist in intelligent batching without
       // exceeding memory limits.
-      const sizeLookups = batch.batch.map((r) => {
-        return {
-          source_table: postgresTableId(r.record.sourceTable.id),
-          /**
-           * Encode to hex in order to pass a jsonb
-           */
-          source_key: storage.serializeReplicaId(r.beforeId).toString('hex')
-        };
-      });
+      // Within this branch the batch stores current_data, so the per-table flag is the
+      // effective value - only look up sizes for tables that actually store current_data.
+      const sizeLookups = batch.batch
+        .filter((r) => r.record.sourceTable.storeCurrentData)
+        .map((r) => {
+          return {
+            source_table: postgresTableId(r.record.sourceTable.id),
+            /**
+             * Encode to hex in order to pass a jsonb
+             */
+            source_key: storage.serializeReplicaId(r.beforeId).toString('hex')
+          };
+        });
 
       sizes = new Map<string, number>();
 
@@ -1006,15 +1010,14 @@ export class PostgresBucketBatch
       }
     }
 
-    if (didFlush) {
-      await this.clearError(db);
-    }
+    // The error itself is cleared by the caller, outside the replication transaction - see flushInner.
+    const clearedError = didFlush && !this.clearedError;
 
     // Don't return empty batches
-    if (resumeBatch?.batch.length) {
-      return resumeBatch;
-    }
-    return null;
+    return {
+      resumeBatch: resumeBatch?.batch.length ? resumeBatch : null,
+      clearedError
+    };
   }
 
   protected async saveOperation(
@@ -1028,6 +1031,9 @@ export class PostgresBucketBatch
     const afterId = operation.afterId;
     let after = record.after;
     const sourceTable = record.sourceTable;
+    // Effective per-record flag: store current_data only if both the batch (source-level)
+    // and the table (e.g. non-FULL replica identity) require it.
+    const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
 
     let existingBuckets: CurrentBucket[] = [];
     let newBuckets: CurrentBucket[] = [];
@@ -1056,7 +1062,7 @@ export class PostgresBucketBatch
         existingLookups = [];
         // Log to help with debugging if there was a consistency issue
 
-        if (this.options.store_current_data) {
+        if (storeCurrentData) {
           if (this.markRecordUnavailable != null) {
             // This will trigger a "resnapshot" of the record.
             // This is not relevant if storeCurrentData is false, since we'll get the full row
@@ -1072,7 +1078,7 @@ export class PostgresBucketBatch
       } else {
         existingBuckets = result.buckets;
         existingLookups = result.lookups;
-        if (this.options.store_current_data) {
+        if (storeCurrentData) {
           const data = storage.deserializeBson(result.data) as sync_rules.SqliteRow;
           after = storage.mergeToast(after!, data);
         }
@@ -1083,7 +1089,9 @@ export class PostgresBucketBatch
         // Not an error if we re-apply a transaction
         existingBuckets = [];
         existingLookups = [];
-        // Log to help with debugging if there was a consistency issue
+        // Log to help with debugging if there was a consistency issue.
+        // Gate on the batch-level flag: FULL tables (per-record flag false) still get a
+        // current_data entry, so a missing record on DELETE is meaningful for them too.
         if (this.options.store_current_data && this.markRecordUnavailable == null) {
           this.logger.warn(
             `Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
@@ -1096,7 +1104,7 @@ export class PostgresBucketBatch
     }
 
     let afterData: Buffer<ArrayBuffer> | undefined;
-    if (afterId != null && !this.options.store_current_data) {
+    if (afterId != null && !storeCurrentData) {
       afterData = storage.serializeBson({});
     } else if (afterId != null) {
       try {
@@ -1118,7 +1126,7 @@ export class PostgresBucketBatch
           {
             level: errors.ErrorSeverity.WARNING,
             metadata: {
-              replication_slot: this.options.slot_name,
+              replication_slot: this.options.replicationStreamName,
               table: record.sourceTable.qualifiedName
             }
           }
@@ -1159,7 +1167,7 @@ export class PostgresBucketBatch
     // However, it will be valid by the end of the transaction.
     //
     // In this case, we don't save the op, but we do save the current data.
-    if (afterId && after && utils.isCompleteRow(this.options.store_current_data, after)) {
+    if (afterId && after && utils.isCompleteRow(storeCurrentData, after)) {
       // Insert or update
       if (sourceTable.syncData) {
         const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({
@@ -1174,7 +1182,7 @@ export class PostgresBucketBatch
             {
               level: errors.ErrorSeverity.WARNING,
               metadata: {
-                replication_slot: this.options.slot_name,
+                replication_slot: this.options.replicationStreamName,
                 table: record.sourceTable.qualifiedName
               }
             }
@@ -1215,7 +1223,7 @@ export class PostgresBucketBatch
             {
               level: errors.ErrorSeverity.WARNING,
               metadata: {
-                replication_slot: this.options.slot_name,
+                replication_slot: this.options.replicationStreamName,
                 table: record.sourceTable.qualifiedName
               }
             }
@@ -1272,14 +1280,13 @@ export class PostgresBucketBatch
    *
    * Called on new commits.
    */
-  private async autoActivate(lsn: string): Promise<void> {
+  private async autoActivate(lsn: string, checkpoint: models.ActiveCheckpointDecoded): Promise<void> {
     if (!this.needsActivation) {
       // Already activated
       return;
     }
 
-    let didActivate = false;
-    await this.db.transaction(async (db) => {
+    const activationResult = await this.db.transaction(async (db) => {
       const syncRulesRow = await db.sql`
         SELECT
           state,
@@ -1313,22 +1320,27 @@ export class PostgresBucketBatch
             )
             AND id != ${{ type: 'int4', value: this.group_id }}
         `.execute();
-        didActivate = true;
-        this.needsActivation = false;
+        await notifySyncRulesUpdate(db, checkpoint);
+        return 'activated';
       } else if (syncRulesRow?.state != storage.SyncRuleState.PROCESSING) {
-        this.needsActivation = false;
+        return 'not-processing';
       }
+      return 'pending';
     });
-    if (didActivate) {
+
+    if (activationResult != 'pending') {
+      this.needsActivation = false;
+    }
+    if (activationResult == 'activated') {
       this.logger.info(`Activated new replication stream at ${lsn}`);
     }
   }
 
   /**
-   * Gets relevant {@link SqlEventDescriptor}s for the given {@link SourceTable}
+   * Gets relevant {@link HydratedEventDescriptor}s for the given {@link SourceTable}
    * TODO maybe share this with an abstract class
    */
-  protected getTableEvents(table: storage.SourceTable): sync_rules.SqlEventDescriptor[] {
+  protected getTableEvents(table: storage.SourceTable): sync_rules.HydratedEventDescriptor[] {
     return this.sync_rules.eventDescriptors.filter((evt) =>
       [...evt.getSourceTables()].some((sourceTable) => sourceTable.matches(table.ref))
     );
@@ -1374,11 +1386,6 @@ export class PostgresBucketBatch
   protected async clearError(
     db: lib_postgres.AbstractPostgresConnection | lib_postgres.DatabaseClient = this.db
   ): Promise<void> {
-    // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
-    if (this.clearedError) {
-      return;
-    }
-
     await db.sql`
       UPDATE sync_rules
       SET
@@ -1386,7 +1393,6 @@ export class PostgresBucketBatch
       WHERE
         id = ${{ type: 'int4', value: this.group_id }}
     `.execute();
-    this.clearedError = true;
   }
 
   private async getLastOpIdSequence(db: lib_postgres.AbstractPostgresConnection) {
@@ -1411,12 +1417,17 @@ export class PostgresBucketBatch
  * Uses Postgres' NOTIFY functionality to update different processes when the
  * active checkpoint has been updated.
  */
-export const notifySyncRulesUpdate = async (db: lib_postgres.DatabaseClient, update: StatefulCheckpointDecoded) => {
-  if (update.state != storage.SyncRuleState.ACTIVE) {
-    return;
-  }
-
-  await db.query({
-    statement: `NOTIFY ${NOTIFICATION_CHANNEL}, '${models.ActiveCheckpointNotification.encode({ active_checkpoint: update })}'`
-  });
+export const notifySyncRulesUpdate = async (
+  db: lib_postgres.AbstractPostgresConnection,
+  update: models.ActiveCheckpointDecoded
+) => {
+  const payload = models.ActiveCheckpointNotification.encode({ active_checkpoint: update });
+
+  await db.sql`
+    SELECT
+      pg_notify (
+        ${{ type: 'varchar', value: NOTIFICATION_CHANNEL }},
+        ${{ type: 'varchar', value: payload }}
+      )
+  `.execute();
 };