@powersync/service-module-mssql 0.0.0-dev-20260225160713 → 0.0.0-dev-20260511080634

package/src/replication/CDCStream.ts

@@ -1,45 +1,43 @@
 import {
   container,
   DatabaseConnectionError,
+  logger as defaultLogger,
   ErrorCode,
   Logger,
-  logger as defaultLogger,
   ReplicationAbortedError,
-  ReplicationAssertionError,
-  ServiceAssertionError
+  ReplicationAssertionError
 } from '@powersync/lib-services-framework';
-import { getUuidReplicaIdentityBson, MetricsEngine, SourceEntityDescriptor, storage } from '@powersync/service-core';
-
 import {
-  SqliteInputRow,
-  SqliteRow,
-  SqlSyncRules,
-  HydratedSyncRules,
-  TablePattern
-} from '@powersync/service-sync-rules';
+  getUuidReplicaIdentityBson,
+  MetricsEngine,
+  ReplicationLagTracker,
+  SourceEntityDescriptor,
+  storage
+} from '@powersync/service-core';
+
+import { HydratedSyncRules, SqliteInputRow, SqliteRow, TablePattern } from '@powersync/service-sync-rules';
 
 import { ReplicationMetric } from '@powersync/service-types';
-import { BatchedSnapshotQuery, MSSQLSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js';
-import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
-import { getReplicationIdentityColumns, getTablesFromPattern, ResolvedTable } from '../utils/schema.js';
+import sql from 'mssql';
+import { CaptureInstance } from '../common/CaptureInstance.js';
+import { LSN } from '../common/LSN.js';
+import { CDCToSqliteRow, toSqliteInputRow } from '../common/mssqls-to-sqlite.js';
+import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
+import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
+import { AdditionalConfig } from '../types/types.js';
 import {
+  checkRetentionThresholds,
   checkSourceConfiguration,
   createCheckpoint,
-  getCaptureInstance,
+  getCaptureInstances,
   getLatestLSN,
   getLatestReplicatedLSN,
-  isIColumnMetadata,
-  isTableEnabledForCDC,
-  isWithinRetentionThreshold,
-  toQualifiedTableName
+  isIColumnMetadata
 } from '../utils/mssql.js';
-import sql from 'mssql';
-import { CDCToSqliteRow, toSqliteInputRow } from '../common/mssqls-to-sqlite.js';
-import { LSN } from '../common/LSN.js';
-import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
-import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
-import { CDCEventHandler, CDCPoller } from './CDCPoller.js';
-import { AdditionalConfig } from '../types/types.js';
+import { getReplicationIdentityColumns, getTablesFromPattern, ResolvedTable } from '../utils/schema.js';
+import { CDCEventHandler, CDCPoller, SchemaChange, SchemaChangeType } from './CDCPoller.js';
+import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
+import { BatchedSnapshotQuery, MSSQLSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js';
 
 export interface CDCStreamOptions {
   connections: MSSQLConnectionManager;
@@ -55,17 +53,30 @@ export interface CDCStreamOptions {
   snapshotBatchSize?: number;
 
   additionalConfig: AdditionalConfig;
+
+  /**
+   * Override schema check interval, defaults is 60 seconds.
+   */
+  schemaCheckIntervalMs?: number;
 }
 
 export enum SnapshotStatus {
-  IN_PROGRESS = 'in-progress',
+  INITIAL = 'initial',
+  RESUME = 'resume',
   DONE = 'done',
-  RESTART_REQUIRED = 'restart-required'
+  LIMITED_RESNAPSHOT = 'limited-resnapshot'
 }
 
 export interface SnapshotStatusResult {
   status: SnapshotStatus;
   snapshotLSN: string | null;
+  /**
+   *  Under certain circumstances it may be necessary to re-snapshot specific tables:
+   *  * A new capture instance has been created for a table.
+   *  * A table has been renamed.
+   *  * The retention threshold has been exceeded for a table.
+   */
+  specificTablesToResnapshot?: MSSQLSourceTable[];
 }
 
 export class CDCConfigurationError extends Error {
@@ -94,18 +105,9 @@ export class CDCStream {
   private readonly abortSignal: AbortSignal;
   private readonly logger: Logger;
 
-  private tableCache = new MSSQLSourceTableCache();
+  public tableCache = new MSSQLSourceTableCache();
 
-  /**
-   * Time of the oldest uncommitted change, according to the source db.
-   * This is used to determine the replication lag.
-   */
-  private oldestUncommittedChange: Date | null = null;
-  /**
-   * Keep track of whether we have done a commit or keepalive yet.
-   * We can only compute replication lag if isStartingReplication == false, or oldestUncommittedChange is present.
-   */
-  public isStartingReplication = true;
+  private replicationLag = new ReplicationLagTracker();
 
   constructor(private options: CDCStreamOptions) {
     this.logger = options.logger ?? defaultLogger;
@@ -123,6 +125,10 @@ export class CDCStream {
     return this.abortSignal.aborted;
   }
 
+  get isStartingReplication() {
+    return this.replicationLag.isStartingReplication;
+  }
+
   get defaultSchema() {
     return this.connections.schema;
   }
@@ -172,14 +178,11 @@ export class CDCStream {
         logger: this.logger,
         zeroLSN: LSN.ZERO,
         defaultSchema: this.defaultSchema,
-        storeCurrentData: true
+        storeCurrentData: false
       },
       async (batch) => {
         for (let tablePattern of sourceTables) {
-          const tables = await this.getQualifiedTableNames(batch, tablePattern);
-          for (const table of tables) {
-            this.tableCache.set(table);
-          }
+          await this.getQualifiedTableNames(batch, tablePattern);
         }
       }
     );
@@ -193,21 +196,12 @@ export class CDCStream {
       return [];
     }
 
+    const captureInstances = await getCaptureInstances({ connectionManager: this.connections });
     const matchedTables: ResolvedTable[] = await getTablesFromPattern(this.connections, tablePattern);
 
     const tables: MSSQLSourceTable[] = [];
     for (const matchedTable of matchedTables) {
-      const isEnabled = await isTableEnabledForCDC({
-        connectionManager: this.connections,
-        table: matchedTable.name,
-        schema: matchedTable.schema
-      });
-
-      if (!isEnabled) {
-        this.logger.info(`Skipping ${matchedTable.schema}.${matchedTable.name} - table is not enabled for CDC.`);
-        continue;
-      }
-
+      const captureInstanceDetails = captureInstances.get(matchedTable.objectId as number);
       // TODO: Check RLS settings for table
 
       const replicaIdColumns = await getReplicationIdentityColumns({
@@ -224,6 +218,7 @@ export class CDCStream {
           objectId: matchedTable.objectId,
           replicaIdColumns: replicaIdColumns.columns
         },
+        captureInstanceDetails?.instances[0] ?? null,
         false
       );
 
@@ -235,6 +230,7 @@ export class CDCStream {
   async processTable(
     batch: storage.BucketStorageBatch,
     table: SourceEntityDescriptor,
+    captureInstance: CaptureInstance | null,
     snapshot: boolean
   ): Promise<MSSQLSourceTable> {
     if (!table.objectId && typeof table.objectId != 'number') {
@@ -247,40 +243,33 @@ export class CDCStream {
       entity_descriptor: table,
       sync_rules: this.syncRules
     });
-    const captureInstance = await getCaptureInstance({
-      connectionManager: this.connections,
-      tableName: resolved.table.name,
-      schema: resolved.table.schema
-    });
+    const resolvedTable = new MSSQLSourceTable(resolved.table);
+
     if (!captureInstance) {
-      throw new ServiceAssertionError(
-        `Missing capture instance for table ${toQualifiedTableName(resolved.table.schema, resolved.table.name)}`
+      this.logger.warn(
+        `Missing capture instance for table ${resolvedTable.toQualifiedName()}. This table will not be replicated until CDC is enabled for it.`
       );
+    } else {
+      resolvedTable.setCaptureInstance(captureInstance);
     }
-    const resolvedTable = new MSSQLSourceTable({
-      sourceTable: resolved.table,
-      captureInstance: captureInstance
-    });
+
+    this.tableCache.set(resolvedTable);
 
     // Drop conflicting tables. This includes for example renamed tables.
     await batch.drop(resolved.dropTables);
 
     // Snapshot if:
-    // 1. Snapshot is requested (false for initial snapshot, since that process handles it elsewhere)
-    // 2. Snapshot is not already done, AND:
-    // 3. The table is used in sync rules.
-    const shouldSnapshot = snapshot && !resolved.table.snapshotComplete && resolved.table.syncAny;
+    // 1. The table is in the sync config and snapshot is requested, or not already done.
+    // 2. AND the table is enabled for CDC with a valid capture instance.
+    const shouldSnapshot =
+      snapshot && !resolved.table.snapshotComplete && resolved.table.syncAny && resolvedTable.enabledForCDC();
 
     if (shouldSnapshot) {
       // Truncate this table in case a previous snapshot was interrupted.
       await batch.truncate([resolved.table]);
 
       // Start the snapshot inside a transaction.
-      try {
-        await this.snapshotTableInTx(batch, resolvedTable);
-      } finally {
-        // TODO Cleanup?
-      }
+      await this.snapshotTableInTx(batch, resolvedTable);
     }
 
     return resolvedTable;
@@ -310,7 +299,10 @@ export class CDCStream {
       const postSnapshotLSN = await getLatestLSN(this.connections);
       // Side note: A ROLLBACK would probably also be fine here, since we only read in this transaction.
       await transaction.commit();
-      const [updatedSourceTable] = await batch.markSnapshotDone([table.sourceTable], postSnapshotLSN.toString());
+      const [updatedSourceTable] = await batch.markTableSnapshotDone([table.sourceTable], postSnapshotLSN.toString());
+      this.logger.info(
+        `Snapshot of ${table.toQualifiedName()} completed. Post-snapshot LSN: ${postSnapshotLSN.toString()}`
+      );
       this.tableCache.updateSourceTable(updatedSourceTable);
     } catch (e) {
       await transaction.rollback();
@@ -341,17 +333,17 @@ export class CDCStream {
       );
       if (table.sourceTable.snapshotStatus?.lastKey != null) {
         this.logger.info(
-          `Replicating ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - resuming from ${orderByKey.name} > ${(query as BatchedSnapshotQuery).lastKey}`
+          `Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - resuming from ${orderByKey.name} > ${(query as BatchedSnapshotQuery).lastKey}`
         );
       } else {
         this.logger.info(
-          `Replicating ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - resumable`
+          `Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - resumable`
         );
       }
     } else {
       // Fallback case - query the entire table
       this.logger.info(
-        `Replicating ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - not resumable`
+        `Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()} - not resumable`
       );
       query = new SimpleSnapshotQuery(transaction, table);
       replicatedCount = 0;
@@ -417,8 +409,6 @@ export class CDCStream {
       });
       this.tableCache.updateSourceTable(updatedSourceTable);
 
-      this.logger.info(`Replicating ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
-
       if (this.abortSignal.aborted) {
         // We only abort after flushing
         throw new ReplicationAbortedError(`Initial replication interrupted`);
@@ -427,6 +417,8 @@ export class CDCStream {
       // When the batch of rows is smaller than the requested batch size we know it is the final batch
       if (batchReplicatedCount < this.snapshotBatchSize) {
         hasRemainingData = false;
+      } else {
+        this.logger.info(`Snapshotting ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
       }
     }
   }
@@ -455,13 +447,7 @@ export class CDCStream {
    * and starts again from scratch.
    */
   async startInitialReplication(snapshotStatus: SnapshotStatusResult) {
-    let { status, snapshotLSN } = snapshotStatus;
-
-    if (status === SnapshotStatus.RESTART_REQUIRED) {
-      this.logger.info(`Snapshot restart required, clearing state.`);
-      // This happens if the last replicated checkpoint LSN is no longer available in the CDC tables.
-      await this.storage.clear({ signal: this.abortSignal });
-    }
+    let { status, snapshotLSN, specificTablesToResnapshot } = snapshotStatus;
 
     await this.storage.startBatch(
       {
@@ -472,14 +458,26 @@ export class CDCStream {
         skipExistingRows: true
       },
       async (batch) => {
-        if (snapshotLSN == null) {
-          // First replication attempt - set the snapshot LSN to the current LSN before starting
-          snapshotLSN = (await getLatestReplicatedLSN(this.connections)).toString();
-          await batch.setResumeLsn(snapshotLSN);
-          const latestLSN = (await getLatestLSN(this.connections)).toString();
-          this.logger.info(`Marking snapshot at ${snapshotLSN}, Latest DB LSN ${latestLSN}.`);
-        } else {
-          this.logger.info(`Resuming snapshot at ${snapshotLSN}.`);
+        switch (status) {
+          case SnapshotStatus.INITIAL:
+            // First replication attempt - set the snapshot LSN to the current LSN before starting
+            snapshotLSN = (await getLatestReplicatedLSN(this.connections)).toString();
+            await batch.setResumeLsn(snapshotLSN);
+            const latestLSN = (await getLatestLSN(this.connections)).toString();
+            this.logger.info(`Marking snapshot at ${snapshotLSN}, Latest DB LSN ${latestLSN}.`);
+            break;
+          case SnapshotStatus.RESUME:
+            this.logger.info(`Resuming snapshot at ${snapshotLSN}.`);
+            break;
+          case SnapshotStatus.LIMITED_RESNAPSHOT:
+            for (const table of specificTablesToResnapshot!) {
+              await batch.drop([table.sourceTable]);
+              // Update table in the table cache
+              await this.processTable(batch, table.sourceTable, table.captureInstance, false);
+            }
+            break;
+          default:
+            throw new ReplicationAssertionError(`Unsupported snapshot status: ${status}`);
         }
 
         const tablesToSnapshot: MSSQLSourceTable[] = [];
@@ -489,14 +487,17 @@ export class CDCStream {
             continue;
           }
 
+          if (!table.enabledForCDC()) {
+            this.logger.info(`Skipping table [${table.toQualifiedName()}] - not enabled for CDC.`);
+            continue;
+          }
+
           const count = await this.estimatedCountNumber(table);
           const updatedSourceTable = await batch.updateTableProgress(table.sourceTable, {
             totalEstimatedCount: count
           });
           this.tableCache.updateSourceTable(updatedSourceTable);
           tablesToSnapshot.push(table);
-
-          this.logger.info(`To replicate: ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
         }
 
         for (const table of tablesToSnapshot) {
@@ -506,11 +507,17 @@ export class CDCStream {
 
         // This will not create a consistent checkpoint yet, but will persist the op.
         // Actual checkpoint will be created when streaming replication caught up.
-        await batch.commit(snapshotLSN);
-
-        this.logger.info(
-          `Snapshot done. Need to replicate from ${snapshotLSN} to ${batch.noCheckpointBeforeLsn} to be consistent`
-        );
+        const postSnapshotLSN = await getLatestLSN(this.connections);
+        await batch.markAllSnapshotDone(postSnapshotLSN.toString());
+        await batch.commit(snapshotLSN!);
+
+        if (tablesToSnapshot.length > 0) {
+          this.logger.info(
+            `All snapshots done. Need to replicate from ${snapshotLSN} to ${postSnapshotLSN} to be consistent.`
+          );
+        } else {
+          this.logger.info(`No tables to snapshot. Need to replicate from ${snapshotLSN}.`);
+        }
       }
     );
   }
@@ -525,6 +532,8 @@ export class CDCStream {
     const snapshotStatus = await this.checkSnapshotStatus();
     if (snapshotStatus.status !== SnapshotStatus.DONE) {
       await this.startInitialReplication(snapshotStatus);
+    } else {
+      this.logger.info(`Initial replication already done`);
     }
   }
 
@@ -534,26 +543,50 @@ export class CDCStream {
    */
   private async checkSnapshotStatus(): Promise<SnapshotStatusResult> {
     const status = await this.storage.getStatus();
+
     if (status.snapshot_done && status.checkpoint_lsn) {
+      const additionalTablesToSnapshot: Set<MSSQLSourceTable> = new Set();
+      const newTables = this.tableCache.getAll().filter((table) => !table.sourceTable.snapshotComplete);
+      if (newTables.length > 0) {
+        this.logger.info(
+          `Detected new table(s) [${newTables.map((table) => table.toQualifiedName()).join(', ')}] that have not been snapshotted yet.`
+        );
+        newTables.forEach((table) => additionalTablesToSnapshot.add(table));
+      }
+
       // Snapshot is done, but we still need to check that the last known checkpoint LSN is still
-      // within the threshold of the CDC tables
-      this.logger.info(`Initial replication already done`);
+      // within the retention threshold of the CDC tables
 
       const lastCheckpointLSN = LSN.fromString(status.checkpoint_lsn);
       // Check that the CDC tables still have valid data
-      const isAvailable = await isWithinRetentionThreshold({
+      const tablesOutsideRetentionThreshold = await checkRetentionThresholds({
         checkpointLSN: lastCheckpointLSN,
         tables: this.tableCache.getAll(),
         connectionManager: this.connections
       });
-      if (!isAvailable) {
+      if (tablesOutsideRetentionThreshold.length > 0) {
         this.logger.warn(
-          `Updates from the last checkpoint are no longer available in the CDC instance, starting initial replication again.`
+          `Updates from the last checkpoint are no longer available in the CDC instances of the following table(s): ${tablesOutsideRetentionThreshold.map((table) => table.toQualifiedName()).join(', ')}.`
         );
+        tablesOutsideRetentionThreshold.forEach((table) => additionalTablesToSnapshot.add(table));
+      }
+      if (additionalTablesToSnapshot.size > 0) {
+        return {
+          status: SnapshotStatus.LIMITED_RESNAPSHOT,
+          snapshotLSN: status.checkpoint_lsn,
+          specificTablesToResnapshot: Array.from(additionalTablesToSnapshot)
+        };
+      } else {
+        return {
+          status: SnapshotStatus.DONE,
+          snapshotLSN: null
+        };
       }
-      return { status: isAvailable ? SnapshotStatus.DONE : SnapshotStatus.RESTART_REQUIRED, snapshotLSN: null };
     } else {
-      return { status: SnapshotStatus.IN_PROGRESS, snapshotLSN: status.snapshot_lsn };
+      return {
+        status: status.snapshot_lsn != null ? SnapshotStatus.RESUME : SnapshotStatus.INITIAL,
+        snapshotLSN: status.snapshot_lsn
+      };
     }
   }
 
@@ -571,16 +604,17 @@ export class CDCStream {
           throw new ReplicationAssertionError(`No LSN found to resume replication from.`);
         }
         const startLSN = LSN.fromString(batch.resumeFromLsn);
-        const sourceTables: MSSQLSourceTable[] = this.tableCache.getAll();
         const eventHandler = this.createEventHandler(batch);
 
         const poller = new CDCPoller({
           connectionManager: this.connections,
           eventHandler,
-          sourceTables,
+          getReplicatedTables: () => this.tableCache.getAll(),
+          sourceTables: this.syncRules.getSourceTables(),
           startLSN,
           logger: this.logger,
-          additionalConfig: this.options.additionalConfig
+          additionalConfig: this.options.additionalConfig,
+          schemaCheckIntervalMs: this.options.schemaCheckIntervalMs
         });
 
         this.abortSignal.addEventListener(
@@ -639,16 +673,125 @@ export class CDCStream {
         this.metrics.getCounter(ReplicationMetric.ROWS_REPLICATED).add(1);
       },
       onCommit: async (lsn: string, transactionCount: number) => {
-        await batch.commit(lsn);
+        const { checkpointBlocked } = await batch.commit(lsn, {
+          oldestUncommittedChange: this.replicationLag.oldestUncommittedChange
+        });
         this.metrics.getCounter(ReplicationMetric.TRANSACTIONS_REPLICATED).add(transactionCount);
-        this.isStartingReplication = false;
+        if (!checkpointBlocked) {
+          this.replicationLag.markCommitted();
+        }
       },
-      onSchemaChange: async () => {
-        // TODO: Handle schema changes
+      onSchemaChange: async (schemaChange: SchemaChange) => {
+        await this.handleSchemaChange(batch, schemaChange);
       }
     };
   }
 
+  async handleSchemaChange(batch: storage.BucketStorageBatch, change: SchemaChange): Promise<void> {
+    let actionedSchemaChange = true;
+
+    switch (change.type) {
+      case SchemaChangeType.TABLE_RENAME:
+        const fromTable = change.table!;
+        this.logger.info(
+          `Table ${fromTable.toQualifiedName()} has been renamed ${change.newTable ? `to [${change.newTable.name}].` : '.'}`
+        );
+
+        // Old table needs to be cleaned up
+        await batch.drop([fromTable.sourceTable]);
+        this.tableCache.delete(fromTable.objectId);
+
+        if (change.newTable) {
+          await this.handleCreateOrUpdateTable(batch, change.newTable, change.newCaptureInstance!);
+        }
+        break;
+      case SchemaChangeType.TABLE_CREATE:
+        await this.handleCreateOrUpdateTable(batch, change.newTable!, change.newCaptureInstance!);
+        break;
+      case SchemaChangeType.TABLE_COLUMN_CHANGES:
+        await this.handleColumnChanges(change.table!, change.newCaptureInstance!);
+        actionedSchemaChange = false;
+        break;
+      case SchemaChangeType.NEW_CAPTURE_INSTANCE:
+        this.logger.info(
+          `New CDC capture instance detected for table ${change.table!.toQualifiedName()}. Re-snapshotting table...`
+        );
+        await batch.drop([change.table!.sourceTable]);
+        this.tableCache.delete(change.table!.objectId);
+
+        await this.handleCreateOrUpdateTable(batch, change.table!.sourceTable, change.newCaptureInstance!);
+        break;
+      case SchemaChangeType.TABLE_DROP:
+        await batch.drop([change.table!.sourceTable]);
+        this.tableCache.delete(change.table!.objectId);
+        break;
+      case SchemaChangeType.MISSING_CAPTURE_INSTANCE:
+        // Stop replication for this table until CDC is re-enabled.
+        this.logger.warn(
+          `Table ${change.table!.toQualifiedName()} has been disabled for CDC. Re-enable CDC to continue replication.`
+        );
+        change.table!.clearCaptureInstance();
+        actionedSchemaChange = false;
+        break;
+      default:
+        throw new ReplicationAssertionError(`Unknown schema change type: ${change.type}`);
+    }
+
+    // Create a new checkpoint after the schema change
+    if (actionedSchemaChange) {
+      await createCheckpoint(this.connections);
+    }
+  }
+
+  private async handleCreateOrUpdateTable(
+    batch: storage.BucketStorageBatch,
+    table: Omit<SourceEntityDescriptor, 'replicaIdColumns'>,
+    captureInstance: CaptureInstance
+  ): Promise<void> {
+    const replicaIdColumns = await getReplicationIdentityColumns({
+      connectionManager: this.connections,
+      tableName: table.name,
+      schema: table.schema
+    });
+
+    await this.processTable(
+      batch,
+      {
+        name: table.name,
+        schema: table.schema,
+        objectId: table.objectId,
+        replicaIdColumns: replicaIdColumns.columns
+      },
+      captureInstance,
+      true
+    );
+  }
+
+  /**
+   * There is very little that can be automatically done to handle the column changes other than to warn the user about the schema drift.
+   *
+   * Due to the way CDC works, users are prevented from making column schema changes that affect the replication identities of a table.
+   * If changes like that are required, CDC has to be disabled and re-enabled for the table. This would then be handled by the detection of the new
+   * capture instance.
+   * @param table
+   * @param captureInstance
+   */
+  private async handleColumnChanges(table: MSSQLSourceTable, captureInstance: CaptureInstance): Promise<void> {
+    // Check there are any new pending schema changes
+    if (
+      table.captureInstance?.objectId === captureInstance.objectId &&
+      table.captureInstance?.pendingSchemaChanges.length === captureInstance.pendingSchemaChanges.length
+    ) {
+      return;
+    }
+
+    // New pending schema changes were detected - warn about those as well.
+    this.logger.warn(
+      `Schema drift detected for table ${table.toQualifiedName()}. To ensure consistency, disable and re-enable CDC for this table.\n Pending schema changes:\n ${captureInstance.pendingSchemaChanges.join(', \n')}`
+    );
+    table.captureInstance = captureInstance;
+  }
+
   /**
    * Convert CDC row data to SqliteRow format.
    * CDC rows include table columns plus CDC metadata columns (__$operation, __$start_lsn, etc.).
@@ -660,17 +803,8 @@ export class CDCStream {
     return this.syncRules.applyRowContext<never>(inputRow);
   }
 
-  async getReplicationLagMillis(): Promise<number | undefined> {
-    if (this.oldestUncommittedChange == null) {
-      if (this.isStartingReplication) {
-        // We don't have anything to compute replication lag with yet.
-        return undefined;
-      } else {
-        // We don't have any uncommitted changes, so replication is up-to-date.
-        return 0;
-      }
-    }
-    return Date.now() - this.oldestUncommittedChange.getTime();
+  getReplicationLagMillis(): number | undefined {
+    return this.replicationLag.getLagMillis();
   }
 
   private touch() {
@@ -678,4 +812,13 @@ export class CDCStream {
       this.logger.error(`Error touching probe`, e);
     });
   }
+
+  /**
+   *  Creates an update in the source database to ensure regular checkpoints via the CDC
+   */
+  public async keepAlive() {
+    if (!this.replicationLag.isStartingReplication && !this.stopped) {
+      await createCheckpoint(this.connections);
+    }
+  }
 }

package/src/replication/MSSQLConnectionManager.ts

@@ -1,8 +1,8 @@
 import { BaseObserver, logger } from '@powersync/lib-services-framework';
-import sql from 'mssql';
-import { NormalizedMSSQLConnectionConfig } from '../types/types.js';
 import { POWERSYNC_VERSION } from '@powersync/service-core';
+import sql from 'mssql';
 import { MSSQLParameter } from '../types/mssql-data-types.js';
+import { NormalizedMSSQLConnectionConfig } from '../types/types.js';
 import { addParameters } from '../utils/mssql.js';
 
 export const DEFAULT_SCHEMA = 'dbo';
@@ -87,13 +87,23 @@ export class MSSQLConnectionManager extends BaseObserver<MSSQLConnectionManagerL
 
   async execute(procedure: string, parameters?: MSSQLParameter[]): Promise<sql.IProcedureResult<any>> {
     await this.ensureConnected();
-    let request = this.pool.request();
-    if (parameters) {
-      if (parameters) {
-        request = addParameters(request, parameters);
+    for (let tries = 2; ; tries--) {
+      try {
+        logger.debug(`Executing procedure: ${procedure}`);
+        let request = this.pool.request();
+        if (parameters) {
+          if (parameters) {
+            request = addParameters(request, parameters);
+          }
+        }
+        return request.execute(procedure);
+      } catch (e) {
+        if (tries == 1) {
+          throw e;
+        }
+        logger.warn(`Error executing stored procedure: ${procedure}, retrying..`, e);
       }
     }
-    return request.execute(procedure);
   }
 
   async end(): Promise<void> {

package/src/replication/MSSQLConnectionManagerFactory.ts

@@ -1,7 +1,7 @@
 import { logger } from '@powersync/lib-services-framework';
+import sql from 'mssql';
 import { ResolvedMSSQLConnectionConfig } from '../types/types.js';
 import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
-import sql from 'mssql';
 
 export class MSSQLConnectionManagerFactory {
   private readonly connectionManagers: Set<MSSQLConnectionManager>;