@powersync/service-module-mssql 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/CHANGELOG.md +148 -6
  2. package/dist/api/MSSQLRouteAPIAdapter.js +26 -34
  3. package/dist/api/MSSQLRouteAPIAdapter.js.map +1 -1
  4. package/dist/common/MSSQLSourceTable.d.ts +12 -2
  5. package/dist/common/MSSQLSourceTable.js +30 -4
  6. package/dist/common/MSSQLSourceTable.js.map +1 -1
  7. package/dist/module/MSSQLModule.d.ts +0 -1
  8. package/dist/module/MSSQLModule.js +2 -2
  9. package/dist/module/MSSQLModule.js.map +1 -1
  10. package/dist/replication/CDCPoller.d.ts +13 -36
  11. package/dist/replication/CDCPoller.js +173 -129
  12. package/dist/replication/CDCPoller.js.map +1 -1
  13. package/dist/replication/CDCStream.d.ts +11 -14
  14. package/dist/replication/CDCStream.js +115 -106
  15. package/dist/replication/CDCStream.js.map +1 -1
  16. package/dist/replication/CaptureReconciler.d.ts +47 -0
  17. package/dist/replication/CaptureReconciler.js +155 -0
  18. package/dist/replication/CaptureReconciler.js.map +1 -0
  19. package/dist/replication/MSSQLConnectionManager.d.ts +1 -0
  20. package/dist/replication/MSSQLConnectionManager.js +19 -14
  21. package/dist/replication/MSSQLConnectionManager.js.map +1 -1
  22. package/dist/replication/MSSQLTableReconciliationContext.d.ts +21 -0
  23. package/dist/replication/MSSQLTableReconciliationContext.js +7 -0
  24. package/dist/replication/MSSQLTableReconciliationContext.js.map +1 -0
  25. package/dist/replication/SchemaChange.d.ts +67 -0
  26. package/dist/replication/SchemaChange.js +31 -0
  27. package/dist/replication/SchemaChange.js.map +1 -0
  28. package/dist/types/types.d.ts +13 -0
  29. package/dist/types/types.js +28 -2
  30. package/dist/types/types.js.map +1 -1
  31. package/dist/utils/mssql.js +1 -1
  32. package/dist/utils/schema.d.ts +14 -1
  33. package/dist/utils/schema.js +40 -45
  34. package/dist/utils/schema.js.map +1 -1
  35. package/package.json +9 -9
  36. package/src/api/MSSQLRouteAPIAdapter.ts +27 -33
  37. package/src/common/MSSQLSourceTable.ts +32 -4
  38. package/src/module/MSSQLModule.ts +2 -3
  39. package/src/replication/CDCPoller.ts +206 -162
  40. package/src/replication/CDCStream.ts +148 -154
  41. package/src/replication/CaptureReconciler.ts +186 -0
  42. package/src/replication/MSSQLConnectionManager.ts +23 -14
  43. package/src/replication/MSSQLTableReconciliationContext.ts +29 -0
  44. package/src/replication/SchemaChange.ts +79 -0
  45. package/src/types/types.ts +47 -2
  46. package/src/utils/mssql.ts +1 -1
  47. package/src/utils/schema.ts +57 -58
  48. package/test/src/CDCPoller.test.ts +382 -0
  49. package/test/src/CDCStream.test.ts +6 -2
  50. package/test/src/CDCStreamTestContext.ts +11 -4
  51. package/test/src/CaptureReconciler.test.ts +171 -0
  52. package/test/src/MSSQLSourceTable.test.ts +30 -0
  53. package/test/src/config.test.ts +71 -0
  54. package/test/src/schema-changes.test.ts +513 -249
  55. package/test/src/schema.test.ts +17 -0
  56. package/test/src/util.ts +70 -6
  57. package/tsconfig.tsbuildinfo +1 -1
@@ -1,5 +1,6 @@
1
1
  import { SourceEntityDescriptor, SourceTable } from '@powersync/service-core';
2
2
  import { ServiceAssertionError } from '@powersync/service-errors';
3
+ import { readCaptureMetadata } from '../replication/CaptureReconciler.js';
3
4
  import { toQualifiedTableName } from '../utils/mssql.js';
4
5
  import { CaptureInstance } from './CaptureInstance.js';
5
6
 
@@ -26,9 +27,15 @@ export class MSSQLSourceTable {
26
27
 
27
28
  public readonly ref: SourceEntityDescriptor;
28
29
 
30
+ /**
31
+ * Decoded once, since the persisted metadata only changes when a SourceTable is replaced.
32
+ */
33
+ private captureObjectId: number | null;
34
+
29
35
  constructor(ref: SourceEntityDescriptor, sourceTables: SourceTable[]) {
30
36
  this.sourceTables = sourceTables;
31
37
  this.ref = ref;
38
+ this.captureObjectId = readPinnedCaptureObjectId(sourceTables);
32
39
  }
33
40
 
34
41
  updateSourceTable(updated: SourceTable): void {
@@ -37,6 +44,7 @@ export class MSSQLSourceTable {
37
44
  throw new ServiceAssertionError(`No SourceTable found for table: ${updated.id}`);
38
45
  }
39
46
  this.sourceTables[index] = updated;
47
+ this.captureObjectId = readPinnedCaptureObjectId(this.sourceTables);
40
48
  }
41
49
 
42
50
  getReplicatedSourceTables(): SourceTable[] {
@@ -47,12 +55,19 @@ export class MSSQLSourceTable {
47
55
  return this.captureInstance !== null;
48
56
  }
49
57
 
50
- setCaptureInstance(captureInstance: CaptureInstance) {
51
- this.captureInstance = captureInstance;
58
+ /**
59
+ * Persisted capture-table object id, or null for a legacy binding.
60
+ */
61
+ get pinnedCaptureObjectId(): number | null {
62
+ return this.captureObjectId;
52
63
  }
53
64
 
54
- clearCaptureInstance() {
55
- this.captureInstance = null;
65
+ /**
66
+ * Bind the available capture instance matching this table's persisted identity.
67
+ */
68
+ setCaptureInstance(availableInstances: readonly CaptureInstance[]): void {
69
+ this.captureInstance =
70
+ availableInstances.find((instance) => instance.objectId === this.pinnedCaptureObjectId) ?? null;
56
71
  }
57
72
 
58
73
  get allChangesFunction() {
@@ -84,3 +99,16 @@ export class MSSQLSourceTable {
84
99
  return toQualifiedTableName(this.ref.schema, this.ref.name);
85
100
  }
86
101
  }
102
+
103
+ /**
104
+ * The pinned capture-table object id shared by these records, or null for a legacy binding.
105
+ */
106
+ function readPinnedCaptureObjectId(sourceTables: SourceTable[]): number | null {
107
+ for (const sourceTable of sourceTables) {
108
+ const metadata = readCaptureMetadata(sourceTable.sourceMetadata);
109
+ if (metadata != null) {
110
+ return metadata.captureTableObjectId;
111
+ }
112
+ }
113
+ return null;
114
+ }
@@ -23,8 +23,6 @@ export class MSSQLModule extends replication.ReplicationModule<types.MSSQLConnec
23
23
  });
24
24
  }
25
25
 
26
- async onInitialized(context: system.ServiceContextContainer): Promise<void> {}
27
-
28
26
  protected createRouteAPIAdapter(): api.RouteAPI {
29
27
  return new MSSQLRouteAPIAdapter(this.resolveConfig(this.decodedConfig!));
30
28
  }
@@ -41,7 +39,8 @@ export class MSSQLModule extends replication.ReplicationModule<types.MSSQLConnec
41
39
  metricsEngine: context.metricsEngine,
42
40
  connectionFactory: connectionFactory,
43
41
  rateLimiter: new MSSQLErrorRateLimiter(),
44
- additionalConfig: normalisedConfig.additionalConfig
42
+ additionalConfig: normalisedConfig.additionalConfig,
43
+ heartbeatIntervalSeconds: normalisedConfig.heartbeat_interval_seconds
45
44
  });
46
45
  }
47
46
 
@@ -5,48 +5,40 @@ import {
5
5
  Logger,
6
6
  ReplicationAssertionError
7
7
  } from '@powersync/lib-services-framework';
8
- import { TablePattern } from '@powersync/service-sync-rules';
9
8
  import sql from 'mssql';
10
9
  import timers from 'timers/promises';
11
- import { CaptureInstance } from '../common/CaptureInstance.js';
12
10
  import { LSN } from '../common/LSN.js';
13
11
  import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
14
12
  import { AdditionalConfig } from '../types/types.js';
15
13
  import { isDeadlockError } from '../utils/deadlock.js';
16
- import { CaptureInstanceDetails, getCaptureInstances, incrementLSN, toQualifiedTableName } from '../utils/mssql.js';
17
- import { SourceTableChangeRef, tableExists } from '../utils/schema.js';
14
+ import { CaptureInstanceDetails, getCaptureInstances, incrementLSN } from '../utils/mssql.js';
15
+ import { tableExists } from '../utils/schema.js';
16
+ import { CaptureInstanceMissingError } from './CaptureReconciler.js';
18
17
  import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
18
+ import { SchemaChange, SchemaChangeType } from './SchemaChange.js';
19
19
 
20
- enum Operation {
21
- DELETE = 1,
22
- INSERT = 2,
23
- UPDATE_BEFORE = 3,
24
- UPDATE_AFTER = 4
20
+ enum LogicalChangeType {
21
+ INSERT = 'INSERT',
22
+ DELETE = 'DELETE',
23
+ UPDATE = 'UPDATE',
24
+ DEFERRED_UPDATE = 'DEFERRED UPDATE'
25
25
  }
26
26
 
27
- export enum SchemaChangeType {
28
- TABLE_RENAME = 'table_rename',
29
- TABLE_DROP = 'table_drop',
30
- TABLE_CREATE = 'table_create',
31
- TABLE_COLUMN_CHANGES = 'table_column_changes',
32
- NEW_CAPTURE_INSTANCE = 'new_capture_instance',
33
- MISSING_CAPTURE_INSTANCE = 'missing_capture_instance'
34
- }
35
-
36
- export interface SchemaChange {
37
- type: SchemaChangeType;
38
- /**
39
- * The table that the schema change applies to. Populated for table drops, renames, new capture instances, and DDL changes.
40
- */
41
- table?: MSSQLSourceTable;
27
+ /**
28
+ * One logical change to a single row, made up of the one or two CDC rows that describe it.
29
+ */
30
+ interface LogicalChange {
31
+ transactionLSN: LSN;
32
+ type: LogicalChangeType;
42
33
  /**
43
- * Populated for new tables or renames, but only if the new table matches a sync config source table.
34
+ * Inserts and Deletes resolve to 1 row.
35
+ * Updates resolve to 2 rows, the row values before and after the update: [rowBefore, rowAfter].
44
36
  */
45
- newTable?: SourceTableChangeRef;
46
-
47
- newCaptureInstance?: CaptureInstance;
37
+ rows: any[];
48
38
  }
49
39
 
40
+ export const DEFAULT_SCHEMA_CHECK_INTERVAL_MS = 60_000;
41
+
50
42
  export interface CDCEventHandler {
51
43
  onInsert: (row: any, table: MSSQLSourceTable, columns: sql.IColumnMetadata) => Promise<void>;
52
44
  onUpdate: (rowAfter: any, rowBefore: any, table: MSSQLSourceTable, columns: sql.IColumnMetadata) => Promise<void>;
@@ -55,17 +47,11 @@ export interface CDCEventHandler {
55
47
  onSchemaChange: (change: SchemaChange) => Promise<void>;
56
48
  }
57
49
 
58
- export const DEFAULT_SCHEMA_CHECK_INTERVAL_MS = 60_000;
59
-
60
50
  export interface CDCPollerOptions {
61
51
  connectionManager: MSSQLConnectionManager;
62
52
  eventHandler: CDCEventHandler;
63
53
  /** CDC enabled source tables from the sync config to replicate */
64
54
  getReplicatedTables: () => MSSQLSourceTable[];
65
- /** All table patterns from the sync config. Can contain tables that need to be replicated
66
- * but do not yet have CDC enabled
67
- */
68
- sourceTables: TablePattern[];
69
55
  startLSN: LSN;
70
56
  logger?: Logger;
71
57
  additionalConfig: AdditionalConfig;
@@ -78,27 +64,30 @@ export interface CDCPollerOptions {
78
64
  }
79
65
 
80
66
  /**
67
+ * Polls SQL Server CDC change tables for changes at a configurable interval.
81
68
  *
69
+ * Processes changes in commit order, groups CDC rows into logical insert,
70
+ * update, and delete operations. It only commits once all the operations recorded in a polling cycle have been processed.
71
+ * Periodically runs checks to detect schema and capture-instance changes.
82
72
  */
83
73
  export class CDCPoller {
84
74
  private connectionManager: MSSQLConnectionManager;
85
75
  private eventHandler: CDCEventHandler;
86
76
  private currentLSN: LSN;
87
77
  private logger: Logger;
88
- private listenerError: Error | null;
89
78
  private captureInstances: Map<number, CaptureInstanceDetails>;
90
79
 
80
+ private pollingError: Error | null = null;
91
81
  private isStopped: boolean = false;
92
82
  private isStopping: boolean = false;
93
83
  private isPolling: boolean = false;
94
84
  private lastSchemaCheckTime: number = 0;
95
85
 
96
- constructor(public options: CDCPollerOptions) {
86
+ constructor(private options: CDCPollerOptions) {
97
87
  this.logger = options.logger ?? defaultLogger;
98
88
  this.connectionManager = options.connectionManager;
99
89
  this.eventHandler = options.eventHandler;
100
90
  this.currentLSN = options.startLSN;
101
- this.listenerError = null;
102
91
  this.captureInstances = new Map<number, CaptureInstanceDetails>();
103
92
  }
104
93
 
@@ -174,7 +163,7 @@ export class CDCPoller {
174
163
  }
175
164
 
176
165
  // Non-recoverable errors
177
- this.listenerError = error as Error;
166
+ this.pollingError = error as Error;
178
167
  this.logger.error('Error during CDC polling:', error);
179
168
  this.stop();
180
169
  }
@@ -182,9 +171,9 @@ export class CDCPoller {
182
171
  }
183
172
  }
184
173
 
185
- if (this.listenerError) {
186
- this.logger.error('CDC polling was stopped due to an error:', this.listenerError);
187
- throw this.listenerError;
174
+ if (this.pollingError) {
175
+ this.logger.error('CDC polling was stopped due to an error:', this.pollingError);
176
+ throw this.pollingError;
188
177
  }
189
178
 
190
179
  this.logger.info(`CDC polling stopped...`);
@@ -218,20 +207,21 @@ export class CDCPoller {
218
207
 
219
208
  this.logger.info(`Polling bounds are ${startLSN} -> ${endLSN} spanning ${results.length} transaction(s).`);
220
209
 
221
- let transactionCount = 0;
210
+ // We poll for batch size transactions, but these include transactions not applicable to our Source Tables.
211
+ // A single transaction can also span several Source Tables, so collect the distinct transaction LSNs
212
+ // that produced changes rather than counting per table, which would either double count the
213
+ // transactions spanning tables or miss the transactions applicable to only one of them.
214
+ let transactionLSNs = new Set<string>();
222
215
  this.logger.debug(
223
216
  `Currently replicating tables: ${this.replicatedTables.map((table) => table.toQualifiedName()).join(', ')}`
224
217
  );
225
218
  for (const table of this.replicatedTables) {
226
219
  if (table.enabledForCDC()) {
227
- const tableTransactionCount = await this.pollTable(table, { startLSN, endLSN });
228
- // We poll for batch size transactions, but these include transactions not applicable to our Source Tables.
229
- // Each Source Table may or may not have transactions that are applicable to it, so just keep track of the highest number of transactions processed for any Source Table.
230
- if (tableTransactionCount > transactionCount) {
231
- transactionCount = tableTransactionCount;
232
- }
220
+ const transactions = await this.pollTable(table, { startLSN, endLSN });
221
+ transactions.forEach((t) => transactionLSNs.add(t));
233
222
  }
234
223
  }
224
+ const transactionCount = transactionLSNs.size;
235
225
 
236
226
  this.logger.info(
237
227
  `Processed ${results.length} transaction(s), including ${transactionCount} Source Table transaction(s). Commited LSN: ${endLSN.toString()}`
@@ -248,68 +238,77 @@ export class CDCPoller {
248
238
  }
249
239
  }
250
240
 
251
- private async pollTable(table: MSSQLSourceTable, bounds: { startLSN: LSN; endLSN: LSN }): Promise<number> {
252
- // Ensure that the startLSN is not before the minimum LSN for the table
253
- const minLSN = this.captureInstances.get(table.objectId)!.instances[0].minLSN;
241
+ /**
242
+ * Processes the changes this table has within the given bounds, and returns the LSNs of the
243
+ * transactions those changes belong to. The LSNs are returned in their string form so that the
244
+ * caller can deduplicate them across tables by value.
245
+ */
246
+ private async pollTable(table: MSSQLSourceTable, bounds: { startLSN: LSN; endLSN: LSN }): Promise<Set<string>> {
247
+ const transactionLSNs = new Set<string>();
248
+
249
+ // CDC cleanup can advance minLSN while the capture-table identity remains unchanged, so use
250
+ // the latest metadata loaded by the schema check rather than the instance bound at startup.
251
+ const availableInstances = this.captureInstances.get(table.objectId)?.instances ?? [];
252
+ table.setCaptureInstance(availableInstances);
253
+ const boundInstance = table.captureInstance;
254
+ if (boundInstance == null) {
255
+ // The pinned instance can be dropped between schema checks.
256
+ throw new CaptureInstanceMissingError(
257
+ `The CDC capture instance for table ${table.toQualifiedName()} (pinned to object id ` +
258
+ `${table.pinnedCaptureObjectId}) is no longer available. Deploy a new sync config to replicate ` +
259
+ `this table against an available capture instance.`
260
+ );
261
+ }
262
+ const minLSN = boundInstance.minLSN;
254
263
  if (minLSN > bounds.endLSN) {
255
- return 0;
264
+ return transactionLSNs;
256
265
  } else if (minLSN >= bounds.startLSN) {
257
266
  bounds.startLSN = minLSN;
258
267
  }
259
268
 
260
269
  try {
261
- const { recordset: results } = await this.connectionManager.query(
262
- `
263
- SELECT * FROM ${table.allChangesFunction}(@from_lsn, @to_lsn, 'all update old') ORDER BY __$start_lsn, __$seqval
264
- `,
265
- [
266
- { name: 'from_lsn', type: sql.VarBinary, value: bounds.startLSN.toBinary() },
267
- { name: 'to_lsn', type: sql.VarBinary, value: bounds.endLSN.toBinary() }
268
- ]
269
- );
270
+ const request = await this.connectionManager.createRequest();
271
+ request.input('from_lsn', sql.VarBinary, bounds.startLSN.toBinary());
272
+ request.input('to_lsn', sql.VarBinary, bounds.endLSN.toBinary());
270
273
 
271
- let transactionCount = 0;
272
- let updateBefore: any = null;
273
- let lastTransactionLSN: LSN | null = null;
274
- for (const row of results) {
275
- const transactionLSN = LSN.fromBinary(row.__$start_lsn);
276
- switch (row.__$operation) {
277
- case Operation.DELETE:
278
- await this.eventHandler.onDelete(row, table, results.columns);
279
- this.logger.info(`Processed DELETE row LSN: ${transactionLSN}`);
280
- break;
281
- case Operation.INSERT:
282
- await this.eventHandler.onInsert(row, table, results.columns);
283
- this.logger.info(`Processed INSERT row LSN: ${transactionLSN}`);
274
+ let columns: sql.IColumnMetadata | null = null;
275
+ request.on('recordset', (recordsetColumns) => {
276
+ columns = recordsetColumns;
277
+ });
278
+ const stream = request.toReadableStream();
279
+ request.query(`
280
+ SELECT * FROM ${table.allChangesFunction}(@from_lsn, @to_lsn, 'all update old') ORDER BY __$start_lsn, __$seqval, __$operation
281
+ `);
282
+
283
+ for await (const { transactionLSN, type, rows } of groupLogicalChanges(stream, table)) {
284
+ if (columns == null) {
285
+ throw new ReplicationAssertionError(
286
+ `Missing CDC column metadata while polling for updates for table ${table.toQualifiedName()}.`
287
+ );
288
+ }
289
+ switch (type) {
290
+ case LogicalChangeType.DELETE:
291
+ await this.eventHandler.onDelete(rows[0], table, columns);
284
292
  break;
285
- case Operation.UPDATE_BEFORE:
286
- updateBefore = row;
287
- this.logger.debug(`Processed UPDATE, before row LSN: ${transactionLSN}`);
293
+ case LogicalChangeType.INSERT:
294
+ await this.eventHandler.onInsert(rows[0], table, columns);
288
295
  break;
289
- case Operation.UPDATE_AFTER:
290
- if (updateBefore === null) {
291
- throw new ReplicationAssertionError('Missing before image for update event.');
292
- }
293
- await this.eventHandler.onUpdate(row, updateBefore, table, results.columns);
294
- updateBefore = null;
295
- this.logger.info(`Processed UPDATE row LSN: ${transactionLSN}`);
296
+ case LogicalChangeType.UPDATE:
297
+ case LogicalChangeType.DEFERRED_UPDATE:
298
+ const [rowBefore, rowAfter] = rows;
299
+ await this.eventHandler.onUpdate(rowAfter, rowBefore, table, columns);
296
300
  break;
297
- default:
298
- this.logger.warn(`Unknown operation type [${row.__$operation}] encountered in CDC changes.`);
299
301
  }
302
+ this.logger.info(`Processed ${type}. Transaction LSN: ${transactionLSN}`);
300
303
 
301
- // Increment transaction count when we encounter a new transaction LSN (except for UPDATE_BEFORE rows)
302
- if (transactionLSN != lastTransactionLSN) {
303
- lastTransactionLSN = transactionLSN;
304
- if (row.__$operation !== Operation.UPDATE_BEFORE) {
305
- transactionCount++;
306
- }
307
- }
304
+ transactionLSNs.add(transactionLSN.toString());
308
305
  }
309
306
 
310
- return transactionCount;
307
+ return transactionLSNs;
311
308
  } catch (error) {
312
- // This Covers both deleted tables and capture instances
309
+ // This Covers both deleted tables and capture instances. Unlike the check above, this cannot
310
+ // tell the two apart, so it stays recoverable: the forced schema check classifies it as a
311
+ // dropped table or a missing capture instance and fails with the matching error.
313
312
  if (error.message.includes(`Invalid object name`)) {
314
313
  throw new DatabaseQueryError(
315
314
  ErrorCode.PSYNC_S1601,
@@ -325,33 +324,14 @@ export class CDCPoller {
325
324
  return Date.now() - this.lastSchemaCheckTime >= this.schemaCheckIntervalMs;
326
325
  }
327
326
 
328
- /**
329
- * Checks the given table for pending schema changes that can lead to inconsistencies in the replicated data if not handled.
330
- * Returns the SchemaChange if any are found, null otherwise.
331
- */
332
327
  private async checkForSchemaChanges(): Promise<SchemaChange[]> {
333
328
  const schemaChanges: SchemaChange[] = [];
334
329
 
335
- const newTables = this.checkForNewTables();
336
- for (const table of newTables) {
337
- this.logger.info(
338
- `New table ${toQualifiedTableName(table.sourceTable.schema, table.sourceTable.name)} matching the sync config has been created. Handling schema change...`
339
- );
340
- schemaChanges.push({
341
- type: SchemaChangeType.TABLE_CREATE,
342
- newTable: {
343
- name: table.sourceTable.name,
344
- schema: table.sourceTable.schema,
345
- objectId: table.sourceTable.objectId
346
- },
347
- newCaptureInstance: table.instances[0]
348
- });
349
- }
350
-
330
+ // Exact table names keep the replicated set fixed until the next deploy.
351
331
  for (const table of this.replicatedTables) {
352
332
  const exists = await tableExists(table.objectId, this.connectionManager);
353
333
  if (!exists) {
354
- this.logger.info(`Table ${table.toQualifiedName()} has been dropped. Handling schema change...`);
334
+ this.logger.info(`Table ${table.toQualifiedName()} has been dropped.`);
355
335
  schemaChanges.push({
356
336
  type: SchemaChangeType.TABLE_DROP,
357
337
  table
@@ -361,85 +341,149 @@ export class CDCPoller {
361
341
 
362
342
  const captureInstanceDetails = this.captureInstances.get(table.objectId);
363
343
  if (!captureInstanceDetails) {
364
- if (table.enabledForCDC()) {
365
- // Table had a capture instance but no longer does.
366
- schemaChanges.push({
367
- type: SchemaChangeType.MISSING_CAPTURE_INSTANCE,
368
- table
369
- });
370
- }
344
+ // The table had a capture instance when the stream started, but no longer does.
345
+ schemaChanges.push({
346
+ type: SchemaChangeType.MISSING_CAPTURE_INSTANCE,
347
+ table
348
+ });
371
349
  continue;
372
350
  }
373
351
 
374
352
  const latestCaptureInstance = captureInstanceDetails.instances[0];
375
- // If the table is not enabled for CDC or the capture instance is different, we need to re-snapshot the source table
376
- if (!table.enabledForCDC() || table.captureInstance!.objectId !== latestCaptureInstance.objectId) {
353
+
354
+ table.setCaptureInstance(captureInstanceDetails.instances);
355
+ const boundInstance = table.captureInstance;
356
+ if (boundInstance == null) {
357
+ // Include the replacement so the error can suggest the right recovery step.
358
+ schemaChanges.push({
359
+ type: SchemaChangeType.MISSING_CAPTURE_INSTANCE,
360
+ table,
361
+ replacementInstance: latestCaptureInstance
362
+ });
363
+ continue;
364
+ }
365
+ if (latestCaptureInstance.objectId !== boundInstance.objectId) {
366
+ // Keep checking for rename and column changes against the pinned instance.
377
367
  schemaChanges.push({
378
368
  type: SchemaChangeType.NEW_CAPTURE_INSTANCE,
379
369
  table,
380
370
  newCaptureInstance: latestCaptureInstance
381
371
  });
382
- continue;
383
372
  }
384
373
 
385
- // One of the replicated tables has been renamed
374
+ // The new name is only used in the error message.
386
375
  if (table.ref.name !== captureInstanceDetails.sourceTable.name) {
387
- const newTable = this.tableMatchesSyncRules(
388
- captureInstanceDetails.sourceTable.schema,
389
- captureInstanceDetails.sourceTable.name
390
- )
391
- ? {
392
- name: captureInstanceDetails.sourceTable.name,
393
- schema: captureInstanceDetails.sourceTable.schema,
394
- objectId: captureInstanceDetails.sourceTable.objectId
395
- }
396
- : undefined;
397
-
398
376
  schemaChanges.push({
399
377
  type: SchemaChangeType.TABLE_RENAME,
400
378
  table,
401
- newTable,
402
- newCaptureInstance: latestCaptureInstance
379
+ newTable: {
380
+ name: captureInstanceDetails.sourceTable.name,
381
+ schema: captureInstanceDetails.sourceTable.schema,
382
+ objectId: captureInstanceDetails.sourceTable.objectId
383
+ }
403
384
  });
404
385
  continue;
405
386
  }
406
387
 
407
- if (latestCaptureInstance.pendingSchemaChanges.length > 0) {
388
+ // Report drift against the capture instance this stream uses.
389
+ if (boundInstance.pendingSchemaChanges.length > 0) {
408
390
  schemaChanges.push({
409
391
  type: SchemaChangeType.TABLE_COLUMN_CHANGES,
410
392
  table,
411
- newCaptureInstance: latestCaptureInstance
393
+ captureInstance: boundInstance
412
394
  });
413
395
  }
414
396
  }
415
397
 
416
398
  return schemaChanges;
417
399
  }
400
+ }
418
401
 
419
- private checkForNewTables(): CaptureInstanceDetails[] {
420
- const newTables: CaptureInstanceDetails[] = [];
421
- for (const [objectId, captureInstanceDetails] of this.captureInstances.entries()) {
422
- // If a source table is not in the replicated tables array, but a capture instance exists for it, it is potentially a new table to replicate.
423
- if (!this.replicatedTables.some((table) => table.objectId === objectId)) {
424
- // Check if the new table matches any of the sync config source tables.
425
- if (
426
- this.tableMatchesSyncRules(captureInstanceDetails.sourceTable.schema, captureInstanceDetails.sourceTable.name)
427
- ) {
428
- newTables.push(captureInstanceDetails);
429
- }
430
- }
402
+ /**
403
+ * Groups CDC change rows into the logical row changes they describe.
404
+ *
405
+ * SQL Server records a logical change as either one row (a plain insert or delete) or two rows that share
406
+ * a `__$seqval`. `__$seqval` represents the ordering of the changes to a row within a transaction.
407
+ * CDC operations that can share a `__$seqval` are:
408
+ * - The before and after operations of an in-place update
409
+ * - The delete and insert operations of a deferred update.
410
+ *
411
+ * This method groups and emits rows in the same transaction based on their `__$seqval`
412
+ */
413
+ async function* groupLogicalChanges(rows: AsyncIterable<any>, table: MSSQLSourceTable): AsyncGenerator<LogicalChange> {
414
+ interface PendingGroup {
415
+ transactionLSN: Buffer;
416
+ sequence: Buffer;
417
+ rows: any[];
418
+ }
419
+
420
+ let current: PendingGroup | null = null;
421
+
422
+ for await (const row of rows) {
423
+ const transactionLSN: Buffer = row.__$start_lsn;
424
+ const sequence: Buffer = row.__$seqval;
425
+
426
+ if (current && !(transactionLSN.equals(current.transactionLSN) && sequence.equals(current.sequence))) {
427
+ yield toLogicalChange(current.rows, current.transactionLSN, table);
428
+ current = null;
431
429
  }
430
+ current ??= {
431
+ transactionLSN,
432
+ sequence,
433
+ rows: []
434
+ };
435
+ current.rows.push(row);
436
+ }
432
437
 
433
- return newTables;
438
+ if (current) {
439
+ yield toLogicalChange(current.rows, current.transactionLSN, table);
434
440
  }
441
+ }
442
+
443
+ function toLogicalChange(rows: any[], startLSN: Buffer, table: MSSQLSourceTable): LogicalChange {
444
+ const transactionLSN = LSN.fromBinary(startLSN);
445
+ return {
446
+ transactionLSN,
447
+ type: resolveLogicalChangeType(rows, transactionLSN, table),
448
+ rows: rows
449
+ };
450
+ }
435
451
 
436
- private tableMatchesSyncRules(schema: string, tableName: string): boolean {
437
- return this.options.sourceTables.some((tablePattern) =>
438
- tablePattern.matches({
439
- connectionTag: this.connectionManager.connectionTag,
440
- schema: schema,
441
- name: tableName
442
- })
452
+ function resolveLogicalChangeType(orderedRows: any[], transactionLSN: LSN, table: MSSQLSourceTable): LogicalChangeType {
453
+ // This matches the actual CDC operation codes:https://learn.microsoft.com/en-us/sql/relational-databases/system-functions/cdc-fn-cdc-get-all-changes-capture-instance-transact-sql?view=sql-server-ver17#table-returned
454
+ enum Operation {
455
+ DELETE = 1,
456
+ INSERT = 2,
457
+ UPDATE_BEFORE = 3,
458
+ UPDATE_AFTER = 4
459
+ }
460
+
461
+ if (orderedRows.length === 1) {
462
+ const operation = orderedRows[0].__$operation;
463
+
464
+ if (operation === Operation.INSERT) {
465
+ return LogicalChangeType.INSERT;
466
+ } else if (operation === Operation.DELETE) {
467
+ return LogicalChangeType.DELETE;
468
+ } else {
469
+ throw new ReplicationAssertionError(
470
+ `Unrecognized operation: ${operation} for table ${table.toQualifiedName()} in transaction LSN ${transactionLSN}.`
471
+ );
472
+ }
473
+ } else if (orderedRows.length === 2) {
474
+ const [first, second] = orderedRows;
475
+ if (first.__$operation === Operation.UPDATE_BEFORE && second.__$operation === Operation.UPDATE_AFTER) {
476
+ return LogicalChangeType.UPDATE;
477
+ } else if (first.__$operation === Operation.DELETE && second.__$operation === Operation.INSERT) {
478
+ return LogicalChangeType.DEFERRED_UPDATE;
479
+ }
480
+
481
+ throw new ReplicationAssertionError(
482
+ `Unexpected CDC operations [${first.__$operation}, ${second.__$operation}] for a single logical change on table ${table.toQualifiedName()} in transaction LSN ${transactionLSN}.`
443
483
  );
444
484
  }
485
+
486
+ throw new ReplicationAssertionError(
487
+ `Unexpected number of CDC operations [${orderedRows.length}] for a single logical change on table ${table.toQualifiedName()} in transaction LSN ${transactionLSN}.`
488
+ );
445
489
  }