@powersync/service-module-mssql 0.0.0-dev-20260803144812 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +130 -6
- package/dist/api/MSSQLRouteAPIAdapter.js +26 -34
- package/dist/api/MSSQLRouteAPIAdapter.js.map +1 -1
- package/dist/common/MSSQLSourceTable.d.ts +12 -2
- package/dist/common/MSSQLSourceTable.js +30 -4
- package/dist/common/MSSQLSourceTable.js.map +1 -1
- package/dist/module/MSSQLModule.js +2 -1
- package/dist/module/MSSQLModule.js.map +1 -1
- package/dist/replication/CDCPoller.d.ts +13 -36
- package/dist/replication/CDCPoller.js +173 -129
- package/dist/replication/CDCPoller.js.map +1 -1
- package/dist/replication/CDCStream.d.ts +11 -14
- package/dist/replication/CDCStream.js +115 -106
- package/dist/replication/CDCStream.js.map +1 -1
- package/dist/replication/CaptureReconciler.d.ts +47 -0
- package/dist/replication/CaptureReconciler.js +155 -0
- package/dist/replication/CaptureReconciler.js.map +1 -0
- package/dist/replication/MSSQLConnectionManager.d.ts +1 -0
- package/dist/replication/MSSQLConnectionManager.js +19 -14
- package/dist/replication/MSSQLConnectionManager.js.map +1 -1
- package/dist/replication/MSSQLTableReconciliationContext.d.ts +21 -0
- package/dist/replication/MSSQLTableReconciliationContext.js +7 -0
- package/dist/replication/MSSQLTableReconciliationContext.js.map +1 -0
- package/dist/replication/SchemaChange.d.ts +67 -0
- package/dist/replication/SchemaChange.js +31 -0
- package/dist/replication/SchemaChange.js.map +1 -0
- package/dist/types/types.d.ts +13 -0
- package/dist/types/types.js +28 -2
- package/dist/types/types.js.map +1 -1
- package/dist/utils/mssql.js +1 -1
- package/dist/utils/schema.d.ts +14 -1
- package/dist/utils/schema.js +40 -45
- package/dist/utils/schema.js.map +1 -1
- package/package.json +9 -9
- package/src/api/MSSQLRouteAPIAdapter.ts +27 -33
- package/src/common/MSSQLSourceTable.ts +32 -4
- package/src/module/MSSQLModule.ts +2 -1
- package/src/replication/CDCPoller.ts +206 -162
- package/src/replication/CDCStream.ts +148 -154
- package/src/replication/CaptureReconciler.ts +186 -0
- package/src/replication/MSSQLConnectionManager.ts +23 -14
- package/src/replication/MSSQLTableReconciliationContext.ts +29 -0
- package/src/replication/SchemaChange.ts +79 -0
- package/src/types/types.ts +47 -2
- package/src/utils/mssql.ts +1 -1
- package/src/utils/schema.ts +57 -58
- package/test/src/CDCPoller.test.ts +382 -0
- package/test/src/CDCStream.test.ts +6 -2
- package/test/src/CDCStreamTestContext.ts +7 -0
- package/test/src/CaptureReconciler.test.ts +171 -0
- package/test/src/MSSQLSourceTable.test.ts +30 -0
- package/test/src/config.test.ts +71 -0
- package/test/src/schema-changes.test.ts +513 -249
- package/test/src/schema.test.ts +17 -0
- package/test/src/util.ts +70 -6
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { SourceEntityDescriptor, SourceTable } from '@powersync/service-core';
|
|
2
2
|
import { ServiceAssertionError } from '@powersync/service-errors';
|
|
3
|
+
import { readCaptureMetadata } from '../replication/CaptureReconciler.js';
|
|
3
4
|
import { toQualifiedTableName } from '../utils/mssql.js';
|
|
4
5
|
import { CaptureInstance } from './CaptureInstance.js';
|
|
5
6
|
|
|
@@ -26,9 +27,15 @@ export class MSSQLSourceTable {
|
|
|
26
27
|
|
|
27
28
|
public readonly ref: SourceEntityDescriptor;
|
|
28
29
|
|
|
30
|
+
/**
|
|
31
|
+
* Decoded once, since the persisted metadata only changes when a SourceTable is replaced.
|
|
32
|
+
*/
|
|
33
|
+
private captureObjectId: number | null;
|
|
34
|
+
|
|
29
35
|
constructor(ref: SourceEntityDescriptor, sourceTables: SourceTable[]) {
|
|
30
36
|
this.sourceTables = sourceTables;
|
|
31
37
|
this.ref = ref;
|
|
38
|
+
this.captureObjectId = readPinnedCaptureObjectId(sourceTables);
|
|
32
39
|
}
|
|
33
40
|
|
|
34
41
|
updateSourceTable(updated: SourceTable): void {
|
|
@@ -37,6 +44,7 @@ export class MSSQLSourceTable {
|
|
|
37
44
|
throw new ServiceAssertionError(`No SourceTable found for table: ${updated.id}`);
|
|
38
45
|
}
|
|
39
46
|
this.sourceTables[index] = updated;
|
|
47
|
+
this.captureObjectId = readPinnedCaptureObjectId(this.sourceTables);
|
|
40
48
|
}
|
|
41
49
|
|
|
42
50
|
getReplicatedSourceTables(): SourceTable[] {
|
|
@@ -47,12 +55,19 @@ export class MSSQLSourceTable {
|
|
|
47
55
|
return this.captureInstance !== null;
|
|
48
56
|
}
|
|
49
57
|
|
|
50
|
-
|
|
51
|
-
|
|
58
|
+
/**
|
|
59
|
+
* Persisted capture-table object id, or null for a legacy binding.
|
|
60
|
+
*/
|
|
61
|
+
get pinnedCaptureObjectId(): number | null {
|
|
62
|
+
return this.captureObjectId;
|
|
52
63
|
}
|
|
53
64
|
|
|
54
|
-
|
|
55
|
-
|
|
65
|
+
/**
|
|
66
|
+
* Bind the available capture instance matching this table's persisted identity.
|
|
67
|
+
*/
|
|
68
|
+
setCaptureInstance(availableInstances: readonly CaptureInstance[]): void {
|
|
69
|
+
this.captureInstance =
|
|
70
|
+
availableInstances.find((instance) => instance.objectId === this.pinnedCaptureObjectId) ?? null;
|
|
56
71
|
}
|
|
57
72
|
|
|
58
73
|
get allChangesFunction() {
|
|
@@ -84,3 +99,16 @@ export class MSSQLSourceTable {
|
|
|
84
99
|
return toQualifiedTableName(this.ref.schema, this.ref.name);
|
|
85
100
|
}
|
|
86
101
|
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* The pinned capture-table object id shared by these records, or null for a legacy binding.
|
|
105
|
+
*/
|
|
106
|
+
function readPinnedCaptureObjectId(sourceTables: SourceTable[]): number | null {
|
|
107
|
+
for (const sourceTable of sourceTables) {
|
|
108
|
+
const metadata = readCaptureMetadata(sourceTable.sourceMetadata);
|
|
109
|
+
if (metadata != null) {
|
|
110
|
+
return metadata.captureTableObjectId;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
@@ -41,7 +41,8 @@ export class MSSQLModule extends replication.ReplicationModule<types.MSSQLConnec
|
|
|
41
41
|
metricsEngine: context.metricsEngine,
|
|
42
42
|
connectionFactory: connectionFactory,
|
|
43
43
|
rateLimiter: new MSSQLErrorRateLimiter(),
|
|
44
|
-
additionalConfig: normalisedConfig.additionalConfig
|
|
44
|
+
additionalConfig: normalisedConfig.additionalConfig,
|
|
45
|
+
heartbeatIntervalSeconds: normalisedConfig.heartbeat_interval_seconds
|
|
45
46
|
});
|
|
46
47
|
}
|
|
47
48
|
|
|
@@ -5,48 +5,40 @@ import {
|
|
|
5
5
|
Logger,
|
|
6
6
|
ReplicationAssertionError
|
|
7
7
|
} from '@powersync/lib-services-framework';
|
|
8
|
-
import { TablePattern } from '@powersync/service-sync-rules';
|
|
9
8
|
import sql from 'mssql';
|
|
10
9
|
import timers from 'timers/promises';
|
|
11
|
-
import { CaptureInstance } from '../common/CaptureInstance.js';
|
|
12
10
|
import { LSN } from '../common/LSN.js';
|
|
13
11
|
import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
|
|
14
12
|
import { AdditionalConfig } from '../types/types.js';
|
|
15
13
|
import { isDeadlockError } from '../utils/deadlock.js';
|
|
16
|
-
import { CaptureInstanceDetails, getCaptureInstances, incrementLSN
|
|
17
|
-
import {
|
|
14
|
+
import { CaptureInstanceDetails, getCaptureInstances, incrementLSN } from '../utils/mssql.js';
|
|
15
|
+
import { tableExists } from '../utils/schema.js';
|
|
16
|
+
import { CaptureInstanceMissingError } from './CaptureReconciler.js';
|
|
18
17
|
import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
|
|
18
|
+
import { SchemaChange, SchemaChangeType } from './SchemaChange.js';
|
|
19
19
|
|
|
20
|
-
enum
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
20
|
+
enum LogicalChangeType {
|
|
21
|
+
INSERT = 'INSERT',
|
|
22
|
+
DELETE = 'DELETE',
|
|
23
|
+
UPDATE = 'UPDATE',
|
|
24
|
+
DEFERRED_UPDATE = 'DEFERRED UPDATE'
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
MISSING_CAPTURE_INSTANCE = 'missing_capture_instance'
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
export interface SchemaChange {
|
|
37
|
-
type: SchemaChangeType;
|
|
38
|
-
/**
|
|
39
|
-
* The table that the schema change applies to. Populated for table drops, renames, new capture instances, and DDL changes.
|
|
40
|
-
*/
|
|
41
|
-
table?: MSSQLSourceTable;
|
|
27
|
+
/**
|
|
28
|
+
* One logical change to a single row, made up of the one or two CDC rows that describe it.
|
|
29
|
+
*/
|
|
30
|
+
interface LogicalChange {
|
|
31
|
+
transactionLSN: LSN;
|
|
32
|
+
type: LogicalChangeType;
|
|
42
33
|
/**
|
|
43
|
-
*
|
|
34
|
+
* Inserts and Deletes resolve to 1 row.
|
|
35
|
+
* Updates resolve to 2 rows, the row values before and after the update: [rowBefore, rowAfter].
|
|
44
36
|
*/
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
newCaptureInstance?: CaptureInstance;
|
|
37
|
+
rows: any[];
|
|
48
38
|
}
|
|
49
39
|
|
|
40
|
+
export const DEFAULT_SCHEMA_CHECK_INTERVAL_MS = 60_000;
|
|
41
|
+
|
|
50
42
|
export interface CDCEventHandler {
|
|
51
43
|
onInsert: (row: any, table: MSSQLSourceTable, columns: sql.IColumnMetadata) => Promise<void>;
|
|
52
44
|
onUpdate: (rowAfter: any, rowBefore: any, table: MSSQLSourceTable, columns: sql.IColumnMetadata) => Promise<void>;
|
|
@@ -55,17 +47,11 @@ export interface CDCEventHandler {
|
|
|
55
47
|
onSchemaChange: (change: SchemaChange) => Promise<void>;
|
|
56
48
|
}
|
|
57
49
|
|
|
58
|
-
export const DEFAULT_SCHEMA_CHECK_INTERVAL_MS = 60_000;
|
|
59
|
-
|
|
60
50
|
export interface CDCPollerOptions {
|
|
61
51
|
connectionManager: MSSQLConnectionManager;
|
|
62
52
|
eventHandler: CDCEventHandler;
|
|
63
53
|
/** CDC enabled source tables from the sync config to replicate */
|
|
64
54
|
getReplicatedTables: () => MSSQLSourceTable[];
|
|
65
|
-
/** All table patterns from the sync config. Can contain tables that need to be replicated
|
|
66
|
-
* but do not yet have CDC enabled
|
|
67
|
-
*/
|
|
68
|
-
sourceTables: TablePattern[];
|
|
69
55
|
startLSN: LSN;
|
|
70
56
|
logger?: Logger;
|
|
71
57
|
additionalConfig: AdditionalConfig;
|
|
@@ -78,27 +64,30 @@ export interface CDCPollerOptions {
|
|
|
78
64
|
}
|
|
79
65
|
|
|
80
66
|
/**
|
|
67
|
+
* Polls SQL Server CDC change tables for changes at a configurable interval.
|
|
81
68
|
*
|
|
69
|
+
* Processes changes in commit order, groups CDC rows into logical insert,
|
|
70
|
+
* update, and delete operations. It only commits once all the operations recorded in a polling cycle have been processed.
|
|
71
|
+
* Periodically runs checks to detect schema and capture-instance changes.
|
|
82
72
|
*/
|
|
83
73
|
export class CDCPoller {
|
|
84
74
|
private connectionManager: MSSQLConnectionManager;
|
|
85
75
|
private eventHandler: CDCEventHandler;
|
|
86
76
|
private currentLSN: LSN;
|
|
87
77
|
private logger: Logger;
|
|
88
|
-
private listenerError: Error | null;
|
|
89
78
|
private captureInstances: Map<number, CaptureInstanceDetails>;
|
|
90
79
|
|
|
80
|
+
private pollingError: Error | null = null;
|
|
91
81
|
private isStopped: boolean = false;
|
|
92
82
|
private isStopping: boolean = false;
|
|
93
83
|
private isPolling: boolean = false;
|
|
94
84
|
private lastSchemaCheckTime: number = 0;
|
|
95
85
|
|
|
96
|
-
constructor(
|
|
86
|
+
constructor(private options: CDCPollerOptions) {
|
|
97
87
|
this.logger = options.logger ?? defaultLogger;
|
|
98
88
|
this.connectionManager = options.connectionManager;
|
|
99
89
|
this.eventHandler = options.eventHandler;
|
|
100
90
|
this.currentLSN = options.startLSN;
|
|
101
|
-
this.listenerError = null;
|
|
102
91
|
this.captureInstances = new Map<number, CaptureInstanceDetails>();
|
|
103
92
|
}
|
|
104
93
|
|
|
@@ -174,7 +163,7 @@ export class CDCPoller {
|
|
|
174
163
|
}
|
|
175
164
|
|
|
176
165
|
// Non-recoverable errors
|
|
177
|
-
this.
|
|
166
|
+
this.pollingError = error as Error;
|
|
178
167
|
this.logger.error('Error during CDC polling:', error);
|
|
179
168
|
this.stop();
|
|
180
169
|
}
|
|
@@ -182,9 +171,9 @@ export class CDCPoller {
|
|
|
182
171
|
}
|
|
183
172
|
}
|
|
184
173
|
|
|
185
|
-
if (this.
|
|
186
|
-
this.logger.error('CDC polling was stopped due to an error:', this.
|
|
187
|
-
throw this.
|
|
174
|
+
if (this.pollingError) {
|
|
175
|
+
this.logger.error('CDC polling was stopped due to an error:', this.pollingError);
|
|
176
|
+
throw this.pollingError;
|
|
188
177
|
}
|
|
189
178
|
|
|
190
179
|
this.logger.info(`CDC polling stopped...`);
|
|
@@ -218,20 +207,21 @@ export class CDCPoller {
|
|
|
218
207
|
|
|
219
208
|
this.logger.info(`Polling bounds are ${startLSN} -> ${endLSN} spanning ${results.length} transaction(s).`);
|
|
220
209
|
|
|
221
|
-
|
|
210
|
+
// We poll for batch size transactions, but these include transactions not applicable to our Source Tables.
|
|
211
|
+
// A single transaction can also span several Source Tables, so collect the distinct transaction LSNs
|
|
212
|
+
// that produced changes rather than counting per table, which would either double count the
|
|
213
|
+
// transactions spanning tables or miss the transactions applicable to only one of them.
|
|
214
|
+
let transactionLSNs = new Set<string>();
|
|
222
215
|
this.logger.debug(
|
|
223
216
|
`Currently replicating tables: ${this.replicatedTables.map((table) => table.toQualifiedName()).join(', ')}`
|
|
224
217
|
);
|
|
225
218
|
for (const table of this.replicatedTables) {
|
|
226
219
|
if (table.enabledForCDC()) {
|
|
227
|
-
const
|
|
228
|
-
|
|
229
|
-
// Each Source Table may or may not have transactions that are applicable to it, so just keep track of the highest number of transactions processed for any Source Table.
|
|
230
|
-
if (tableTransactionCount > transactionCount) {
|
|
231
|
-
transactionCount = tableTransactionCount;
|
|
232
|
-
}
|
|
220
|
+
const transactions = await this.pollTable(table, { startLSN, endLSN });
|
|
221
|
+
transactions.forEach((t) => transactionLSNs.add(t));
|
|
233
222
|
}
|
|
234
223
|
}
|
|
224
|
+
const transactionCount = transactionLSNs.size;
|
|
235
225
|
|
|
236
226
|
this.logger.info(
|
|
237
227
|
`Processed ${results.length} transaction(s), including ${transactionCount} Source Table transaction(s). Commited LSN: ${endLSN.toString()}`
|
|
@@ -248,68 +238,77 @@ export class CDCPoller {
|
|
|
248
238
|
}
|
|
249
239
|
}
|
|
250
240
|
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
241
|
+
/**
|
|
242
|
+
* Processes the changes this table has within the given bounds, and returns the LSNs of the
|
|
243
|
+
* transactions those changes belong to. The LSNs are returned in their string form so that the
|
|
244
|
+
* caller can deduplicate them across tables by value.
|
|
245
|
+
*/
|
|
246
|
+
private async pollTable(table: MSSQLSourceTable, bounds: { startLSN: LSN; endLSN: LSN }): Promise<Set<string>> {
|
|
247
|
+
const transactionLSNs = new Set<string>();
|
|
248
|
+
|
|
249
|
+
// CDC cleanup can advance minLSN while the capture-table identity remains unchanged, so use
|
|
250
|
+
// the latest metadata loaded by the schema check rather than the instance bound at startup.
|
|
251
|
+
const availableInstances = this.captureInstances.get(table.objectId)?.instances ?? [];
|
|
252
|
+
table.setCaptureInstance(availableInstances);
|
|
253
|
+
const boundInstance = table.captureInstance;
|
|
254
|
+
if (boundInstance == null) {
|
|
255
|
+
// The pinned instance can be dropped between schema checks.
|
|
256
|
+
throw new CaptureInstanceMissingError(
|
|
257
|
+
`The CDC capture instance for table ${table.toQualifiedName()} (pinned to object id ` +
|
|
258
|
+
`${table.pinnedCaptureObjectId}) is no longer available. Deploy a new sync config to replicate ` +
|
|
259
|
+
`this table against an available capture instance.`
|
|
260
|
+
);
|
|
261
|
+
}
|
|
262
|
+
const minLSN = boundInstance.minLSN;
|
|
254
263
|
if (minLSN > bounds.endLSN) {
|
|
255
|
-
return
|
|
264
|
+
return transactionLSNs;
|
|
256
265
|
} else if (minLSN >= bounds.startLSN) {
|
|
257
266
|
bounds.startLSN = minLSN;
|
|
258
267
|
}
|
|
259
268
|
|
|
260
269
|
try {
|
|
261
|
-
const
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
`,
|
|
265
|
-
[
|
|
266
|
-
{ name: 'from_lsn', type: sql.VarBinary, value: bounds.startLSN.toBinary() },
|
|
267
|
-
{ name: 'to_lsn', type: sql.VarBinary, value: bounds.endLSN.toBinary() }
|
|
268
|
-
]
|
|
269
|
-
);
|
|
270
|
+
const request = await this.connectionManager.createRequest();
|
|
271
|
+
request.input('from_lsn', sql.VarBinary, bounds.startLSN.toBinary());
|
|
272
|
+
request.input('to_lsn', sql.VarBinary, bounds.endLSN.toBinary());
|
|
270
273
|
|
|
271
|
-
let
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
274
|
+
let columns: sql.IColumnMetadata | null = null;
|
|
275
|
+
request.on('recordset', (recordsetColumns) => {
|
|
276
|
+
columns = recordsetColumns;
|
|
277
|
+
});
|
|
278
|
+
const stream = request.toReadableStream();
|
|
279
|
+
request.query(`
|
|
280
|
+
SELECT * FROM ${table.allChangesFunction}(@from_lsn, @to_lsn, 'all update old') ORDER BY __$start_lsn, __$seqval, __$operation
|
|
281
|
+
`);
|
|
282
|
+
|
|
283
|
+
for await (const { transactionLSN, type, rows } of groupLogicalChanges(stream, table)) {
|
|
284
|
+
if (columns == null) {
|
|
285
|
+
throw new ReplicationAssertionError(
|
|
286
|
+
`Missing CDC column metadata while polling for updates for table ${table.toQualifiedName()}.`
|
|
287
|
+
);
|
|
288
|
+
}
|
|
289
|
+
switch (type) {
|
|
290
|
+
case LogicalChangeType.DELETE:
|
|
291
|
+
await this.eventHandler.onDelete(rows[0], table, columns);
|
|
284
292
|
break;
|
|
285
|
-
case
|
|
286
|
-
|
|
287
|
-
this.logger.debug(`Processed UPDATE, before row LSN: ${transactionLSN}`);
|
|
293
|
+
case LogicalChangeType.INSERT:
|
|
294
|
+
await this.eventHandler.onInsert(rows[0], table, columns);
|
|
288
295
|
break;
|
|
289
|
-
case
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
await this.eventHandler.onUpdate(row, updateBefore, table, results.columns);
|
|
294
|
-
updateBefore = null;
|
|
295
|
-
this.logger.info(`Processed UPDATE row LSN: ${transactionLSN}`);
|
|
296
|
+
case LogicalChangeType.UPDATE:
|
|
297
|
+
case LogicalChangeType.DEFERRED_UPDATE:
|
|
298
|
+
const [rowBefore, rowAfter] = rows;
|
|
299
|
+
await this.eventHandler.onUpdate(rowAfter, rowBefore, table, columns);
|
|
296
300
|
break;
|
|
297
|
-
default:
|
|
298
|
-
this.logger.warn(`Unknown operation type [${row.__$operation}] encountered in CDC changes.`);
|
|
299
301
|
}
|
|
302
|
+
this.logger.info(`Processed ${type}. Transaction LSN: ${transactionLSN}`);
|
|
300
303
|
|
|
301
|
-
|
|
302
|
-
if (transactionLSN != lastTransactionLSN) {
|
|
303
|
-
lastTransactionLSN = transactionLSN;
|
|
304
|
-
if (row.__$operation !== Operation.UPDATE_BEFORE) {
|
|
305
|
-
transactionCount++;
|
|
306
|
-
}
|
|
307
|
-
}
|
|
304
|
+
transactionLSNs.add(transactionLSN.toString());
|
|
308
305
|
}
|
|
309
306
|
|
|
310
|
-
return
|
|
307
|
+
return transactionLSNs;
|
|
311
308
|
} catch (error) {
|
|
312
|
-
// This Covers both deleted tables and capture instances
|
|
309
|
+
// This Covers both deleted tables and capture instances. Unlike the check above, this cannot
|
|
310
|
+
// tell the two apart, so it stays recoverable: the forced schema check classifies it as a
|
|
311
|
+
// dropped table or a missing capture instance and fails with the matching error.
|
|
313
312
|
if (error.message.includes(`Invalid object name`)) {
|
|
314
313
|
throw new DatabaseQueryError(
|
|
315
314
|
ErrorCode.PSYNC_S1601,
|
|
@@ -325,33 +324,14 @@ export class CDCPoller {
|
|
|
325
324
|
return Date.now() - this.lastSchemaCheckTime >= this.schemaCheckIntervalMs;
|
|
326
325
|
}
|
|
327
326
|
|
|
328
|
-
/**
|
|
329
|
-
* Checks the given table for pending schema changes that can lead to inconsistencies in the replicated data if not handled.
|
|
330
|
-
* Returns the SchemaChange if any are found, null otherwise.
|
|
331
|
-
*/
|
|
332
327
|
private async checkForSchemaChanges(): Promise<SchemaChange[]> {
|
|
333
328
|
const schemaChanges: SchemaChange[] = [];
|
|
334
329
|
|
|
335
|
-
|
|
336
|
-
for (const table of newTables) {
|
|
337
|
-
this.logger.info(
|
|
338
|
-
`New table ${toQualifiedTableName(table.sourceTable.schema, table.sourceTable.name)} matching the sync config has been created. Handling schema change...`
|
|
339
|
-
);
|
|
340
|
-
schemaChanges.push({
|
|
341
|
-
type: SchemaChangeType.TABLE_CREATE,
|
|
342
|
-
newTable: {
|
|
343
|
-
name: table.sourceTable.name,
|
|
344
|
-
schema: table.sourceTable.schema,
|
|
345
|
-
objectId: table.sourceTable.objectId
|
|
346
|
-
},
|
|
347
|
-
newCaptureInstance: table.instances[0]
|
|
348
|
-
});
|
|
349
|
-
}
|
|
350
|
-
|
|
330
|
+
// Exact table names keep the replicated set fixed until the next deploy.
|
|
351
331
|
for (const table of this.replicatedTables) {
|
|
352
332
|
const exists = await tableExists(table.objectId, this.connectionManager);
|
|
353
333
|
if (!exists) {
|
|
354
|
-
this.logger.info(`Table ${table.toQualifiedName()} has been dropped
|
|
334
|
+
this.logger.info(`Table ${table.toQualifiedName()} has been dropped.`);
|
|
355
335
|
schemaChanges.push({
|
|
356
336
|
type: SchemaChangeType.TABLE_DROP,
|
|
357
337
|
table
|
|
@@ -361,85 +341,149 @@ export class CDCPoller {
|
|
|
361
341
|
|
|
362
342
|
const captureInstanceDetails = this.captureInstances.get(table.objectId);
|
|
363
343
|
if (!captureInstanceDetails) {
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
});
|
|
370
|
-
}
|
|
344
|
+
// The table had a capture instance when the stream started, but no longer does.
|
|
345
|
+
schemaChanges.push({
|
|
346
|
+
type: SchemaChangeType.MISSING_CAPTURE_INSTANCE,
|
|
347
|
+
table
|
|
348
|
+
});
|
|
371
349
|
continue;
|
|
372
350
|
}
|
|
373
351
|
|
|
374
352
|
const latestCaptureInstance = captureInstanceDetails.instances[0];
|
|
375
|
-
|
|
376
|
-
|
|
353
|
+
|
|
354
|
+
table.setCaptureInstance(captureInstanceDetails.instances);
|
|
355
|
+
const boundInstance = table.captureInstance;
|
|
356
|
+
if (boundInstance == null) {
|
|
357
|
+
// Include the replacement so the error can suggest the right recovery step.
|
|
358
|
+
schemaChanges.push({
|
|
359
|
+
type: SchemaChangeType.MISSING_CAPTURE_INSTANCE,
|
|
360
|
+
table,
|
|
361
|
+
replacementInstance: latestCaptureInstance
|
|
362
|
+
});
|
|
363
|
+
continue;
|
|
364
|
+
}
|
|
365
|
+
if (latestCaptureInstance.objectId !== boundInstance.objectId) {
|
|
366
|
+
// Keep checking for rename and column changes against the pinned instance.
|
|
377
367
|
schemaChanges.push({
|
|
378
368
|
type: SchemaChangeType.NEW_CAPTURE_INSTANCE,
|
|
379
369
|
table,
|
|
380
370
|
newCaptureInstance: latestCaptureInstance
|
|
381
371
|
});
|
|
382
|
-
continue;
|
|
383
372
|
}
|
|
384
373
|
|
|
385
|
-
//
|
|
374
|
+
// The new name is only used in the error message.
|
|
386
375
|
if (table.ref.name !== captureInstanceDetails.sourceTable.name) {
|
|
387
|
-
const newTable = this.tableMatchesSyncRules(
|
|
388
|
-
captureInstanceDetails.sourceTable.schema,
|
|
389
|
-
captureInstanceDetails.sourceTable.name
|
|
390
|
-
)
|
|
391
|
-
? {
|
|
392
|
-
name: captureInstanceDetails.sourceTable.name,
|
|
393
|
-
schema: captureInstanceDetails.sourceTable.schema,
|
|
394
|
-
objectId: captureInstanceDetails.sourceTable.objectId
|
|
395
|
-
}
|
|
396
|
-
: undefined;
|
|
397
|
-
|
|
398
376
|
schemaChanges.push({
|
|
399
377
|
type: SchemaChangeType.TABLE_RENAME,
|
|
400
378
|
table,
|
|
401
|
-
newTable
|
|
402
|
-
|
|
379
|
+
newTable: {
|
|
380
|
+
name: captureInstanceDetails.sourceTable.name,
|
|
381
|
+
schema: captureInstanceDetails.sourceTable.schema,
|
|
382
|
+
objectId: captureInstanceDetails.sourceTable.objectId
|
|
383
|
+
}
|
|
403
384
|
});
|
|
404
385
|
continue;
|
|
405
386
|
}
|
|
406
387
|
|
|
407
|
-
|
|
388
|
+
// Report drift against the capture instance this stream uses.
|
|
389
|
+
if (boundInstance.pendingSchemaChanges.length > 0) {
|
|
408
390
|
schemaChanges.push({
|
|
409
391
|
type: SchemaChangeType.TABLE_COLUMN_CHANGES,
|
|
410
392
|
table,
|
|
411
|
-
|
|
393
|
+
captureInstance: boundInstance
|
|
412
394
|
});
|
|
413
395
|
}
|
|
414
396
|
}
|
|
415
397
|
|
|
416
398
|
return schemaChanges;
|
|
417
399
|
}
|
|
400
|
+
}
|
|
418
401
|
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
402
|
+
/**
|
|
403
|
+
* Groups CDC change rows into the logical row changes they describe.
|
|
404
|
+
*
|
|
405
|
+
* SQL Server records a logical change as either one row (a plain insert or delete) or two rows that share
|
|
406
|
+
* a `__$seqval`. `__$seqval` represents the ordering of the changes to a row within a transaction.
|
|
407
|
+
* CDC operations that can share a `__$seqval` are:
|
|
408
|
+
* - The before and after operations of an in-place update
|
|
409
|
+
* - The delete and insert operations of a deferred update.
|
|
410
|
+
*
|
|
411
|
+
* This method groups and emits rows in the same transaction based on their `__$seqval`
|
|
412
|
+
*/
|
|
413
|
+
async function* groupLogicalChanges(rows: AsyncIterable<any>, table: MSSQLSourceTable): AsyncGenerator<LogicalChange> {
|
|
414
|
+
interface PendingGroup {
|
|
415
|
+
transactionLSN: Buffer;
|
|
416
|
+
sequence: Buffer;
|
|
417
|
+
rows: any[];
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
let current: PendingGroup | null = null;
|
|
421
|
+
|
|
422
|
+
for await (const row of rows) {
|
|
423
|
+
const transactionLSN: Buffer = row.__$start_lsn;
|
|
424
|
+
const sequence: Buffer = row.__$seqval;
|
|
425
|
+
|
|
426
|
+
if (current && !(transactionLSN.equals(current.transactionLSN) && sequence.equals(current.sequence))) {
|
|
427
|
+
yield toLogicalChange(current.rows, current.transactionLSN, table);
|
|
428
|
+
current = null;
|
|
431
429
|
}
|
|
430
|
+
current ??= {
|
|
431
|
+
transactionLSN,
|
|
432
|
+
sequence,
|
|
433
|
+
rows: []
|
|
434
|
+
};
|
|
435
|
+
current.rows.push(row);
|
|
436
|
+
}
|
|
432
437
|
|
|
433
|
-
|
|
438
|
+
if (current) {
|
|
439
|
+
yield toLogicalChange(current.rows, current.transactionLSN, table);
|
|
434
440
|
}
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
function toLogicalChange(rows: any[], startLSN: Buffer, table: MSSQLSourceTable): LogicalChange {
|
|
444
|
+
const transactionLSN = LSN.fromBinary(startLSN);
|
|
445
|
+
return {
|
|
446
|
+
transactionLSN,
|
|
447
|
+
type: resolveLogicalChangeType(rows, transactionLSN, table),
|
|
448
|
+
rows: rows
|
|
449
|
+
};
|
|
450
|
+
}
|
|
435
451
|
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
452
|
+
function resolveLogicalChangeType(orderedRows: any[], transactionLSN: LSN, table: MSSQLSourceTable): LogicalChangeType {
|
|
453
|
+
// This matches the actual CDC operation codes:https://learn.microsoft.com/en-us/sql/relational-databases/system-functions/cdc-fn-cdc-get-all-changes-capture-instance-transact-sql?view=sql-server-ver17#table-returned
|
|
454
|
+
enum Operation {
|
|
455
|
+
DELETE = 1,
|
|
456
|
+
INSERT = 2,
|
|
457
|
+
UPDATE_BEFORE = 3,
|
|
458
|
+
UPDATE_AFTER = 4
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
if (orderedRows.length === 1) {
|
|
462
|
+
const operation = orderedRows[0].__$operation;
|
|
463
|
+
|
|
464
|
+
if (operation === Operation.INSERT) {
|
|
465
|
+
return LogicalChangeType.INSERT;
|
|
466
|
+
} else if (operation === Operation.DELETE) {
|
|
467
|
+
return LogicalChangeType.DELETE;
|
|
468
|
+
} else {
|
|
469
|
+
throw new ReplicationAssertionError(
|
|
470
|
+
`Unrecognized operation: ${operation} for table ${table.toQualifiedName()} in transaction LSN ${transactionLSN}.`
|
|
471
|
+
);
|
|
472
|
+
}
|
|
473
|
+
} else if (orderedRows.length === 2) {
|
|
474
|
+
const [first, second] = orderedRows;
|
|
475
|
+
if (first.__$operation === Operation.UPDATE_BEFORE && second.__$operation === Operation.UPDATE_AFTER) {
|
|
476
|
+
return LogicalChangeType.UPDATE;
|
|
477
|
+
} else if (first.__$operation === Operation.DELETE && second.__$operation === Operation.INSERT) {
|
|
478
|
+
return LogicalChangeType.DEFERRED_UPDATE;
|
|
479
|
+
}
|
|
480
|
+
|
|
481
|
+
throw new ReplicationAssertionError(
|
|
482
|
+
`Unexpected CDC operations [${first.__$operation}, ${second.__$operation}] for a single logical change on table ${table.toQualifiedName()} in transaction LSN ${transactionLSN}.`
|
|
443
483
|
);
|
|
444
484
|
}
|
|
485
|
+
|
|
486
|
+
throw new ReplicationAssertionError(
|
|
487
|
+
`Unexpected number of CDC operations [${orderedRows.length}] for a single logical change on table ${table.toQualifiedName()} in transaction LSN ${transactionLSN}.`
|
|
488
|
+
);
|
|
445
489
|
}
|