@powersync/service-module-mssql 0.9.0 → 0.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +113 -0
- package/dist/api/MSSQLRouteAPIAdapter.js +26 -34
- package/dist/api/MSSQLRouteAPIAdapter.js.map +1 -1
- package/dist/common/MSSQLSourceTable.d.ts +12 -2
- package/dist/common/MSSQLSourceTable.js +30 -4
- package/dist/common/MSSQLSourceTable.js.map +1 -1
- package/dist/replication/CDCPoller.d.ts +13 -36
- package/dist/replication/CDCPoller.js +173 -129
- package/dist/replication/CDCPoller.js.map +1 -1
- package/dist/replication/CDCStream.d.ts +11 -14
- package/dist/replication/CDCStream.js +115 -106
- package/dist/replication/CDCStream.js.map +1 -1
- package/dist/replication/CaptureReconciler.d.ts +47 -0
- package/dist/replication/CaptureReconciler.js +155 -0
- package/dist/replication/CaptureReconciler.js.map +1 -0
- package/dist/replication/MSSQLConnectionManager.d.ts +1 -0
- package/dist/replication/MSSQLConnectionManager.js +19 -14
- package/dist/replication/MSSQLConnectionManager.js.map +1 -1
- package/dist/replication/MSSQLTableReconciliationContext.d.ts +21 -0
- package/dist/replication/MSSQLTableReconciliationContext.js +7 -0
- package/dist/replication/MSSQLTableReconciliationContext.js.map +1 -0
- package/dist/replication/SchemaChange.d.ts +67 -0
- package/dist/replication/SchemaChange.js +31 -0
- package/dist/replication/SchemaChange.js.map +1 -0
- package/dist/types/types.d.ts +8 -0
- package/dist/types/types.js +10 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/mssql.js +1 -1
- package/dist/utils/schema.d.ts +14 -1
- package/dist/utils/schema.js +40 -45
- package/dist/utils/schema.js.map +1 -1
- package/package.json +8 -8
- package/src/api/MSSQLRouteAPIAdapter.ts +27 -33
- package/src/common/MSSQLSourceTable.ts +32 -4
- package/src/replication/CDCPoller.ts +206 -162
- package/src/replication/CDCStream.ts +148 -154
- package/src/replication/CaptureReconciler.ts +186 -0
- package/src/replication/MSSQLConnectionManager.ts +23 -14
- package/src/replication/MSSQLTableReconciliationContext.ts +29 -0
- package/src/replication/SchemaChange.ts +79 -0
- package/src/types/types.ts +22 -1
- package/src/utils/mssql.ts +1 -1
- package/src/utils/schema.ts +57 -58
- package/test/src/CDCPoller.test.ts +382 -0
- package/test/src/CDCStream.test.ts +6 -2
- package/test/src/CDCStreamTestContext.ts +7 -0
- package/test/src/CaptureReconciler.test.ts +171 -0
- package/test/src/MSSQLSourceTable.test.ts +30 -0
- package/test/src/config.test.ts +37 -0
- package/test/src/schema-changes.test.ts +513 -249
- package/test/src/schema.test.ts +17 -0
- package/test/src/util.ts +70 -6
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -26,6 +26,7 @@ import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
|
|
|
26
26
|
import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
|
|
27
27
|
import { AdditionalConfig } from '../types/types.js';
|
|
28
28
|
import {
|
|
29
|
+
CaptureInstanceDetails,
|
|
29
30
|
checkRetentionThresholds,
|
|
30
31
|
checkSourceConfiguration,
|
|
31
32
|
createCheckpoint,
|
|
@@ -34,15 +35,21 @@ import {
|
|
|
34
35
|
getLatestReplicatedLSN,
|
|
35
36
|
isIColumnMetadata
|
|
36
37
|
} from '../utils/mssql.js';
|
|
38
|
+
import { getReplicationIdentityColumns, getTablesFromPattern, ResolvedTable } from '../utils/schema.js';
|
|
37
39
|
import {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
} from '
|
|
43
|
-
import { CDCEventHandler, CDCPoller, SchemaChange, SchemaChangeType } from './CDCPoller.js';
|
|
40
|
+
CaptureInstanceMissingError,
|
|
41
|
+
createCaptureReconciler,
|
|
42
|
+
SourceTableUnavailableError
|
|
43
|
+
} from './CaptureReconciler.js';
|
|
44
|
+
import { CDCEventHandler, CDCPoller } from './CDCPoller.js';
|
|
44
45
|
import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
|
|
45
46
|
import { BatchedSnapshotQuery, MSSQLSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js';
|
|
47
|
+
import type {
|
|
48
|
+
MSSQLReadyTableReconciliationContext,
|
|
49
|
+
MSSQLUnavailableTableReconciliationContext
|
|
50
|
+
} from './MSSQLTableReconciliationContext.js';
|
|
51
|
+
import { MSSQLTableReconciliationState } from './MSSQLTableReconciliationContext.js';
|
|
52
|
+
import { SchemaChange, SchemaChangeType } from './SchemaChange.js';
|
|
46
53
|
|
|
47
54
|
export interface CDCStreamOptions {
|
|
48
55
|
connections: MSSQLConnectionManager;
|
|
@@ -114,6 +121,10 @@ export class CDCStream {
|
|
|
114
121
|
|
|
115
122
|
private replicationLag = new ReplicationLagTracker();
|
|
116
123
|
|
|
124
|
+
// Latest capture instance warning emitted for each source table.
|
|
125
|
+
private warnedNewerCaptureObjectId = new Map<number, number>();
|
|
126
|
+
private warnedSchemaChangeCount = new Map<number, number>();
|
|
127
|
+
|
|
117
128
|
constructor(private options: CDCStreamOptions) {
|
|
118
129
|
this.logger = options.logger ?? defaultLogger;
|
|
119
130
|
this.storage = options.storage;
|
|
@@ -178,16 +189,19 @@ export class CDCStream {
|
|
|
178
189
|
|
|
179
190
|
async populateTableCache() {
|
|
180
191
|
const sourceTables = this.syncRules.getSourceTables();
|
|
192
|
+
// Loaded once for all patterns - this reads the CDC metadata for the entire database.
|
|
193
|
+
const captureInstances = await getCaptureInstances({ connectionManager: this.connections });
|
|
181
194
|
await this.storage.startBatch(
|
|
182
195
|
{
|
|
183
196
|
logger: this.logger,
|
|
184
197
|
zeroLSN: LSN.ZERO,
|
|
185
198
|
defaultSchema: this.defaultSchema,
|
|
186
|
-
storeCurrentData: false
|
|
199
|
+
storeCurrentData: false,
|
|
200
|
+
signal: this.abortSignal
|
|
187
201
|
},
|
|
188
202
|
async (batch) => {
|
|
189
203
|
for (let tablePattern of sourceTables) {
|
|
190
|
-
await this.getQualifiedTableNames(batch, tablePattern);
|
|
204
|
+
await this.getQualifiedTableNames(batch, tablePattern, captureInstances);
|
|
191
205
|
}
|
|
192
206
|
}
|
|
193
207
|
);
|
|
@@ -195,28 +209,31 @@ export class CDCStream {
|
|
|
195
209
|
|
|
196
210
|
async getQualifiedTableNames(
|
|
197
211
|
batch: storage.BucketStorageBatch,
|
|
198
|
-
tablePattern: TablePattern
|
|
212
|
+
tablePattern: TablePattern,
|
|
213
|
+
captureInstances: Map<number, CaptureInstanceDetails>
|
|
199
214
|
): Promise<MSSQLSourceTable[]> {
|
|
200
215
|
if (tablePattern.connectionTag != this.connections.connectionTag) {
|
|
201
216
|
return [];
|
|
202
217
|
}
|
|
203
218
|
|
|
204
|
-
const captureInstances = await getCaptureInstances({ connectionManager: this.connections });
|
|
205
219
|
const matchedTables: ResolvedTable[] = await getTablesFromPattern(this.connections, tablePattern);
|
|
206
220
|
|
|
221
|
+
if (matchedTables.length == 0) {
|
|
222
|
+
await this.reportUnavailableTable(batch, {
|
|
223
|
+
state: MSSQLTableReconciliationState.TABLE_MISSING,
|
|
224
|
+
source: {
|
|
225
|
+
connectionTag: this.connectionTag,
|
|
226
|
+
name: tablePattern.name,
|
|
227
|
+
schema: tablePattern.schema,
|
|
228
|
+
objectId: undefined,
|
|
229
|
+
// Replica identity metadata is unavailable when the physical table does not exist.
|
|
230
|
+
replicaIdColumns: []
|
|
231
|
+
}
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
|
|
207
235
|
const tables: MSSQLSourceTable[] = [];
|
|
208
236
|
for (const matchedTable of matchedTables) {
|
|
209
|
-
const captureInstanceDetails = captureInstances.get(matchedTable.objectId as number);
|
|
210
|
-
if (!captureInstanceDetails) {
|
|
211
|
-
// Match Postgres publication handling: If the source table cannot stream changes yet,
|
|
212
|
-
// don't resolve it into storage. Once CDC is enabled, the schema poller detects the
|
|
213
|
-
// new capture instance and resolves/snapshots it then.
|
|
214
|
-
this.logger.info(
|
|
215
|
-
`Skipping ${tablePattern.schema}.${matchedTable.name} - not enabled for CDC. This table will not be replicated until CDC is enabled for it.`
|
|
216
|
-
);
|
|
217
|
-
continue;
|
|
218
|
-
}
|
|
219
|
-
|
|
220
237
|
// TODO: Check RLS settings for table
|
|
221
238
|
|
|
222
239
|
const replicaIdColumns = await getReplicationIdentityColumns({
|
|
@@ -225,45 +242,54 @@ export class CDCStream {
|
|
|
225
242
|
schema: matchedTable.schema
|
|
226
243
|
});
|
|
227
244
|
|
|
228
|
-
const
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
245
|
+
const source: SourceEntityDescriptor = {
|
|
246
|
+
connectionTag: this.connectionTag,
|
|
247
|
+
name: matchedTable.name,
|
|
248
|
+
schema: matchedTable.schema,
|
|
249
|
+
objectId: matchedTable.objectId,
|
|
250
|
+
replicaIdColumns: replicaIdColumns.columns
|
|
251
|
+
};
|
|
252
|
+
const captureInstanceDetails = captureInstances.get(matchedTable.objectId);
|
|
253
|
+
if (!captureInstanceDetails?.instances.length) {
|
|
254
|
+
await this.reportUnavailableTable(batch, {
|
|
255
|
+
state: MSSQLTableReconciliationState.CDC_DISABLED,
|
|
256
|
+
source
|
|
257
|
+
});
|
|
258
|
+
} else {
|
|
259
|
+
const table = await this.processTable(batch, {
|
|
260
|
+
state: MSSQLTableReconciliationState.READY,
|
|
261
|
+
source,
|
|
262
|
+
captureInstances: captureInstanceDetails.instances
|
|
263
|
+
});
|
|
240
264
|
|
|
241
|
-
|
|
265
|
+
tables.push(table);
|
|
266
|
+
}
|
|
242
267
|
}
|
|
243
268
|
return tables;
|
|
244
269
|
}
|
|
245
270
|
|
|
246
271
|
async processTable(
|
|
247
272
|
batch: storage.BucketStorageBatch,
|
|
248
|
-
|
|
249
|
-
captureInstance: CaptureInstance | null,
|
|
250
|
-
snapshot: boolean
|
|
273
|
+
context: MSSQLReadyTableReconciliationContext
|
|
251
274
|
): Promise<MSSQLSourceTable> {
|
|
252
|
-
|
|
253
|
-
|
|
275
|
+
const { source } = context;
|
|
276
|
+
if (!source.objectId && typeof source.objectId != 'number') {
|
|
277
|
+
throw new ReplicationAssertionError(`objectId expected, got ${typeof source.objectId}`);
|
|
254
278
|
}
|
|
279
|
+
// Load all source metadata before resolving so the storage transaction does no source I/O.
|
|
255
280
|
const resolved = await batch.resolveTables({
|
|
256
281
|
connection_id: this.connectionId,
|
|
257
|
-
source
|
|
282
|
+
source,
|
|
283
|
+
reconcileSourceTables: createCaptureReconciler(context)
|
|
258
284
|
});
|
|
259
|
-
const resolvedTable = new MSSQLSourceTable(table, resolved.tables);
|
|
260
285
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
286
|
+
const resolvedTable = new MSSQLSourceTable(source, resolved.tables);
|
|
287
|
+
|
|
288
|
+
resolvedTable.setCaptureInstance(context.captureInstances);
|
|
289
|
+
if (!resolvedTable.enabledForCDC()) {
|
|
290
|
+
throw new ReplicationAssertionError(
|
|
291
|
+
`No capture instance matching the persisted binding is available for table ${resolvedTable.toQualifiedName()}`
|
|
264
292
|
);
|
|
265
|
-
} else {
|
|
266
|
-
resolvedTable.setCaptureInstance(captureInstance);
|
|
267
293
|
}
|
|
268
294
|
|
|
269
295
|
this.tableCache.set(resolvedTable);
|
|
@@ -271,27 +297,26 @@ export class CDCStream {
|
|
|
271
297
|
// Drop conflicting tables. This includes for example renamed tables.
|
|
272
298
|
await batch.drop(resolved.dropTables);
|
|
273
299
|
|
|
274
|
-
//
|
|
275
|
-
//
|
|
276
|
-
// 2. AND the table is enabled for CDC with a valid capture instance.
|
|
277
|
-
const snapshotCandidates = resolved.tables.filter(
|
|
278
|
-
(candidate) => snapshot && !candidate.snapshotComplete && candidate.syncAny
|
|
279
|
-
);
|
|
280
|
-
const shouldSnapshot = snapshotCandidates.length > 0 && resolvedTable.enabledForCDC();
|
|
281
|
-
|
|
282
|
-
if (shouldSnapshot) {
|
|
283
|
-
// Truncate tables in case a previous snapshot was interrupted.
|
|
284
|
-
await batch.truncate(snapshotCandidates);
|
|
285
|
-
|
|
286
|
-
// Start the snapshot(s) inside a transaction.
|
|
287
|
-
for (const candidate of snapshotCandidates) {
|
|
288
|
-
await this.snapshotTableInTx(batch, resolvedTable, candidate);
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
|
|
300
|
+
// Snapshotting is driven by startInitialReplication, which picks up any source table that is
|
|
301
|
+
// not snapshot-complete after the table cache has been populated.
|
|
292
302
|
return resolvedTable;
|
|
293
303
|
}
|
|
294
304
|
|
|
305
|
+
private async reportUnavailableTable(
|
|
306
|
+
batch: storage.BucketStorageBatch,
|
|
307
|
+
context: MSSQLUnavailableTableReconciliationContext
|
|
308
|
+
): Promise<never> {
|
|
309
|
+
await batch.resolveTables({
|
|
310
|
+
connection_id: this.connectionId,
|
|
311
|
+
source: context.source,
|
|
312
|
+
reconcileSourceTables: createCaptureReconciler(context)
|
|
313
|
+
});
|
|
314
|
+
// Unavailable-state reconciliation must reject; this enforces the Promise<never> contract if it returns.
|
|
315
|
+
throw new ReplicationAssertionError(
|
|
316
|
+
`Unavailable source-table reconciliation unexpectedly completed for ${context.source.schema}.${context.source.name}`
|
|
317
|
+
);
|
|
318
|
+
}
|
|
319
|
+
|
|
295
320
|
private async snapshotTableInTx(
|
|
296
321
|
batch: storage.BucketStorageBatch,
|
|
297
322
|
physicalTable: MSSQLSourceTable,
|
|
@@ -479,7 +504,8 @@ export class CDCStream {
|
|
|
479
504
|
zeroLSN: LSN.ZERO,
|
|
480
505
|
defaultSchema: this.defaultSchema,
|
|
481
506
|
storeCurrentData: false,
|
|
482
|
-
skipExistingRows: true
|
|
507
|
+
skipExistingRows: true,
|
|
508
|
+
signal: this.abortSignal
|
|
483
509
|
},
|
|
484
510
|
async (batch) => {
|
|
485
511
|
switch (status) {
|
|
@@ -495,9 +521,18 @@ export class CDCStream {
|
|
|
495
521
|
break;
|
|
496
522
|
case SnapshotStatus.LIMITED_RESNAPSHOT:
|
|
497
523
|
for (const table of specificTablesToResnapshot!) {
|
|
524
|
+
if (table.captureInstance == null) {
|
|
525
|
+
throw new ReplicationAssertionError(
|
|
526
|
+
`No capture instance bound for table ${table.toQualifiedName()} while re-snapshotting`
|
|
527
|
+
);
|
|
528
|
+
}
|
|
498
529
|
await batch.drop(table.getReplicatedSourceTables());
|
|
499
|
-
//
|
|
500
|
-
await this.processTable(batch,
|
|
530
|
+
// Re-resolve against the bound instance only, so the new record keeps the same pin.
|
|
531
|
+
await this.processTable(batch, {
|
|
532
|
+
state: MSSQLTableReconciliationState.READY,
|
|
533
|
+
source: table.ref,
|
|
534
|
+
captureInstances: [table.captureInstance]
|
|
535
|
+
});
|
|
501
536
|
}
|
|
502
537
|
break;
|
|
503
538
|
default:
|
|
@@ -626,7 +661,8 @@ export class CDCStream {
|
|
|
626
661
|
zeroLSN: LSN.ZERO,
|
|
627
662
|
defaultSchema: this.defaultSchema,
|
|
628
663
|
storeCurrentData: false,
|
|
629
|
-
skipExistingRows: false
|
|
664
|
+
skipExistingRows: false,
|
|
665
|
+
signal: this.abortSignal
|
|
630
666
|
},
|
|
631
667
|
async (batch) => {
|
|
632
668
|
if (batch.resumeFromLsn == null) {
|
|
@@ -639,7 +675,6 @@ export class CDCStream {
|
|
|
639
675
|
connectionManager: this.connections,
|
|
640
676
|
eventHandler,
|
|
641
677
|
getReplicatedTables: () => this.tableCache.getAll(),
|
|
642
|
-
sourceTables: this.syncRules.getSourceTables(),
|
|
643
678
|
startLSN,
|
|
644
679
|
logger: this.logger,
|
|
645
680
|
additionalConfig: this.options.additionalConfig,
|
|
@@ -723,110 +758,69 @@ export class CDCStream {
|
|
|
723
758
|
}
|
|
724
759
|
|
|
725
760
|
async handleSchemaChange(batch: storage.BucketStorageBatch, change: SchemaChange): Promise<void> {
|
|
726
|
-
let actionedSchemaChange = true;
|
|
727
|
-
|
|
728
761
|
switch (change.type) {
|
|
729
762
|
case SchemaChangeType.TABLE_RENAME:
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
`Table ${
|
|
763
|
+
// Continuing could commit past unread changes because the rename has no known LSN.
|
|
764
|
+
throw new SourceTableUnavailableError(
|
|
765
|
+
`Table ${change.table.toQualifiedName()} has been renamed to [${change.newTable.name}]. ` +
|
|
766
|
+
`Deploy a new sync config that uses the new name to replicate it again. ` +
|
|
767
|
+
`Its already-replicated data is retained until then.`
|
|
733
768
|
);
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
if (change.newTable) {
|
|
741
|
-
await this.handleCreateOrUpdateTable(batch, change.newTable, change.newCaptureInstance!);
|
|
742
|
-
}
|
|
743
|
-
break;
|
|
744
|
-
case SchemaChangeType.TABLE_CREATE:
|
|
745
|
-
await this.handleCreateOrUpdateTable(batch, change.newTable!, change.newCaptureInstance!);
|
|
746
|
-
break;
|
|
747
|
-
case SchemaChangeType.TABLE_COLUMN_CHANGES:
|
|
748
|
-
await this.handleColumnChanges(change.table!, change.newCaptureInstance!);
|
|
749
|
-
actionedSchemaChange = false;
|
|
750
|
-
break;
|
|
751
|
-
case SchemaChangeType.NEW_CAPTURE_INSTANCE:
|
|
752
|
-
this.logger.info(
|
|
753
|
-
`New CDC capture instance detected for table ${change.table!.toQualifiedName()}. Re-snapshotting table...`
|
|
769
|
+
case SchemaChangeType.TABLE_DROP:
|
|
770
|
+
throw new SourceTableUnavailableError(
|
|
771
|
+
`Table ${change.table.toQualifiedName()} has been dropped from the source. Deploy a new sync config ` +
|
|
772
|
+
`that no longer includes it. Its already-replicated data is retained until the new sync config ` +
|
|
773
|
+
`becomes active.`
|
|
754
774
|
);
|
|
755
|
-
|
|
756
|
-
this.
|
|
757
|
-
|
|
758
|
-
await this.handleCreateOrUpdateTable(batch, change.table!.ref, change.newCaptureInstance!);
|
|
775
|
+
case SchemaChangeType.TABLE_COLUMN_CHANGES:
|
|
776
|
+
this.warnColumnChanges(change.table, change.captureInstance);
|
|
759
777
|
break;
|
|
760
|
-
case SchemaChangeType.
|
|
761
|
-
|
|
762
|
-
|
|
778
|
+
case SchemaChangeType.NEW_CAPTURE_INSTANCE: {
|
|
779
|
+
// Warn once per newer instance and keep polling the pinned one.
|
|
780
|
+
const { table, newCaptureInstance } = change;
|
|
781
|
+
if (this.warnedNewerCaptureObjectId.get(table.objectId) !== newCaptureInstance.objectId) {
|
|
782
|
+
this.logger.warn(
|
|
783
|
+
`A newer CDC capture instance (object id ${newCaptureInstance.objectId}) is available for table ${table.toQualifiedName()}, ` +
|
|
784
|
+
`but the current replication process remains pinned to capture instance object id ${table.pinnedCaptureObjectId}. ` +
|
|
785
|
+
`Deploy a new sync config to adopt the new capture instance, and keep the current instance ` +
|
|
786
|
+
`available until that sync config becomes active.`
|
|
787
|
+
);
|
|
788
|
+
this.warnedNewerCaptureObjectId.set(table.objectId, newCaptureInstance.objectId);
|
|
789
|
+
}
|
|
763
790
|
break;
|
|
791
|
+
}
|
|
764
792
|
case SchemaChangeType.MISSING_CAPTURE_INSTANCE:
|
|
765
|
-
//
|
|
766
|
-
|
|
767
|
-
`
|
|
793
|
+
// A replacement may have a different schema, so never adopt it in place.
|
|
794
|
+
throw new CaptureInstanceMissingError(
|
|
795
|
+
`The CDC capture instance for table ${change.table.toQualifiedName()} (pinned to object id ` +
|
|
796
|
+
`${change.table.pinnedCaptureObjectId}) is no longer available. ` +
|
|
797
|
+
`PowerSync cannot continue replicating the table - a dropped capture instance cannot be ` +
|
|
798
|
+
`restored, because re-enabling CDC creates a new one with a different object id. ` +
|
|
799
|
+
(change.replacementInstance
|
|
800
|
+
? `Deploy a new sync config to adopt the replacement.`
|
|
801
|
+
: `Re-enable CDC for this table, then deploy a new sync config.`)
|
|
768
802
|
);
|
|
769
|
-
change.table!.clearCaptureInstance();
|
|
770
|
-
actionedSchemaChange = false;
|
|
771
|
-
break;
|
|
772
803
|
default:
|
|
773
|
-
throw new ReplicationAssertionError(`Unknown schema change type: ${change.type}`);
|
|
774
|
-
}
|
|
775
|
-
|
|
776
|
-
// Create a new checkpoint after the schema change
|
|
777
|
-
if (actionedSchemaChange) {
|
|
778
|
-
await createCheckpoint(this.connections);
|
|
804
|
+
throw new ReplicationAssertionError(`Unknown schema change type: ${(change as SchemaChange).type}`);
|
|
779
805
|
}
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
private async handleCreateOrUpdateTable(
|
|
783
|
-
batch: storage.BucketStorageBatch,
|
|
784
|
-
table: SourceTableChangeRef,
|
|
785
|
-
captureInstance: CaptureInstance
|
|
786
|
-
): Promise<void> {
|
|
787
|
-
const replicaIdColumns = await getReplicationIdentityColumns({
|
|
788
|
-
connectionManager: this.connections,
|
|
789
|
-
tableName: table.name,
|
|
790
|
-
schema: table.schema
|
|
791
|
-
});
|
|
792
|
-
|
|
793
|
-
await this.processTable(
|
|
794
|
-
batch,
|
|
795
|
-
{
|
|
796
|
-
connectionTag: this.connectionTag,
|
|
797
|
-
name: table.name,
|
|
798
|
-
schema: table.schema,
|
|
799
|
-
objectId: table.objectId,
|
|
800
|
-
replicaIdColumns: replicaIdColumns.columns
|
|
801
|
-
},
|
|
802
|
-
captureInstance,
|
|
803
|
-
true
|
|
804
|
-
);
|
|
806
|
+
// Warning-only changes do not need a checkpoint.
|
|
805
807
|
}
|
|
806
808
|
|
|
807
809
|
/**
|
|
808
|
-
*
|
|
809
|
-
*
|
|
810
|
-
* Due to the way CDC works, users are prevented from making column schema changes that affect the replication identities of a table.
|
|
811
|
-
* If changes like that are required, CDC has to be disabled and re-enabled for the table. This would then be handled by the detection of the new
|
|
812
|
-
* capture instance.
|
|
813
|
-
* @param table
|
|
814
|
-
* @param captureInstance
|
|
810
|
+
* Warn about column drift while continuing with the pinned capture schema.
|
|
815
811
|
*/
|
|
816
|
-
private
|
|
817
|
-
|
|
818
|
-
if (
|
|
819
|
-
table.captureInstance?.objectId === captureInstance.objectId &&
|
|
820
|
-
table.captureInstance?.pendingSchemaChanges.length === captureInstance.pendingSchemaChanges.length
|
|
821
|
-
) {
|
|
812
|
+
private warnColumnChanges(table: MSSQLSourceTable, captureInstance: CaptureInstance): void {
|
|
813
|
+
const changeCount = captureInstance.pendingSchemaChanges.length;
|
|
814
|
+
if (this.warnedSchemaChangeCount.get(table.objectId) === changeCount) {
|
|
822
815
|
return;
|
|
823
816
|
}
|
|
824
817
|
|
|
825
|
-
// New pending schema changes were detected - warn about those as well.
|
|
826
818
|
this.logger.warn(
|
|
827
|
-
`Schema drift detected for table ${table.toQualifiedName()}.
|
|
819
|
+
`Schema drift detected for table ${table.toQualifiedName()}. Until a new sync config becomes active, ` +
|
|
820
|
+
`replication continues against the captured schema, so these changes are not replicated. Deploy a new ` +
|
|
821
|
+
`sync config to pick them up.\n Pending schema changes:\n ${captureInstance.pendingSchemaChanges.join(', \n')}`
|
|
828
822
|
);
|
|
829
|
-
table.
|
|
823
|
+
this.warnedSchemaChangeCount.set(table.objectId, changeCount);
|
|
830
824
|
}
|
|
831
825
|
|
|
832
826
|
/**
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import { ErrorCode, ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { JsonValue, SourceTableCandidate, storage } from '@powersync/service-core';
|
|
3
|
+
import * as t from 'ts-codec';
|
|
4
|
+
import { toQualifiedTableName } from '../utils/mssql.js';
|
|
5
|
+
import type { MSSQLTableReconciliationContext } from './MSSQLTableReconciliationContext.js';
|
|
6
|
+
import { MSSQLTableReconciliationState } from './MSSQLTableReconciliationContext.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The pinned capture instance was dropped and cannot be replaced in this stream.
|
|
10
|
+
*/
|
|
11
|
+
export class CaptureInstanceMissingError extends ServiceError {
|
|
12
|
+
constructor(message: string) {
|
|
13
|
+
super(ErrorCode.PSYNC_S1601, message);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* A configured table does not exist or does not have CDC enabled yet.
|
|
19
|
+
*/
|
|
20
|
+
export class SourceTableNotReadyError extends ServiceError {
|
|
21
|
+
constructor(message: string) {
|
|
22
|
+
super(ErrorCode.PSYNC_S1602, message);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* A replicated table is unavailable or no longer matches its persisted source binding.
|
|
28
|
+
*/
|
|
29
|
+
export class SourceTableUnavailableError extends ServiceError {
|
|
30
|
+
constructor(message: string) {
|
|
31
|
+
super(ErrorCode.PSYNC_S1603, message);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Persisted capture-table identity. Names are not enough because SQL Server can reuse them.
|
|
37
|
+
*/
|
|
38
|
+
export const MSSQLSourceMetadata = t.object({
|
|
39
|
+
captureTableObjectId: t.number
|
|
40
|
+
});
|
|
41
|
+
export type MSSQLSourceMetadata = t.Decoded<typeof MSSQLSourceMetadata>;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Parse persisted capture metadata, returning null for legacy records.
|
|
45
|
+
*/
|
|
46
|
+
export function readCaptureMetadata(value: JsonValue): MSSQLSourceMetadata | null {
|
|
47
|
+
if (value == null) {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
return MSSQLSourceMetadata.decode(value as t.Encoded<typeof MSSQLSourceMetadata>);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Reconcile a discovered source-table state against persisted bindings. Ready tables reconcile
|
|
55
|
+
* against capture instances ordered newest first. New and legacy bindings use the newest instance;
|
|
56
|
+
* pinned bindings keep their instance or fail if it is gone.
|
|
57
|
+
*/
|
|
58
|
+
export function createCaptureReconciler(context: MSSQLTableReconciliationContext) {
|
|
59
|
+
return (({ source, candidates }) => {
|
|
60
|
+
if (context.state === MSSQLTableReconciliationState.TABLE_MISSING) {
|
|
61
|
+
if (candidates.length > 0) {
|
|
62
|
+
// The table is missing, but we already recorded it. Display the corresponding error.
|
|
63
|
+
throw sourceTableUnavailableError(source);
|
|
64
|
+
}
|
|
65
|
+
throw new SourceTableNotReadyError(
|
|
66
|
+
`Source table ${toQualifiedTableName(source.schema, source.name)} from the sync configuration does not ` +
|
|
67
|
+
`exist. Create the table and enable CDC for it.`
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Apply capture pinning after the shared identity check.
|
|
72
|
+
const compatible: SourceTableCandidate[] = [];
|
|
73
|
+
const incompatibleTables: SourceTableCandidate[] = [];
|
|
74
|
+
for (const candidate of candidates) {
|
|
75
|
+
if (storage.sourceIdentityCompatible(source, candidate)) {
|
|
76
|
+
compatible.push(candidate);
|
|
77
|
+
} else {
|
|
78
|
+
incompatibleTables.push(candidate);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if (compatible.length === 0) {
|
|
83
|
+
if (candidates.length > 0) {
|
|
84
|
+
// An overlapping persisted binding with a different source identity is not a new table.
|
|
85
|
+
// This can happen when a table is dropped and recreated while replication is stopped.
|
|
86
|
+
// Reusing the old snapshot or replacing it during a normal job restart would silently
|
|
87
|
+
// adopt a different physical table without a sync config deployment boundary.
|
|
88
|
+
throw sourceTableUnavailableError(source);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
if (context.state === MSSQLTableReconciliationState.CDC_DISABLED) {
|
|
92
|
+
throw new SourceTableNotReadyError(
|
|
93
|
+
`CDC is not enabled for source table ${toQualifiedTableName(source.schema, source.name)}, which ` +
|
|
94
|
+
`matches the sync configuration. Enable CDC for this table to start replicating it.`
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Pin a new binding to the newest instance.
|
|
99
|
+
const newest = context.captureInstances[0];
|
|
100
|
+
return {
|
|
101
|
+
compatibleTables: [],
|
|
102
|
+
incompatibleTables,
|
|
103
|
+
newTableValues: { sourceMetadata: captureMetadata(newest.objectId) }
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// At least one compatible candidate anchors this process to the discovered source identity.
|
|
108
|
+
// Any remaining incompatible candidates are stale overlapping records, so storage may remove
|
|
109
|
+
// them without adopting a replacement table or changing the binding this process uses.
|
|
110
|
+
if (context.state === MSSQLTableReconciliationState.CDC_DISABLED) {
|
|
111
|
+
throw new CaptureInstanceMissingError(
|
|
112
|
+
`CDC is no longer enabled for source table ${toQualifiedTableName(source.schema, source.name)}. ` +
|
|
113
|
+
`Re-enable CDC, then redeploy the sync config to adopt the replacement capture instance. Its ` +
|
|
114
|
+
`already-replicated data is retained until the redeployed sync config becomes active.`
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const availableInstances = context.captureInstances;
|
|
119
|
+
|
|
120
|
+
const pinnedObjectIds = new Set<number>();
|
|
121
|
+
let metadataFreeCount = 0;
|
|
122
|
+
for (const candidate of compatible) {
|
|
123
|
+
const metadata = readCaptureMetadata(candidate.sourceMetadata);
|
|
124
|
+
if (metadata == null) {
|
|
125
|
+
metadataFreeCount += 1;
|
|
126
|
+
} else {
|
|
127
|
+
pinnedObjectIds.add(metadata.captureTableObjectId);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Never guess which capture schema existing snapshots belong to.
|
|
132
|
+
if (pinnedObjectIds.size > 0 && metadataFreeCount > 0) {
|
|
133
|
+
throw new ReplicationAssertionError(
|
|
134
|
+
`Source table ${toQualifiedTableName(source.schema, source.name)} has a mixture of legacy (metadata-free) and ` +
|
|
135
|
+
`capture-instance-pinned records. This is invalid persisted state.`
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
// All records for a physical table must use the same capture instance.
|
|
139
|
+
if (pinnedObjectIds.size > 1) {
|
|
140
|
+
throw new ReplicationAssertionError(
|
|
141
|
+
`Source table ${toQualifiedTableName(source.schema, source.name)} has multiple persisted capture identities ` +
|
|
142
|
+
`(${[...pinnedObjectIds].join(', ')}). This is invalid persisted state.`
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
if (pinnedObjectIds.size === 0) {
|
|
147
|
+
// Backfill legacy records with the instance the old code would have selected.
|
|
148
|
+
const newest = availableInstances[0];
|
|
149
|
+
const sourceMetadata = captureMetadata(newest.objectId);
|
|
150
|
+
return {
|
|
151
|
+
compatibleTables: compatible.map((candidate) => candidate.withSourceMetadata(sourceMetadata)),
|
|
152
|
+
incompatibleTables,
|
|
153
|
+
newTableValues: { sourceMetadata }
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const [captureTableObjectId] = [...pinnedObjectIds];
|
|
158
|
+
const available = availableInstances.some((instance) => instance.objectId === captureTableObjectId);
|
|
159
|
+
if (!available) {
|
|
160
|
+
// A replacement may capture a different schema, so do not adopt it in place.
|
|
161
|
+
throw new CaptureInstanceMissingError(
|
|
162
|
+
`The CDC capture instance (object id ${captureTableObjectId}) for source table ` +
|
|
163
|
+
`${toQualifiedTableName(source.schema, source.name)} is no longer available. Deploy a new sync config to ` +
|
|
164
|
+
`replicate this table against a new capture instance.`
|
|
165
|
+
);
|
|
166
|
+
}
|
|
167
|
+
return {
|
|
168
|
+
compatibleTables: compatible,
|
|
169
|
+
incompatibleTables,
|
|
170
|
+
newTableValues: { sourceMetadata: captureMetadata(captureTableObjectId) }
|
|
171
|
+
};
|
|
172
|
+
}) satisfies storage.SourceTableCandidateReconciler;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function captureMetadata(captureTableObjectId: number): MSSQLSourceMetadata {
|
|
176
|
+
return { captureTableObjectId };
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function sourceTableUnavailableError(source: { schema: string; name: string }): SourceTableUnavailableError {
|
|
180
|
+
return new SourceTableUnavailableError(
|
|
181
|
+
`Table ${toQualifiedTableName(source.schema, source.name)} no longer matches the source table binding ` +
|
|
182
|
+
`selected by this replication process. It may have been dropped and recreated, renamed, or had its ` +
|
|
183
|
+
`replication identity changed. Redeploy the sync config to adopt the replacement. Its already-replicated ` +
|
|
184
|
+
`data is retained until the redeployed sync config becomes active.`
|
|
185
|
+
);
|
|
186
|
+
}
|
|
@@ -11,6 +11,28 @@ export interface MSSQLConnectionManagerListener {
|
|
|
11
11
|
onEnded(): void;
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
+
export function createConnectionPoolConfig(
|
|
15
|
+
options: NormalizedMSSQLConnectionConfig,
|
|
16
|
+
poolOptions: sql.PoolOpts<sql.Connection>
|
|
17
|
+
): sql.config {
|
|
18
|
+
return {
|
|
19
|
+
authentication: options.authentication,
|
|
20
|
+
user: options.username,
|
|
21
|
+
password: options.password,
|
|
22
|
+
server: options.hostname,
|
|
23
|
+
port: options.port,
|
|
24
|
+
database: options.database,
|
|
25
|
+
pool: poolOptions,
|
|
26
|
+
options: {
|
|
27
|
+
appName: `powersync/${POWERSYNC_VERSION}`,
|
|
28
|
+
encrypt: true, // Required for Azure
|
|
29
|
+
trustServerCertificate: options.additionalConfig.trustServerCertificate,
|
|
30
|
+
...(options.tls_servername ? { serverName: options.tls_servername } : {}),
|
|
31
|
+
...(options.cacert ? { cryptoCredentialsDetails: { ca: options.cacert } } : {})
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
|
|
14
36
|
export class MSSQLConnectionManager extends BaseObserver<MSSQLConnectionManagerListener> {
|
|
15
37
|
private readonly pool: sql.ConnectionPool;
|
|
16
38
|
|
|
@@ -20,20 +42,7 @@ export class MSSQLConnectionManager extends BaseObserver<MSSQLConnectionManagerL
|
|
|
20
42
|
) {
|
|
21
43
|
super();
|
|
22
44
|
// The pool is lazy - no connections are opened until a query is performed.
|
|
23
|
-
this.pool = new sql.ConnectionPool(
|
|
24
|
-
authentication: options.authentication,
|
|
25
|
-
user: options.username,
|
|
26
|
-
password: options.password,
|
|
27
|
-
server: options.hostname,
|
|
28
|
-
port: options.port,
|
|
29
|
-
database: options.database,
|
|
30
|
-
pool: poolOptions,
|
|
31
|
-
options: {
|
|
32
|
-
appName: `powersync/${POWERSYNC_VERSION}`,
|
|
33
|
-
encrypt: true, // Required for Azure
|
|
34
|
-
trustServerCertificate: options.additionalConfig.trustServerCertificate
|
|
35
|
-
}
|
|
36
|
-
});
|
|
45
|
+
this.pool = new sql.ConnectionPool(createConnectionPoolConfig(options, poolOptions));
|
|
37
46
|
}
|
|
38
47
|
|
|
39
48
|
public get connectionTag() {
|