@powersync/service-module-mssql 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +84 -0
- package/dist/api/MSSQLRouteAPIAdapter.js +26 -34
- package/dist/api/MSSQLRouteAPIAdapter.js.map +1 -1
- package/dist/common/MSSQLSourceTable.d.ts +12 -2
- package/dist/common/MSSQLSourceTable.js +30 -4
- package/dist/common/MSSQLSourceTable.js.map +1 -1
- package/dist/replication/CDCPoller.d.ts +13 -36
- package/dist/replication/CDCPoller.js +173 -129
- package/dist/replication/CDCPoller.js.map +1 -1
- package/dist/replication/CDCStream.d.ts +11 -14
- package/dist/replication/CDCStream.js +109 -103
- package/dist/replication/CDCStream.js.map +1 -1
- package/dist/replication/CaptureReconciler.d.ts +47 -0
- package/dist/replication/CaptureReconciler.js +155 -0
- package/dist/replication/CaptureReconciler.js.map +1 -0
- package/dist/replication/MSSQLConnectionManager.d.ts +1 -0
- package/dist/replication/MSSQLConnectionManager.js +19 -14
- package/dist/replication/MSSQLConnectionManager.js.map +1 -1
- package/dist/replication/MSSQLTableReconciliationContext.d.ts +21 -0
- package/dist/replication/MSSQLTableReconciliationContext.js +7 -0
- package/dist/replication/MSSQLTableReconciliationContext.js.map +1 -0
- package/dist/replication/SchemaChange.d.ts +67 -0
- package/dist/replication/SchemaChange.js +31 -0
- package/dist/replication/SchemaChange.js.map +1 -0
- package/dist/types/types.d.ts +8 -0
- package/dist/types/types.js +9 -0
- package/dist/types/types.js.map +1 -1
- package/dist/utils/mssql.js +1 -1
- package/dist/utils/schema.d.ts +14 -1
- package/dist/utils/schema.js +40 -45
- package/dist/utils/schema.js.map +1 -1
- package/package.json +4 -4
- package/src/api/MSSQLRouteAPIAdapter.ts +27 -33
- package/src/common/MSSQLSourceTable.ts +32 -4
- package/src/replication/CDCPoller.ts +206 -162
- package/src/replication/CDCStream.ts +142 -151
- package/src/replication/CaptureReconciler.ts +186 -0
- package/src/replication/MSSQLConnectionManager.ts +23 -14
- package/src/replication/MSSQLTableReconciliationContext.ts +29 -0
- package/src/replication/SchemaChange.ts +79 -0
- package/src/types/types.ts +21 -0
- package/src/utils/mssql.ts +1 -1
- package/src/utils/schema.ts +57 -58
- package/test/src/CDCPoller.test.ts +382 -0
- package/test/src/CDCStream.test.ts +6 -2
- package/test/src/CDCStreamTestContext.ts +7 -0
- package/test/src/CaptureReconciler.test.ts +171 -0
- package/test/src/MSSQLSourceTable.test.ts +30 -0
- package/test/src/config.test.ts +37 -0
- package/test/src/schema-changes.test.ts +513 -249
- package/test/src/schema.test.ts +17 -0
- package/test/src/util.ts +70 -6
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -26,6 +26,7 @@ import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
|
|
|
26
26
|
import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
|
|
27
27
|
import { AdditionalConfig } from '../types/types.js';
|
|
28
28
|
import {
|
|
29
|
+
CaptureInstanceDetails,
|
|
29
30
|
checkRetentionThresholds,
|
|
30
31
|
checkSourceConfiguration,
|
|
31
32
|
createCheckpoint,
|
|
@@ -34,15 +35,21 @@ import {
|
|
|
34
35
|
getLatestReplicatedLSN,
|
|
35
36
|
isIColumnMetadata
|
|
36
37
|
} from '../utils/mssql.js';
|
|
38
|
+
import { getReplicationIdentityColumns, getTablesFromPattern, ResolvedTable } from '../utils/schema.js';
|
|
37
39
|
import {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
} from '
|
|
43
|
-
import { CDCEventHandler, CDCPoller, SchemaChange, SchemaChangeType } from './CDCPoller.js';
|
|
40
|
+
CaptureInstanceMissingError,
|
|
41
|
+
createCaptureReconciler,
|
|
42
|
+
SourceTableUnavailableError
|
|
43
|
+
} from './CaptureReconciler.js';
|
|
44
|
+
import { CDCEventHandler, CDCPoller } from './CDCPoller.js';
|
|
44
45
|
import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
|
|
45
46
|
import { BatchedSnapshotQuery, MSSQLSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js';
|
|
47
|
+
import type {
|
|
48
|
+
MSSQLReadyTableReconciliationContext,
|
|
49
|
+
MSSQLUnavailableTableReconciliationContext
|
|
50
|
+
} from './MSSQLTableReconciliationContext.js';
|
|
51
|
+
import { MSSQLTableReconciliationState } from './MSSQLTableReconciliationContext.js';
|
|
52
|
+
import { SchemaChange, SchemaChangeType } from './SchemaChange.js';
|
|
46
53
|
|
|
47
54
|
export interface CDCStreamOptions {
|
|
48
55
|
connections: MSSQLConnectionManager;
|
|
@@ -114,6 +121,10 @@ export class CDCStream {
|
|
|
114
121
|
|
|
115
122
|
private replicationLag = new ReplicationLagTracker();
|
|
116
123
|
|
|
124
|
+
// Latest capture instance warning emitted for each source table.
|
|
125
|
+
private warnedNewerCaptureObjectId = new Map<number, number>();
|
|
126
|
+
private warnedSchemaChangeCount = new Map<number, number>();
|
|
127
|
+
|
|
117
128
|
constructor(private options: CDCStreamOptions) {
|
|
118
129
|
this.logger = options.logger ?? defaultLogger;
|
|
119
130
|
this.storage = options.storage;
|
|
@@ -178,6 +189,8 @@ export class CDCStream {
|
|
|
178
189
|
|
|
179
190
|
async populateTableCache() {
|
|
180
191
|
const sourceTables = this.syncRules.getSourceTables();
|
|
192
|
+
// Loaded once for all patterns - this reads the CDC metadata for the entire database.
|
|
193
|
+
const captureInstances = await getCaptureInstances({ connectionManager: this.connections });
|
|
181
194
|
await this.storage.startBatch(
|
|
182
195
|
{
|
|
183
196
|
logger: this.logger,
|
|
@@ -187,7 +200,7 @@ export class CDCStream {
|
|
|
187
200
|
},
|
|
188
201
|
async (batch) => {
|
|
189
202
|
for (let tablePattern of sourceTables) {
|
|
190
|
-
await this.getQualifiedTableNames(batch, tablePattern);
|
|
203
|
+
await this.getQualifiedTableNames(batch, tablePattern, captureInstances);
|
|
191
204
|
}
|
|
192
205
|
}
|
|
193
206
|
);
|
|
@@ -195,28 +208,31 @@ export class CDCStream {
|
|
|
195
208
|
|
|
196
209
|
async getQualifiedTableNames(
|
|
197
210
|
batch: storage.BucketStorageBatch,
|
|
198
|
-
tablePattern: TablePattern
|
|
211
|
+
tablePattern: TablePattern,
|
|
212
|
+
captureInstances: Map<number, CaptureInstanceDetails>
|
|
199
213
|
): Promise<MSSQLSourceTable[]> {
|
|
200
214
|
if (tablePattern.connectionTag != this.connections.connectionTag) {
|
|
201
215
|
return [];
|
|
202
216
|
}
|
|
203
217
|
|
|
204
|
-
const captureInstances = await getCaptureInstances({ connectionManager: this.connections });
|
|
205
218
|
const matchedTables: ResolvedTable[] = await getTablesFromPattern(this.connections, tablePattern);
|
|
206
219
|
|
|
220
|
+
if (matchedTables.length == 0) {
|
|
221
|
+
await this.reportUnavailableTable(batch, {
|
|
222
|
+
state: MSSQLTableReconciliationState.TABLE_MISSING,
|
|
223
|
+
source: {
|
|
224
|
+
connectionTag: this.connectionTag,
|
|
225
|
+
name: tablePattern.name,
|
|
226
|
+
schema: tablePattern.schema,
|
|
227
|
+
objectId: undefined,
|
|
228
|
+
// Replica identity metadata is unavailable when the physical table does not exist.
|
|
229
|
+
replicaIdColumns: []
|
|
230
|
+
}
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
|
|
207
234
|
const tables: MSSQLSourceTable[] = [];
|
|
208
235
|
for (const matchedTable of matchedTables) {
|
|
209
|
-
const captureInstanceDetails = captureInstances.get(matchedTable.objectId as number);
|
|
210
|
-
if (!captureInstanceDetails) {
|
|
211
|
-
// Match Postgres publication handling: If the source table cannot stream changes yet,
|
|
212
|
-
// don't resolve it into storage. Once CDC is enabled, the schema poller detects the
|
|
213
|
-
// new capture instance and resolves/snapshots it then.
|
|
214
|
-
this.logger.info(
|
|
215
|
-
`Skipping ${tablePattern.schema}.${matchedTable.name} - not enabled for CDC. This table will not be replicated until CDC is enabled for it.`
|
|
216
|
-
);
|
|
217
|
-
continue;
|
|
218
|
-
}
|
|
219
|
-
|
|
220
236
|
// TODO: Check RLS settings for table
|
|
221
237
|
|
|
222
238
|
const replicaIdColumns = await getReplicationIdentityColumns({
|
|
@@ -225,45 +241,54 @@ export class CDCStream {
|
|
|
225
241
|
schema: matchedTable.schema
|
|
226
242
|
});
|
|
227
243
|
|
|
228
|
-
const
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
244
|
+
const source: SourceEntityDescriptor = {
|
|
245
|
+
connectionTag: this.connectionTag,
|
|
246
|
+
name: matchedTable.name,
|
|
247
|
+
schema: matchedTable.schema,
|
|
248
|
+
objectId: matchedTable.objectId,
|
|
249
|
+
replicaIdColumns: replicaIdColumns.columns
|
|
250
|
+
};
|
|
251
|
+
const captureInstanceDetails = captureInstances.get(matchedTable.objectId);
|
|
252
|
+
if (!captureInstanceDetails?.instances.length) {
|
|
253
|
+
await this.reportUnavailableTable(batch, {
|
|
254
|
+
state: MSSQLTableReconciliationState.CDC_DISABLED,
|
|
255
|
+
source
|
|
256
|
+
});
|
|
257
|
+
} else {
|
|
258
|
+
const table = await this.processTable(batch, {
|
|
259
|
+
state: MSSQLTableReconciliationState.READY,
|
|
260
|
+
source,
|
|
261
|
+
captureInstances: captureInstanceDetails.instances
|
|
262
|
+
});
|
|
240
263
|
|
|
241
|
-
|
|
264
|
+
tables.push(table);
|
|
265
|
+
}
|
|
242
266
|
}
|
|
243
267
|
return tables;
|
|
244
268
|
}
|
|
245
269
|
|
|
246
270
|
async processTable(
|
|
247
271
|
batch: storage.BucketStorageBatch,
|
|
248
|
-
|
|
249
|
-
captureInstance: CaptureInstance | null,
|
|
250
|
-
snapshot: boolean
|
|
272
|
+
context: MSSQLReadyTableReconciliationContext
|
|
251
273
|
): Promise<MSSQLSourceTable> {
|
|
252
|
-
|
|
253
|
-
|
|
274
|
+
const { source } = context;
|
|
275
|
+
if (!source.objectId && typeof source.objectId != 'number') {
|
|
276
|
+
throw new ReplicationAssertionError(`objectId expected, got ${typeof source.objectId}`);
|
|
254
277
|
}
|
|
278
|
+
// Load all source metadata before resolving so the storage transaction does no source I/O.
|
|
255
279
|
const resolved = await batch.resolveTables({
|
|
256
280
|
connection_id: this.connectionId,
|
|
257
|
-
source
|
|
281
|
+
source,
|
|
282
|
+
reconcileSourceTables: createCaptureReconciler(context)
|
|
258
283
|
});
|
|
259
|
-
const resolvedTable = new MSSQLSourceTable(table, resolved.tables);
|
|
260
284
|
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
285
|
+
const resolvedTable = new MSSQLSourceTable(source, resolved.tables);
|
|
286
|
+
|
|
287
|
+
resolvedTable.setCaptureInstance(context.captureInstances);
|
|
288
|
+
if (!resolvedTable.enabledForCDC()) {
|
|
289
|
+
throw new ReplicationAssertionError(
|
|
290
|
+
`No capture instance matching the persisted binding is available for table ${resolvedTable.toQualifiedName()}`
|
|
264
291
|
);
|
|
265
|
-
} else {
|
|
266
|
-
resolvedTable.setCaptureInstance(captureInstance);
|
|
267
292
|
}
|
|
268
293
|
|
|
269
294
|
this.tableCache.set(resolvedTable);
|
|
@@ -271,27 +296,26 @@ export class CDCStream {
|
|
|
271
296
|
// Drop conflicting tables. This includes for example renamed tables.
|
|
272
297
|
await batch.drop(resolved.dropTables);
|
|
273
298
|
|
|
274
|
-
//
|
|
275
|
-
//
|
|
276
|
-
// 2. AND the table is enabled for CDC with a valid capture instance.
|
|
277
|
-
const snapshotCandidates = resolved.tables.filter(
|
|
278
|
-
(candidate) => snapshot && !candidate.snapshotComplete && candidate.syncAny
|
|
279
|
-
);
|
|
280
|
-
const shouldSnapshot = snapshotCandidates.length > 0 && resolvedTable.enabledForCDC();
|
|
281
|
-
|
|
282
|
-
if (shouldSnapshot) {
|
|
283
|
-
// Truncate tables in case a previous snapshot was interrupted.
|
|
284
|
-
await batch.truncate(snapshotCandidates);
|
|
285
|
-
|
|
286
|
-
// Start the snapshot(s) inside a transaction.
|
|
287
|
-
for (const candidate of snapshotCandidates) {
|
|
288
|
-
await this.snapshotTableInTx(batch, resolvedTable, candidate);
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
|
|
299
|
+
// Snapshotting is driven by startInitialReplication, which picks up any source table that is
|
|
300
|
+
// not snapshot-complete after the table cache has been populated.
|
|
292
301
|
return resolvedTable;
|
|
293
302
|
}
|
|
294
303
|
|
|
304
|
+
private async reportUnavailableTable(
|
|
305
|
+
batch: storage.BucketStorageBatch,
|
|
306
|
+
context: MSSQLUnavailableTableReconciliationContext
|
|
307
|
+
): Promise<never> {
|
|
308
|
+
await batch.resolveTables({
|
|
309
|
+
connection_id: this.connectionId,
|
|
310
|
+
source: context.source,
|
|
311
|
+
reconcileSourceTables: createCaptureReconciler(context)
|
|
312
|
+
});
|
|
313
|
+
// Unavailable-state reconciliation must reject; this enforces the Promise<never> contract if it returns.
|
|
314
|
+
throw new ReplicationAssertionError(
|
|
315
|
+
`Unavailable source-table reconciliation unexpectedly completed for ${context.source.schema}.${context.source.name}`
|
|
316
|
+
);
|
|
317
|
+
}
|
|
318
|
+
|
|
295
319
|
private async snapshotTableInTx(
|
|
296
320
|
batch: storage.BucketStorageBatch,
|
|
297
321
|
physicalTable: MSSQLSourceTable,
|
|
@@ -495,9 +519,18 @@ export class CDCStream {
|
|
|
495
519
|
break;
|
|
496
520
|
case SnapshotStatus.LIMITED_RESNAPSHOT:
|
|
497
521
|
for (const table of specificTablesToResnapshot!) {
|
|
522
|
+
if (table.captureInstance == null) {
|
|
523
|
+
throw new ReplicationAssertionError(
|
|
524
|
+
`No capture instance bound for table ${table.toQualifiedName()} while re-snapshotting`
|
|
525
|
+
);
|
|
526
|
+
}
|
|
498
527
|
await batch.drop(table.getReplicatedSourceTables());
|
|
499
|
-
//
|
|
500
|
-
await this.processTable(batch,
|
|
528
|
+
// Re-resolve against the bound instance only, so the new record keeps the same pin.
|
|
529
|
+
await this.processTable(batch, {
|
|
530
|
+
state: MSSQLTableReconciliationState.READY,
|
|
531
|
+
source: table.ref,
|
|
532
|
+
captureInstances: [table.captureInstance]
|
|
533
|
+
});
|
|
501
534
|
}
|
|
502
535
|
break;
|
|
503
536
|
default:
|
|
@@ -639,7 +672,6 @@ export class CDCStream {
|
|
|
639
672
|
connectionManager: this.connections,
|
|
640
673
|
eventHandler,
|
|
641
674
|
getReplicatedTables: () => this.tableCache.getAll(),
|
|
642
|
-
sourceTables: this.syncRules.getSourceTables(),
|
|
643
675
|
startLSN,
|
|
644
676
|
logger: this.logger,
|
|
645
677
|
additionalConfig: this.options.additionalConfig,
|
|
@@ -723,110 +755,69 @@ export class CDCStream {
|
|
|
723
755
|
}
|
|
724
756
|
|
|
725
757
|
async handleSchemaChange(batch: storage.BucketStorageBatch, change: SchemaChange): Promise<void> {
|
|
726
|
-
let actionedSchemaChange = true;
|
|
727
|
-
|
|
728
758
|
switch (change.type) {
|
|
729
759
|
case SchemaChangeType.TABLE_RENAME:
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
`Table ${
|
|
760
|
+
// Continuing could commit past unread changes because the rename has no known LSN.
|
|
761
|
+
throw new SourceTableUnavailableError(
|
|
762
|
+
`Table ${change.table.toQualifiedName()} has been renamed to [${change.newTable.name}]. ` +
|
|
763
|
+
`Deploy a new sync config that uses the new name to replicate it again. ` +
|
|
764
|
+
`Its already-replicated data is retained until then.`
|
|
733
765
|
);
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
if (change.newTable) {
|
|
741
|
-
await this.handleCreateOrUpdateTable(batch, change.newTable, change.newCaptureInstance!);
|
|
742
|
-
}
|
|
743
|
-
break;
|
|
744
|
-
case SchemaChangeType.TABLE_CREATE:
|
|
745
|
-
await this.handleCreateOrUpdateTable(batch, change.newTable!, change.newCaptureInstance!);
|
|
746
|
-
break;
|
|
747
|
-
case SchemaChangeType.TABLE_COLUMN_CHANGES:
|
|
748
|
-
await this.handleColumnChanges(change.table!, change.newCaptureInstance!);
|
|
749
|
-
actionedSchemaChange = false;
|
|
750
|
-
break;
|
|
751
|
-
case SchemaChangeType.NEW_CAPTURE_INSTANCE:
|
|
752
|
-
this.logger.info(
|
|
753
|
-
`New CDC capture instance detected for table ${change.table!.toQualifiedName()}. Re-snapshotting table...`
|
|
766
|
+
case SchemaChangeType.TABLE_DROP:
|
|
767
|
+
throw new SourceTableUnavailableError(
|
|
768
|
+
`Table ${change.table.toQualifiedName()} has been dropped from the source. Deploy a new sync config ` +
|
|
769
|
+
`that no longer includes it. Its already-replicated data is retained until the new sync config ` +
|
|
770
|
+
`becomes active.`
|
|
754
771
|
);
|
|
755
|
-
|
|
756
|
-
this.
|
|
757
|
-
|
|
758
|
-
await this.handleCreateOrUpdateTable(batch, change.table!.ref, change.newCaptureInstance!);
|
|
772
|
+
case SchemaChangeType.TABLE_COLUMN_CHANGES:
|
|
773
|
+
this.warnColumnChanges(change.table, change.captureInstance);
|
|
759
774
|
break;
|
|
760
|
-
case SchemaChangeType.
|
|
761
|
-
|
|
762
|
-
|
|
775
|
+
case SchemaChangeType.NEW_CAPTURE_INSTANCE: {
|
|
776
|
+
// Warn once per newer instance and keep polling the pinned one.
|
|
777
|
+
const { table, newCaptureInstance } = change;
|
|
778
|
+
if (this.warnedNewerCaptureObjectId.get(table.objectId) !== newCaptureInstance.objectId) {
|
|
779
|
+
this.logger.warn(
|
|
780
|
+
`A newer CDC capture instance (object id ${newCaptureInstance.objectId}) is available for table ${table.toQualifiedName()}, ` +
|
|
781
|
+
`but the current replication process remains pinned to capture instance object id ${table.pinnedCaptureObjectId}. ` +
|
|
782
|
+
`Deploy a new sync config to adopt the new capture instance, and keep the current instance ` +
|
|
783
|
+
`available until that sync config becomes active.`
|
|
784
|
+
);
|
|
785
|
+
this.warnedNewerCaptureObjectId.set(table.objectId, newCaptureInstance.objectId);
|
|
786
|
+
}
|
|
763
787
|
break;
|
|
788
|
+
}
|
|
764
789
|
case SchemaChangeType.MISSING_CAPTURE_INSTANCE:
|
|
765
|
-
//
|
|
766
|
-
|
|
767
|
-
`
|
|
790
|
+
// A replacement may have a different schema, so never adopt it in place.
|
|
791
|
+
throw new CaptureInstanceMissingError(
|
|
792
|
+
`The CDC capture instance for table ${change.table.toQualifiedName()} (pinned to object id ` +
|
|
793
|
+
`${change.table.pinnedCaptureObjectId}) is no longer available. ` +
|
|
794
|
+
`PowerSync cannot continue replicating the table - a dropped capture instance cannot be ` +
|
|
795
|
+
`restored, because re-enabling CDC creates a new one with a different object id. ` +
|
|
796
|
+
(change.replacementInstance
|
|
797
|
+
? `Deploy a new sync config to adopt the replacement.`
|
|
798
|
+
: `Re-enable CDC for this table, then deploy a new sync config.`)
|
|
768
799
|
);
|
|
769
|
-
change.table!.clearCaptureInstance();
|
|
770
|
-
actionedSchemaChange = false;
|
|
771
|
-
break;
|
|
772
800
|
default:
|
|
773
|
-
throw new ReplicationAssertionError(`Unknown schema change type: ${change.type}`);
|
|
774
|
-
}
|
|
775
|
-
|
|
776
|
-
// Create a new checkpoint after the schema change
|
|
777
|
-
if (actionedSchemaChange) {
|
|
778
|
-
await createCheckpoint(this.connections);
|
|
801
|
+
throw new ReplicationAssertionError(`Unknown schema change type: ${(change as SchemaChange).type}`);
|
|
779
802
|
}
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
private async handleCreateOrUpdateTable(
|
|
783
|
-
batch: storage.BucketStorageBatch,
|
|
784
|
-
table: SourceTableChangeRef,
|
|
785
|
-
captureInstance: CaptureInstance
|
|
786
|
-
): Promise<void> {
|
|
787
|
-
const replicaIdColumns = await getReplicationIdentityColumns({
|
|
788
|
-
connectionManager: this.connections,
|
|
789
|
-
tableName: table.name,
|
|
790
|
-
schema: table.schema
|
|
791
|
-
});
|
|
792
|
-
|
|
793
|
-
await this.processTable(
|
|
794
|
-
batch,
|
|
795
|
-
{
|
|
796
|
-
connectionTag: this.connectionTag,
|
|
797
|
-
name: table.name,
|
|
798
|
-
schema: table.schema,
|
|
799
|
-
objectId: table.objectId,
|
|
800
|
-
replicaIdColumns: replicaIdColumns.columns
|
|
801
|
-
},
|
|
802
|
-
captureInstance,
|
|
803
|
-
true
|
|
804
|
-
);
|
|
803
|
+
// Warning-only changes do not need a checkpoint.
|
|
805
804
|
}
|
|
806
805
|
|
|
807
806
|
/**
|
|
808
|
-
*
|
|
809
|
-
*
|
|
810
|
-
* Due to the way CDC works, users are prevented from making column schema changes that affect the replication identities of a table.
|
|
811
|
-
* If changes like that are required, CDC has to be disabled and re-enabled for the table. This would then be handled by the detection of the new
|
|
812
|
-
* capture instance.
|
|
813
|
-
* @param table
|
|
814
|
-
* @param captureInstance
|
|
807
|
+
* Warn about column drift while continuing with the pinned capture schema.
|
|
815
808
|
*/
|
|
816
|
-
private
|
|
817
|
-
|
|
818
|
-
if (
|
|
819
|
-
table.captureInstance?.objectId === captureInstance.objectId &&
|
|
820
|
-
table.captureInstance?.pendingSchemaChanges.length === captureInstance.pendingSchemaChanges.length
|
|
821
|
-
) {
|
|
809
|
+
private warnColumnChanges(table: MSSQLSourceTable, captureInstance: CaptureInstance): void {
|
|
810
|
+
const changeCount = captureInstance.pendingSchemaChanges.length;
|
|
811
|
+
if (this.warnedSchemaChangeCount.get(table.objectId) === changeCount) {
|
|
822
812
|
return;
|
|
823
813
|
}
|
|
824
814
|
|
|
825
|
-
// New pending schema changes were detected - warn about those as well.
|
|
826
815
|
this.logger.warn(
|
|
827
|
-
`Schema drift detected for table ${table.toQualifiedName()}.
|
|
816
|
+
`Schema drift detected for table ${table.toQualifiedName()}. Until a new sync config becomes active, ` +
|
|
817
|
+
`replication continues against the captured schema, so these changes are not replicated. Deploy a new ` +
|
|
818
|
+
`sync config to pick them up.\n Pending schema changes:\n ${captureInstance.pendingSchemaChanges.join(', \n')}`
|
|
828
819
|
);
|
|
829
|
-
table.
|
|
820
|
+
this.warnedSchemaChangeCount.set(table.objectId, changeCount);
|
|
830
821
|
}
|
|
831
822
|
|
|
832
823
|
/**
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import { ErrorCode, ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { JsonValue, SourceTableCandidate, storage } from '@powersync/service-core';
|
|
3
|
+
import * as t from 'ts-codec';
|
|
4
|
+
import { toQualifiedTableName } from '../utils/mssql.js';
|
|
5
|
+
import type { MSSQLTableReconciliationContext } from './MSSQLTableReconciliationContext.js';
|
|
6
|
+
import { MSSQLTableReconciliationState } from './MSSQLTableReconciliationContext.js';
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The pinned capture instance was dropped and cannot be replaced in this stream.
|
|
10
|
+
*/
|
|
11
|
+
export class CaptureInstanceMissingError extends ServiceError {
|
|
12
|
+
constructor(message: string) {
|
|
13
|
+
super(ErrorCode.PSYNC_S1601, message);
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* A configured table does not exist or does not have CDC enabled yet.
|
|
19
|
+
*/
|
|
20
|
+
export class SourceTableNotReadyError extends ServiceError {
|
|
21
|
+
constructor(message: string) {
|
|
22
|
+
super(ErrorCode.PSYNC_S1602, message);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* A replicated table is unavailable or no longer matches its persisted source binding.
|
|
28
|
+
*/
|
|
29
|
+
export class SourceTableUnavailableError extends ServiceError {
|
|
30
|
+
constructor(message: string) {
|
|
31
|
+
super(ErrorCode.PSYNC_S1603, message);
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Persisted capture-table identity. Names are not enough because SQL Server can reuse them.
|
|
37
|
+
*/
|
|
38
|
+
export const MSSQLSourceMetadata = t.object({
|
|
39
|
+
captureTableObjectId: t.number
|
|
40
|
+
});
|
|
41
|
+
export type MSSQLSourceMetadata = t.Decoded<typeof MSSQLSourceMetadata>;
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Parse persisted capture metadata, returning null for legacy records.
|
|
45
|
+
*/
|
|
46
|
+
export function readCaptureMetadata(value: JsonValue): MSSQLSourceMetadata | null {
|
|
47
|
+
if (value == null) {
|
|
48
|
+
return null;
|
|
49
|
+
}
|
|
50
|
+
return MSSQLSourceMetadata.decode(value as t.Encoded<typeof MSSQLSourceMetadata>);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Reconcile a discovered source-table state against persisted bindings. Ready tables reconcile
|
|
55
|
+
* against capture instances ordered newest first. New and legacy bindings use the newest instance;
|
|
56
|
+
* pinned bindings keep their instance or fail if it is gone.
|
|
57
|
+
*/
|
|
58
|
+
export function createCaptureReconciler(context: MSSQLTableReconciliationContext) {
|
|
59
|
+
return (({ source, candidates }) => {
|
|
60
|
+
if (context.state === MSSQLTableReconciliationState.TABLE_MISSING) {
|
|
61
|
+
if (candidates.length > 0) {
|
|
62
|
+
// The table is missing, but we already recorded it. Display the corresponding error.
|
|
63
|
+
throw sourceTableUnavailableError(source);
|
|
64
|
+
}
|
|
65
|
+
throw new SourceTableNotReadyError(
|
|
66
|
+
`Source table ${toQualifiedTableName(source.schema, source.name)} from the sync configuration does not ` +
|
|
67
|
+
`exist. Create the table and enable CDC for it.`
|
|
68
|
+
);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Apply capture pinning after the shared identity check.
|
|
72
|
+
const compatible: SourceTableCandidate[] = [];
|
|
73
|
+
const incompatibleTables: SourceTableCandidate[] = [];
|
|
74
|
+
for (const candidate of candidates) {
|
|
75
|
+
if (storage.sourceIdentityCompatible(source, candidate)) {
|
|
76
|
+
compatible.push(candidate);
|
|
77
|
+
} else {
|
|
78
|
+
incompatibleTables.push(candidate);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
if (compatible.length === 0) {
|
|
83
|
+
if (candidates.length > 0) {
|
|
84
|
+
// An overlapping persisted binding with a different source identity is not a new table.
|
|
85
|
+
// This can happen when a table is dropped and recreated while replication is stopped.
|
|
86
|
+
// Reusing the old snapshot or replacing it during a normal job restart would silently
|
|
87
|
+
// adopt a different physical table without a sync config deployment boundary.
|
|
88
|
+
throw sourceTableUnavailableError(source);
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
if (context.state === MSSQLTableReconciliationState.CDC_DISABLED) {
|
|
92
|
+
throw new SourceTableNotReadyError(
|
|
93
|
+
`CDC is not enabled for source table ${toQualifiedTableName(source.schema, source.name)}, which ` +
|
|
94
|
+
`matches the sync configuration. Enable CDC for this table to start replicating it.`
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Pin a new binding to the newest instance.
|
|
99
|
+
const newest = context.captureInstances[0];
|
|
100
|
+
return {
|
|
101
|
+
compatibleTables: [],
|
|
102
|
+
incompatibleTables,
|
|
103
|
+
newTableValues: { sourceMetadata: captureMetadata(newest.objectId) }
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
// At least one compatible candidate anchors this process to the discovered source identity.
|
|
108
|
+
// Any remaining incompatible candidates are stale overlapping records, so storage may remove
|
|
109
|
+
// them without adopting a replacement table or changing the binding this process uses.
|
|
110
|
+
if (context.state === MSSQLTableReconciliationState.CDC_DISABLED) {
|
|
111
|
+
throw new CaptureInstanceMissingError(
|
|
112
|
+
`CDC is no longer enabled for source table ${toQualifiedTableName(source.schema, source.name)}. ` +
|
|
113
|
+
`Re-enable CDC, then redeploy the sync config to adopt the replacement capture instance. Its ` +
|
|
114
|
+
`already-replicated data is retained until the redeployed sync config becomes active.`
|
|
115
|
+
);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const availableInstances = context.captureInstances;
|
|
119
|
+
|
|
120
|
+
const pinnedObjectIds = new Set<number>();
|
|
121
|
+
let metadataFreeCount = 0;
|
|
122
|
+
for (const candidate of compatible) {
|
|
123
|
+
const metadata = readCaptureMetadata(candidate.sourceMetadata);
|
|
124
|
+
if (metadata == null) {
|
|
125
|
+
metadataFreeCount += 1;
|
|
126
|
+
} else {
|
|
127
|
+
pinnedObjectIds.add(metadata.captureTableObjectId);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// Never guess which capture schema existing snapshots belong to.
|
|
132
|
+
if (pinnedObjectIds.size > 0 && metadataFreeCount > 0) {
|
|
133
|
+
throw new ReplicationAssertionError(
|
|
134
|
+
`Source table ${toQualifiedTableName(source.schema, source.name)} has a mixture of legacy (metadata-free) and ` +
|
|
135
|
+
`capture-instance-pinned records. This is invalid persisted state.`
|
|
136
|
+
);
|
|
137
|
+
}
|
|
138
|
+
// All records for a physical table must use the same capture instance.
|
|
139
|
+
if (pinnedObjectIds.size > 1) {
|
|
140
|
+
throw new ReplicationAssertionError(
|
|
141
|
+
`Source table ${toQualifiedTableName(source.schema, source.name)} has multiple persisted capture identities ` +
|
|
142
|
+
`(${[...pinnedObjectIds].join(', ')}). This is invalid persisted state.`
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
if (pinnedObjectIds.size === 0) {
|
|
147
|
+
// Backfill legacy records with the instance the old code would have selected.
|
|
148
|
+
const newest = availableInstances[0];
|
|
149
|
+
const sourceMetadata = captureMetadata(newest.objectId);
|
|
150
|
+
return {
|
|
151
|
+
compatibleTables: compatible.map((candidate) => candidate.withSourceMetadata(sourceMetadata)),
|
|
152
|
+
incompatibleTables,
|
|
153
|
+
newTableValues: { sourceMetadata }
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const [captureTableObjectId] = [...pinnedObjectIds];
|
|
158
|
+
const available = availableInstances.some((instance) => instance.objectId === captureTableObjectId);
|
|
159
|
+
if (!available) {
|
|
160
|
+
// A replacement may capture a different schema, so do not adopt it in place.
|
|
161
|
+
throw new CaptureInstanceMissingError(
|
|
162
|
+
`The CDC capture instance (object id ${captureTableObjectId}) for source table ` +
|
|
163
|
+
`${toQualifiedTableName(source.schema, source.name)} is no longer available. Deploy a new sync config to ` +
|
|
164
|
+
`replicate this table against a new capture instance.`
|
|
165
|
+
);
|
|
166
|
+
}
|
|
167
|
+
return {
|
|
168
|
+
compatibleTables: compatible,
|
|
169
|
+
incompatibleTables,
|
|
170
|
+
newTableValues: { sourceMetadata: captureMetadata(captureTableObjectId) }
|
|
171
|
+
};
|
|
172
|
+
}) satisfies storage.SourceTableCandidateReconciler;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function captureMetadata(captureTableObjectId: number): MSSQLSourceMetadata {
|
|
176
|
+
return { captureTableObjectId };
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
function sourceTableUnavailableError(source: { schema: string; name: string }): SourceTableUnavailableError {
|
|
180
|
+
return new SourceTableUnavailableError(
|
|
181
|
+
`Table ${toQualifiedTableName(source.schema, source.name)} no longer matches the source table binding ` +
|
|
182
|
+
`selected by this replication process. It may have been dropped and recreated, renamed, or had its ` +
|
|
183
|
+
`replication identity changed. Redeploy the sync config to adopt the replacement. Its already-replicated ` +
|
|
184
|
+
`data is retained until the redeployed sync config becomes active.`
|
|
185
|
+
);
|
|
186
|
+
}
|
|
@@ -11,6 +11,28 @@ export interface MSSQLConnectionManagerListener {
|
|
|
11
11
|
onEnded(): void;
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
+
export function createConnectionPoolConfig(
|
|
15
|
+
options: NormalizedMSSQLConnectionConfig,
|
|
16
|
+
poolOptions: sql.PoolOpts<sql.Connection>
|
|
17
|
+
): sql.config {
|
|
18
|
+
return {
|
|
19
|
+
authentication: options.authentication,
|
|
20
|
+
user: options.username,
|
|
21
|
+
password: options.password,
|
|
22
|
+
server: options.hostname,
|
|
23
|
+
port: options.port,
|
|
24
|
+
database: options.database,
|
|
25
|
+
pool: poolOptions,
|
|
26
|
+
options: {
|
|
27
|
+
appName: `powersync/${POWERSYNC_VERSION}`,
|
|
28
|
+
encrypt: true, // Required for Azure
|
|
29
|
+
trustServerCertificate: options.additionalConfig.trustServerCertificate,
|
|
30
|
+
...(options.tls_servername ? { serverName: options.tls_servername } : {}),
|
|
31
|
+
...(options.cacert ? { cryptoCredentialsDetails: { ca: options.cacert } } : {})
|
|
32
|
+
}
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
|
|
14
36
|
export class MSSQLConnectionManager extends BaseObserver<MSSQLConnectionManagerListener> {
|
|
15
37
|
private readonly pool: sql.ConnectionPool;
|
|
16
38
|
|
|
@@ -20,20 +42,7 @@ export class MSSQLConnectionManager extends BaseObserver<MSSQLConnectionManagerL
|
|
|
20
42
|
) {
|
|
21
43
|
super();
|
|
22
44
|
// The pool is lazy - no connections are opened until a query is performed.
|
|
23
|
-
this.pool = new sql.ConnectionPool(
|
|
24
|
-
authentication: options.authentication,
|
|
25
|
-
user: options.username,
|
|
26
|
-
password: options.password,
|
|
27
|
-
server: options.hostname,
|
|
28
|
-
port: options.port,
|
|
29
|
-
database: options.database,
|
|
30
|
-
pool: poolOptions,
|
|
31
|
-
options: {
|
|
32
|
-
appName: `powersync/${POWERSYNC_VERSION}`,
|
|
33
|
-
encrypt: true, // Required for Azure
|
|
34
|
-
trustServerCertificate: options.additionalConfig.trustServerCertificate
|
|
35
|
-
}
|
|
36
|
-
});
|
|
45
|
+
this.pool = new sql.ConnectionPool(createConnectionPoolConfig(options, poolOptions));
|
|
37
46
|
}
|
|
38
47
|
|
|
39
48
|
public get connectionTag() {
|