@powersync/service-module-mssql 0.9.0 → 0.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. package/CHANGELOG.md +113 -0
  2. package/dist/api/MSSQLRouteAPIAdapter.js +26 -34
  3. package/dist/api/MSSQLRouteAPIAdapter.js.map +1 -1
  4. package/dist/common/MSSQLSourceTable.d.ts +12 -2
  5. package/dist/common/MSSQLSourceTable.js +30 -4
  6. package/dist/common/MSSQLSourceTable.js.map +1 -1
  7. package/dist/replication/CDCPoller.d.ts +13 -36
  8. package/dist/replication/CDCPoller.js +173 -129
  9. package/dist/replication/CDCPoller.js.map +1 -1
  10. package/dist/replication/CDCStream.d.ts +11 -14
  11. package/dist/replication/CDCStream.js +115 -106
  12. package/dist/replication/CDCStream.js.map +1 -1
  13. package/dist/replication/CaptureReconciler.d.ts +47 -0
  14. package/dist/replication/CaptureReconciler.js +155 -0
  15. package/dist/replication/CaptureReconciler.js.map +1 -0
  16. package/dist/replication/MSSQLConnectionManager.d.ts +1 -0
  17. package/dist/replication/MSSQLConnectionManager.js +19 -14
  18. package/dist/replication/MSSQLConnectionManager.js.map +1 -1
  19. package/dist/replication/MSSQLTableReconciliationContext.d.ts +21 -0
  20. package/dist/replication/MSSQLTableReconciliationContext.js +7 -0
  21. package/dist/replication/MSSQLTableReconciliationContext.js.map +1 -0
  22. package/dist/replication/SchemaChange.d.ts +67 -0
  23. package/dist/replication/SchemaChange.js +31 -0
  24. package/dist/replication/SchemaChange.js.map +1 -0
  25. package/dist/types/types.d.ts +8 -0
  26. package/dist/types/types.js +10 -1
  27. package/dist/types/types.js.map +1 -1
  28. package/dist/utils/mssql.js +1 -1
  29. package/dist/utils/schema.d.ts +14 -1
  30. package/dist/utils/schema.js +40 -45
  31. package/dist/utils/schema.js.map +1 -1
  32. package/package.json +8 -8
  33. package/src/api/MSSQLRouteAPIAdapter.ts +27 -33
  34. package/src/common/MSSQLSourceTable.ts +32 -4
  35. package/src/replication/CDCPoller.ts +206 -162
  36. package/src/replication/CDCStream.ts +148 -154
  37. package/src/replication/CaptureReconciler.ts +186 -0
  38. package/src/replication/MSSQLConnectionManager.ts +23 -14
  39. package/src/replication/MSSQLTableReconciliationContext.ts +29 -0
  40. package/src/replication/SchemaChange.ts +79 -0
  41. package/src/types/types.ts +22 -1
  42. package/src/utils/mssql.ts +1 -1
  43. package/src/utils/schema.ts +57 -58
  44. package/test/src/CDCPoller.test.ts +382 -0
  45. package/test/src/CDCStream.test.ts +6 -2
  46. package/test/src/CDCStreamTestContext.ts +7 -0
  47. package/test/src/CaptureReconciler.test.ts +171 -0
  48. package/test/src/MSSQLSourceTable.test.ts +30 -0
  49. package/test/src/config.test.ts +37 -0
  50. package/test/src/schema-changes.test.ts +513 -249
  51. package/test/src/schema.test.ts +17 -0
  52. package/test/src/util.ts +70 -6
  53. package/tsconfig.tsbuildinfo +1 -1
@@ -26,6 +26,7 @@ import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
26
26
  import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
27
27
  import { AdditionalConfig } from '../types/types.js';
28
28
  import {
29
+ CaptureInstanceDetails,
29
30
  checkRetentionThresholds,
30
31
  checkSourceConfiguration,
31
32
  createCheckpoint,
@@ -34,15 +35,21 @@ import {
34
35
  getLatestReplicatedLSN,
35
36
  isIColumnMetadata
36
37
  } from '../utils/mssql.js';
38
+ import { getReplicationIdentityColumns, getTablesFromPattern, ResolvedTable } from '../utils/schema.js';
37
39
  import {
38
- getReplicationIdentityColumns,
39
- getTablesFromPattern,
40
- ResolvedTable,
41
- SourceTableChangeRef
42
- } from '../utils/schema.js';
43
- import { CDCEventHandler, CDCPoller, SchemaChange, SchemaChangeType } from './CDCPoller.js';
40
+ CaptureInstanceMissingError,
41
+ createCaptureReconciler,
42
+ SourceTableUnavailableError
43
+ } from './CaptureReconciler.js';
44
+ import { CDCEventHandler, CDCPoller } from './CDCPoller.js';
44
45
  import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
45
46
  import { BatchedSnapshotQuery, MSSQLSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js';
47
+ import type {
48
+ MSSQLReadyTableReconciliationContext,
49
+ MSSQLUnavailableTableReconciliationContext
50
+ } from './MSSQLTableReconciliationContext.js';
51
+ import { MSSQLTableReconciliationState } from './MSSQLTableReconciliationContext.js';
52
+ import { SchemaChange, SchemaChangeType } from './SchemaChange.js';
46
53
 
47
54
  export interface CDCStreamOptions {
48
55
  connections: MSSQLConnectionManager;
@@ -114,6 +121,10 @@ export class CDCStream {
114
121
 
115
122
  private replicationLag = new ReplicationLagTracker();
116
123
 
124
+ // Latest capture instance warning emitted for each source table.
125
+ private warnedNewerCaptureObjectId = new Map<number, number>();
126
+ private warnedSchemaChangeCount = new Map<number, number>();
127
+
117
128
  constructor(private options: CDCStreamOptions) {
118
129
  this.logger = options.logger ?? defaultLogger;
119
130
  this.storage = options.storage;
@@ -178,16 +189,19 @@ export class CDCStream {
178
189
 
179
190
  async populateTableCache() {
180
191
  const sourceTables = this.syncRules.getSourceTables();
192
+ // Loaded once for all patterns - this reads the CDC metadata for the entire database.
193
+ const captureInstances = await getCaptureInstances({ connectionManager: this.connections });
181
194
  await this.storage.startBatch(
182
195
  {
183
196
  logger: this.logger,
184
197
  zeroLSN: LSN.ZERO,
185
198
  defaultSchema: this.defaultSchema,
186
- storeCurrentData: false
199
+ storeCurrentData: false,
200
+ signal: this.abortSignal
187
201
  },
188
202
  async (batch) => {
189
203
  for (let tablePattern of sourceTables) {
190
- await this.getQualifiedTableNames(batch, tablePattern);
204
+ await this.getQualifiedTableNames(batch, tablePattern, captureInstances);
191
205
  }
192
206
  }
193
207
  );
@@ -195,28 +209,31 @@ export class CDCStream {
195
209
 
196
210
  async getQualifiedTableNames(
197
211
  batch: storage.BucketStorageBatch,
198
- tablePattern: TablePattern
212
+ tablePattern: TablePattern,
213
+ captureInstances: Map<number, CaptureInstanceDetails>
199
214
  ): Promise<MSSQLSourceTable[]> {
200
215
  if (tablePattern.connectionTag != this.connections.connectionTag) {
201
216
  return [];
202
217
  }
203
218
 
204
- const captureInstances = await getCaptureInstances({ connectionManager: this.connections });
205
219
  const matchedTables: ResolvedTable[] = await getTablesFromPattern(this.connections, tablePattern);
206
220
 
221
+ if (matchedTables.length == 0) {
222
+ await this.reportUnavailableTable(batch, {
223
+ state: MSSQLTableReconciliationState.TABLE_MISSING,
224
+ source: {
225
+ connectionTag: this.connectionTag,
226
+ name: tablePattern.name,
227
+ schema: tablePattern.schema,
228
+ objectId: undefined,
229
+ // Replica identity metadata is unavailable when the physical table does not exist.
230
+ replicaIdColumns: []
231
+ }
232
+ });
233
+ }
234
+
207
235
  const tables: MSSQLSourceTable[] = [];
208
236
  for (const matchedTable of matchedTables) {
209
- const captureInstanceDetails = captureInstances.get(matchedTable.objectId as number);
210
- if (!captureInstanceDetails) {
211
- // Match Postgres publication handling: If the source table cannot stream changes yet,
212
- // don't resolve it into storage. Once CDC is enabled, the schema poller detects the
213
- // new capture instance and resolves/snapshots it then.
214
- this.logger.info(
215
- `Skipping ${tablePattern.schema}.${matchedTable.name} - not enabled for CDC. This table will not be replicated until CDC is enabled for it.`
216
- );
217
- continue;
218
- }
219
-
220
237
  // TODO: Check RLS settings for table
221
238
 
222
239
  const replicaIdColumns = await getReplicationIdentityColumns({
@@ -225,45 +242,54 @@ export class CDCStream {
225
242
  schema: matchedTable.schema
226
243
  });
227
244
 
228
- const table = await this.processTable(
229
- batch,
230
- {
231
- connectionTag: this.connectionTag,
232
- name: matchedTable.name,
233
- schema: matchedTable.schema,
234
- objectId: matchedTable.objectId,
235
- replicaIdColumns: replicaIdColumns.columns
236
- },
237
- captureInstanceDetails.instances[0],
238
- false
239
- );
245
+ const source: SourceEntityDescriptor = {
246
+ connectionTag: this.connectionTag,
247
+ name: matchedTable.name,
248
+ schema: matchedTable.schema,
249
+ objectId: matchedTable.objectId,
250
+ replicaIdColumns: replicaIdColumns.columns
251
+ };
252
+ const captureInstanceDetails = captureInstances.get(matchedTable.objectId);
253
+ if (!captureInstanceDetails?.instances.length) {
254
+ await this.reportUnavailableTable(batch, {
255
+ state: MSSQLTableReconciliationState.CDC_DISABLED,
256
+ source
257
+ });
258
+ } else {
259
+ const table = await this.processTable(batch, {
260
+ state: MSSQLTableReconciliationState.READY,
261
+ source,
262
+ captureInstances: captureInstanceDetails.instances
263
+ });
240
264
 
241
- tables.push(table);
265
+ tables.push(table);
266
+ }
242
267
  }
243
268
  return tables;
244
269
  }
245
270
 
246
271
  async processTable(
247
272
  batch: storage.BucketStorageBatch,
248
- table: SourceEntityDescriptor,
249
- captureInstance: CaptureInstance | null,
250
- snapshot: boolean
273
+ context: MSSQLReadyTableReconciliationContext
251
274
  ): Promise<MSSQLSourceTable> {
252
- if (!table.objectId && typeof table.objectId != 'number') {
253
- throw new ReplicationAssertionError(`objectId expected, got ${typeof table.objectId}`);
275
+ const { source } = context;
276
+ if (!source.objectId && typeof source.objectId != 'number') {
277
+ throw new ReplicationAssertionError(`objectId expected, got ${typeof source.objectId}`);
254
278
  }
279
+ // Load all source metadata before resolving so the storage transaction does no source I/O.
255
280
  const resolved = await batch.resolveTables({
256
281
  connection_id: this.connectionId,
257
- source: table
282
+ source,
283
+ reconcileSourceTables: createCaptureReconciler(context)
258
284
  });
259
- const resolvedTable = new MSSQLSourceTable(table, resolved.tables);
260
285
 
261
- if (!captureInstance) {
262
- this.logger.warn(
263
- `Missing capture instance for table ${resolvedTable.toQualifiedName()}. This table will not be replicated until CDC is enabled for it.`
286
+ const resolvedTable = new MSSQLSourceTable(source, resolved.tables);
287
+
288
+ resolvedTable.setCaptureInstance(context.captureInstances);
289
+ if (!resolvedTable.enabledForCDC()) {
290
+ throw new ReplicationAssertionError(
291
+ `No capture instance matching the persisted binding is available for table ${resolvedTable.toQualifiedName()}`
264
292
  );
265
- } else {
266
- resolvedTable.setCaptureInstance(captureInstance);
267
293
  }
268
294
 
269
295
  this.tableCache.set(resolvedTable);
@@ -271,27 +297,26 @@ export class CDCStream {
271
297
  // Drop conflicting tables. This includes for example renamed tables.
272
298
  await batch.drop(resolved.dropTables);
273
299
 
274
- // Snapshot if:
275
- // 1. The table is in the sync config and snapshot is requested, or not already done.
276
- // 2. AND the table is enabled for CDC with a valid capture instance.
277
- const snapshotCandidates = resolved.tables.filter(
278
- (candidate) => snapshot && !candidate.snapshotComplete && candidate.syncAny
279
- );
280
- const shouldSnapshot = snapshotCandidates.length > 0 && resolvedTable.enabledForCDC();
281
-
282
- if (shouldSnapshot) {
283
- // Truncate tables in case a previous snapshot was interrupted.
284
- await batch.truncate(snapshotCandidates);
285
-
286
- // Start the snapshot(s) inside a transaction.
287
- for (const candidate of snapshotCandidates) {
288
- await this.snapshotTableInTx(batch, resolvedTable, candidate);
289
- }
290
- }
291
-
300
+ // Snapshotting is driven by startInitialReplication, which picks up any source table that is
301
+ // not snapshot-complete after the table cache has been populated.
292
302
  return resolvedTable;
293
303
  }
294
304
 
305
+ private async reportUnavailableTable(
306
+ batch: storage.BucketStorageBatch,
307
+ context: MSSQLUnavailableTableReconciliationContext
308
+ ): Promise<never> {
309
+ await batch.resolveTables({
310
+ connection_id: this.connectionId,
311
+ source: context.source,
312
+ reconcileSourceTables: createCaptureReconciler(context)
313
+ });
314
+ // Unavailable-state reconciliation must reject; this enforces the Promise<never> contract if it returns.
315
+ throw new ReplicationAssertionError(
316
+ `Unavailable source-table reconciliation unexpectedly completed for ${context.source.schema}.${context.source.name}`
317
+ );
318
+ }
319
+
295
320
  private async snapshotTableInTx(
296
321
  batch: storage.BucketStorageBatch,
297
322
  physicalTable: MSSQLSourceTable,
@@ -479,7 +504,8 @@ export class CDCStream {
479
504
  zeroLSN: LSN.ZERO,
480
505
  defaultSchema: this.defaultSchema,
481
506
  storeCurrentData: false,
482
- skipExistingRows: true
507
+ skipExistingRows: true,
508
+ signal: this.abortSignal
483
509
  },
484
510
  async (batch) => {
485
511
  switch (status) {
@@ -495,9 +521,18 @@ export class CDCStream {
495
521
  break;
496
522
  case SnapshotStatus.LIMITED_RESNAPSHOT:
497
523
  for (const table of specificTablesToResnapshot!) {
524
+ if (table.captureInstance == null) {
525
+ throw new ReplicationAssertionError(
526
+ `No capture instance bound for table ${table.toQualifiedName()} while re-snapshotting`
527
+ );
528
+ }
498
529
  await batch.drop(table.getReplicatedSourceTables());
499
- // Update table in the table cache
500
- await this.processTable(batch, table.ref, table.captureInstance, false);
530
+ // Re-resolve against the bound instance only, so the new record keeps the same pin.
531
+ await this.processTable(batch, {
532
+ state: MSSQLTableReconciliationState.READY,
533
+ source: table.ref,
534
+ captureInstances: [table.captureInstance]
535
+ });
501
536
  }
502
537
  break;
503
538
  default:
@@ -626,7 +661,8 @@ export class CDCStream {
626
661
  zeroLSN: LSN.ZERO,
627
662
  defaultSchema: this.defaultSchema,
628
663
  storeCurrentData: false,
629
- skipExistingRows: false
664
+ skipExistingRows: false,
665
+ signal: this.abortSignal
630
666
  },
631
667
  async (batch) => {
632
668
  if (batch.resumeFromLsn == null) {
@@ -639,7 +675,6 @@ export class CDCStream {
639
675
  connectionManager: this.connections,
640
676
  eventHandler,
641
677
  getReplicatedTables: () => this.tableCache.getAll(),
642
- sourceTables: this.syncRules.getSourceTables(),
643
678
  startLSN,
644
679
  logger: this.logger,
645
680
  additionalConfig: this.options.additionalConfig,
@@ -723,110 +758,69 @@ export class CDCStream {
723
758
  }
724
759
 
725
760
  async handleSchemaChange(batch: storage.BucketStorageBatch, change: SchemaChange): Promise<void> {
726
- let actionedSchemaChange = true;
727
-
728
761
  switch (change.type) {
729
762
  case SchemaChangeType.TABLE_RENAME:
730
- const fromTable = change.table!;
731
- this.logger.info(
732
- `Table ${fromTable.toQualifiedName()} has been renamed ${change.newTable ? `to [${change.newTable.name}].` : '.'}`
763
+ // Continuing could commit past unread changes because the rename has no known LSN.
764
+ throw new SourceTableUnavailableError(
765
+ `Table ${change.table.toQualifiedName()} has been renamed to [${change.newTable.name}]. ` +
766
+ `Deploy a new sync config that uses the new name to replicate it again. ` +
767
+ `Its already-replicated data is retained until then.`
733
768
  );
734
-
735
- // Old table needs to be cleaned up
736
- const fromTables = fromTable.getReplicatedSourceTables();
737
- await batch.drop(fromTables);
738
- this.tableCache.delete(fromTable.objectId);
739
-
740
- if (change.newTable) {
741
- await this.handleCreateOrUpdateTable(batch, change.newTable, change.newCaptureInstance!);
742
- }
743
- break;
744
- case SchemaChangeType.TABLE_CREATE:
745
- await this.handleCreateOrUpdateTable(batch, change.newTable!, change.newCaptureInstance!);
746
- break;
747
- case SchemaChangeType.TABLE_COLUMN_CHANGES:
748
- await this.handleColumnChanges(change.table!, change.newCaptureInstance!);
749
- actionedSchemaChange = false;
750
- break;
751
- case SchemaChangeType.NEW_CAPTURE_INSTANCE:
752
- this.logger.info(
753
- `New CDC capture instance detected for table ${change.table!.toQualifiedName()}. Re-snapshotting table...`
769
+ case SchemaChangeType.TABLE_DROP:
770
+ throw new SourceTableUnavailableError(
771
+ `Table ${change.table.toQualifiedName()} has been dropped from the source. Deploy a new sync config ` +
772
+ `that no longer includes it. Its already-replicated data is retained until the new sync config ` +
773
+ `becomes active.`
754
774
  );
755
- await batch.drop(change.table!.getReplicatedSourceTables());
756
- this.tableCache.delete(change.table!.objectId);
757
-
758
- await this.handleCreateOrUpdateTable(batch, change.table!.ref, change.newCaptureInstance!);
775
+ case SchemaChangeType.TABLE_COLUMN_CHANGES:
776
+ this.warnColumnChanges(change.table, change.captureInstance);
759
777
  break;
760
- case SchemaChangeType.TABLE_DROP:
761
- await batch.drop(change.table!.getReplicatedSourceTables());
762
- this.tableCache.delete(change.table!.objectId);
778
+ case SchemaChangeType.NEW_CAPTURE_INSTANCE: {
779
+ // Warn once per newer instance and keep polling the pinned one.
780
+ const { table, newCaptureInstance } = change;
781
+ if (this.warnedNewerCaptureObjectId.get(table.objectId) !== newCaptureInstance.objectId) {
782
+ this.logger.warn(
783
+ `A newer CDC capture instance (object id ${newCaptureInstance.objectId}) is available for table ${table.toQualifiedName()}, ` +
784
+ `but the current replication process remains pinned to capture instance object id ${table.pinnedCaptureObjectId}. ` +
785
+ `Deploy a new sync config to adopt the new capture instance, and keep the current instance ` +
786
+ `available until that sync config becomes active.`
787
+ );
788
+ this.warnedNewerCaptureObjectId.set(table.objectId, newCaptureInstance.objectId);
789
+ }
763
790
  break;
791
+ }
764
792
  case SchemaChangeType.MISSING_CAPTURE_INSTANCE:
765
- // Stop replication for this table until CDC is re-enabled.
766
- this.logger.warn(
767
- `Table ${change.table!.toQualifiedName()} has been disabled for CDC. Re-enable CDC to continue replication.`
793
+ // A replacement may have a different schema, so never adopt it in place.
794
+ throw new CaptureInstanceMissingError(
795
+ `The CDC capture instance for table ${change.table.toQualifiedName()} (pinned to object id ` +
796
+ `${change.table.pinnedCaptureObjectId}) is no longer available. ` +
797
+ `PowerSync cannot continue replicating the table - a dropped capture instance cannot be ` +
798
+ `restored, because re-enabling CDC creates a new one with a different object id. ` +
799
+ (change.replacementInstance
800
+ ? `Deploy a new sync config to adopt the replacement.`
801
+ : `Re-enable CDC for this table, then deploy a new sync config.`)
768
802
  );
769
- change.table!.clearCaptureInstance();
770
- actionedSchemaChange = false;
771
- break;
772
803
  default:
773
- throw new ReplicationAssertionError(`Unknown schema change type: ${change.type}`);
774
- }
775
-
776
- // Create a new checkpoint after the schema change
777
- if (actionedSchemaChange) {
778
- await createCheckpoint(this.connections);
804
+ throw new ReplicationAssertionError(`Unknown schema change type: ${(change as SchemaChange).type}`);
779
805
  }
780
- }
781
-
782
- private async handleCreateOrUpdateTable(
783
- batch: storage.BucketStorageBatch,
784
- table: SourceTableChangeRef,
785
- captureInstance: CaptureInstance
786
- ): Promise<void> {
787
- const replicaIdColumns = await getReplicationIdentityColumns({
788
- connectionManager: this.connections,
789
- tableName: table.name,
790
- schema: table.schema
791
- });
792
-
793
- await this.processTable(
794
- batch,
795
- {
796
- connectionTag: this.connectionTag,
797
- name: table.name,
798
- schema: table.schema,
799
- objectId: table.objectId,
800
- replicaIdColumns: replicaIdColumns.columns
801
- },
802
- captureInstance,
803
- true
804
- );
806
+ // Warning-only changes do not need a checkpoint.
805
807
  }
806
808
 
807
809
  /**
808
- * There is very little that can be automatically done to handle the column changes other than to warn the user about the schema drift.
809
- *
810
- * Due to the way CDC works, users are prevented from making column schema changes that affect the replication identities of a table.
811
- * If changes like that are required, CDC has to be disabled and re-enabled for the table. This would then be handled by the detection of the new
812
- * capture instance.
813
- * @param table
814
- * @param captureInstance
810
+ * Warn about column drift while continuing with the pinned capture schema.
815
811
  */
816
- private async handleColumnChanges(table: MSSQLSourceTable, captureInstance: CaptureInstance): Promise<void> {
817
- // Check there are any new pending schema changes
818
- if (
819
- table.captureInstance?.objectId === captureInstance.objectId &&
820
- table.captureInstance?.pendingSchemaChanges.length === captureInstance.pendingSchemaChanges.length
821
- ) {
812
+ private warnColumnChanges(table: MSSQLSourceTable, captureInstance: CaptureInstance): void {
813
+ const changeCount = captureInstance.pendingSchemaChanges.length;
814
+ if (this.warnedSchemaChangeCount.get(table.objectId) === changeCount) {
822
815
  return;
823
816
  }
824
817
 
825
- // New pending schema changes were detected - warn about those as well.
826
818
  this.logger.warn(
827
- `Schema drift detected for table ${table.toQualifiedName()}. To ensure consistency, disable and re-enable CDC for this table.\n Pending schema changes:\n ${captureInstance.pendingSchemaChanges.join(', \n')}`
819
+ `Schema drift detected for table ${table.toQualifiedName()}. Until a new sync config becomes active, ` +
820
+ `replication continues against the captured schema, so these changes are not replicated. Deploy a new ` +
821
+ `sync config to pick them up.\n Pending schema changes:\n ${captureInstance.pendingSchemaChanges.join(', \n')}`
828
822
  );
829
- table.captureInstance = captureInstance;
823
+ this.warnedSchemaChangeCount.set(table.objectId, changeCount);
830
824
  }
831
825
 
832
826
  /**
@@ -0,0 +1,186 @@
1
+ import { ErrorCode, ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework';
2
+ import { JsonValue, SourceTableCandidate, storage } from '@powersync/service-core';
3
+ import * as t from 'ts-codec';
4
+ import { toQualifiedTableName } from '../utils/mssql.js';
5
+ import type { MSSQLTableReconciliationContext } from './MSSQLTableReconciliationContext.js';
6
+ import { MSSQLTableReconciliationState } from './MSSQLTableReconciliationContext.js';
7
+
8
+ /**
9
+ * The pinned capture instance was dropped and cannot be replaced in this stream.
10
+ */
11
+ export class CaptureInstanceMissingError extends ServiceError {
12
+ constructor(message: string) {
13
+ super(ErrorCode.PSYNC_S1601, message);
14
+ }
15
+ }
16
+
17
+ /**
18
+ * A configured table does not exist or does not have CDC enabled yet.
19
+ */
20
+ export class SourceTableNotReadyError extends ServiceError {
21
+ constructor(message: string) {
22
+ super(ErrorCode.PSYNC_S1602, message);
23
+ }
24
+ }
25
+
26
+ /**
27
+ * A replicated table is unavailable or no longer matches its persisted source binding.
28
+ */
29
+ export class SourceTableUnavailableError extends ServiceError {
30
+ constructor(message: string) {
31
+ super(ErrorCode.PSYNC_S1603, message);
32
+ }
33
+ }
34
+
35
+ /**
36
+ * Persisted capture-table identity. Names are not enough because SQL Server can reuse them.
37
+ */
38
+ export const MSSQLSourceMetadata = t.object({
39
+ captureTableObjectId: t.number
40
+ });
41
+ export type MSSQLSourceMetadata = t.Decoded<typeof MSSQLSourceMetadata>;
42
+
43
+ /**
44
+ * Parse persisted capture metadata, returning null for legacy records.
45
+ */
46
+ export function readCaptureMetadata(value: JsonValue): MSSQLSourceMetadata | null {
47
+ if (value == null) {
48
+ return null;
49
+ }
50
+ return MSSQLSourceMetadata.decode(value as t.Encoded<typeof MSSQLSourceMetadata>);
51
+ }
52
+
53
+ /**
54
+ * Reconcile a discovered source-table state against persisted bindings. Ready tables reconcile
55
+ * against capture instances ordered newest first. New and legacy bindings use the newest instance;
56
+ * pinned bindings keep their instance or fail if it is gone.
57
+ */
58
+ export function createCaptureReconciler(context: MSSQLTableReconciliationContext) {
59
+ return (({ source, candidates }) => {
60
+ if (context.state === MSSQLTableReconciliationState.TABLE_MISSING) {
61
+ if (candidates.length > 0) {
62
+ // The table is missing, but we already recorded it. Display the corresponding error.
63
+ throw sourceTableUnavailableError(source);
64
+ }
65
+ throw new SourceTableNotReadyError(
66
+ `Source table ${toQualifiedTableName(source.schema, source.name)} from the sync configuration does not ` +
67
+ `exist. Create the table and enable CDC for it.`
68
+ );
69
+ }
70
+
71
+ // Apply capture pinning after the shared identity check.
72
+ const compatible: SourceTableCandidate[] = [];
73
+ const incompatibleTables: SourceTableCandidate[] = [];
74
+ for (const candidate of candidates) {
75
+ if (storage.sourceIdentityCompatible(source, candidate)) {
76
+ compatible.push(candidate);
77
+ } else {
78
+ incompatibleTables.push(candidate);
79
+ }
80
+ }
81
+
82
+ if (compatible.length === 0) {
83
+ if (candidates.length > 0) {
84
+ // An overlapping persisted binding with a different source identity is not a new table.
85
+ // This can happen when a table is dropped and recreated while replication is stopped.
86
+ // Reusing the old snapshot or replacing it during a normal job restart would silently
87
+ // adopt a different physical table without a sync config deployment boundary.
88
+ throw sourceTableUnavailableError(source);
89
+ }
90
+
91
+ if (context.state === MSSQLTableReconciliationState.CDC_DISABLED) {
92
+ throw new SourceTableNotReadyError(
93
+ `CDC is not enabled for source table ${toQualifiedTableName(source.schema, source.name)}, which ` +
94
+ `matches the sync configuration. Enable CDC for this table to start replicating it.`
95
+ );
96
+ }
97
+
98
+ // Pin a new binding to the newest instance.
99
+ const newest = context.captureInstances[0];
100
+ return {
101
+ compatibleTables: [],
102
+ incompatibleTables,
103
+ newTableValues: { sourceMetadata: captureMetadata(newest.objectId) }
104
+ };
105
+ }
106
+
107
+ // At least one compatible candidate anchors this process to the discovered source identity.
108
+ // Any remaining incompatible candidates are stale overlapping records, so storage may remove
109
+ // them without adopting a replacement table or changing the binding this process uses.
110
+ if (context.state === MSSQLTableReconciliationState.CDC_DISABLED) {
111
+ throw new CaptureInstanceMissingError(
112
+ `CDC is no longer enabled for source table ${toQualifiedTableName(source.schema, source.name)}. ` +
113
+ `Re-enable CDC, then redeploy the sync config to adopt the replacement capture instance. Its ` +
114
+ `already-replicated data is retained until the redeployed sync config becomes active.`
115
+ );
116
+ }
117
+
118
+ const availableInstances = context.captureInstances;
119
+
120
+ const pinnedObjectIds = new Set<number>();
121
+ let metadataFreeCount = 0;
122
+ for (const candidate of compatible) {
123
+ const metadata = readCaptureMetadata(candidate.sourceMetadata);
124
+ if (metadata == null) {
125
+ metadataFreeCount += 1;
126
+ } else {
127
+ pinnedObjectIds.add(metadata.captureTableObjectId);
128
+ }
129
+ }
130
+
131
+ // Never guess which capture schema existing snapshots belong to.
132
+ if (pinnedObjectIds.size > 0 && metadataFreeCount > 0) {
133
+ throw new ReplicationAssertionError(
134
+ `Source table ${toQualifiedTableName(source.schema, source.name)} has a mixture of legacy (metadata-free) and ` +
135
+ `capture-instance-pinned records. This is invalid persisted state.`
136
+ );
137
+ }
138
+ // All records for a physical table must use the same capture instance.
139
+ if (pinnedObjectIds.size > 1) {
140
+ throw new ReplicationAssertionError(
141
+ `Source table ${toQualifiedTableName(source.schema, source.name)} has multiple persisted capture identities ` +
142
+ `(${[...pinnedObjectIds].join(', ')}). This is invalid persisted state.`
143
+ );
144
+ }
145
+
146
+ if (pinnedObjectIds.size === 0) {
147
+ // Backfill legacy records with the instance the old code would have selected.
148
+ const newest = availableInstances[0];
149
+ const sourceMetadata = captureMetadata(newest.objectId);
150
+ return {
151
+ compatibleTables: compatible.map((candidate) => candidate.withSourceMetadata(sourceMetadata)),
152
+ incompatibleTables,
153
+ newTableValues: { sourceMetadata }
154
+ };
155
+ }
156
+
157
+ const [captureTableObjectId] = [...pinnedObjectIds];
158
+ const available = availableInstances.some((instance) => instance.objectId === captureTableObjectId);
159
+ if (!available) {
160
+ // A replacement may capture a different schema, so do not adopt it in place.
161
+ throw new CaptureInstanceMissingError(
162
+ `The CDC capture instance (object id ${captureTableObjectId}) for source table ` +
163
+ `${toQualifiedTableName(source.schema, source.name)} is no longer available. Deploy a new sync config to ` +
164
+ `replicate this table against a new capture instance.`
165
+ );
166
+ }
167
+ return {
168
+ compatibleTables: compatible,
169
+ incompatibleTables,
170
+ newTableValues: { sourceMetadata: captureMetadata(captureTableObjectId) }
171
+ };
172
+ }) satisfies storage.SourceTableCandidateReconciler;
173
+ }
174
+
175
+ function captureMetadata(captureTableObjectId: number): MSSQLSourceMetadata {
176
+ return { captureTableObjectId };
177
+ }
178
+
179
+ function sourceTableUnavailableError(source: { schema: string; name: string }): SourceTableUnavailableError {
180
+ return new SourceTableUnavailableError(
181
+ `Table ${toQualifiedTableName(source.schema, source.name)} no longer matches the source table binding ` +
182
+ `selected by this replication process. It may have been dropped and recreated, renamed, or had its ` +
183
+ `replication identity changed. Redeploy the sync config to adopt the replacement. Its already-replicated ` +
184
+ `data is retained until the redeployed sync config becomes active.`
185
+ );
186
+ }
@@ -11,6 +11,28 @@ export interface MSSQLConnectionManagerListener {
11
11
  onEnded(): void;
12
12
  }
13
13
 
14
+ export function createConnectionPoolConfig(
15
+ options: NormalizedMSSQLConnectionConfig,
16
+ poolOptions: sql.PoolOpts<sql.Connection>
17
+ ): sql.config {
18
+ return {
19
+ authentication: options.authentication,
20
+ user: options.username,
21
+ password: options.password,
22
+ server: options.hostname,
23
+ port: options.port,
24
+ database: options.database,
25
+ pool: poolOptions,
26
+ options: {
27
+ appName: `powersync/${POWERSYNC_VERSION}`,
28
+ encrypt: true, // Required for Azure
29
+ trustServerCertificate: options.additionalConfig.trustServerCertificate,
30
+ ...(options.tls_servername ? { serverName: options.tls_servername } : {}),
31
+ ...(options.cacert ? { cryptoCredentialsDetails: { ca: options.cacert } } : {})
32
+ }
33
+ };
34
+ }
35
+
14
36
  export class MSSQLConnectionManager extends BaseObserver<MSSQLConnectionManagerListener> {
15
37
  private readonly pool: sql.ConnectionPool;
16
38
 
@@ -20,20 +42,7 @@ export class MSSQLConnectionManager extends BaseObserver<MSSQLConnectionManagerL
20
42
  ) {
21
43
  super();
22
44
  // The pool is lazy - no connections are opened until a query is performed.
23
- this.pool = new sql.ConnectionPool({
24
- authentication: options.authentication,
25
- user: options.username,
26
- password: options.password,
27
- server: options.hostname,
28
- port: options.port,
29
- database: options.database,
30
- pool: poolOptions,
31
- options: {
32
- appName: `powersync/${POWERSYNC_VERSION}`,
33
- encrypt: true, // Required for Azure
34
- trustServerCertificate: options.additionalConfig.trustServerCertificate
35
- }
36
- });
45
+ this.pool = new sql.ConnectionPool(createConnectionPoolConfig(options, poolOptions));
37
46
  }
38
47
 
39
48
  public get connectionTag() {