@powersync/service-module-mssql 0.0.1 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/dist/api/MSSQLRouteAPIAdapter.js +16 -51
  3. package/dist/api/MSSQLRouteAPIAdapter.js.map +1 -1
  4. package/dist/common/LSN.js +2 -2
  5. package/dist/common/LSN.js.map +1 -1
  6. package/dist/common/MSSQLSourceTable.js +4 -4
  7. package/dist/common/MSSQLSourceTable.js.map +1 -1
  8. package/dist/common/mssqls-to-sqlite.js +4 -4
  9. package/dist/common/mssqls-to-sqlite.js.map +1 -1
  10. package/dist/module/MSSQLModule.js +1 -1
  11. package/dist/module/MSSQLModule.js.map +1 -1
  12. package/dist/replication/CDCPoller.d.ts +2 -2
  13. package/dist/replication/CDCPoller.js +12 -8
  14. package/dist/replication/CDCPoller.js.map +1 -1
  15. package/dist/replication/CDCReplicationJob.d.ts +2 -2
  16. package/dist/replication/CDCReplicationJob.js +1 -1
  17. package/dist/replication/CDCReplicationJob.js.map +1 -1
  18. package/dist/replication/CDCReplicator.d.ts +2 -2
  19. package/dist/replication/CDCReplicator.js +1 -1
  20. package/dist/replication/CDCReplicator.js.map +1 -1
  21. package/dist/replication/CDCStream.d.ts +2 -2
  22. package/dist/replication/CDCStream.js +32 -37
  23. package/dist/replication/CDCStream.js.map +1 -1
  24. package/dist/replication/MSSQLConnectionManager.js +1 -1
  25. package/dist/replication/MSSQLConnectionManager.js.map +1 -1
  26. package/dist/replication/MSSQLSnapshotQuery.d.ts +0 -17
  27. package/dist/replication/MSSQLSnapshotQuery.js +0 -47
  28. package/dist/replication/MSSQLSnapshotQuery.js.map +1 -1
  29. package/dist/types/types.d.ts +80 -23
  30. package/dist/types/types.js +24 -24
  31. package/dist/types/types.js.map +1 -1
  32. package/dist/utils/mssql.js +28 -14
  33. package/dist/utils/mssql.js.map +1 -1
  34. package/dist/utils/schema.js +31 -15
  35. package/dist/utils/schema.js.map +1 -1
  36. package/package.json +10 -10
  37. package/src/api/MSSQLRouteAPIAdapter.ts +16 -51
  38. package/src/common/LSN.ts +2 -2
  39. package/src/common/MSSQLSourceTable.ts +4 -4
  40. package/src/common/mssqls-to-sqlite.ts +11 -4
  41. package/src/module/MSSQLModule.ts +1 -1
  42. package/src/replication/CDCPoller.ts +15 -10
  43. package/src/replication/CDCReplicationJob.ts +3 -3
  44. package/src/replication/CDCReplicator.ts +3 -3
  45. package/src/replication/CDCStream.ts +36 -49
  46. package/src/replication/MSSQLConnectionManager.ts +1 -1
  47. package/src/replication/MSSQLSnapshotQuery.ts +0 -54
  48. package/src/types/types.ts +41 -45
  49. package/src/utils/mssql.ts +29 -16
  50. package/src/utils/schema.ts +31 -17
  51. package/test/src/CDCStream.test.ts +9 -17
  52. package/test/src/CDCStreamTestContext.ts +5 -5
  53. package/test/src/CDCStream_resumable_snapshot.test.ts +15 -9
  54. package/test/src/mssql-to-sqlite.test.ts +28 -27
  55. package/test/src/util.ts +27 -14
  56. package/tsconfig.tsbuildinfo +1 -1
  57. package/ci/init-mssql.sql +0 -50
  58. package/test/tsconfig.tsbuildinfo +0 -1
@@ -13,13 +13,7 @@ import { getUuidReplicaIdentityBson, MetricsEngine, SourceEntityDescriptor, stor
13
13
  import { SqliteInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
14
14
 
15
15
  import { ReplicationMetric } from '@powersync/service-types';
16
- import {
17
- BatchedSnapshotQuery,
18
- IdSnapshotQuery,
19
- MSSQLSnapshotQuery,
20
- PrimaryKeyValue,
21
- SimpleSnapshotQuery
22
- } from './MSSQLSnapshotQuery.js';
16
+ import { BatchedSnapshotQuery, MSSQLSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js';
23
17
  import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
24
18
  import { getReplicationIdentityColumns, getTablesFromPattern, ResolvedTable } from '../utils/schema.js';
25
19
  import {
@@ -39,7 +33,7 @@ import { LSN } from '../common/LSN.js';
39
33
  import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
40
34
  import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
41
35
  import { CDCEventHandler, CDCPoller } from './CDCPoller.js';
42
- import { CDCPollingOptions } from '../types/types.js';
36
+ import { AdditionalConfig } from '../types/types.js';
43
37
 
44
38
  export interface CDCStreamOptions {
45
39
  connections: MSSQLConnectionManager;
@@ -54,7 +48,7 @@ export interface CDCStreamOptions {
54
48
  */
55
49
  snapshotBatchSize?: number;
56
50
 
57
- pollingOptions: CDCPollingOptions;
51
+ additionalConfig: AdditionalConfig;
58
52
  }
59
53
 
60
54
  export enum SnapshotStatus {
@@ -247,7 +241,11 @@ export class CDCStream {
247
241
  entity_descriptor: table,
248
242
  sync_rules: this.syncRules
249
243
  });
250
- const captureInstance = await getCaptureInstance({ connectionManager: this.connections, tableName: resolved.table.name, schema: resolved.table.schema });
244
+ const captureInstance = await getCaptureInstance({
245
+ connectionManager: this.connections,
246
+ tableName: resolved.table.name,
247
+ schema: resolved.table.schema
248
+ });
251
249
  if (!captureInstance) {
252
250
  throw new ServiceAssertionError(
253
251
  `Missing capture instance for table ${toQualifiedTableName(resolved.table.schema, resolved.table.name)}`
@@ -282,18 +280,14 @@ export class CDCStream {
282
280
  return resolvedTable;
283
281
  }
284
282
 
285
- private async snapshotTableInTx(
286
- batch: storage.BucketStorageBatch,
287
- table: MSSQLSourceTable,
288
- limited?: PrimaryKeyValue[]
289
- ): Promise<void> {
283
+ private async snapshotTableInTx(batch: storage.BucketStorageBatch, table: MSSQLSourceTable): Promise<void> {
290
284
  // Note: We use the "Read Committed" isolation level here, not snapshot isolation.
291
285
  // The data may change during the transaction, but that is compensated for in the streaming
292
286
  // replication afterward.
293
287
  const transaction = await this.connections.createTransaction();
294
288
  await transaction.begin(sql.ISOLATION_LEVEL.READ_COMMITTED);
295
289
  try {
296
- await this.snapshotTable(batch, transaction, table, limited);
290
+ await this.snapshotTable(batch, transaction, table);
297
291
 
298
292
  // Get the current LSN.
299
293
  // The data will only be consistent once incremental replication has passed that point.
@@ -321,8 +315,7 @@ export class CDCStream {
321
315
  private async snapshotTable(
322
316
  batch: storage.BucketStorageBatch,
323
317
  transaction: sql.Transaction,
324
- table: MSSQLSourceTable,
325
- limited?: PrimaryKeyValue[]
318
+ table: MSSQLSourceTable
326
319
  ) {
327
320
  let totalEstimatedCount = table.sourceTable.snapshotStatus?.totalEstimatedCount;
328
321
  let replicatedCount = table.sourceTable.snapshotStatus?.replicatedCount ?? 0;
@@ -331,9 +324,7 @@ export class CDCStream {
331
324
  // We do streaming on two levels:
332
325
  // 1. Coarse select from the entire table, stream rows 1 by one
333
326
  // 2. Fine level: Stream batches of rows with each fetch call
334
- if (limited) {
335
- query = new IdSnapshotQuery(transaction, table, limited);
336
- } else if (BatchedSnapshotQuery.supports(table)) {
327
+ if (BatchedSnapshotQuery.supports(table)) {
337
328
  // Single primary key - we can use the primary key for chunking
338
329
  const orderByKey = table.sourceTable.replicaIdColumns[0];
339
330
  query = new BatchedSnapshotQuery(
@@ -361,7 +352,6 @@ export class CDCStream {
361
352
  }
362
353
  await query.initialize();
363
354
 
364
- let columns: sql.IColumnMetadata | null = null;
365
355
  let hasRemainingData = true;
366
356
  while (hasRemainingData) {
367
357
  // Fetch 10k at a time.
@@ -369,6 +359,7 @@ export class CDCStream {
369
359
  // and not spending too much time on each FETCH call.
370
360
  // We aim for a couple of seconds on each FETCH call.
371
361
  let batchReplicatedCount = 0;
362
+ let columns: sql.IColumnMetadata | null = null;
372
363
  const cursor = query.next();
373
364
  for await (const result of cursor) {
374
365
  if (columns == null && isIColumnMetadata(result)) {
@@ -400,30 +391,27 @@ export class CDCStream {
400
391
 
401
392
  // Important: flush before marking progress
402
393
  await batch.flush();
403
- if (limited == null) {
404
- let lastKey: Uint8Array | undefined;
405
- if (query instanceof BatchedSnapshotQuery) {
406
- lastKey = query.getLastKeySerialized();
407
- }
408
- if (lastCountTime < performance.now() - 10 * 60 * 1000) {
409
- // Even though we're doing the snapshot inside a transaction, the transaction uses
410
- // the default "Read Committed" isolation level. This means we can get new data
411
- // within the transaction, so we re-estimate the count every 10 minutes when replicating
412
- // large tables.
413
- totalEstimatedCount = await this.estimatedCountNumber(table, transaction);
414
- lastCountTime = performance.now();
415
- }
416
- const updatedSourceTable = await batch.updateTableProgress(table.sourceTable, {
417
- lastKey: lastKey,
418
- replicatedCount: replicatedCount,
419
- totalEstimatedCount: totalEstimatedCount
420
- });
421
- this.tableCache.updateSourceTable(updatedSourceTable);
422
394
 
423
- this.logger.info(`Replicating ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
424
- } else {
425
- this.logger.info(`Replicating ${table.toQualifiedName()} ${replicatedCount}/${limited.length} for resnapshot`);
395
+ let lastKey: Uint8Array | undefined;
396
+ if (query instanceof BatchedSnapshotQuery) {
397
+ lastKey = query.getLastKeySerialized();
398
+ }
399
+ if (lastCountTime < performance.now() - 10 * 60 * 1000) {
400
+ // Even though we're doing the snapshot inside a transaction, the transaction uses
401
+ // the default "Read Committed" isolation level. This means we can get new data
402
+ // within the transaction, so we re-estimate the count every 10 minutes when replicating
403
+ // large tables.
404
+ totalEstimatedCount = await this.estimatedCountNumber(table, transaction);
405
+ lastCountTime = performance.now();
426
406
  }
407
+ const updatedSourceTable = await batch.updateTableProgress(table.sourceTable, {
408
+ lastKey: lastKey,
409
+ replicatedCount: replicatedCount,
410
+ totalEstimatedCount: totalEstimatedCount
411
+ });
412
+ this.tableCache.updateSourceTable(updatedSourceTable);
413
+
414
+ this.logger.info(`Replicating ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
427
415
 
428
416
  if (this.abortSignal.aborted) {
429
417
  // We only abort after flushing
@@ -445,13 +433,12 @@ export class CDCStream {
445
433
  */
446
434
  async estimatedCountNumber(table: MSSQLSourceTable, transaction?: sql.Transaction): Promise<number> {
447
435
  const request = transaction ? transaction.request() : await this.connections.createRequest();
448
- const { recordset: result } = await request.query(
436
+ const { recordset: result } = await request.input('tableName', table.toQualifiedName()).query(
449
437
  `SELECT SUM(row_count) AS total_rows
450
438
  FROM sys.dm_db_partition_stats
451
- WHERE object_id = OBJECT_ID('${table.toQualifiedName()}')
439
+ WHERE object_id = OBJECT_ID(@tableName)
452
440
  AND index_id < 2;`
453
441
  );
454
- // TODO Fallback query in case user does not have permission?
455
442
  return result[0].total_rows ?? -1;
456
443
  }
457
444
 
@@ -586,8 +573,8 @@ export class CDCStream {
586
573
  eventHandler,
587
574
  sourceTables,
588
575
  startLSN,
589
- pollingOptions: this.options.pollingOptions,
590
- logger: this.logger
576
+ logger: this.logger,
577
+ additionalConfig: this.options.additionalConfig
591
578
  });
592
579
 
593
580
  this.abortSignal.addEventListener(
@@ -31,7 +31,7 @@ export class MSSQLConnectionManager extends BaseObserver<MSSQLConnectionManagerL
31
31
  options: {
32
32
  appName: `powersync/${POWERSYNC_VERSION}`,
33
33
  encrypt: true, // Required for Azure
34
- trustServerCertificate: options.trustServerCertificate
34
+ trustServerCertificate: options.additionalConfig.trustServerCertificate
35
35
  }
36
36
  });
37
37
  }
@@ -1,5 +1,4 @@
1
1
  import { bson, ColumnDescriptor, SourceTable } from '@powersync/service-core';
2
- import { SqliteValue } from '@powersync/service-sync-rules';
3
2
  import { ServiceAssertionError } from '@powersync/lib-services-framework';
4
3
  import { MSSQLBaseType } from '../types/mssql-data-types.js';
5
4
  import sql from 'mssql';
@@ -15,8 +14,6 @@ export interface MSSQLSnapshotQuery {
15
14
  next(): AsyncIterableIterator<sql.IColumnMetadata | sql.IRecordSet<any>>;
16
15
  }
17
16
 
18
- export type PrimaryKeyValue = Record<string, SqliteValue>;
19
-
20
17
  /**
21
18
  * Snapshot query using a plain SELECT * FROM table
22
19
  *
@@ -177,54 +174,3 @@ export class BatchedSnapshotQuery implements MSSQLSnapshotQuery {
177
174
  return decoded[this.key.name];
178
175
  }
179
176
  }
180
-
181
- /**
182
- * This performs a snapshot query using a list of primary keys.
183
- *
184
- * This is not used for general snapshots, but is used when we need to re-fetch specific rows
185
- * during streaming replication.
186
- */
187
- export class IdSnapshotQuery implements MSSQLSnapshotQuery {
188
- static supports(table: SourceTable | MSSQLSourceTable) {
189
- // We have the same requirements as BatchedSnapshotQuery.
190
- // This is typically only used as a fallback when ChunkedSnapshotQuery
191
- // skipped some rows.
192
- return BatchedSnapshotQuery.supports(table);
193
- }
194
-
195
- public constructor(
196
- private readonly transaction: sql.Transaction,
197
- private readonly table: MSSQLSourceTable,
198
- private readonly keys: PrimaryKeyValue[]
199
- ) {}
200
-
201
- public async initialize(): Promise<void> {
202
- // No-op
203
- }
204
-
205
- public async *next(): AsyncIterableIterator<sql.IColumnMetadata | sql.IRecordSet<any>> {
206
- const metadataRequest = this.transaction.request();
207
- metadataRequest.stream = true;
208
- const metadataPromise = new Promise<sql.IColumnMetadata>((resolve, reject) => {
209
- metadataRequest.on('recordset', resolve);
210
- metadataRequest.on('error', reject);
211
- });
212
- metadataRequest.query(`SELECT TOP(0) * FROM ${this.table.toQualifiedName()}`);
213
- const columnMetadata: sql.IColumnMetadata = await metadataPromise;
214
- yield columnMetadata;
215
-
216
- const keyDefinition = this.table.sourceTable.replicaIdColumns[0];
217
- const ids = this.keys.map((record) => record[keyDefinition.name]);
218
-
219
- const request = this.transaction.request();
220
- const stream = request.toReadableStream();
221
- request
222
- .input('ids', ids)
223
- .query(`SELECT * FROM ${this.table.toQualifiedName()} WHERE ${escapeIdentifier(keyDefinition.name)} = @ids`);
224
-
225
- // MSSQL only streams one row at a time
226
- for await (const row of stream) {
227
- yield row;
228
- }
229
- }
230
- }
@@ -63,21 +63,42 @@ export const DefaultAuthentication = t.object({
63
63
  });
64
64
  export type DefaultAuthentication = t.Decoded<typeof DefaultAuthentication>;
65
65
 
66
- export type AuthenticationType =
67
- | DefaultAuthentication
68
- | AzureActiveDirectoryPasswordAuthentication
69
- | AzureActiveDirectoryServicePrincipalSecret;
66
+ export const Authentication = DefaultAuthentication.or(AzureActiveDirectoryPasswordAuthentication).or(
67
+ AzureActiveDirectoryServicePrincipalSecret
68
+ );
69
+ export type Authentication = t.Decoded<typeof Authentication>;
70
70
 
71
-
72
- export interface CDCPollingOptions {
71
+ export const AdditionalConfig = t.object({
72
+ /**
73
+ * Interval in milliseconds to wait between polling cycles. Defaults to 1000 milliseconds.
74
+ */
75
+ pollingIntervalMs: t.number.optional(),
73
76
  /**
74
77
  * Maximum number of transactions to poll per polling cycle. Defaults to 10.
75
78
  */
76
- batchSize: number;
79
+ pollingBatchSize: t.number.optional(),
80
+
77
81
  /**
78
- * Interval in milliseconds to wait between polling cycles. Defaults to 1 second.
82
+ * Whether to trust the server certificate. Set to true for local development and self-signed certificates.
83
+ * Default is false.
79
84
  */
80
- intervalMs: number;
85
+ trustServerCertificate: t.boolean.optional()
86
+ });
87
+
88
+ export interface AdditionalConfig {
89
+ /**
90
+ * Interval in milliseconds to wait between polling cycles. Defaults to 1000 milliseconds.
91
+ */
92
+ pollingIntervalMs: number;
93
+ /**
94
+ * Maximum number of transactions to poll per polling cycle. Defaults to 10.
95
+ */
96
+ pollingBatchSize: number;
97
+ /**
98
+ * Whether to trust the server certificate. Set to true for local development and self-signed certificates.
99
+ * Default is false.
100
+ */
101
+ trustServerCertificate: boolean;
81
102
  }
82
103
 
83
104
  export interface NormalizedMSSQLConnectionConfig {
@@ -91,16 +112,11 @@ export interface NormalizedMSSQLConnectionConfig {
91
112
  database: string;
92
113
  schema?: string;
93
114
 
94
- authentication?: AuthenticationType;
95
-
96
- cdcPollingOptions: CDCPollingOptions;
115
+ authentication?: Authentication;
97
116
 
98
- /**
99
- * Whether to trust the server certificate. Set to true for local development and self-signed certificates.
100
- * Default is false.
101
- */
102
- trustServerCertificate: boolean;
103
117
  lookup?: LookupFunction;
118
+
119
+ additionalConfig: AdditionalConfig;
104
120
  }
105
121
 
106
122
  export const MSSQLConnectionConfig = service_types.configFile.DataSourceConfig.and(
@@ -114,22 +130,10 @@ export const MSSQLConnectionConfig = service_types.configFile.DataSourceConfig.a
114
130
  hostname: t.string.optional(),
115
131
  port: service_types.configFile.portCodec.optional(),
116
132
 
117
- authentication: DefaultAuthentication.or(AzureActiveDirectoryPasswordAuthentication)
118
- .or(AzureActiveDirectoryServicePrincipalSecret)
119
- .optional(),
120
-
121
- cdcPollingOptions: t.object({
122
- batchSize: t.number.optional(),
123
- intervalMs: t.number.optional()
124
- }).optional(),
125
-
126
- /**
127
- * Whether to trust the server certificate. Set to true for local development and self-signed certificates.
128
- * Default is false.
129
- */
130
- trustServerCertificate: t.boolean.optional(),
133
+ authentication: Authentication.optional(),
131
134
 
132
- reject_ip_ranges: t.array(t.string).optional()
135
+ reject_ip_ranges: t.array(t.string).optional(),
136
+ additionalConfig: AdditionalConfig.optional()
133
137
  })
134
138
  );
135
139
 
@@ -203,19 +207,11 @@ export function normalizeConnectionConfig(options: MSSQLConnectionConfig): Norma
203
207
  lookup,
204
208
  authentication: options.authentication,
205
209
 
206
- cdcPollingOptions: {
207
- /**
208
- * Maximum number of transactions to poll per polling cycle. Defaults to 10.
209
- */
210
- batchSize: options.cdcPollingOptions?.batchSize ?? 10,
211
-
212
- /**
213
- * Interval in milliseconds to wait between polling cycles. Defaults to 1 second.
214
- */
215
- intervalMs: options.cdcPollingOptions?.intervalMs ?? 1000,
216
- },
217
-
218
- trustServerCertificate: options.trustServerCertificate ?? false,
210
+ additionalConfig: {
211
+ pollingIntervalMs: options.additionalConfig?.pollingIntervalMs ?? 1000,
212
+ pollingBatchSize: options.additionalConfig?.pollingBatchSize ?? 10,
213
+ trustServerCertificate: options.additionalConfig?.trustServerCertificate ?? false
214
+ }
219
215
  } satisfies NormalizedMSSQLConnectionConfig;
220
216
  }
221
217
 
@@ -89,8 +89,11 @@ export async function ensurePowerSyncCheckpointsTable(connectionManager: MSSQLCo
89
89
  try {
90
90
  // check if the dbo_powersync_checkpoints table exists
91
91
  const { recordset: checkpointsResult } = await connectionManager.query(`
92
- SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '${connectionManager.schema}' AND TABLE_NAME = '${POWERSYNC_CHECKPOINTS_TABLE}';
93
- `);
92
+ SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = @schema AND TABLE_NAME = @tableName;
93
+ `, [
94
+ { name: 'schema', type: sql.VarChar(sql.MAX), value: connectionManager.schema },
95
+ { name: 'tableName', type: sql.VarChar(sql.MAX), value: POWERSYNC_CHECKPOINTS_TABLE },
96
+ ]);
94
97
  if (checkpointsResult.length > 0) {
95
98
  // Table already exists, check if CDC is enabled
96
99
  const isEnabled = await isTableEnabledForCDC({
@@ -114,7 +117,7 @@ export async function ensurePowerSyncCheckpointsTable(connectionManager: MSSQLCo
114
117
  // Try to create the table
115
118
  try {
116
119
  await connectionManager.query(`
117
- CREATE TABLE ${connectionManager.schema}.${POWERSYNC_CHECKPOINTS_TABLE} (
120
+ CREATE TABLE ${escapeIdentifier(connectionManager.schema)}.${escapeIdentifier(POWERSYNC_CHECKPOINTS_TABLE)} (
118
121
  id INT IDENTITY PRIMARY KEY,
119
122
  last_updated DATETIME NOT NULL DEFAULT (GETDATE())
120
123
  )`);
@@ -137,7 +140,7 @@ export async function ensurePowerSyncCheckpointsTable(connectionManager: MSSQLCo
137
140
 
138
141
  export async function createCheckpoint(connectionManager: MSSQLConnectionManager): Promise<void> {
139
142
  await connectionManager.query(`
140
- MERGE ${connectionManager.schema}.${POWERSYNC_CHECKPOINTS_TABLE} AS target
143
+ MERGE ${escapeIdentifier(connectionManager.schema)}.${escapeIdentifier(POWERSYNC_CHECKPOINTS_TABLE)} AS target
141
144
  USING (SELECT 1 AS id) AS source
142
145
  ON target.id = source.id
143
146
  WHEN MATCHED THEN
@@ -164,10 +167,12 @@ export async function isTableEnabledForCDC(options: IsTableEnabledForCDCOptions)
164
167
  SELECT 1 FROM cdc.change_tables ct
165
168
  JOIN sys.tables AS tbl ON tbl.object_id = ct.source_object_id
166
169
  JOIN sys.schemas AS sch ON sch.schema_id = tbl.schema_id
167
- WHERE sch.name = '${schema}'
168
- AND tbl.name = '${table}'
169
- `
170
- );
170
+ WHERE sch.name = @schema
171
+ AND tbl.name = @tableName
172
+ `, [
173
+ { name: 'schema', type: sql.VarChar(sql.MAX), value: schema },
174
+ { name: 'tableName', type: sql.VarChar(sql.MAX), value: table },
175
+ ]);
171
176
  return checkResult.length > 0;
172
177
  }
173
178
 
@@ -227,7 +232,8 @@ export async function isWithinRetentionThreshold(options: IsWithinRetentionThres
227
232
 
228
233
  export async function getMinLSN(connectionManager: MSSQLConnectionManager, captureInstance: string): Promise<LSN> {
229
234
  const { recordset: result } = await connectionManager.query(
230
- `SELECT sys.fn_cdc_get_min_lsn('${captureInstance}') AS min_lsn`
235
+ `SELECT sys.fn_cdc_get_min_lsn(@captureInstance) AS min_lsn`,
236
+ [{ name: 'captureInstance', type: sql.VarChar(sql.MAX), value: captureInstance }]
231
237
  );
232
238
  const rawMinLSN: Buffer = result[0].min_lsn;
233
239
  return LSN.fromBinary(rawMinLSN);
@@ -258,11 +264,13 @@ export async function getCaptureInstance(options: GetCaptureInstanceOptions): Pr
258
264
  sys.tables tbl
259
265
  INNER JOIN sys.schemas sch ON tbl.schema_id = sch.schema_id
260
266
  INNER JOIN cdc.change_tables ct ON ct.source_object_id = tbl.object_id
261
- WHERE sch.name = '${schema}'
262
- AND tbl.name = '${tableName}'
267
+ WHERE sch.name = @schema
268
+ AND tbl.name = @tableName
263
269
  AND ct.end_lsn IS NULL;
264
- `
265
- );
270
+ `, [
271
+ { name: 'schema', type: sql.VarChar(sql.MAX), value: schema },
272
+ { name: 'tableName', type: sql.VarChar(sql.MAX), value: tableName },
273
+ ]);
266
274
 
267
275
  if (result.length === 0) {
268
276
  return null;
@@ -301,7 +309,10 @@ export async function getLatestReplicatedLSN(connectionManager: MSSQLConnectionM
301
309
  * @param identifier
302
310
  */
303
311
  export function escapeIdentifier(identifier: string): string {
304
- return `[${identifier}]`;
312
+ // 1. Replace existing closing brackets ] with ]] to escape them
313
+ // 2. Replace dots . with ].[ to handle qualified names
314
+ // 3. Wrap the whole result in [ ]
315
+ return `[${identifier.replace(/]/g, ']]').replace(/\./g, '].[')}]`;
305
316
  }
306
317
 
307
318
  export function toQualifiedTableName(schema: string, tableName: string): string {
@@ -319,7 +330,7 @@ export function isIColumnMetadata(obj: any): obj is sql.IColumnMetadata {
319
330
  propertiesMatched =
320
331
  typeof property.index === 'number' &&
321
332
  typeof property.name === 'string' &&
322
- typeof property.length === 'number' &&
333
+ (typeof property.length === 'number' || typeof property.length === 'undefined') &&
323
334
  (typeof property.type === 'function' || typeof property.type === 'object') &&
324
335
  typeof property.nullable === 'boolean' &&
325
336
  typeof property.caseSensitive === 'boolean' &&
@@ -383,7 +394,9 @@ export async function getDebugTableInfo(options: GetDebugTableInfoOptions): Prom
383
394
 
384
395
  let selectError: service_types.ReplicationError | null = null;
385
396
  try {
386
- await connectionManager.query(`SELECT TOP 1 * FROM [${toQualifiedTableName(schema, table.name)}]`);
397
+ await connectionManager.query(`SELECT TOP 1 * FROM @tableName`, [
398
+ { name: 'tableName', type: sql.VarChar(sql.MAX), value: toQualifiedTableName(schema, table.name) },
399
+ ]);
387
400
  } catch (e) {
388
401
  selectError = { level: 'fatal', message: e.message };
389
402
  }
@@ -2,7 +2,7 @@ import { SourceEntityDescriptor } from '@powersync/service-core';
2
2
  import { TablePattern } from '@powersync/service-sync-rules';
3
3
  import { MSSQLConnectionManager } from '../replication/MSSQLConnectionManager.js';
4
4
  import { MSSQLColumnDescriptor } from '../types/mssql-data-types.js';
5
- import { escapeIdentifier } from './mssql.js';
5
+ import sql from 'mssql';
6
6
 
7
7
  export interface GetColumnsOptions {
8
8
  connectionManager: MSSQLConnectionManager;
@@ -23,10 +23,13 @@ async function getColumns(options: GetColumnsOptions): Promise<MSSQLColumnDescri
23
23
  JOIN sys.tables AS tbl ON tbl.object_id = col.object_id
24
24
  JOIN sys.schemas AS sch ON sch.schema_id = tbl.schema_id
25
25
  JOIN sys.types AS typ ON typ.user_type_id = col.user_type_id
26
- WHERE sch.name = '${schema}'
27
- AND tbl.name = '${tableName}'
26
+ WHERE sch.name = @schema
27
+ AND tbl.name = @tableName
28
28
  ORDER BY col.column_id;
29
- `);
29
+ `, [
30
+ { name: 'schema', type: sql.VarChar(sql.MAX), value: schema },
31
+ { name: 'tableName', type: sql.VarChar(sql.MAX), value: tableName },
32
+ ]);
30
33
 
31
34
  return columnResults.map((row) => {
32
35
  return {
@@ -65,10 +68,13 @@ export async function getReplicationIdentityColumns(
65
68
  JOIN sys.index_columns AS idx_col ON idx_col.object_id = idx.object_id AND idx_col.index_id = idx.index_id
66
69
  JOIN sys.columns AS col ON col.object_id = idx_col.object_id AND col.column_id = idx_col.column_id
67
70
  JOIN sys.types AS typ ON typ.user_type_id = col.user_type_id
68
- WHERE sch.name = '${schema}'
69
- AND tbl.name = '${tableName}'
71
+ WHERE sch.name = @schema
72
+ AND tbl.name = @tableName
70
73
  ORDER BY idx_col.key_ordinal;
71
- `);
74
+ `, [
75
+ { name: 'schema', type: sql.VarChar(sql.MAX), value: schema },
76
+ { name: 'tableName', type: sql.VarChar(sql.MAX), value: tableName },
77
+ ]);
72
78
 
73
79
  if (primaryKeyColumns.length > 0) {
74
80
  return {
@@ -95,10 +101,13 @@ export async function getReplicationIdentityColumns(
95
101
  JOIN sys.index_columns AS idx_col ON idx_col.object_id = idx.object_id AND idx_col.index_id = idx.index_id
96
102
  JOIN sys.columns AS col ON col.object_id = idx_col.object_id AND col.column_id = idx_col.column_id
97
103
  JOIN sys.types AS typ ON typ.user_type_id = col.user_type_id
98
- WHERE sch.name = '${schema}'
99
- AND tbl.name = '${tableName}'
104
+ WHERE sch.name = @schema
105
+ AND tbl.name = @tableName
100
106
  ORDER BY idx_col.key_ordinal;
101
- `);
107
+ `, [
108
+ { name: 'schema', type: sql.VarChar(sql.MAX), value: schema },
109
+ { name: 'tableName', type: sql.VarChar(sql.MAX), value: tableName },
110
+ ]);
102
111
 
103
112
  if (uniqueKeyColumns.length > 0) {
104
113
  return {
@@ -134,9 +143,12 @@ export async function getTablesFromPattern(
134
143
  tbl.object_id AS object_id
135
144
  FROM sys.tables tbl
136
145
  JOIN sys.schemas sch ON tbl.schema_id = sch.schema_id
137
- WHERE sch.name = '${tablePattern.schema}'
138
- AND tbl.name LIKE '${tablePattern.tablePattern}'
139
- `);
146
+ WHERE sch.name = @schema
147
+ AND tbl.name LIKE @tablePattern
148
+ `, [
149
+ { name: 'schema', type: sql.VarChar(sql.MAX), value: tablePattern.schema },
150
+ { name: 'tablePattern', type: sql.VarChar(sql.MAX), value: tablePattern.tablePattern },
151
+ ]);
140
152
 
141
153
  return tableResults
142
154
  .map((row) => {
@@ -156,10 +168,12 @@ export async function getTablesFromPattern(
156
168
  tbl.object_id AS object_id
157
169
  FROM sys.tables tbl
158
170
  JOIN sys.schemas sch ON tbl.schema_id = sch.schema_id
159
- WHERE sch.name = '${tablePattern.schema}'
160
- AND tbl.name = '${tablePattern.name}'
161
- `
162
- );
171
+ WHERE sch.name = @schema
172
+ AND tbl.name = @tablePattern
173
+ `, [
174
+ { name: 'schema', type: sql.VarChar(sql.MAX), value: tablePattern.schema },
175
+ { name: 'tablePattern', type: sql.VarChar(sql.MAX), value: tablePattern.tablePattern },
176
+ ]);
163
177
 
164
178
  return tableResults.map((row) => {
165
179
  return {
@@ -1,16 +1,10 @@
1
1
  import { describe, expect, test } from 'vitest';
2
2
  import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests';
3
3
  import { ReplicationMetric } from '@powersync/service-types';
4
- import {
5
- createTestTable,
6
- describeWithStorage,
7
- INITIALIZED_MONGO_STORAGE_FACTORY,
8
- insertTestData,
9
- waitForPendingCDCChanges
10
- } from './util.js';
4
+ import { createTestTable, describeWithStorage, insertTestData, waitForPendingCDCChanges } from './util.js';
11
5
  import { storage } from '@powersync/service-core';
12
6
  import { CDCStreamTestContext } from './CDCStreamTestContext.js';
13
- import { getLatestReplicatedLSN } from '@module/utils/mssql.js';
7
+ import { getLatestLSN } from '@module/utils/mssql.js';
14
8
  import sql from 'mssql';
15
9
 
16
10
  const BASIC_SYNC_RULES = `
@@ -20,11 +14,9 @@ bucket_definitions:
20
14
  - SELECT id, description FROM "test_data"
21
15
  `;
22
16
 
23
- // describe('CDCStream tests', () => {
24
- // describeWithStorage({ timeout: 20_000 }, defineCDCStreamTests);
25
- // });
26
-
27
- defineCDCStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY);
17
+ describe('CDCStream tests', () => {
18
+ describeWithStorage({ timeout: 20_000 }, defineCDCStreamTests);
19
+ });
28
20
 
29
21
  function defineCDCStreamTests(factory: storage.TestStorageFactory) {
30
22
  test('Initial snapshot sync', async () => {
@@ -33,7 +25,7 @@ function defineCDCStreamTests(factory: storage.TestStorageFactory) {
33
25
  await context.updateSyncRules(BASIC_SYNC_RULES);
34
26
 
35
27
  await createTestTable(connectionManager, 'test_data');
36
- const beforeLSN = await getLatestReplicatedLSN(connectionManager);
28
+ const beforeLSN = await getLatestLSN(connectionManager);
37
29
  const testData = await insertTestData(connectionManager, 'test_data');
38
30
  await waitForPendingCDCChanges(beforeLSN, connectionManager);
39
31
  const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
@@ -77,7 +69,7 @@ function defineCDCStreamTests(factory: storage.TestStorageFactory) {
77
69
  await context.updateSyncRules(BASIC_SYNC_RULES);
78
70
 
79
71
  await createTestTable(connectionManager, 'test_data');
80
- const beforeLSN = await getLatestReplicatedLSN(connectionManager);
72
+ const beforeLSN = await getLatestLSN(connectionManager);
81
73
  const testData = await insertTestData(connectionManager, 'test_data');
82
74
  await waitForPendingCDCChanges(beforeLSN, connectionManager);
83
75
  await context.replicateSnapshot();
@@ -101,7 +93,7 @@ function defineCDCStreamTests(factory: storage.TestStorageFactory) {
101
93
  await context.updateSyncRules(BASIC_SYNC_RULES);
102
94
 
103
95
  await createTestTable(connectionManager, 'test_data');
104
- const beforeLSN = await getLatestReplicatedLSN(connectionManager);
96
+ const beforeLSN = await getLatestLSN(connectionManager);
105
97
  const testData = await insertTestData(connectionManager, 'test_data');
106
98
  await waitForPendingCDCChanges(beforeLSN, connectionManager);
107
99
  await context.replicateSnapshot();
@@ -128,8 +120,8 @@ function defineCDCStreamTests(factory: storage.TestStorageFactory) {
128
120
  await createTestTable(connectionManager, 'test_data_1');
129
121
  await createTestTable(connectionManager, 'test_data_2');
130
122
 
131
- const beforeLSN = await getLatestReplicatedLSN(connectionManager);
132
123
  const testData11 = await insertTestData(connectionManager, 'test_data_1');
124
+ const beforeLSN = await getLatestLSN(connectionManager);
133
125
  const testData21 = await insertTestData(connectionManager, 'test_data_2');
134
126
  await waitForPendingCDCChanges(beforeLSN, connectionManager);
135
127
 
@@ -12,7 +12,6 @@ import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './uti
12
12
  import { CDCStream, CDCStreamOptions } from '@module/replication/CDCStream.js';
13
13
  import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
14
14
  import timers from 'timers/promises';
15
- import { CDCPollingOptions } from '@module/types/types.js';
16
15
 
17
16
  /**
18
17
  * Tests operating on the change data capture need to configure the stream and manage asynchronous
@@ -110,10 +109,11 @@ export class CDCStreamTestContext implements AsyncDisposable {
110
109
  metrics: METRICS_HELPER.metricsEngine,
111
110
  connections: this.connectionManager,
112
111
  abortSignal: this.abortController.signal,
113
- pollingOptions: {
114
- batchSize: 10,
115
- intervalMs: 1000
116
- } satisfies CDCPollingOptions,
112
+ additionalConfig: {
113
+ pollingBatchSize: 10,
114
+ pollingIntervalMs: 1000,
115
+ trustServerCertificate: true
116
+ },
117
117
  ...this.cdcStreamOptions
118
118
  };
119
119
  this._cdcStream = new CDCStream(options);