@powersync/service-module-mssql 0.0.1 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/api/MSSQLRouteAPIAdapter.js +16 -51
- package/dist/api/MSSQLRouteAPIAdapter.js.map +1 -1
- package/dist/common/LSN.js +2 -2
- package/dist/common/LSN.js.map +1 -1
- package/dist/common/MSSQLSourceTable.js +4 -4
- package/dist/common/MSSQLSourceTable.js.map +1 -1
- package/dist/common/mssqls-to-sqlite.js +4 -4
- package/dist/common/mssqls-to-sqlite.js.map +1 -1
- package/dist/module/MSSQLModule.js +1 -1
- package/dist/module/MSSQLModule.js.map +1 -1
- package/dist/replication/CDCPoller.d.ts +2 -2
- package/dist/replication/CDCPoller.js +12 -8
- package/dist/replication/CDCPoller.js.map +1 -1
- package/dist/replication/CDCReplicationJob.d.ts +2 -2
- package/dist/replication/CDCReplicationJob.js +1 -1
- package/dist/replication/CDCReplicationJob.js.map +1 -1
- package/dist/replication/CDCReplicator.d.ts +2 -2
- package/dist/replication/CDCReplicator.js +1 -1
- package/dist/replication/CDCReplicator.js.map +1 -1
- package/dist/replication/CDCStream.d.ts +2 -2
- package/dist/replication/CDCStream.js +32 -37
- package/dist/replication/CDCStream.js.map +1 -1
- package/dist/replication/MSSQLConnectionManager.js +1 -1
- package/dist/replication/MSSQLConnectionManager.js.map +1 -1
- package/dist/replication/MSSQLSnapshotQuery.d.ts +0 -17
- package/dist/replication/MSSQLSnapshotQuery.js +0 -47
- package/dist/replication/MSSQLSnapshotQuery.js.map +1 -1
- package/dist/types/types.d.ts +80 -23
- package/dist/types/types.js +24 -24
- package/dist/types/types.js.map +1 -1
- package/dist/utils/mssql.js +28 -14
- package/dist/utils/mssql.js.map +1 -1
- package/dist/utils/schema.js +31 -15
- package/dist/utils/schema.js.map +1 -1
- package/package.json +10 -10
- package/src/api/MSSQLRouteAPIAdapter.ts +16 -51
- package/src/common/LSN.ts +2 -2
- package/src/common/MSSQLSourceTable.ts +4 -4
- package/src/common/mssqls-to-sqlite.ts +11 -4
- package/src/module/MSSQLModule.ts +1 -1
- package/src/replication/CDCPoller.ts +15 -10
- package/src/replication/CDCReplicationJob.ts +3 -3
- package/src/replication/CDCReplicator.ts +3 -3
- package/src/replication/CDCStream.ts +36 -49
- package/src/replication/MSSQLConnectionManager.ts +1 -1
- package/src/replication/MSSQLSnapshotQuery.ts +0 -54
- package/src/types/types.ts +41 -45
- package/src/utils/mssql.ts +29 -16
- package/src/utils/schema.ts +31 -17
- package/test/src/CDCStream.test.ts +9 -17
- package/test/src/CDCStreamTestContext.ts +5 -5
- package/test/src/CDCStream_resumable_snapshot.test.ts +15 -9
- package/test/src/mssql-to-sqlite.test.ts +28 -27
- package/test/src/util.ts +27 -14
- package/tsconfig.tsbuildinfo +1 -1
- package/ci/init-mssql.sql +0 -50
- package/test/tsconfig.tsbuildinfo +0 -1
|
@@ -13,13 +13,7 @@ import { getUuidReplicaIdentityBson, MetricsEngine, SourceEntityDescriptor, stor
|
|
|
13
13
|
import { SqliteInputRow, SqliteRow, SqlSyncRules, TablePattern } from '@powersync/service-sync-rules';
|
|
14
14
|
|
|
15
15
|
import { ReplicationMetric } from '@powersync/service-types';
|
|
16
|
-
import {
|
|
17
|
-
BatchedSnapshotQuery,
|
|
18
|
-
IdSnapshotQuery,
|
|
19
|
-
MSSQLSnapshotQuery,
|
|
20
|
-
PrimaryKeyValue,
|
|
21
|
-
SimpleSnapshotQuery
|
|
22
|
-
} from './MSSQLSnapshotQuery.js';
|
|
16
|
+
import { BatchedSnapshotQuery, MSSQLSnapshotQuery, SimpleSnapshotQuery } from './MSSQLSnapshotQuery.js';
|
|
23
17
|
import { MSSQLConnectionManager } from './MSSQLConnectionManager.js';
|
|
24
18
|
import { getReplicationIdentityColumns, getTablesFromPattern, ResolvedTable } from '../utils/schema.js';
|
|
25
19
|
import {
|
|
@@ -39,7 +33,7 @@ import { LSN } from '../common/LSN.js';
|
|
|
39
33
|
import { MSSQLSourceTable } from '../common/MSSQLSourceTable.js';
|
|
40
34
|
import { MSSQLSourceTableCache } from '../common/MSSQLSourceTableCache.js';
|
|
41
35
|
import { CDCEventHandler, CDCPoller } from './CDCPoller.js';
|
|
42
|
-
import {
|
|
36
|
+
import { AdditionalConfig } from '../types/types.js';
|
|
43
37
|
|
|
44
38
|
export interface CDCStreamOptions {
|
|
45
39
|
connections: MSSQLConnectionManager;
|
|
@@ -54,7 +48,7 @@ export interface CDCStreamOptions {
|
|
|
54
48
|
*/
|
|
55
49
|
snapshotBatchSize?: number;
|
|
56
50
|
|
|
57
|
-
|
|
51
|
+
additionalConfig: AdditionalConfig;
|
|
58
52
|
}
|
|
59
53
|
|
|
60
54
|
export enum SnapshotStatus {
|
|
@@ -247,7 +241,11 @@ export class CDCStream {
|
|
|
247
241
|
entity_descriptor: table,
|
|
248
242
|
sync_rules: this.syncRules
|
|
249
243
|
});
|
|
250
|
-
const captureInstance = await getCaptureInstance({
|
|
244
|
+
const captureInstance = await getCaptureInstance({
|
|
245
|
+
connectionManager: this.connections,
|
|
246
|
+
tableName: resolved.table.name,
|
|
247
|
+
schema: resolved.table.schema
|
|
248
|
+
});
|
|
251
249
|
if (!captureInstance) {
|
|
252
250
|
throw new ServiceAssertionError(
|
|
253
251
|
`Missing capture instance for table ${toQualifiedTableName(resolved.table.schema, resolved.table.name)}`
|
|
@@ -282,18 +280,14 @@ export class CDCStream {
|
|
|
282
280
|
return resolvedTable;
|
|
283
281
|
}
|
|
284
282
|
|
|
285
|
-
private async snapshotTableInTx(
|
|
286
|
-
batch: storage.BucketStorageBatch,
|
|
287
|
-
table: MSSQLSourceTable,
|
|
288
|
-
limited?: PrimaryKeyValue[]
|
|
289
|
-
): Promise<void> {
|
|
283
|
+
private async snapshotTableInTx(batch: storage.BucketStorageBatch, table: MSSQLSourceTable): Promise<void> {
|
|
290
284
|
// Note: We use the "Read Committed" isolation level here, not snapshot isolation.
|
|
291
285
|
// The data may change during the transaction, but that is compensated for in the streaming
|
|
292
286
|
// replication afterward.
|
|
293
287
|
const transaction = await this.connections.createTransaction();
|
|
294
288
|
await transaction.begin(sql.ISOLATION_LEVEL.READ_COMMITTED);
|
|
295
289
|
try {
|
|
296
|
-
await this.snapshotTable(batch, transaction, table
|
|
290
|
+
await this.snapshotTable(batch, transaction, table);
|
|
297
291
|
|
|
298
292
|
// Get the current LSN.
|
|
299
293
|
// The data will only be consistent once incremental replication has passed that point.
|
|
@@ -321,8 +315,7 @@ export class CDCStream {
|
|
|
321
315
|
private async snapshotTable(
|
|
322
316
|
batch: storage.BucketStorageBatch,
|
|
323
317
|
transaction: sql.Transaction,
|
|
324
|
-
table: MSSQLSourceTable
|
|
325
|
-
limited?: PrimaryKeyValue[]
|
|
318
|
+
table: MSSQLSourceTable
|
|
326
319
|
) {
|
|
327
320
|
let totalEstimatedCount = table.sourceTable.snapshotStatus?.totalEstimatedCount;
|
|
328
321
|
let replicatedCount = table.sourceTable.snapshotStatus?.replicatedCount ?? 0;
|
|
@@ -331,9 +324,7 @@ export class CDCStream {
|
|
|
331
324
|
// We do streaming on two levels:
|
|
332
325
|
// 1. Coarse select from the entire table, stream rows 1 by one
|
|
333
326
|
// 2. Fine level: Stream batches of rows with each fetch call
|
|
334
|
-
if (
|
|
335
|
-
query = new IdSnapshotQuery(transaction, table, limited);
|
|
336
|
-
} else if (BatchedSnapshotQuery.supports(table)) {
|
|
327
|
+
if (BatchedSnapshotQuery.supports(table)) {
|
|
337
328
|
// Single primary key - we can use the primary key for chunking
|
|
338
329
|
const orderByKey = table.sourceTable.replicaIdColumns[0];
|
|
339
330
|
query = new BatchedSnapshotQuery(
|
|
@@ -361,7 +352,6 @@ export class CDCStream {
|
|
|
361
352
|
}
|
|
362
353
|
await query.initialize();
|
|
363
354
|
|
|
364
|
-
let columns: sql.IColumnMetadata | null = null;
|
|
365
355
|
let hasRemainingData = true;
|
|
366
356
|
while (hasRemainingData) {
|
|
367
357
|
// Fetch 10k at a time.
|
|
@@ -369,6 +359,7 @@ export class CDCStream {
|
|
|
369
359
|
// and not spending too much time on each FETCH call.
|
|
370
360
|
// We aim for a couple of seconds on each FETCH call.
|
|
371
361
|
let batchReplicatedCount = 0;
|
|
362
|
+
let columns: sql.IColumnMetadata | null = null;
|
|
372
363
|
const cursor = query.next();
|
|
373
364
|
for await (const result of cursor) {
|
|
374
365
|
if (columns == null && isIColumnMetadata(result)) {
|
|
@@ -400,30 +391,27 @@ export class CDCStream {
|
|
|
400
391
|
|
|
401
392
|
// Important: flush before marking progress
|
|
402
393
|
await batch.flush();
|
|
403
|
-
if (limited == null) {
|
|
404
|
-
let lastKey: Uint8Array | undefined;
|
|
405
|
-
if (query instanceof BatchedSnapshotQuery) {
|
|
406
|
-
lastKey = query.getLastKeySerialized();
|
|
407
|
-
}
|
|
408
|
-
if (lastCountTime < performance.now() - 10 * 60 * 1000) {
|
|
409
|
-
// Even though we're doing the snapshot inside a transaction, the transaction uses
|
|
410
|
-
// the default "Read Committed" isolation level. This means we can get new data
|
|
411
|
-
// within the transaction, so we re-estimate the count every 10 minutes when replicating
|
|
412
|
-
// large tables.
|
|
413
|
-
totalEstimatedCount = await this.estimatedCountNumber(table, transaction);
|
|
414
|
-
lastCountTime = performance.now();
|
|
415
|
-
}
|
|
416
|
-
const updatedSourceTable = await batch.updateTableProgress(table.sourceTable, {
|
|
417
|
-
lastKey: lastKey,
|
|
418
|
-
replicatedCount: replicatedCount,
|
|
419
|
-
totalEstimatedCount: totalEstimatedCount
|
|
420
|
-
});
|
|
421
|
-
this.tableCache.updateSourceTable(updatedSourceTable);
|
|
422
394
|
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
395
|
+
let lastKey: Uint8Array | undefined;
|
|
396
|
+
if (query instanceof BatchedSnapshotQuery) {
|
|
397
|
+
lastKey = query.getLastKeySerialized();
|
|
398
|
+
}
|
|
399
|
+
if (lastCountTime < performance.now() - 10 * 60 * 1000) {
|
|
400
|
+
// Even though we're doing the snapshot inside a transaction, the transaction uses
|
|
401
|
+
// the default "Read Committed" isolation level. This means we can get new data
|
|
402
|
+
// within the transaction, so we re-estimate the count every 10 minutes when replicating
|
|
403
|
+
// large tables.
|
|
404
|
+
totalEstimatedCount = await this.estimatedCountNumber(table, transaction);
|
|
405
|
+
lastCountTime = performance.now();
|
|
426
406
|
}
|
|
407
|
+
const updatedSourceTable = await batch.updateTableProgress(table.sourceTable, {
|
|
408
|
+
lastKey: lastKey,
|
|
409
|
+
replicatedCount: replicatedCount,
|
|
410
|
+
totalEstimatedCount: totalEstimatedCount
|
|
411
|
+
});
|
|
412
|
+
this.tableCache.updateSourceTable(updatedSourceTable);
|
|
413
|
+
|
|
414
|
+
this.logger.info(`Replicating ${table.toQualifiedName()} ${table.sourceTable.formatSnapshotProgress()}`);
|
|
427
415
|
|
|
428
416
|
if (this.abortSignal.aborted) {
|
|
429
417
|
// We only abort after flushing
|
|
@@ -445,13 +433,12 @@ export class CDCStream {
|
|
|
445
433
|
*/
|
|
446
434
|
async estimatedCountNumber(table: MSSQLSourceTable, transaction?: sql.Transaction): Promise<number> {
|
|
447
435
|
const request = transaction ? transaction.request() : await this.connections.createRequest();
|
|
448
|
-
const { recordset: result } = await request.query(
|
|
436
|
+
const { recordset: result } = await request.input('tableName', table.toQualifiedName()).query(
|
|
449
437
|
`SELECT SUM(row_count) AS total_rows
|
|
450
438
|
FROM sys.dm_db_partition_stats
|
|
451
|
-
WHERE object_id = OBJECT_ID(
|
|
439
|
+
WHERE object_id = OBJECT_ID(@tableName)
|
|
452
440
|
AND index_id < 2;`
|
|
453
441
|
);
|
|
454
|
-
// TODO Fallback query in case user does not have permission?
|
|
455
442
|
return result[0].total_rows ?? -1;
|
|
456
443
|
}
|
|
457
444
|
|
|
@@ -586,8 +573,8 @@ export class CDCStream {
|
|
|
586
573
|
eventHandler,
|
|
587
574
|
sourceTables,
|
|
588
575
|
startLSN,
|
|
589
|
-
|
|
590
|
-
|
|
576
|
+
logger: this.logger,
|
|
577
|
+
additionalConfig: this.options.additionalConfig
|
|
591
578
|
});
|
|
592
579
|
|
|
593
580
|
this.abortSignal.addEventListener(
|
|
@@ -31,7 +31,7 @@ export class MSSQLConnectionManager extends BaseObserver<MSSQLConnectionManagerL
|
|
|
31
31
|
options: {
|
|
32
32
|
appName: `powersync/${POWERSYNC_VERSION}`,
|
|
33
33
|
encrypt: true, // Required for Azure
|
|
34
|
-
trustServerCertificate: options.trustServerCertificate
|
|
34
|
+
trustServerCertificate: options.additionalConfig.trustServerCertificate
|
|
35
35
|
}
|
|
36
36
|
});
|
|
37
37
|
}
|
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { bson, ColumnDescriptor, SourceTable } from '@powersync/service-core';
|
|
2
|
-
import { SqliteValue } from '@powersync/service-sync-rules';
|
|
3
2
|
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
4
3
|
import { MSSQLBaseType } from '../types/mssql-data-types.js';
|
|
5
4
|
import sql from 'mssql';
|
|
@@ -15,8 +14,6 @@ export interface MSSQLSnapshotQuery {
|
|
|
15
14
|
next(): AsyncIterableIterator<sql.IColumnMetadata | sql.IRecordSet<any>>;
|
|
16
15
|
}
|
|
17
16
|
|
|
18
|
-
export type PrimaryKeyValue = Record<string, SqliteValue>;
|
|
19
|
-
|
|
20
17
|
/**
|
|
21
18
|
* Snapshot query using a plain SELECT * FROM table
|
|
22
19
|
*
|
|
@@ -177,54 +174,3 @@ export class BatchedSnapshotQuery implements MSSQLSnapshotQuery {
|
|
|
177
174
|
return decoded[this.key.name];
|
|
178
175
|
}
|
|
179
176
|
}
|
|
180
|
-
|
|
181
|
-
/**
|
|
182
|
-
* This performs a snapshot query using a list of primary keys.
|
|
183
|
-
*
|
|
184
|
-
* This is not used for general snapshots, but is used when we need to re-fetch specific rows
|
|
185
|
-
* during streaming replication.
|
|
186
|
-
*/
|
|
187
|
-
export class IdSnapshotQuery implements MSSQLSnapshotQuery {
|
|
188
|
-
static supports(table: SourceTable | MSSQLSourceTable) {
|
|
189
|
-
// We have the same requirements as BatchedSnapshotQuery.
|
|
190
|
-
// This is typically only used as a fallback when ChunkedSnapshotQuery
|
|
191
|
-
// skipped some rows.
|
|
192
|
-
return BatchedSnapshotQuery.supports(table);
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
public constructor(
|
|
196
|
-
private readonly transaction: sql.Transaction,
|
|
197
|
-
private readonly table: MSSQLSourceTable,
|
|
198
|
-
private readonly keys: PrimaryKeyValue[]
|
|
199
|
-
) {}
|
|
200
|
-
|
|
201
|
-
public async initialize(): Promise<void> {
|
|
202
|
-
// No-op
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
public async *next(): AsyncIterableIterator<sql.IColumnMetadata | sql.IRecordSet<any>> {
|
|
206
|
-
const metadataRequest = this.transaction.request();
|
|
207
|
-
metadataRequest.stream = true;
|
|
208
|
-
const metadataPromise = new Promise<sql.IColumnMetadata>((resolve, reject) => {
|
|
209
|
-
metadataRequest.on('recordset', resolve);
|
|
210
|
-
metadataRequest.on('error', reject);
|
|
211
|
-
});
|
|
212
|
-
metadataRequest.query(`SELECT TOP(0) * FROM ${this.table.toQualifiedName()}`);
|
|
213
|
-
const columnMetadata: sql.IColumnMetadata = await metadataPromise;
|
|
214
|
-
yield columnMetadata;
|
|
215
|
-
|
|
216
|
-
const keyDefinition = this.table.sourceTable.replicaIdColumns[0];
|
|
217
|
-
const ids = this.keys.map((record) => record[keyDefinition.name]);
|
|
218
|
-
|
|
219
|
-
const request = this.transaction.request();
|
|
220
|
-
const stream = request.toReadableStream();
|
|
221
|
-
request
|
|
222
|
-
.input('ids', ids)
|
|
223
|
-
.query(`SELECT * FROM ${this.table.toQualifiedName()} WHERE ${escapeIdentifier(keyDefinition.name)} = @ids`);
|
|
224
|
-
|
|
225
|
-
// MSSQL only streams one row at a time
|
|
226
|
-
for await (const row of stream) {
|
|
227
|
-
yield row;
|
|
228
|
-
}
|
|
229
|
-
}
|
|
230
|
-
}
|
package/src/types/types.ts
CHANGED
|
@@ -63,21 +63,42 @@ export const DefaultAuthentication = t.object({
|
|
|
63
63
|
});
|
|
64
64
|
export type DefaultAuthentication = t.Decoded<typeof DefaultAuthentication>;
|
|
65
65
|
|
|
66
|
-
export
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
66
|
+
export const Authentication = DefaultAuthentication.or(AzureActiveDirectoryPasswordAuthentication).or(
|
|
67
|
+
AzureActiveDirectoryServicePrincipalSecret
|
|
68
|
+
);
|
|
69
|
+
export type Authentication = t.Decoded<typeof Authentication>;
|
|
70
70
|
|
|
71
|
-
|
|
72
|
-
|
|
71
|
+
export const AdditionalConfig = t.object({
|
|
72
|
+
/**
|
|
73
|
+
* Interval in milliseconds to wait between polling cycles. Defaults to 1000 milliseconds.
|
|
74
|
+
*/
|
|
75
|
+
pollingIntervalMs: t.number.optional(),
|
|
73
76
|
/**
|
|
74
77
|
* Maximum number of transactions to poll per polling cycle. Defaults to 10.
|
|
75
78
|
*/
|
|
76
|
-
|
|
79
|
+
pollingBatchSize: t.number.optional(),
|
|
80
|
+
|
|
77
81
|
/**
|
|
78
|
-
*
|
|
82
|
+
* Whether to trust the server certificate. Set to true for local development and self-signed certificates.
|
|
83
|
+
* Default is false.
|
|
79
84
|
*/
|
|
80
|
-
|
|
85
|
+
trustServerCertificate: t.boolean.optional()
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
export interface AdditionalConfig {
|
|
89
|
+
/**
|
|
90
|
+
* Interval in milliseconds to wait between polling cycles. Defaults to 1000 milliseconds.
|
|
91
|
+
*/
|
|
92
|
+
pollingIntervalMs: number;
|
|
93
|
+
/**
|
|
94
|
+
* Maximum number of transactions to poll per polling cycle. Defaults to 10.
|
|
95
|
+
*/
|
|
96
|
+
pollingBatchSize: number;
|
|
97
|
+
/**
|
|
98
|
+
* Whether to trust the server certificate. Set to true for local development and self-signed certificates.
|
|
99
|
+
* Default is false.
|
|
100
|
+
*/
|
|
101
|
+
trustServerCertificate: boolean;
|
|
81
102
|
}
|
|
82
103
|
|
|
83
104
|
export interface NormalizedMSSQLConnectionConfig {
|
|
@@ -91,16 +112,11 @@ export interface NormalizedMSSQLConnectionConfig {
|
|
|
91
112
|
database: string;
|
|
92
113
|
schema?: string;
|
|
93
114
|
|
|
94
|
-
authentication?:
|
|
95
|
-
|
|
96
|
-
cdcPollingOptions: CDCPollingOptions;
|
|
115
|
+
authentication?: Authentication;
|
|
97
116
|
|
|
98
|
-
/**
|
|
99
|
-
* Whether to trust the server certificate. Set to true for local development and self-signed certificates.
|
|
100
|
-
* Default is false.
|
|
101
|
-
*/
|
|
102
|
-
trustServerCertificate: boolean;
|
|
103
117
|
lookup?: LookupFunction;
|
|
118
|
+
|
|
119
|
+
additionalConfig: AdditionalConfig;
|
|
104
120
|
}
|
|
105
121
|
|
|
106
122
|
export const MSSQLConnectionConfig = service_types.configFile.DataSourceConfig.and(
|
|
@@ -114,22 +130,10 @@ export const MSSQLConnectionConfig = service_types.configFile.DataSourceConfig.a
|
|
|
114
130
|
hostname: t.string.optional(),
|
|
115
131
|
port: service_types.configFile.portCodec.optional(),
|
|
116
132
|
|
|
117
|
-
authentication:
|
|
118
|
-
.or(AzureActiveDirectoryServicePrincipalSecret)
|
|
119
|
-
.optional(),
|
|
120
|
-
|
|
121
|
-
cdcPollingOptions: t.object({
|
|
122
|
-
batchSize: t.number.optional(),
|
|
123
|
-
intervalMs: t.number.optional()
|
|
124
|
-
}).optional(),
|
|
125
|
-
|
|
126
|
-
/**
|
|
127
|
-
* Whether to trust the server certificate. Set to true for local development and self-signed certificates.
|
|
128
|
-
* Default is false.
|
|
129
|
-
*/
|
|
130
|
-
trustServerCertificate: t.boolean.optional(),
|
|
133
|
+
authentication: Authentication.optional(),
|
|
131
134
|
|
|
132
|
-
reject_ip_ranges: t.array(t.string).optional()
|
|
135
|
+
reject_ip_ranges: t.array(t.string).optional(),
|
|
136
|
+
additionalConfig: AdditionalConfig.optional()
|
|
133
137
|
})
|
|
134
138
|
);
|
|
135
139
|
|
|
@@ -203,19 +207,11 @@ export function normalizeConnectionConfig(options: MSSQLConnectionConfig): Norma
|
|
|
203
207
|
lookup,
|
|
204
208
|
authentication: options.authentication,
|
|
205
209
|
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
/**
|
|
213
|
-
* Interval in milliseconds to wait between polling cycles. Defaults to 1 second.
|
|
214
|
-
*/
|
|
215
|
-
intervalMs: options.cdcPollingOptions?.intervalMs ?? 1000,
|
|
216
|
-
},
|
|
217
|
-
|
|
218
|
-
trustServerCertificate: options.trustServerCertificate ?? false,
|
|
210
|
+
additionalConfig: {
|
|
211
|
+
pollingIntervalMs: options.additionalConfig?.pollingIntervalMs ?? 1000,
|
|
212
|
+
pollingBatchSize: options.additionalConfig?.pollingBatchSize ?? 10,
|
|
213
|
+
trustServerCertificate: options.additionalConfig?.trustServerCertificate ?? false
|
|
214
|
+
}
|
|
219
215
|
} satisfies NormalizedMSSQLConnectionConfig;
|
|
220
216
|
}
|
|
221
217
|
|
package/src/utils/mssql.ts
CHANGED
|
@@ -89,8 +89,11 @@ export async function ensurePowerSyncCheckpointsTable(connectionManager: MSSQLCo
|
|
|
89
89
|
try {
|
|
90
90
|
// check if the dbo_powersync_checkpoints table exists
|
|
91
91
|
const { recordset: checkpointsResult } = await connectionManager.query(`
|
|
92
|
-
SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA =
|
|
93
|
-
|
|
92
|
+
SELECT * FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = @schema AND TABLE_NAME = @tableName;
|
|
93
|
+
`, [
|
|
94
|
+
{ name: 'schema', type: sql.VarChar(sql.MAX), value: connectionManager.schema },
|
|
95
|
+
{ name: 'tableName', type: sql.VarChar(sql.MAX), value: POWERSYNC_CHECKPOINTS_TABLE },
|
|
96
|
+
]);
|
|
94
97
|
if (checkpointsResult.length > 0) {
|
|
95
98
|
// Table already exists, check if CDC is enabled
|
|
96
99
|
const isEnabled = await isTableEnabledForCDC({
|
|
@@ -114,7 +117,7 @@ export async function ensurePowerSyncCheckpointsTable(connectionManager: MSSQLCo
|
|
|
114
117
|
// Try to create the table
|
|
115
118
|
try {
|
|
116
119
|
await connectionManager.query(`
|
|
117
|
-
CREATE TABLE ${connectionManager.schema}.${POWERSYNC_CHECKPOINTS_TABLE} (
|
|
120
|
+
CREATE TABLE ${escapeIdentifier(connectionManager.schema)}.${escapeIdentifier(POWERSYNC_CHECKPOINTS_TABLE)} (
|
|
118
121
|
id INT IDENTITY PRIMARY KEY,
|
|
119
122
|
last_updated DATETIME NOT NULL DEFAULT (GETDATE())
|
|
120
123
|
)`);
|
|
@@ -137,7 +140,7 @@ export async function ensurePowerSyncCheckpointsTable(connectionManager: MSSQLCo
|
|
|
137
140
|
|
|
138
141
|
export async function createCheckpoint(connectionManager: MSSQLConnectionManager): Promise<void> {
|
|
139
142
|
await connectionManager.query(`
|
|
140
|
-
MERGE ${connectionManager.schema}.${POWERSYNC_CHECKPOINTS_TABLE} AS target
|
|
143
|
+
MERGE ${escapeIdentifier(connectionManager.schema)}.${escapeIdentifier(POWERSYNC_CHECKPOINTS_TABLE)} AS target
|
|
141
144
|
USING (SELECT 1 AS id) AS source
|
|
142
145
|
ON target.id = source.id
|
|
143
146
|
WHEN MATCHED THEN
|
|
@@ -164,10 +167,12 @@ export async function isTableEnabledForCDC(options: IsTableEnabledForCDCOptions)
|
|
|
164
167
|
SELECT 1 FROM cdc.change_tables ct
|
|
165
168
|
JOIN sys.tables AS tbl ON tbl.object_id = ct.source_object_id
|
|
166
169
|
JOIN sys.schemas AS sch ON sch.schema_id = tbl.schema_id
|
|
167
|
-
WHERE sch.name =
|
|
168
|
-
AND tbl.name =
|
|
169
|
-
|
|
170
|
-
|
|
170
|
+
WHERE sch.name = @schema
|
|
171
|
+
AND tbl.name = @tableName
|
|
172
|
+
`, [
|
|
173
|
+
{ name: 'schema', type: sql.VarChar(sql.MAX), value: schema },
|
|
174
|
+
{ name: 'tableName', type: sql.VarChar(sql.MAX), value: table },
|
|
175
|
+
]);
|
|
171
176
|
return checkResult.length > 0;
|
|
172
177
|
}
|
|
173
178
|
|
|
@@ -227,7 +232,8 @@ export async function isWithinRetentionThreshold(options: IsWithinRetentionThres
|
|
|
227
232
|
|
|
228
233
|
export async function getMinLSN(connectionManager: MSSQLConnectionManager, captureInstance: string): Promise<LSN> {
|
|
229
234
|
const { recordset: result } = await connectionManager.query(
|
|
230
|
-
`SELECT sys.fn_cdc_get_min_lsn(
|
|
235
|
+
`SELECT sys.fn_cdc_get_min_lsn(@captureInstance) AS min_lsn`,
|
|
236
|
+
[{ name: 'captureInstance', type: sql.VarChar(sql.MAX), value: captureInstance }]
|
|
231
237
|
);
|
|
232
238
|
const rawMinLSN: Buffer = result[0].min_lsn;
|
|
233
239
|
return LSN.fromBinary(rawMinLSN);
|
|
@@ -258,11 +264,13 @@ export async function getCaptureInstance(options: GetCaptureInstanceOptions): Pr
|
|
|
258
264
|
sys.tables tbl
|
|
259
265
|
INNER JOIN sys.schemas sch ON tbl.schema_id = sch.schema_id
|
|
260
266
|
INNER JOIN cdc.change_tables ct ON ct.source_object_id = tbl.object_id
|
|
261
|
-
WHERE sch.name =
|
|
262
|
-
AND tbl.name =
|
|
267
|
+
WHERE sch.name = @schema
|
|
268
|
+
AND tbl.name = @tableName
|
|
263
269
|
AND ct.end_lsn IS NULL;
|
|
264
|
-
|
|
265
|
-
|
|
270
|
+
`, [
|
|
271
|
+
{ name: 'schema', type: sql.VarChar(sql.MAX), value: schema },
|
|
272
|
+
{ name: 'tableName', type: sql.VarChar(sql.MAX), value: tableName },
|
|
273
|
+
]);
|
|
266
274
|
|
|
267
275
|
if (result.length === 0) {
|
|
268
276
|
return null;
|
|
@@ -301,7 +309,10 @@ export async function getLatestReplicatedLSN(connectionManager: MSSQLConnectionM
|
|
|
301
309
|
* @param identifier
|
|
302
310
|
*/
|
|
303
311
|
export function escapeIdentifier(identifier: string): string {
|
|
304
|
-
|
|
312
|
+
// 1. Replace existing closing brackets ] with ]] to escape them
|
|
313
|
+
// 2. Replace dots . with ].[ to handle qualified names
|
|
314
|
+
// 3. Wrap the whole result in [ ]
|
|
315
|
+
return `[${identifier.replace(/]/g, ']]').replace(/\./g, '].[')}]`;
|
|
305
316
|
}
|
|
306
317
|
|
|
307
318
|
export function toQualifiedTableName(schema: string, tableName: string): string {
|
|
@@ -319,7 +330,7 @@ export function isIColumnMetadata(obj: any): obj is sql.IColumnMetadata {
|
|
|
319
330
|
propertiesMatched =
|
|
320
331
|
typeof property.index === 'number' &&
|
|
321
332
|
typeof property.name === 'string' &&
|
|
322
|
-
typeof property.length === 'number' &&
|
|
333
|
+
(typeof property.length === 'number' || typeof property.length === 'undefined') &&
|
|
323
334
|
(typeof property.type === 'function' || typeof property.type === 'object') &&
|
|
324
335
|
typeof property.nullable === 'boolean' &&
|
|
325
336
|
typeof property.caseSensitive === 'boolean' &&
|
|
@@ -383,7 +394,9 @@ export async function getDebugTableInfo(options: GetDebugTableInfoOptions): Prom
|
|
|
383
394
|
|
|
384
395
|
let selectError: service_types.ReplicationError | null = null;
|
|
385
396
|
try {
|
|
386
|
-
await connectionManager.query(`SELECT TOP 1 * FROM [
|
|
397
|
+
await connectionManager.query(`SELECT TOP 1 * FROM @tableName`, [
|
|
398
|
+
{ name: 'tableName', type: sql.VarChar(sql.MAX), value: toQualifiedTableName(schema, table.name) },
|
|
399
|
+
]);
|
|
387
400
|
} catch (e) {
|
|
388
401
|
selectError = { level: 'fatal', message: e.message };
|
|
389
402
|
}
|
package/src/utils/schema.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { SourceEntityDescriptor } from '@powersync/service-core';
|
|
|
2
2
|
import { TablePattern } from '@powersync/service-sync-rules';
|
|
3
3
|
import { MSSQLConnectionManager } from '../replication/MSSQLConnectionManager.js';
|
|
4
4
|
import { MSSQLColumnDescriptor } from '../types/mssql-data-types.js';
|
|
5
|
-
import
|
|
5
|
+
import sql from 'mssql';
|
|
6
6
|
|
|
7
7
|
export interface GetColumnsOptions {
|
|
8
8
|
connectionManager: MSSQLConnectionManager;
|
|
@@ -23,10 +23,13 @@ async function getColumns(options: GetColumnsOptions): Promise<MSSQLColumnDescri
|
|
|
23
23
|
JOIN sys.tables AS tbl ON tbl.object_id = col.object_id
|
|
24
24
|
JOIN sys.schemas AS sch ON sch.schema_id = tbl.schema_id
|
|
25
25
|
JOIN sys.types AS typ ON typ.user_type_id = col.user_type_id
|
|
26
|
-
WHERE sch.name =
|
|
27
|
-
AND tbl.name =
|
|
26
|
+
WHERE sch.name = @schema
|
|
27
|
+
AND tbl.name = @tableName
|
|
28
28
|
ORDER BY col.column_id;
|
|
29
|
-
|
|
29
|
+
`, [
|
|
30
|
+
{ name: 'schema', type: sql.VarChar(sql.MAX), value: schema },
|
|
31
|
+
{ name: 'tableName', type: sql.VarChar(sql.MAX), value: tableName },
|
|
32
|
+
]);
|
|
30
33
|
|
|
31
34
|
return columnResults.map((row) => {
|
|
32
35
|
return {
|
|
@@ -65,10 +68,13 @@ export async function getReplicationIdentityColumns(
|
|
|
65
68
|
JOIN sys.index_columns AS idx_col ON idx_col.object_id = idx.object_id AND idx_col.index_id = idx.index_id
|
|
66
69
|
JOIN sys.columns AS col ON col.object_id = idx_col.object_id AND col.column_id = idx_col.column_id
|
|
67
70
|
JOIN sys.types AS typ ON typ.user_type_id = col.user_type_id
|
|
68
|
-
WHERE sch.name =
|
|
69
|
-
AND tbl.name =
|
|
71
|
+
WHERE sch.name = @schema
|
|
72
|
+
AND tbl.name = @tableName
|
|
70
73
|
ORDER BY idx_col.key_ordinal;
|
|
71
|
-
|
|
74
|
+
`, [
|
|
75
|
+
{ name: 'schema', type: sql.VarChar(sql.MAX), value: schema },
|
|
76
|
+
{ name: 'tableName', type: sql.VarChar(sql.MAX), value: tableName },
|
|
77
|
+
]);
|
|
72
78
|
|
|
73
79
|
if (primaryKeyColumns.length > 0) {
|
|
74
80
|
return {
|
|
@@ -95,10 +101,13 @@ export async function getReplicationIdentityColumns(
|
|
|
95
101
|
JOIN sys.index_columns AS idx_col ON idx_col.object_id = idx.object_id AND idx_col.index_id = idx.index_id
|
|
96
102
|
JOIN sys.columns AS col ON col.object_id = idx_col.object_id AND col.column_id = idx_col.column_id
|
|
97
103
|
JOIN sys.types AS typ ON typ.user_type_id = col.user_type_id
|
|
98
|
-
WHERE sch.name =
|
|
99
|
-
AND tbl.name =
|
|
104
|
+
WHERE sch.name = @schema
|
|
105
|
+
AND tbl.name = @tableName
|
|
100
106
|
ORDER BY idx_col.key_ordinal;
|
|
101
|
-
|
|
107
|
+
`, [
|
|
108
|
+
{ name: 'schema', type: sql.VarChar(sql.MAX), value: schema },
|
|
109
|
+
{ name: 'tableName', type: sql.VarChar(sql.MAX), value: tableName },
|
|
110
|
+
]);
|
|
102
111
|
|
|
103
112
|
if (uniqueKeyColumns.length > 0) {
|
|
104
113
|
return {
|
|
@@ -134,9 +143,12 @@ export async function getTablesFromPattern(
|
|
|
134
143
|
tbl.object_id AS object_id
|
|
135
144
|
FROM sys.tables tbl
|
|
136
145
|
JOIN sys.schemas sch ON tbl.schema_id = sch.schema_id
|
|
137
|
-
WHERE sch.name =
|
|
138
|
-
AND tbl.name LIKE
|
|
139
|
-
|
|
146
|
+
WHERE sch.name = @schema
|
|
147
|
+
AND tbl.name LIKE @tablePattern
|
|
148
|
+
`, [
|
|
149
|
+
{ name: 'schema', type: sql.VarChar(sql.MAX), value: tablePattern.schema },
|
|
150
|
+
{ name: 'tablePattern', type: sql.VarChar(sql.MAX), value: tablePattern.tablePattern },
|
|
151
|
+
]);
|
|
140
152
|
|
|
141
153
|
return tableResults
|
|
142
154
|
.map((row) => {
|
|
@@ -156,10 +168,12 @@ export async function getTablesFromPattern(
|
|
|
156
168
|
tbl.object_id AS object_id
|
|
157
169
|
FROM sys.tables tbl
|
|
158
170
|
JOIN sys.schemas sch ON tbl.schema_id = sch.schema_id
|
|
159
|
-
|
|
160
|
-
AND tbl.name =
|
|
161
|
-
|
|
162
|
-
|
|
171
|
+
WHERE sch.name = @schema
|
|
172
|
+
AND tbl.name = @tablePattern
|
|
173
|
+
`, [
|
|
174
|
+
{ name: 'schema', type: sql.VarChar(sql.MAX), value: tablePattern.schema },
|
|
175
|
+
{ name: 'tablePattern', type: sql.VarChar(sql.MAX), value: tablePattern.tablePattern },
|
|
176
|
+
]);
|
|
163
177
|
|
|
164
178
|
return tableResults.map((row) => {
|
|
165
179
|
return {
|
|
@@ -1,16 +1,10 @@
|
|
|
1
1
|
import { describe, expect, test } from 'vitest';
|
|
2
2
|
import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests';
|
|
3
3
|
import { ReplicationMetric } from '@powersync/service-types';
|
|
4
|
-
import {
|
|
5
|
-
createTestTable,
|
|
6
|
-
describeWithStorage,
|
|
7
|
-
INITIALIZED_MONGO_STORAGE_FACTORY,
|
|
8
|
-
insertTestData,
|
|
9
|
-
waitForPendingCDCChanges
|
|
10
|
-
} from './util.js';
|
|
4
|
+
import { createTestTable, describeWithStorage, insertTestData, waitForPendingCDCChanges } from './util.js';
|
|
11
5
|
import { storage } from '@powersync/service-core';
|
|
12
6
|
import { CDCStreamTestContext } from './CDCStreamTestContext.js';
|
|
13
|
-
import {
|
|
7
|
+
import { getLatestLSN } from '@module/utils/mssql.js';
|
|
14
8
|
import sql from 'mssql';
|
|
15
9
|
|
|
16
10
|
const BASIC_SYNC_RULES = `
|
|
@@ -20,11 +14,9 @@ bucket_definitions:
|
|
|
20
14
|
- SELECT id, description FROM "test_data"
|
|
21
15
|
`;
|
|
22
16
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
defineCDCStreamTests(INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
17
|
+
describe('CDCStream tests', () => {
|
|
18
|
+
describeWithStorage({ timeout: 20_000 }, defineCDCStreamTests);
|
|
19
|
+
});
|
|
28
20
|
|
|
29
21
|
function defineCDCStreamTests(factory: storage.TestStorageFactory) {
|
|
30
22
|
test('Initial snapshot sync', async () => {
|
|
@@ -33,7 +25,7 @@ function defineCDCStreamTests(factory: storage.TestStorageFactory) {
|
|
|
33
25
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
34
26
|
|
|
35
27
|
await createTestTable(connectionManager, 'test_data');
|
|
36
|
-
const beforeLSN = await
|
|
28
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
37
29
|
const testData = await insertTestData(connectionManager, 'test_data');
|
|
38
30
|
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
39
31
|
const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0;
|
|
@@ -77,7 +69,7 @@ function defineCDCStreamTests(factory: storage.TestStorageFactory) {
|
|
|
77
69
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
78
70
|
|
|
79
71
|
await createTestTable(connectionManager, 'test_data');
|
|
80
|
-
const beforeLSN = await
|
|
72
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
81
73
|
const testData = await insertTestData(connectionManager, 'test_data');
|
|
82
74
|
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
83
75
|
await context.replicateSnapshot();
|
|
@@ -101,7 +93,7 @@ function defineCDCStreamTests(factory: storage.TestStorageFactory) {
|
|
|
101
93
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
102
94
|
|
|
103
95
|
await createTestTable(connectionManager, 'test_data');
|
|
104
|
-
const beforeLSN = await
|
|
96
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
105
97
|
const testData = await insertTestData(connectionManager, 'test_data');
|
|
106
98
|
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
107
99
|
await context.replicateSnapshot();
|
|
@@ -128,8 +120,8 @@ function defineCDCStreamTests(factory: storage.TestStorageFactory) {
|
|
|
128
120
|
await createTestTable(connectionManager, 'test_data_1');
|
|
129
121
|
await createTestTable(connectionManager, 'test_data_2');
|
|
130
122
|
|
|
131
|
-
const beforeLSN = await getLatestReplicatedLSN(connectionManager);
|
|
132
123
|
const testData11 = await insertTestData(connectionManager, 'test_data_1');
|
|
124
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
133
125
|
const testData21 = await insertTestData(connectionManager, 'test_data_2');
|
|
134
126
|
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
135
127
|
|
|
@@ -12,7 +12,6 @@ import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './uti
|
|
|
12
12
|
import { CDCStream, CDCStreamOptions } from '@module/replication/CDCStream.js';
|
|
13
13
|
import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
|
|
14
14
|
import timers from 'timers/promises';
|
|
15
|
-
import { CDCPollingOptions } from '@module/types/types.js';
|
|
16
15
|
|
|
17
16
|
/**
|
|
18
17
|
* Tests operating on the change data capture need to configure the stream and manage asynchronous
|
|
@@ -110,10 +109,11 @@ export class CDCStreamTestContext implements AsyncDisposable {
|
|
|
110
109
|
metrics: METRICS_HELPER.metricsEngine,
|
|
111
110
|
connections: this.connectionManager,
|
|
112
111
|
abortSignal: this.abortController.signal,
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
112
|
+
additionalConfig: {
|
|
113
|
+
pollingBatchSize: 10,
|
|
114
|
+
pollingIntervalMs: 1000,
|
|
115
|
+
trustServerCertificate: true
|
|
116
|
+
},
|
|
117
117
|
...this.cdcStreamOptions
|
|
118
118
|
};
|
|
119
119
|
this._cdcStream = new CDCStream(options);
|