@powersync/service-module-mssql 0.5.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +32 -0
- package/dist/common/CaptureInstance.d.ts +14 -0
- package/dist/common/CaptureInstance.js +2 -0
- package/dist/common/CaptureInstance.js.map +1 -0
- package/dist/common/MSSQLSourceTable.d.ts +16 -14
- package/dist/common/MSSQLSourceTable.js +35 -16
- package/dist/common/MSSQLSourceTable.js.map +1 -1
- package/dist/replication/CDCPoller.d.ts +42 -20
- package/dist/replication/CDCPoller.js +200 -60
- package/dist/replication/CDCPoller.js.map +1 -1
- package/dist/replication/CDCReplicationJob.js +9 -1
- package/dist/replication/CDCReplicationJob.js.map +1 -1
- package/dist/replication/CDCStream.d.ts +35 -4
- package/dist/replication/CDCStream.js +181 -74
- package/dist/replication/CDCStream.js.map +1 -1
- package/dist/replication/MSSQLConnectionManager.js +16 -5
- package/dist/replication/MSSQLConnectionManager.js.map +1 -1
- package/dist/types/types.d.ts +4 -56
- package/dist/types/types.js +5 -24
- package/dist/types/types.js.map +1 -1
- package/dist/utils/deadlock.d.ts +9 -0
- package/dist/utils/deadlock.js +40 -0
- package/dist/utils/deadlock.js.map +1 -0
- package/dist/utils/mssql.d.ts +33 -15
- package/dist/utils/mssql.js +101 -99
- package/dist/utils/mssql.js.map +1 -1
- package/dist/utils/schema.d.ts +9 -0
- package/dist/utils/schema.js +34 -0
- package/dist/utils/schema.js.map +1 -1
- package/package.json +8 -8
- package/src/common/CaptureInstance.ts +15 -0
- package/src/common/MSSQLSourceTable.ts +33 -24
- package/src/replication/CDCPoller.ts +272 -72
- package/src/replication/CDCReplicationJob.ts +8 -1
- package/src/replication/CDCStream.ts +237 -90
- package/src/replication/MSSQLConnectionManager.ts +15 -5
- package/src/types/types.ts +5 -28
- package/src/utils/deadlock.ts +44 -0
- package/src/utils/mssql.ts +159 -124
- package/src/utils/schema.ts +43 -0
- package/test/src/CDCStreamTestContext.ts +9 -2
- package/test/src/env.ts +1 -1
- package/test/src/mssql-to-sqlite.test.ts +18 -10
- package/test/src/schema-changes.test.ts +470 -0
- package/test/src/util.ts +75 -12
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -0,0 +1,470 @@
|
|
|
1
|
+
import { putOp, removeOp } from '@powersync/service-core-tests';
|
|
2
|
+
import { describe, expect, test, vi } from 'vitest';
|
|
3
|
+
import { storage } from '@powersync/service-core';
|
|
4
|
+
import sql from 'mssql';
|
|
5
|
+
|
|
6
|
+
import { CDCStreamTestContext } from './CDCStreamTestContext.js';
|
|
7
|
+
import {
|
|
8
|
+
createTestTableWithBasicId,
|
|
9
|
+
describeWithStorage,
|
|
10
|
+
disableCDCForTable,
|
|
11
|
+
dropTestTable,
|
|
12
|
+
enableCDCForTable,
|
|
13
|
+
insertBasicIdTestData,
|
|
14
|
+
renameTable,
|
|
15
|
+
waitForPendingCDCChanges
|
|
16
|
+
} from './util.js';
|
|
17
|
+
import { getLatestLSN, toQualifiedTableName } from '@module/utils/mssql.js';
|
|
18
|
+
import { SchemaChangeType } from '@module/replication/CDCPoller.js';
|
|
19
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
20
|
+
|
|
21
|
+
describe('MSSQL Schema Changes Tests', () => {
|
|
22
|
+
describeWithStorage({ timeout: 60_000 }, defineSchemaChangesTests);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
const BASIC_SYNC_RULES = `
|
|
26
|
+
bucket_definitions:
|
|
27
|
+
global:
|
|
28
|
+
data:
|
|
29
|
+
- SELECT id, description FROM "test_data"
|
|
30
|
+
`;
|
|
31
|
+
|
|
32
|
+
function defineSchemaChangesTests(config: storage.TestStorageConfig) {
|
|
33
|
+
const { factory } = config;
|
|
34
|
+
|
|
35
|
+
test('Create table: New table in the sync rules', async () => {
|
|
36
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
37
|
+
const { connectionManager } = context;
|
|
38
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
39
|
+
|
|
40
|
+
await context.replicateSnapshot();
|
|
41
|
+
await context.startStreaming();
|
|
42
|
+
|
|
43
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
44
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
45
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
46
|
+
|
|
47
|
+
const data = await context.getFinalBucketState('global[]');
|
|
48
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test('Create table: New table created while PowerSync is stopped', async () => {
|
|
52
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
53
|
+
const { connectionManager } = context;
|
|
54
|
+
await context.updateSyncRules(`
|
|
55
|
+
bucket_definitions:
|
|
56
|
+
global:
|
|
57
|
+
data:
|
|
58
|
+
- SELECT id, description FROM "test_data%"
|
|
59
|
+
`);
|
|
60
|
+
|
|
61
|
+
await createTestTableWithBasicId(connectionManager, 'test_data1');
|
|
62
|
+
const testData = await insertBasicIdTestData(connectionManager, 'test_data1');
|
|
63
|
+
|
|
64
|
+
await context.replicateSnapshot();
|
|
65
|
+
await context.startStreaming();
|
|
66
|
+
|
|
67
|
+
await context.dispose();
|
|
68
|
+
|
|
69
|
+
await createTestTableWithBasicId(connectionManager, 'test_data2');
|
|
70
|
+
|
|
71
|
+
await using newContext = await CDCStreamTestContext.open(factory, { doNotClear: true });
|
|
72
|
+
await newContext.loadActiveSyncRules();
|
|
73
|
+
|
|
74
|
+
await newContext.replicateSnapshot();
|
|
75
|
+
await newContext.startStreaming();
|
|
76
|
+
|
|
77
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data2');
|
|
78
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data2');
|
|
79
|
+
|
|
80
|
+
const finalState = await newContext.getFinalBucketState('global[]');
|
|
81
|
+
expect(finalState).toMatchObject([
|
|
82
|
+
putOp('test_data1', testData),
|
|
83
|
+
putOp('test_data2', testData1),
|
|
84
|
+
putOp('test_data2', testData2)
|
|
85
|
+
]);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test('Create table: New table not in the sync rules', async () => {
|
|
89
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
90
|
+
const { connectionManager } = context;
|
|
91
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
92
|
+
|
|
93
|
+
await context.replicateSnapshot();
|
|
94
|
+
await context.startStreaming();
|
|
95
|
+
|
|
96
|
+
await createTestTableWithBasicId(connectionManager, 'test_data_ignored');
|
|
97
|
+
await insertBasicIdTestData(connectionManager, 'test_data_ignored');
|
|
98
|
+
|
|
99
|
+
const data = await context.getBucketData('global[]');
|
|
100
|
+
expect(data).toMatchObject([]);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test('Drop table: Table in the sync rules', async () => {
|
|
104
|
+
await using context = await CDCStreamTestContext.open(factory, {
|
|
105
|
+
cdcStreamOptions: { schemaCheckIntervalMs: 5000 }
|
|
106
|
+
});
|
|
107
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
108
|
+
|
|
109
|
+
const { connectionManager } = context;
|
|
110
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
111
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
112
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
113
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
114
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
115
|
+
|
|
116
|
+
await context.replicateSnapshot();
|
|
117
|
+
await context.startStreaming();
|
|
118
|
+
|
|
119
|
+
let data = await context.getBucketData('global[]');
|
|
120
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
121
|
+
await dropTestTable(connectionManager, 'test_data');
|
|
122
|
+
|
|
123
|
+
data = await context.getFinalBucketState('global[]');
|
|
124
|
+
expect(data).toMatchObject([]);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test('Re-create table', async () => {
|
|
128
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
129
|
+
const { connectionManager } = context;
|
|
130
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
131
|
+
|
|
132
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
133
|
+
|
|
134
|
+
await context.replicateSnapshot();
|
|
135
|
+
await context.startStreaming();
|
|
136
|
+
|
|
137
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
138
|
+
let data = await context.getBucketData('global[]');
|
|
139
|
+
expect(data).toMatchObject([putOp('test_data', testData1)]);
|
|
140
|
+
|
|
141
|
+
let schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
142
|
+
await dropTestTable(connectionManager, 'test_data');
|
|
143
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.TABLE_DROP);
|
|
144
|
+
|
|
145
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
146
|
+
|
|
147
|
+
const testData = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
148
|
+
|
|
149
|
+
data = await context.getFinalBucketState('global[]');
|
|
150
|
+
expect(data).toMatchObject([putOp('test_data', testData)]);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
test('Rename table: Table not in the sync rules to one in the sync rules', async () => {
|
|
154
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
155
|
+
const { connectionManager } = context;
|
|
156
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
157
|
+
|
|
158
|
+
await createTestTableWithBasicId(connectionManager, 'test_data_old');
|
|
159
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
160
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data_old');
|
|
161
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
162
|
+
|
|
163
|
+
await context.replicateSnapshot();
|
|
164
|
+
await context.startStreaming();
|
|
165
|
+
|
|
166
|
+
const schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
167
|
+
await renameTable(connectionManager, 'test_data_old', 'test_data');
|
|
168
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.TABLE_CREATE);
|
|
169
|
+
|
|
170
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
171
|
+
const data = await context.getFinalBucketState('global[]');
|
|
172
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
test('Rename table: Table in the sync rules to another table in the sync rules', async () => {
|
|
176
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
177
|
+
const { connectionManager } = context;
|
|
178
|
+
|
|
179
|
+
await context.updateSyncRules(`
|
|
180
|
+
bucket_definitions:
|
|
181
|
+
global:
|
|
182
|
+
data:
|
|
183
|
+
- SELECT id, description FROM "test_data%"
|
|
184
|
+
`);
|
|
185
|
+
|
|
186
|
+
await createTestTableWithBasicId(connectionManager, 'test_data1');
|
|
187
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
188
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data1');
|
|
189
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
190
|
+
|
|
191
|
+
await context.replicateSnapshot();
|
|
192
|
+
await context.startStreaming();
|
|
193
|
+
|
|
194
|
+
const schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
195
|
+
await renameTable(connectionManager, 'test_data1', 'test_data2');
|
|
196
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.TABLE_RENAME);
|
|
197
|
+
|
|
198
|
+
const data = await context.getBucketData('global[]');
|
|
199
|
+
expect(data.slice(0, 2)).toMatchObject([
|
|
200
|
+
// Initial replication
|
|
201
|
+
putOp('test_data1', testData1),
|
|
202
|
+
// Initial truncate
|
|
203
|
+
removeOp('test_data1', testData1.id)
|
|
204
|
+
]);
|
|
205
|
+
|
|
206
|
+
const finalState = await context.getFinalBucketState('global[]');
|
|
207
|
+
expect(finalState).toMatchObject([putOp('test_data2', testData1)]);
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
test('Rename table: Table renamed while PowerSync is stopped', async () => {
|
|
211
|
+
let context = await CDCStreamTestContext.open(factory);
|
|
212
|
+
let { connectionManager } = context;
|
|
213
|
+
|
|
214
|
+
await context.updateSyncRules(`
|
|
215
|
+
bucket_definitions:
|
|
216
|
+
global:
|
|
217
|
+
data:
|
|
218
|
+
- SELECT id, description FROM "test_data%"
|
|
219
|
+
`);
|
|
220
|
+
|
|
221
|
+
await createTestTableWithBasicId(connectionManager, 'test_data1');
|
|
222
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
223
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data1');
|
|
224
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
225
|
+
|
|
226
|
+
await context.replicateSnapshot();
|
|
227
|
+
await context.startStreaming();
|
|
228
|
+
|
|
229
|
+
let data = await context.getBucketData('global[]');
|
|
230
|
+
expect(data).toMatchObject([putOp('test_data1', testData1)]);
|
|
231
|
+
|
|
232
|
+
await context.dispose();
|
|
233
|
+
await renameTable(connectionManager, 'test_data1', 'test_data2');
|
|
234
|
+
|
|
235
|
+
await using newContext = await CDCStreamTestContext.open(factory, { doNotClear: true });
|
|
236
|
+
await newContext.loadActiveSyncRules();
|
|
237
|
+
|
|
238
|
+
await newContext.replicateSnapshot();
|
|
239
|
+
await newContext.startStreaming();
|
|
240
|
+
|
|
241
|
+
const finalState = await newContext.getFinalBucketState('global[]');
|
|
242
|
+
expect(finalState).toMatchObject([putOp('test_data2', testData1)]);
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
test('Rename table: Table in the sync rules to not in the sync rules', async () => {
|
|
246
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
247
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
248
|
+
|
|
249
|
+
const { connectionManager } = context;
|
|
250
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
251
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
252
|
+
const testData = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
253
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
254
|
+
|
|
255
|
+
await context.replicateSnapshot();
|
|
256
|
+
await context.startStreaming();
|
|
257
|
+
|
|
258
|
+
let data = await context.getBucketData('global[]');
|
|
259
|
+
expect(data).toMatchObject([putOp('test_data', testData)]);
|
|
260
|
+
|
|
261
|
+
const schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
262
|
+
await renameTable(connectionManager, 'test_data', 'test_data_ignored');
|
|
263
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.TABLE_RENAME);
|
|
264
|
+
|
|
265
|
+
data = await context.getBucketData('global[]');
|
|
266
|
+
expect(data).toMatchObject([
|
|
267
|
+
// Initial replication
|
|
268
|
+
putOp('test_data', testData),
|
|
269
|
+
// Truncate
|
|
270
|
+
removeOp('test_data', testData.id)
|
|
271
|
+
]);
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
test('New capture instance created for replicating table triggers re-snapshot', async () => {
|
|
275
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
276
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
277
|
+
|
|
278
|
+
const { connectionManager } = context;
|
|
279
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
280
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
281
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
282
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
283
|
+
|
|
284
|
+
await context.replicateSnapshot();
|
|
285
|
+
await context.startStreaming();
|
|
286
|
+
|
|
287
|
+
await enableCDCForTable({ connectionManager, table: 'test_data', captureInstance: 'capture_instance_new' });
|
|
288
|
+
|
|
289
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
290
|
+
|
|
291
|
+
const data = await context.getFinalBucketState('global[]');
|
|
292
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
test('New capture instance created for replicating table while PowerSync is stopped', async () => {
|
|
296
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
297
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
298
|
+
const { connectionManager } = context;
|
|
299
|
+
|
|
300
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
301
|
+
let beforeLSN = await getLatestLSN(connectionManager);
|
|
302
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
303
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
304
|
+
|
|
305
|
+
await context.replicateSnapshot();
|
|
306
|
+
await context.startStreaming();
|
|
307
|
+
|
|
308
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
309
|
+
let data = await context.getBucketData('global[]');
|
|
310
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
311
|
+
|
|
312
|
+
await context.dispose();
|
|
313
|
+
await enableCDCForTable({ connectionManager, table: 'test_data', captureInstance: 'capture_instance_new' });
|
|
314
|
+
|
|
315
|
+
await using newContext = await CDCStreamTestContext.open(factory, { doNotClear: true });
|
|
316
|
+
await newContext.loadActiveSyncRules();
|
|
317
|
+
|
|
318
|
+
await newContext.replicateSnapshot();
|
|
319
|
+
await newContext.startStreaming();
|
|
320
|
+
|
|
321
|
+
const testData3 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
322
|
+
|
|
323
|
+
const finalState = await newContext.getFinalBucketState('global[]');
|
|
324
|
+
expect(finalState).toMatchObject([
|
|
325
|
+
putOp('test_data', testData1),
|
|
326
|
+
putOp('test_data', testData2),
|
|
327
|
+
putOp('test_data', testData3)
|
|
328
|
+
]);
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
test('Capture instance created for a sync rule table without a capture instance', async () => {
|
|
332
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
333
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
334
|
+
const { connectionManager } = context;
|
|
335
|
+
|
|
336
|
+
await createTestTableWithBasicId(connectionManager, 'test_data', false);
|
|
337
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
338
|
+
|
|
339
|
+
await context.replicateSnapshot();
|
|
340
|
+
await context.startStreaming();
|
|
341
|
+
|
|
342
|
+
const schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
343
|
+
await enableCDCForTable({ connectionManager, table: 'test_data' });
|
|
344
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.NEW_CAPTURE_INSTANCE);
|
|
345
|
+
|
|
346
|
+
let data = await context.getBucketData('global[]');
|
|
347
|
+
expect(data).toMatchObject([putOp('test_data', testData1)]);
|
|
348
|
+
|
|
349
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
350
|
+
|
|
351
|
+
data = await context.getFinalBucketState('global[]');
|
|
352
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
353
|
+
});
|
|
354
|
+
|
|
355
|
+
test('Capture instance removed for an actively replicating table', async () => {
|
|
356
|
+
await using context = await CDCStreamTestContext.open(factory, {
|
|
357
|
+
cdcStreamOptions: { schemaCheckIntervalMs: 5000 }
|
|
358
|
+
});
|
|
359
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
360
|
+
const { connectionManager } = context;
|
|
361
|
+
|
|
362
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
363
|
+
let beforeLSN = await getLatestLSN(connectionManager);
|
|
364
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
365
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
366
|
+
|
|
367
|
+
await context.replicateSnapshot();
|
|
368
|
+
await context.startStreaming();
|
|
369
|
+
|
|
370
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
371
|
+
let data = await context.getBucketData('global[]');
|
|
372
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
373
|
+
|
|
374
|
+
const schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
375
|
+
await disableCDCForTable(connectionManager, 'test_data');
|
|
376
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.MISSING_CAPTURE_INSTANCE);
|
|
377
|
+
|
|
378
|
+
data = await context.getBucketData('global[]');
|
|
379
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
test('Capture instance removed, and then re-added', async () => {
|
|
383
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
384
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
385
|
+
const { connectionManager } = context;
|
|
386
|
+
|
|
387
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
388
|
+
|
|
389
|
+
await context.replicateSnapshot();
|
|
390
|
+
await context.startStreaming();
|
|
391
|
+
|
|
392
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
393
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
394
|
+
let data = await context.getBucketData('global[]');
|
|
395
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
396
|
+
|
|
397
|
+
let schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
398
|
+
await disableCDCForTable(connectionManager, 'test_data');
|
|
399
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.MISSING_CAPTURE_INSTANCE);
|
|
400
|
+
|
|
401
|
+
schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
402
|
+
await enableCDCForTable({ connectionManager, table: 'test_data' });
|
|
403
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.NEW_CAPTURE_INSTANCE);
|
|
404
|
+
|
|
405
|
+
const testData3 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
406
|
+
const testData4 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
407
|
+
|
|
408
|
+
const finalState = await context.getFinalBucketState('global[]');
|
|
409
|
+
expect(finalState).toMatchObject([
|
|
410
|
+
putOp('test_data', testData1),
|
|
411
|
+
putOp('test_data', testData2),
|
|
412
|
+
putOp('test_data', testData3),
|
|
413
|
+
putOp('test_data', testData4)
|
|
414
|
+
]);
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
test('Column schema changes continue replication, but with warning.', async () => {
|
|
418
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
419
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
420
|
+
const { connectionManager } = context;
|
|
421
|
+
|
|
422
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
423
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
424
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
425
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
426
|
+
|
|
427
|
+
await context.replicateSnapshot();
|
|
428
|
+
await context.startStreaming();
|
|
429
|
+
const schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
430
|
+
await connectionManager.query(`ALTER TABLE test_data ADD new_column INT`);
|
|
431
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.TABLE_COLUMN_CHANGES);
|
|
432
|
+
|
|
433
|
+
const { recordset: result } = await connectionManager.query(
|
|
434
|
+
`
|
|
435
|
+
INSERT INTO ${toQualifiedTableName(connectionManager.schema, 'test_data')} (description, new_column)
|
|
436
|
+
OUTPUT INSERTED.id, INSERTED.description
|
|
437
|
+
VALUES (@description, @new_column)
|
|
438
|
+
`,
|
|
439
|
+
[
|
|
440
|
+
{ name: 'description', type: sql.NVarChar(sql.MAX), value: 'new_column_description' },
|
|
441
|
+
{ name: 'new_column', type: sql.Int, value: 1 }
|
|
442
|
+
]
|
|
443
|
+
);
|
|
444
|
+
|
|
445
|
+
const testData2 = { id: result[0].id, description: result[0].description };
|
|
446
|
+
|
|
447
|
+
const data = await context.getBucketData('global[]');
|
|
448
|
+
// Capture instances do not reflect most schema changes until the capture instance is re-created
|
|
449
|
+
// So testData2 will be replicated but without the new column
|
|
450
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
451
|
+
|
|
452
|
+
expect(
|
|
453
|
+
context.cdcStream.tableCache
|
|
454
|
+
.getAll()
|
|
455
|
+
.every((t) => t.captureInstance && t.captureInstance.pendingSchemaChanges.length > 0)
|
|
456
|
+
).toBe(true);
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
async function expectedSchemaChange(spy: any, type: SchemaChangeType) {
|
|
461
|
+
logger.info(`Test Assertion: Waiting for schema change: ${type}`);
|
|
462
|
+
await vi.waitFor(() => expect(spy).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ type })), {
|
|
463
|
+
timeout: 20000
|
|
464
|
+
});
|
|
465
|
+
|
|
466
|
+
const promises = spy.mock.results.filter((r: any) => r.type === 'return').map((r: any) => r.value);
|
|
467
|
+
|
|
468
|
+
await Promise.all(promises.map((p: Promise<unknown>) => expect(p).resolves.toBeUndefined()));
|
|
469
|
+
logger.info(`Test Assertion: Received expected schema change: ${type}`);
|
|
470
|
+
}
|
package/test/src/util.ts
CHANGED
|
@@ -14,7 +14,7 @@ import * as postgres_storage from '@powersync/service-module-postgres-storage';
|
|
|
14
14
|
import { describe, TestOptions } from 'vitest';
|
|
15
15
|
import { env } from './env.js';
|
|
16
16
|
import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
|
|
17
|
-
import { createCheckpoint,
|
|
17
|
+
import { createCheckpoint, escapeIdentifier, getLatestLSN, toQualifiedTableName } from '@module/utils/mssql.js';
|
|
18
18
|
import sql from 'mssql';
|
|
19
19
|
import { v4 as uuid } from 'uuid';
|
|
20
20
|
import { LSN } from '@module/common/LSN.js';
|
|
@@ -45,7 +45,7 @@ export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
|
|
|
45
45
|
uri: TEST_URI,
|
|
46
46
|
additionalConfig: {
|
|
47
47
|
pollingBatchSize: 10,
|
|
48
|
-
pollingIntervalMs:
|
|
48
|
+
pollingIntervalMs: 100,
|
|
49
49
|
trustServerCertificate: true
|
|
50
50
|
}
|
|
51
51
|
});
|
|
@@ -72,13 +72,12 @@ export async function clearTestDb(connectionManager: MSSQLConnectionManager) {
|
|
|
72
72
|
}
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
-
export async function
|
|
75
|
+
export async function dropTestTable(connectionManager: MSSQLConnectionManager, tableName: string) {
|
|
76
76
|
await connectionManager.execute('sys.sp_cdc_disable_table', [
|
|
77
77
|
{ name: 'source_schema', value: connectionManager.schema },
|
|
78
78
|
{ name: 'source_name', value: tableName },
|
|
79
79
|
{ name: 'capture_instance', value: 'all' }
|
|
80
80
|
]);
|
|
81
|
-
|
|
82
81
|
await connectionManager.query(`DROP TABLE [${tableName}]`);
|
|
83
82
|
}
|
|
84
83
|
|
|
@@ -98,19 +97,26 @@ export async function createTestDb(connectionManager: MSSQLConnectionManager, db
|
|
|
98
97
|
GO`);
|
|
99
98
|
}
|
|
100
99
|
|
|
101
|
-
export async function createTestTable(
|
|
100
|
+
export async function createTestTable(
|
|
101
|
+
connectionManager: MSSQLConnectionManager,
|
|
102
|
+
tableName: string,
|
|
103
|
+
withCaptureInstance: boolean = true
|
|
104
|
+
): Promise<void> {
|
|
102
105
|
await connectionManager.query(`
|
|
103
106
|
CREATE TABLE ${escapeIdentifier(connectionManager.schema)}.${escapeIdentifier(tableName)} (
|
|
104
107
|
id UNIQUEIDENTIFIER PRIMARY KEY,
|
|
105
108
|
description VARCHAR(MAX)
|
|
106
109
|
)
|
|
107
110
|
`);
|
|
108
|
-
|
|
111
|
+
if (withCaptureInstance) {
|
|
112
|
+
await enableCDCForTable({ connectionManager, table: tableName });
|
|
113
|
+
}
|
|
109
114
|
}
|
|
110
115
|
|
|
111
116
|
export async function createTestTableWithBasicId(
|
|
112
117
|
connectionManager: MSSQLConnectionManager,
|
|
113
|
-
tableName: string
|
|
118
|
+
tableName: string,
|
|
119
|
+
withCaptureInstance: boolean = true
|
|
114
120
|
): Promise<void> {
|
|
115
121
|
await connectionManager.query(`
|
|
116
122
|
CREATE TABLE ${escapeIdentifier(connectionManager.schema)}.${escapeIdentifier(tableName)} (
|
|
@@ -118,7 +124,9 @@ export async function createTestTableWithBasicId(
|
|
|
118
124
|
description VARCHAR(MAX)
|
|
119
125
|
)
|
|
120
126
|
`);
|
|
121
|
-
|
|
127
|
+
if (withCaptureInstance) {
|
|
128
|
+
await enableCDCForTable({ connectionManager, table: tableName });
|
|
129
|
+
}
|
|
122
130
|
}
|
|
123
131
|
|
|
124
132
|
export interface TestData {
|
|
@@ -141,6 +149,24 @@ export async function insertTestData(connectionManager: MSSQLConnectionManager,
|
|
|
141
149
|
return { id, description };
|
|
142
150
|
}
|
|
143
151
|
|
|
152
|
+
export async function insertBasicIdTestData(
|
|
153
|
+
connectionManager: MSSQLConnectionManager,
|
|
154
|
+
tableName: string
|
|
155
|
+
): Promise<TestData> {
|
|
156
|
+
const description = `description_${Math.floor(Math.random() * 1000000)}`;
|
|
157
|
+
const { recordset: result } = await connectionManager.query(
|
|
158
|
+
`
|
|
159
|
+
INSERT INTO ${toQualifiedTableName(connectionManager.schema, tableName)} (description)
|
|
160
|
+
OUTPUT INSERTED.id
|
|
161
|
+
VALUES (@description)
|
|
162
|
+
`,
|
|
163
|
+
[{ name: 'description', type: sql.NVarChar(sql.MAX), value: description }]
|
|
164
|
+
);
|
|
165
|
+
const id = result[0].id;
|
|
166
|
+
|
|
167
|
+
return { id, description };
|
|
168
|
+
}
|
|
169
|
+
|
|
144
170
|
export async function waitForPendingCDCChanges(
|
|
145
171
|
beforeLSN: LSN,
|
|
146
172
|
connectionManager: MSSQLConnectionManager
|
|
@@ -157,10 +183,14 @@ export async function waitForPendingCDCChanges(
|
|
|
157
183
|
);
|
|
158
184
|
|
|
159
185
|
if (result.length === 0) {
|
|
160
|
-
logger.info(
|
|
186
|
+
logger.info(
|
|
187
|
+
`Test Assertion: CDC changes pending. Waiting for a transaction newer than: ${beforeLSN.toString()} for 200ms...`
|
|
188
|
+
);
|
|
161
189
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
162
190
|
} else {
|
|
163
|
-
logger.info(
|
|
191
|
+
logger.info(
|
|
192
|
+
`Test Assertion: Expected CDC change found with LSN: ${LSN.fromBinary(result[0].start_lsn).toString()}`
|
|
193
|
+
);
|
|
164
194
|
return;
|
|
165
195
|
}
|
|
166
196
|
}
|
|
@@ -182,14 +212,14 @@ export async function getClientCheckpoint(
|
|
|
182
212
|
const timeout = options?.timeout ?? 50_000;
|
|
183
213
|
let lastCp: ReplicationCheckpoint | null = null;
|
|
184
214
|
|
|
185
|
-
logger.info(`Waiting for LSN checkpoint: ${lsn}`);
|
|
215
|
+
logger.info(`Test Assertion: Waiting for LSN checkpoint: ${lsn}`);
|
|
186
216
|
while (Date.now() - start < timeout) {
|
|
187
217
|
const storage = await storageFactory.getActiveStorage();
|
|
188
218
|
const cp = await storage?.getCheckpoint();
|
|
189
219
|
if (cp != null) {
|
|
190
220
|
lastCp = cp;
|
|
191
221
|
if (cp.lsn != null && cp.lsn >= lsn.toString()) {
|
|
192
|
-
logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
|
|
222
|
+
logger.info(`Test Assertion: Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
|
|
193
223
|
return cp.checkpoint;
|
|
194
224
|
}
|
|
195
225
|
}
|
|
@@ -206,3 +236,36 @@ export async function getClientCheckpoint(
|
|
|
206
236
|
export function createUpperCaseUUID(): string {
|
|
207
237
|
return uuid().toUpperCase();
|
|
208
238
|
}
|
|
239
|
+
|
|
240
|
+
export async function renameTable(connectionManager: MSSQLConnectionManager, fromTable: string, toTable: string) {
|
|
241
|
+
await connectionManager.execute('sp_rename', [
|
|
242
|
+
{ name: 'objname', value: toQualifiedTableName(connectionManager.schema, fromTable) },
|
|
243
|
+
{ name: 'newname', value: toTable }
|
|
244
|
+
]);
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
export interface EnableCDCForTableOptions {
|
|
248
|
+
connectionManager: MSSQLConnectionManager;
|
|
249
|
+
table: string;
|
|
250
|
+
captureInstance?: string;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
export async function enableCDCForTable(options: EnableCDCForTableOptions): Promise<void> {
|
|
254
|
+
const { connectionManager, table, captureInstance } = options;
|
|
255
|
+
|
|
256
|
+
await connectionManager.execute('sys.sp_cdc_enable_table', [
|
|
257
|
+
{ name: 'source_schema', value: connectionManager.schema },
|
|
258
|
+
{ name: 'source_name', value: table },
|
|
259
|
+
{ name: 'role_name', value: 'cdc_reader' },
|
|
260
|
+
{ name: 'supports_net_changes', value: 0 },
|
|
261
|
+
...(captureInstance !== undefined ? [{ name: 'capture_instance', value: captureInstance }] : [])
|
|
262
|
+
]);
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
export async function disableCDCForTable(connectionManager: MSSQLConnectionManager, tableName: string) {
|
|
266
|
+
await connectionManager.execute('sys.sp_cdc_disable_table', [
|
|
267
|
+
{ name: 'source_schema', value: connectionManager.schema },
|
|
268
|
+
{ name: 'source_name', value: tableName },
|
|
269
|
+
{ name: 'capture_instance', value: 'all' }
|
|
270
|
+
]);
|
|
271
|
+
}
|