@powersync/service-module-mssql 0.0.0-dev-20260225160713 → 0.0.0-dev-20260511080634
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +113 -6
- package/dist/api/MSSQLRouteAPIAdapter.d.ts +2 -2
- package/dist/api/MSSQLRouteAPIAdapter.js +3 -3
- package/dist/api/MSSQLRouteAPIAdapter.js.map +1 -1
- package/dist/common/CaptureInstance.d.ts +14 -0
- package/dist/common/CaptureInstance.js +2 -0
- package/dist/common/CaptureInstance.js.map +1 -0
- package/dist/common/MSSQLSourceTable.d.ts +16 -14
- package/dist/common/MSSQLSourceTable.js +35 -16
- package/dist/common/MSSQLSourceTable.js.map +1 -1
- package/dist/common/MSSQLSourceTableCache.js.map +1 -1
- package/dist/common/mssqls-to-sqlite.d.ts +1 -1
- package/dist/common/mssqls-to-sqlite.js +1 -1
- package/dist/common/mssqls-to-sqlite.js.map +1 -1
- package/dist/module/MSSQLModule.js +4 -4
- package/dist/module/MSSQLModule.js.map +1 -1
- package/dist/replication/CDCPoller.d.ts +45 -23
- package/dist/replication/CDCPoller.js +201 -61
- package/dist/replication/CDCPoller.js.map +1 -1
- package/dist/replication/CDCReplicationJob.d.ts +2 -2
- package/dist/replication/CDCReplicationJob.js +12 -4
- package/dist/replication/CDCReplicationJob.js.map +1 -1
- package/dist/replication/CDCReplicator.d.ts +2 -3
- package/dist/replication/CDCReplicator.js +1 -24
- package/dist/replication/CDCReplicator.js.map +1 -1
- package/dist/replication/CDCStream.d.ts +39 -16
- package/dist/replication/CDCStream.js +200 -103
- package/dist/replication/CDCStream.js.map +1 -1
- package/dist/replication/MSSQLConnectionManager.d.ts +1 -1
- package/dist/replication/MSSQLConnectionManager.js +17 -6
- package/dist/replication/MSSQLConnectionManager.js.map +1 -1
- package/dist/replication/MSSQLConnectionManagerFactory.d.ts +1 -1
- package/dist/replication/MSSQLConnectionManagerFactory.js.map +1 -1
- package/dist/replication/MSSQLSnapshotQuery.d.ts +1 -1
- package/dist/replication/MSSQLSnapshotQuery.js +2 -2
- package/dist/replication/MSSQLSnapshotQuery.js.map +1 -1
- package/dist/types/types.d.ts +4 -56
- package/dist/types/types.js +5 -24
- package/dist/types/types.js.map +1 -1
- package/dist/utils/deadlock.d.ts +9 -0
- package/dist/utils/deadlock.js +40 -0
- package/dist/utils/deadlock.js.map +1 -0
- package/dist/utils/mssql.d.ts +36 -18
- package/dist/utils/mssql.js +102 -100
- package/dist/utils/mssql.js.map +1 -1
- package/dist/utils/schema.d.ts +9 -0
- package/dist/utils/schema.js +34 -0
- package/dist/utils/schema.js.map +1 -1
- package/package.json +14 -14
- package/src/api/MSSQLRouteAPIAdapter.ts +4 -4
- package/src/common/CaptureInstance.ts +15 -0
- package/src/common/MSSQLSourceTable.ts +33 -24
- package/src/common/MSSQLSourceTableCache.ts +1 -1
- package/src/common/mssqls-to-sqlite.ts +1 -1
- package/src/module/MSSQLModule.ts +4 -4
- package/src/replication/CDCPoller.ts +275 -75
- package/src/replication/CDCReplicationJob.ts +13 -6
- package/src/replication/CDCReplicator.ts +2 -28
- package/src/replication/CDCStream.ts +271 -128
- package/src/replication/MSSQLConnectionManager.ts +17 -7
- package/src/replication/MSSQLConnectionManagerFactory.ts +1 -1
- package/src/replication/MSSQLSnapshotQuery.ts +3 -3
- package/src/types/types.ts +5 -28
- package/src/utils/deadlock.ts +44 -0
- package/src/utils/mssql.ts +163 -128
- package/src/utils/schema.ts +44 -1
- package/test/src/CDCStream.test.ts +10 -8
- package/test/src/CDCStreamTestContext.ts +32 -12
- package/test/src/CDCStream_resumable_snapshot.test.ts +14 -12
- package/test/src/LSN.test.ts +1 -1
- package/test/src/env.ts +1 -1
- package/test/src/mssql-to-sqlite.test.ts +25 -17
- package/test/src/schema-changes.test.ts +474 -0
- package/test/src/util.ts +81 -18
- package/test/tsconfig.json +0 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -0,0 +1,474 @@
|
|
|
1
|
+
import { storage } from '@powersync/service-core';
|
|
2
|
+
import { putOp, removeOp } from '@powersync/service-core-tests';
|
|
3
|
+
import sql from 'mssql';
|
|
4
|
+
import { describe, expect, test, vi } from 'vitest';
|
|
5
|
+
|
|
6
|
+
import { SchemaChangeType } from '@module/replication/CDCPoller.js';
|
|
7
|
+
import { getLatestLSN, toQualifiedTableName } from '@module/utils/mssql.js';
|
|
8
|
+
import { logger } from '@powersync/lib-services-framework';
|
|
9
|
+
import { CDCStreamTestContext } from './CDCStreamTestContext.js';
|
|
10
|
+
import {
|
|
11
|
+
createTestTableWithBasicId,
|
|
12
|
+
describeWithStorage,
|
|
13
|
+
disableCDCForTable,
|
|
14
|
+
dropTestTable,
|
|
15
|
+
enableCDCForTable,
|
|
16
|
+
insertBasicIdTestData,
|
|
17
|
+
renameTable,
|
|
18
|
+
waitForPendingCDCChanges
|
|
19
|
+
} from './util.js';
|
|
20
|
+
|
|
21
|
+
describe('MSSQL Schema Changes Tests', () => {
|
|
22
|
+
describeWithStorage({ timeout: 60_000 }, defineSchemaChangesTests);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
const BASIC_SYNC_RULES = `
|
|
26
|
+
bucket_definitions:
|
|
27
|
+
global:
|
|
28
|
+
data:
|
|
29
|
+
- SELECT id, description FROM "test_data"
|
|
30
|
+
`;
|
|
31
|
+
|
|
32
|
+
function defineSchemaChangesTests(config: storage.TestStorageConfig) {
|
|
33
|
+
const { factory } = config;
|
|
34
|
+
|
|
35
|
+
test('Create table: New table in the sync rules', async () => {
|
|
36
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
37
|
+
const { connectionManager } = context;
|
|
38
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
39
|
+
|
|
40
|
+
await context.replicateSnapshot();
|
|
41
|
+
await context.startStreaming();
|
|
42
|
+
|
|
43
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
44
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
45
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
46
|
+
|
|
47
|
+
const data = await context.getFinalBucketState('global[]');
|
|
48
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test('Create table: New table created while PowerSync is stopped', async () => {
|
|
52
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
53
|
+
const { connectionManager } = context;
|
|
54
|
+
await context.updateSyncRules(`
|
|
55
|
+
bucket_definitions:
|
|
56
|
+
global:
|
|
57
|
+
data:
|
|
58
|
+
- SELECT id, description FROM "test_data%"
|
|
59
|
+
`);
|
|
60
|
+
|
|
61
|
+
await createTestTableWithBasicId(connectionManager, 'test_data1');
|
|
62
|
+
const testData = await insertBasicIdTestData(connectionManager, 'test_data1');
|
|
63
|
+
|
|
64
|
+
await context.replicateSnapshot();
|
|
65
|
+
await context.startStreaming();
|
|
66
|
+
|
|
67
|
+
await context.dispose();
|
|
68
|
+
|
|
69
|
+
await createTestTableWithBasicId(connectionManager, 'test_data2');
|
|
70
|
+
|
|
71
|
+
await using newContext = await CDCStreamTestContext.open(factory, { doNotClear: true });
|
|
72
|
+
await newContext.loadActiveSyncRules();
|
|
73
|
+
|
|
74
|
+
await newContext.replicateSnapshot();
|
|
75
|
+
await newContext.startStreaming();
|
|
76
|
+
|
|
77
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data2');
|
|
78
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data2');
|
|
79
|
+
|
|
80
|
+
const finalState = await newContext.getFinalBucketState('global[]');
|
|
81
|
+
expect(finalState).toMatchObject([
|
|
82
|
+
putOp('test_data1', testData),
|
|
83
|
+
putOp('test_data2', testData1),
|
|
84
|
+
putOp('test_data2', testData2)
|
|
85
|
+
]);
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test('Create table: New table not in the sync rules', async () => {
|
|
89
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
90
|
+
const { connectionManager } = context;
|
|
91
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
92
|
+
|
|
93
|
+
await context.replicateSnapshot();
|
|
94
|
+
await context.startStreaming();
|
|
95
|
+
|
|
96
|
+
await createTestTableWithBasicId(connectionManager, 'test_data_ignored');
|
|
97
|
+
await insertBasicIdTestData(connectionManager, 'test_data_ignored');
|
|
98
|
+
|
|
99
|
+
const data = await context.getBucketData('global[]');
|
|
100
|
+
expect(data).toMatchObject([]);
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test('Drop table: Table in the sync rules', async () => {
|
|
104
|
+
await using context = await CDCStreamTestContext.open(factory, {
|
|
105
|
+
cdcStreamOptions: { schemaCheckIntervalMs: 5000 }
|
|
106
|
+
});
|
|
107
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
108
|
+
|
|
109
|
+
const { connectionManager } = context;
|
|
110
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
111
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
112
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
113
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
114
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
115
|
+
|
|
116
|
+
await context.replicateSnapshot();
|
|
117
|
+
await context.startStreaming();
|
|
118
|
+
|
|
119
|
+
let data = await context.getBucketData('global[]');
|
|
120
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
121
|
+
await dropTestTable(connectionManager, 'test_data');
|
|
122
|
+
|
|
123
|
+
data = await context.getFinalBucketState('global[]');
|
|
124
|
+
expect(data).toMatchObject([]);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test('Re-create table', async () => {
|
|
128
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
129
|
+
const { connectionManager } = context;
|
|
130
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
131
|
+
|
|
132
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
133
|
+
|
|
134
|
+
await context.replicateSnapshot();
|
|
135
|
+
await context.startStreaming();
|
|
136
|
+
|
|
137
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
138
|
+
let data = await context.getBucketData('global[]');
|
|
139
|
+
expect(data).toMatchObject([putOp('test_data', testData1)]);
|
|
140
|
+
|
|
141
|
+
let schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
142
|
+
await dropTestTable(connectionManager, 'test_data');
|
|
143
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.TABLE_DROP);
|
|
144
|
+
|
|
145
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
146
|
+
|
|
147
|
+
const testData = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
148
|
+
|
|
149
|
+
data = await context.getFinalBucketState('global[]');
|
|
150
|
+
expect(data).toMatchObject([putOp('test_data', testData)]);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
test('Rename table: Table not in the sync rules to one in the sync rules', async () => {
|
|
154
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
155
|
+
const { connectionManager } = context;
|
|
156
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
157
|
+
|
|
158
|
+
await createTestTableWithBasicId(connectionManager, 'test_data_old');
|
|
159
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
160
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data_old');
|
|
161
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
162
|
+
|
|
163
|
+
await context.replicateSnapshot();
|
|
164
|
+
await context.startStreaming();
|
|
165
|
+
|
|
166
|
+
const schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
167
|
+
await renameTable(connectionManager, 'test_data_old', 'test_data');
|
|
168
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.TABLE_CREATE);
|
|
169
|
+
|
|
170
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
171
|
+
const data = await context.getFinalBucketState('global[]');
|
|
172
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
test('Rename table: Table in the sync rules to another table in the sync rules', async () => {
|
|
176
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
177
|
+
const { connectionManager } = context;
|
|
178
|
+
|
|
179
|
+
await context.updateSyncRules(`
|
|
180
|
+
bucket_definitions:
|
|
181
|
+
global:
|
|
182
|
+
data:
|
|
183
|
+
- SELECT id, description FROM "test_data%"
|
|
184
|
+
`);
|
|
185
|
+
|
|
186
|
+
await createTestTableWithBasicId(connectionManager, 'test_data1');
|
|
187
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
188
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data1');
|
|
189
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
190
|
+
|
|
191
|
+
await context.replicateSnapshot();
|
|
192
|
+
await context.startStreaming();
|
|
193
|
+
|
|
194
|
+
const schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
195
|
+
await renameTable(connectionManager, 'test_data1', 'test_data2');
|
|
196
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.TABLE_RENAME);
|
|
197
|
+
|
|
198
|
+
const data = await context.getBucketData('global[]');
|
|
199
|
+
expect(data.slice(0, 2)).toMatchObject([
|
|
200
|
+
// Initial replication
|
|
201
|
+
putOp('test_data1', testData1),
|
|
202
|
+
// Initial truncate
|
|
203
|
+
removeOp('test_data1', testData1.id)
|
|
204
|
+
]);
|
|
205
|
+
|
|
206
|
+
const finalState = await context.getFinalBucketState('global[]');
|
|
207
|
+
expect(finalState).toMatchObject([putOp('test_data2', testData1)]);
|
|
208
|
+
});
|
|
209
|
+
|
|
210
|
+
test('Rename table: Table renamed while PowerSync is stopped', async () => {
|
|
211
|
+
let context = await CDCStreamTestContext.open(factory);
|
|
212
|
+
let { connectionManager } = context;
|
|
213
|
+
|
|
214
|
+
await context.updateSyncRules(`
|
|
215
|
+
bucket_definitions:
|
|
216
|
+
global:
|
|
217
|
+
data:
|
|
218
|
+
- SELECT id, description FROM "test_data%"
|
|
219
|
+
`);
|
|
220
|
+
|
|
221
|
+
await createTestTableWithBasicId(connectionManager, 'test_data1');
|
|
222
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
223
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data1');
|
|
224
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
225
|
+
|
|
226
|
+
await context.replicateSnapshot();
|
|
227
|
+
await context.startStreaming();
|
|
228
|
+
|
|
229
|
+
let data = await context.getBucketData('global[]');
|
|
230
|
+
expect(data).toMatchObject([putOp('test_data1', testData1)]);
|
|
231
|
+
|
|
232
|
+
await context.dispose();
|
|
233
|
+
await renameTable(connectionManager, 'test_data1', 'test_data2');
|
|
234
|
+
|
|
235
|
+
await using newContext = await CDCStreamTestContext.open(factory, { doNotClear: true });
|
|
236
|
+
await newContext.loadActiveSyncRules();
|
|
237
|
+
|
|
238
|
+
await newContext.replicateSnapshot();
|
|
239
|
+
await newContext.startStreaming();
|
|
240
|
+
|
|
241
|
+
const finalState = await newContext.getFinalBucketState('global[]');
|
|
242
|
+
expect(finalState).toMatchObject([putOp('test_data2', testData1)]);
|
|
243
|
+
});
|
|
244
|
+
|
|
245
|
+
test('Rename table: Table in the sync rules to not in the sync rules', async () => {
|
|
246
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
247
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
248
|
+
|
|
249
|
+
const { connectionManager } = context;
|
|
250
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
251
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
252
|
+
const testData = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
253
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
254
|
+
|
|
255
|
+
await context.replicateSnapshot();
|
|
256
|
+
await context.startStreaming();
|
|
257
|
+
|
|
258
|
+
let data = await context.getBucketData('global[]');
|
|
259
|
+
expect(data).toMatchObject([putOp('test_data', testData)]);
|
|
260
|
+
|
|
261
|
+
const schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
262
|
+
await renameTable(connectionManager, 'test_data', 'test_data_ignored');
|
|
263
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.TABLE_RENAME);
|
|
264
|
+
|
|
265
|
+
data = await context.getBucketData('global[]');
|
|
266
|
+
expect(data).toMatchObject([
|
|
267
|
+
// Initial replication
|
|
268
|
+
putOp('test_data', testData),
|
|
269
|
+
// Truncate
|
|
270
|
+
removeOp('test_data', testData.id)
|
|
271
|
+
]);
|
|
272
|
+
});
|
|
273
|
+
|
|
274
|
+
test('New capture instance created for replicating table triggers re-snapshot', async () => {
|
|
275
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
276
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
277
|
+
|
|
278
|
+
const { connectionManager } = context;
|
|
279
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
280
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
281
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
282
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
283
|
+
|
|
284
|
+
await context.replicateSnapshot();
|
|
285
|
+
await context.startStreaming();
|
|
286
|
+
|
|
287
|
+
await enableCDCForTable({ connectionManager, table: 'test_data', captureInstance: 'capture_instance_new' });
|
|
288
|
+
|
|
289
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
290
|
+
|
|
291
|
+
const data = await context.getFinalBucketState('global[]');
|
|
292
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
test('New capture instance created for replicating table while PowerSync is stopped', async () => {
|
|
296
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
297
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
298
|
+
const { connectionManager } = context;
|
|
299
|
+
|
|
300
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
301
|
+
let beforeLSN = await getLatestLSN(connectionManager);
|
|
302
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
303
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
304
|
+
|
|
305
|
+
await context.replicateSnapshot();
|
|
306
|
+
await context.startStreaming();
|
|
307
|
+
|
|
308
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
309
|
+
let data = await context.getBucketData('global[]');
|
|
310
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
311
|
+
|
|
312
|
+
await context.dispose();
|
|
313
|
+
await enableCDCForTable({ connectionManager, table: 'test_data', captureInstance: 'capture_instance_new' });
|
|
314
|
+
|
|
315
|
+
await using newContext = await CDCStreamTestContext.open(factory, { doNotClear: true });
|
|
316
|
+
await newContext.loadActiveSyncRules();
|
|
317
|
+
|
|
318
|
+
await newContext.replicateSnapshot();
|
|
319
|
+
await newContext.startStreaming();
|
|
320
|
+
|
|
321
|
+
const testData3 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
322
|
+
|
|
323
|
+
const finalState = await newContext.getFinalBucketState('global[]');
|
|
324
|
+
expect(finalState).toMatchObject([
|
|
325
|
+
putOp('test_data', testData1),
|
|
326
|
+
putOp('test_data', testData2),
|
|
327
|
+
putOp('test_data', testData3)
|
|
328
|
+
]);
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
test('Capture instance created for a sync rule table without a capture instance', async () => {
|
|
332
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
333
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
334
|
+
const { connectionManager } = context;
|
|
335
|
+
|
|
336
|
+
await createTestTableWithBasicId(connectionManager, 'test_data', false);
|
|
337
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
338
|
+
|
|
339
|
+
await context.replicateSnapshot();
|
|
340
|
+
await context.startStreaming();
|
|
341
|
+
|
|
342
|
+
const schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
343
|
+
await enableCDCForTable({ connectionManager, table: 'test_data' });
|
|
344
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.NEW_CAPTURE_INSTANCE);
|
|
345
|
+
|
|
346
|
+
let data = await context.getBucketData('global[]');
|
|
347
|
+
expect(data).toMatchObject([putOp('test_data', testData1)]);
|
|
348
|
+
|
|
349
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
350
|
+
|
|
351
|
+
data = await context.getFinalBucketState('global[]');
|
|
352
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
353
|
+
});
|
|
354
|
+
|
|
355
|
+
test('Capture instance removed for an actively replicating table', async () => {
|
|
356
|
+
await using context = await CDCStreamTestContext.open(factory, {
|
|
357
|
+
cdcStreamOptions: { schemaCheckIntervalMs: 5000 }
|
|
358
|
+
});
|
|
359
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
360
|
+
const { connectionManager } = context;
|
|
361
|
+
|
|
362
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
363
|
+
let beforeLSN = await getLatestLSN(connectionManager);
|
|
364
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
365
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
366
|
+
|
|
367
|
+
await context.replicateSnapshot();
|
|
368
|
+
await context.startStreaming();
|
|
369
|
+
|
|
370
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
371
|
+
let data = await context.getBucketData('global[]');
|
|
372
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
373
|
+
|
|
374
|
+
const schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
375
|
+
await disableCDCForTable(connectionManager, 'test_data');
|
|
376
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.MISSING_CAPTURE_INSTANCE);
|
|
377
|
+
|
|
378
|
+
data = await context.getBucketData('global[]');
|
|
379
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
test('Capture instance removed, and then re-added', async () => {
|
|
383
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
384
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
385
|
+
const { connectionManager } = context;
|
|
386
|
+
|
|
387
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
388
|
+
|
|
389
|
+
await context.replicateSnapshot();
|
|
390
|
+
await context.startStreaming();
|
|
391
|
+
|
|
392
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
393
|
+
const testData2 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
394
|
+
let data = await context.getBucketData('global[]');
|
|
395
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
396
|
+
|
|
397
|
+
let schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
398
|
+
await disableCDCForTable(connectionManager, 'test_data');
|
|
399
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.MISSING_CAPTURE_INSTANCE);
|
|
400
|
+
|
|
401
|
+
schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
402
|
+
await enableCDCForTable({ connectionManager, table: 'test_data' });
|
|
403
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.NEW_CAPTURE_INSTANCE);
|
|
404
|
+
|
|
405
|
+
const testData3 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
406
|
+
const testData4 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
407
|
+
|
|
408
|
+
const finalState = await context.getFinalBucketState('global[]');
|
|
409
|
+
expect(finalState).toMatchObject([
|
|
410
|
+
putOp('test_data', testData1),
|
|
411
|
+
putOp('test_data', testData2),
|
|
412
|
+
putOp('test_data', testData3),
|
|
413
|
+
putOp('test_data', testData4)
|
|
414
|
+
]);
|
|
415
|
+
});
|
|
416
|
+
|
|
417
|
+
test('Column schema changes continue replication, but with warning.', async () => {
|
|
418
|
+
await using context = await CDCStreamTestContext.open(factory);
|
|
419
|
+
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
420
|
+
const { connectionManager } = context;
|
|
421
|
+
|
|
422
|
+
await createTestTableWithBasicId(connectionManager, 'test_data');
|
|
423
|
+
const beforeLSN = await getLatestLSN(connectionManager);
|
|
424
|
+
const testData1 = await insertBasicIdTestData(connectionManager, 'test_data');
|
|
425
|
+
await waitForPendingCDCChanges(beforeLSN, connectionManager);
|
|
426
|
+
|
|
427
|
+
await context.replicateSnapshot();
|
|
428
|
+
await context.startStreaming();
|
|
429
|
+
const schemaSpy = vi.spyOn(context.cdcStream, 'handleSchemaChange');
|
|
430
|
+
await connectionManager.query(`ALTER TABLE test_data ADD new_column INT`);
|
|
431
|
+
await expectedSchemaChange(schemaSpy, SchemaChangeType.TABLE_COLUMN_CHANGES);
|
|
432
|
+
|
|
433
|
+
const { recordset: result } = await connectionManager.query(
|
|
434
|
+
`
|
|
435
|
+
INSERT INTO ${toQualifiedTableName(connectionManager.schema, 'test_data')} (description, new_column)
|
|
436
|
+
OUTPUT INSERTED.id, INSERTED.description
|
|
437
|
+
VALUES (@description, @new_column)
|
|
438
|
+
`,
|
|
439
|
+
[
|
|
440
|
+
{ name: 'description', type: sql.NVarChar(sql.MAX), value: 'new_column_description' },
|
|
441
|
+
{ name: 'new_column', type: sql.Int, value: 1 }
|
|
442
|
+
]
|
|
443
|
+
);
|
|
444
|
+
|
|
445
|
+
const testData2 = { id: result[0].id, description: result[0].description };
|
|
446
|
+
|
|
447
|
+
const data = await context.getBucketData('global[]');
|
|
448
|
+
// Capture instances do not reflect most schema changes until the capture instance is re-created
|
|
449
|
+
// So testData2 will be replicated but without the new column
|
|
450
|
+
expect(data).toMatchObject([putOp('test_data', testData1), putOp('test_data', testData2)]);
|
|
451
|
+
|
|
452
|
+
expect(
|
|
453
|
+
context.cdcStream.tableCache
|
|
454
|
+
.getAll()
|
|
455
|
+
.every((t) => t.captureInstance && t.captureInstance.pendingSchemaChanges.length > 0)
|
|
456
|
+
).toBe(true);
|
|
457
|
+
});
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
async function expectedSchemaChange(spy: any, type: SchemaChangeType) {
|
|
461
|
+
logger.info(`Test Assertion: Waiting for schema change: ${type}`);
|
|
462
|
+
try {
|
|
463
|
+
await vi.waitFor(() => expect(spy).toHaveBeenCalledWith(expect.anything(), expect.objectContaining({ type })), {
|
|
464
|
+
timeout: 55000
|
|
465
|
+
});
|
|
466
|
+
} catch (error) {
|
|
467
|
+
// The error message thrown here is extremely verbose and not particularly helpful
|
|
468
|
+
throw new Error(`Test Assertion: Timeout waiting for schema change: ${type}`);
|
|
469
|
+
}
|
|
470
|
+
const promises = spy.mock.results.filter((r: any) => r.type === 'return').map((r: any) => r.value);
|
|
471
|
+
|
|
472
|
+
await Promise.all(promises.map((p: Promise<unknown>) => expect(p).resolves.toBeUndefined()));
|
|
473
|
+
logger.info(`Test Assertion: Received expected schema change: ${type}`);
|
|
474
|
+
}
|
package/test/src/util.ts
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
import * as types from '@module/types/types.js';
|
|
2
2
|
import { logger } from '@powersync/lib-services-framework';
|
|
3
|
-
import { BucketStorageFactory, InternalOpId, ReplicationCheckpoint,
|
|
3
|
+
import { BucketStorageFactory, InternalOpId, ReplicationCheckpoint, TestStorageConfig } from '@powersync/service-core';
|
|
4
4
|
|
|
5
5
|
import * as mongo_storage from '@powersync/service-module-mongodb-storage';
|
|
6
6
|
import * as postgres_storage from '@powersync/service-module-postgres-storage';
|
|
7
7
|
|
|
8
|
-
import {
|
|
9
|
-
import { env } from './env.js';
|
|
8
|
+
import { LSN } from '@module/common/LSN.js';
|
|
10
9
|
import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
|
|
11
|
-
import { createCheckpoint,
|
|
10
|
+
import { createCheckpoint, escapeIdentifier, getLatestLSN, toQualifiedTableName } from '@module/utils/mssql.js';
|
|
12
11
|
import sql from 'mssql';
|
|
13
12
|
import { v4 as uuid } from 'uuid';
|
|
14
|
-
import {
|
|
13
|
+
import { describe, TestOptions } from 'vitest';
|
|
14
|
+
import { env } from './env.js';
|
|
15
15
|
|
|
16
16
|
export const TEST_URI = env.MSSQL_TEST_URI;
|
|
17
17
|
|
|
@@ -20,11 +20,11 @@ export const INITIALIZED_MONGO_STORAGE_FACTORY = mongo_storage.test_utils.mongoT
|
|
|
20
20
|
isCI: env.CI
|
|
21
21
|
});
|
|
22
22
|
|
|
23
|
-
export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils.
|
|
23
|
+
export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils.postgresTestSetup({
|
|
24
24
|
url: env.PG_STORAGE_TEST_URL
|
|
25
25
|
});
|
|
26
26
|
|
|
27
|
-
export function describeWithStorage(options: TestOptions, fn: (
|
|
27
|
+
export function describeWithStorage(options: TestOptions, fn: (config: TestStorageConfig) => void) {
|
|
28
28
|
describe.skipIf(!env.TEST_MONGO_STORAGE)(`mongodb storage`, options, function () {
|
|
29
29
|
fn(INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
30
30
|
});
|
|
@@ -39,7 +39,7 @@ export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
|
|
|
39
39
|
uri: TEST_URI,
|
|
40
40
|
additionalConfig: {
|
|
41
41
|
pollingBatchSize: 10,
|
|
42
|
-
pollingIntervalMs:
|
|
42
|
+
pollingIntervalMs: 100,
|
|
43
43
|
trustServerCertificate: true
|
|
44
44
|
}
|
|
45
45
|
});
|
|
@@ -66,13 +66,12 @@ export async function clearTestDb(connectionManager: MSSQLConnectionManager) {
|
|
|
66
66
|
}
|
|
67
67
|
}
|
|
68
68
|
|
|
69
|
-
export async function
|
|
69
|
+
export async function dropTestTable(connectionManager: MSSQLConnectionManager, tableName: string) {
|
|
70
70
|
await connectionManager.execute('sys.sp_cdc_disable_table', [
|
|
71
71
|
{ name: 'source_schema', value: connectionManager.schema },
|
|
72
72
|
{ name: 'source_name', value: tableName },
|
|
73
73
|
{ name: 'capture_instance', value: 'all' }
|
|
74
74
|
]);
|
|
75
|
-
|
|
76
75
|
await connectionManager.query(`DROP TABLE [${tableName}]`);
|
|
77
76
|
}
|
|
78
77
|
|
|
@@ -92,19 +91,26 @@ export async function createTestDb(connectionManager: MSSQLConnectionManager, db
|
|
|
92
91
|
GO`);
|
|
93
92
|
}
|
|
94
93
|
|
|
95
|
-
export async function createTestTable(
|
|
94
|
+
export async function createTestTable(
|
|
95
|
+
connectionManager: MSSQLConnectionManager,
|
|
96
|
+
tableName: string,
|
|
97
|
+
withCaptureInstance: boolean = true
|
|
98
|
+
): Promise<void> {
|
|
96
99
|
await connectionManager.query(`
|
|
97
100
|
CREATE TABLE ${escapeIdentifier(connectionManager.schema)}.${escapeIdentifier(tableName)} (
|
|
98
101
|
id UNIQUEIDENTIFIER PRIMARY KEY,
|
|
99
102
|
description VARCHAR(MAX)
|
|
100
103
|
)
|
|
101
104
|
`);
|
|
102
|
-
|
|
105
|
+
if (withCaptureInstance) {
|
|
106
|
+
await enableCDCForTable({ connectionManager, table: tableName });
|
|
107
|
+
}
|
|
103
108
|
}
|
|
104
109
|
|
|
105
110
|
export async function createTestTableWithBasicId(
|
|
106
111
|
connectionManager: MSSQLConnectionManager,
|
|
107
|
-
tableName: string
|
|
112
|
+
tableName: string,
|
|
113
|
+
withCaptureInstance: boolean = true
|
|
108
114
|
): Promise<void> {
|
|
109
115
|
await connectionManager.query(`
|
|
110
116
|
CREATE TABLE ${escapeIdentifier(connectionManager.schema)}.${escapeIdentifier(tableName)} (
|
|
@@ -112,7 +118,9 @@ export async function createTestTableWithBasicId(
|
|
|
112
118
|
description VARCHAR(MAX)
|
|
113
119
|
)
|
|
114
120
|
`);
|
|
115
|
-
|
|
121
|
+
if (withCaptureInstance) {
|
|
122
|
+
await enableCDCForTable({ connectionManager, table: tableName });
|
|
123
|
+
}
|
|
116
124
|
}
|
|
117
125
|
|
|
118
126
|
export interface TestData {
|
|
@@ -135,6 +143,24 @@ export async function insertTestData(connectionManager: MSSQLConnectionManager,
|
|
|
135
143
|
return { id, description };
|
|
136
144
|
}
|
|
137
145
|
|
|
146
|
+
export async function insertBasicIdTestData(
|
|
147
|
+
connectionManager: MSSQLConnectionManager,
|
|
148
|
+
tableName: string
|
|
149
|
+
): Promise<TestData> {
|
|
150
|
+
const description = `description_${Math.floor(Math.random() * 1000000)}`;
|
|
151
|
+
const { recordset: result } = await connectionManager.query(
|
|
152
|
+
`
|
|
153
|
+
INSERT INTO ${toQualifiedTableName(connectionManager.schema, tableName)} (description)
|
|
154
|
+
OUTPUT INSERTED.id
|
|
155
|
+
VALUES (@description)
|
|
156
|
+
`,
|
|
157
|
+
[{ name: 'description', type: sql.NVarChar(sql.MAX), value: description }]
|
|
158
|
+
);
|
|
159
|
+
const id = result[0].id;
|
|
160
|
+
|
|
161
|
+
return { id, description };
|
|
162
|
+
}
|
|
163
|
+
|
|
138
164
|
export async function waitForPendingCDCChanges(
|
|
139
165
|
beforeLSN: LSN,
|
|
140
166
|
connectionManager: MSSQLConnectionManager
|
|
@@ -151,10 +177,14 @@ export async function waitForPendingCDCChanges(
|
|
|
151
177
|
);
|
|
152
178
|
|
|
153
179
|
if (result.length === 0) {
|
|
154
|
-
logger.info(
|
|
180
|
+
logger.info(
|
|
181
|
+
`Test Assertion: CDC changes pending. Waiting for a transaction newer than: ${beforeLSN.toString()} for 200ms...`
|
|
182
|
+
);
|
|
155
183
|
await new Promise((resolve) => setTimeout(resolve, 200));
|
|
156
184
|
} else {
|
|
157
|
-
logger.info(
|
|
185
|
+
logger.info(
|
|
186
|
+
`Test Assertion: Expected CDC change found with LSN: ${LSN.fromBinary(result[0].start_lsn).toString()}`
|
|
187
|
+
);
|
|
158
188
|
return;
|
|
159
189
|
}
|
|
160
190
|
}
|
|
@@ -176,14 +206,14 @@ export async function getClientCheckpoint(
|
|
|
176
206
|
const timeout = options?.timeout ?? 50_000;
|
|
177
207
|
let lastCp: ReplicationCheckpoint | null = null;
|
|
178
208
|
|
|
179
|
-
logger.info(`Waiting for LSN checkpoint: ${lsn}`);
|
|
209
|
+
logger.info(`Test Assertion: Waiting for LSN checkpoint: ${lsn}`);
|
|
180
210
|
while (Date.now() - start < timeout) {
|
|
181
211
|
const storage = await storageFactory.getActiveStorage();
|
|
182
212
|
const cp = await storage?.getCheckpoint();
|
|
183
213
|
if (cp != null) {
|
|
184
214
|
lastCp = cp;
|
|
185
215
|
if (cp.lsn != null && cp.lsn >= lsn.toString()) {
|
|
186
|
-
logger.info(`Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
|
|
216
|
+
logger.info(`Test Assertion: Got write checkpoint: ${lsn} : ${cp.checkpoint}`);
|
|
187
217
|
return cp.checkpoint;
|
|
188
218
|
}
|
|
189
219
|
}
|
|
@@ -200,3 +230,36 @@ export async function getClientCheckpoint(
|
|
|
200
230
|
export function createUpperCaseUUID(): string {
|
|
201
231
|
return uuid().toUpperCase();
|
|
202
232
|
}
|
|
233
|
+
|
|
234
|
+
export async function renameTable(connectionManager: MSSQLConnectionManager, fromTable: string, toTable: string) {
|
|
235
|
+
await connectionManager.execute('sp_rename', [
|
|
236
|
+
{ name: 'objname', value: toQualifiedTableName(connectionManager.schema, fromTable) },
|
|
237
|
+
{ name: 'newname', value: toTable }
|
|
238
|
+
]);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
export interface EnableCDCForTableOptions {
|
|
242
|
+
connectionManager: MSSQLConnectionManager;
|
|
243
|
+
table: string;
|
|
244
|
+
captureInstance?: string;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
export async function enableCDCForTable(options: EnableCDCForTableOptions): Promise<void> {
|
|
248
|
+
const { connectionManager, table, captureInstance } = options;
|
|
249
|
+
|
|
250
|
+
await connectionManager.execute('sys.sp_cdc_enable_table', [
|
|
251
|
+
{ name: 'source_schema', value: connectionManager.schema },
|
|
252
|
+
{ name: 'source_name', value: table },
|
|
253
|
+
{ name: 'role_name', value: 'cdc_reader' },
|
|
254
|
+
{ name: 'supports_net_changes', value: 0 },
|
|
255
|
+
...(captureInstance !== undefined ? [{ name: 'capture_instance', value: captureInstance }] : [])
|
|
256
|
+
]);
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
export async function disableCDCForTable(connectionManager: MSSQLConnectionManager, tableName: string) {
|
|
260
|
+
await connectionManager.execute('sys.sp_cdc_disable_table', [
|
|
261
|
+
{ name: 'source_schema', value: connectionManager.schema },
|
|
262
|
+
{ name: 'source_name', value: tableName },
|
|
263
|
+
{ name: 'capture_instance', value: 'all' }
|
|
264
|
+
]);
|
|
265
|
+
}
|