@powersync/service-module-mssql 0.8.3 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +113 -0
- package/dist/api/MSSQLRouteAPIAdapter.d.ts +2 -2
- package/dist/api/MSSQLRouteAPIAdapter.js +33 -39
- package/dist/api/MSSQLRouteAPIAdapter.js.map +1 -1
- package/dist/common/MSSQLSourceTable.d.ts +12 -2
- package/dist/common/MSSQLSourceTable.js +30 -4
- package/dist/common/MSSQLSourceTable.js.map +1 -1
- package/dist/module/MSSQLModule.js +2 -1
- package/dist/module/MSSQLModule.js.map +1 -1
- package/dist/replication/CDCPoller.d.ts +13 -36
- package/dist/replication/CDCPoller.js +173 -129
- package/dist/replication/CDCPoller.js.map +1 -1
- package/dist/replication/CDCStream.d.ts +11 -14
- package/dist/replication/CDCStream.js +109 -103
- package/dist/replication/CDCStream.js.map +1 -1
- package/dist/replication/CaptureReconciler.d.ts +47 -0
- package/dist/replication/CaptureReconciler.js +155 -0
- package/dist/replication/CaptureReconciler.js.map +1 -0
- package/dist/replication/MSSQLConnectionManager.d.ts +1 -0
- package/dist/replication/MSSQLConnectionManager.js +19 -14
- package/dist/replication/MSSQLConnectionManager.js.map +1 -1
- package/dist/replication/MSSQLTableReconciliationContext.d.ts +21 -0
- package/dist/replication/MSSQLTableReconciliationContext.js +7 -0
- package/dist/replication/MSSQLTableReconciliationContext.js.map +1 -0
- package/dist/replication/SchemaChange.d.ts +67 -0
- package/dist/replication/SchemaChange.js +31 -0
- package/dist/replication/SchemaChange.js.map +1 -0
- package/dist/types/types.d.ts +13 -0
- package/dist/types/types.js +28 -2
- package/dist/types/types.js.map +1 -1
- package/dist/utils/mssql.js +1 -1
- package/dist/utils/schema.d.ts +14 -1
- package/dist/utils/schema.js +43 -45
- package/dist/utils/schema.js.map +1 -1
- package/package.json +9 -9
- package/src/api/MSSQLRouteAPIAdapter.ts +36 -46
- package/src/common/MSSQLSourceTable.ts +32 -4
- package/src/module/MSSQLModule.ts +2 -1
- package/src/replication/CDCPoller.ts +206 -162
- package/src/replication/CDCStream.ts +142 -151
- package/src/replication/CaptureReconciler.ts +186 -0
- package/src/replication/MSSQLConnectionManager.ts +23 -14
- package/src/replication/MSSQLTableReconciliationContext.ts +29 -0
- package/src/replication/SchemaChange.ts +79 -0
- package/src/types/types.ts +47 -2
- package/src/utils/mssql.ts +1 -1
- package/src/utils/schema.ts +60 -54
- package/test/src/CDCPoller.test.ts +382 -0
- package/test/src/CDCStream.test.ts +6 -2
- package/test/src/CDCStreamTestContext.ts +9 -2
- package/test/src/CaptureReconciler.test.ts +171 -0
- package/test/src/MSSQLSourceTable.test.ts +30 -0
- package/test/src/config.test.ts +71 -0
- package/test/src/schema-changes.test.ts +513 -249
- package/test/src/schema.test.ts +17 -0
- package/test/src/util.ts +70 -6
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -0,0 +1,382 @@
|
|
|
1
|
+
import { LSN } from '@module/common/LSN.js';
|
|
2
|
+
import { MSSQLSourceTable } from '@module/common/MSSQLSourceTable.js';
|
|
3
|
+
import { CDCEventHandler, CDCPoller } from '@module/replication/CDCPoller.js';
|
|
4
|
+
import { MSSQLConnectionManager } from '@module/replication/MSSQLConnectionManager.js';
|
|
5
|
+
import {
|
|
6
|
+
createCheckpoint,
|
|
7
|
+
escapeIdentifier,
|
|
8
|
+
getCaptureInstances,
|
|
9
|
+
getLatestLSN,
|
|
10
|
+
getLatestReplicatedLSN,
|
|
11
|
+
toQualifiedTableName
|
|
12
|
+
} from '@module/utils/mssql.js';
|
|
13
|
+
import { getReplicationIdentityColumns } from '@module/utils/schema.js';
|
|
14
|
+
import { SourceTable } from '@powersync/service-core';
|
|
15
|
+
import sql from 'mssql';
|
|
16
|
+
import timers from 'timers/promises';
|
|
17
|
+
import { afterEach, beforeEach, describe, expect, test } from 'vitest';
|
|
18
|
+
import {
|
|
19
|
+
clearTestDb,
|
|
20
|
+
createTestTableWithBasicId,
|
|
21
|
+
enableCDCForTable,
|
|
22
|
+
insertBasicIdTestData,
|
|
23
|
+
TEST_CONNECTION_OPTIONS,
|
|
24
|
+
waitForPendingCDCChanges
|
|
25
|
+
} from './util.js';
|
|
26
|
+
|
|
27
|
+
describe('CDCPoller tests', { timeout: 60_000 }, () => {
|
|
28
|
+
let connectionManager: MSSQLConnectionManager;
|
|
29
|
+
|
|
30
|
+
beforeEach(async () => {
|
|
31
|
+
connectionManager = new MSSQLConnectionManager(TEST_CONNECTION_OPTIONS, {});
|
|
32
|
+
await clearTestDb(connectionManager);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
afterEach(async () => {
|
|
36
|
+
await connectionManager.end();
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test('Deferred updates are collapsed into a single UPDATE', async () => {
|
|
40
|
+
const tableName = 'test_deferred_update';
|
|
41
|
+
await createDeferredUpdateTestTable(connectionManager, tableName);
|
|
42
|
+
const qualifiedName = toQualifiedTableName(connectionManager.schema, tableName);
|
|
43
|
+
|
|
44
|
+
const beforeInsertLSN = await getLatestLSN(connectionManager);
|
|
45
|
+
await connectionManager.query(`INSERT INTO ${qualifiedName} (id, code) VALUES (1, 'V1'), (2, 'V2')`);
|
|
46
|
+
await waitForPendingCDCChanges(beforeInsertLSN, connectionManager);
|
|
47
|
+
|
|
48
|
+
// Only replicate changes made after the initial insert.
|
|
49
|
+
const startLSN = await getLatestReplicatedLSN(connectionManager);
|
|
50
|
+
|
|
51
|
+
const beforeUpdateLSN = await getLatestLSN(connectionManager);
|
|
52
|
+
// Updating a column with a unique index, using the actual column value creates deferred updates
|
|
53
|
+
await connectionManager.query(`UPDATE ${qualifiedName} SET code = code + 'A'`);
|
|
54
|
+
await waitForPendingCDCChanges(beforeUpdateLSN, connectionManager);
|
|
55
|
+
|
|
56
|
+
const table = await resolveSourceTable(connectionManager, tableName);
|
|
57
|
+
const { operations } = await collectChanges({
|
|
58
|
+
connectionManager,
|
|
59
|
+
tables: [table],
|
|
60
|
+
startLSN,
|
|
61
|
+
expectedOperationCount: 2
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
expect(operations).toMatchObject([
|
|
65
|
+
{ operation: 'update', rowBefore: { id: 1, code: 'V1' }, rowAfter: { id: 1, code: 'V1A' } },
|
|
66
|
+
{ operation: 'update', rowBefore: { id: 2, code: 'V2' }, rowAfter: { id: 2, code: 'V2A' } }
|
|
67
|
+
]);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
test('Normal deletes and inserts in the same transaction are not collapsed into updates', async () => {
|
|
71
|
+
const tableName = 'test_mixed_transaction';
|
|
72
|
+
await createTestTableWithBasicId(connectionManager, tableName);
|
|
73
|
+
const qualifiedName = toQualifiedTableName(connectionManager.schema, tableName);
|
|
74
|
+
|
|
75
|
+
const beforeInsertLSN = await getLatestLSN(connectionManager);
|
|
76
|
+
const initial = await insertBasicIdTestData(connectionManager, tableName);
|
|
77
|
+
await waitForPendingCDCChanges(beforeInsertLSN, connectionManager);
|
|
78
|
+
|
|
79
|
+
const startLSN = await getLatestReplicatedLSN(connectionManager);
|
|
80
|
+
|
|
81
|
+
const beforeUpdateLSN = await getLatestLSN(connectionManager);
|
|
82
|
+
const replacementDescription = 'replacement';
|
|
83
|
+
const { recordset } = await connectionManager.query(
|
|
84
|
+
`
|
|
85
|
+
BEGIN TRAN;
|
|
86
|
+
DELETE FROM ${qualifiedName} WHERE id = @id;
|
|
87
|
+
INSERT INTO ${qualifiedName} (description)
|
|
88
|
+
OUTPUT INSERTED.id, INSERTED.description
|
|
89
|
+
VALUES (@description);
|
|
90
|
+
COMMIT;
|
|
91
|
+
`,
|
|
92
|
+
[
|
|
93
|
+
{ name: 'id', type: sql.Int, value: initial.id },
|
|
94
|
+
{ name: 'description', type: sql.NVarChar(sql.MAX), value: replacementDescription }
|
|
95
|
+
]
|
|
96
|
+
);
|
|
97
|
+
const replacement = recordset[0];
|
|
98
|
+
await waitForPendingCDCChanges(beforeUpdateLSN, connectionManager);
|
|
99
|
+
|
|
100
|
+
const table = await resolveSourceTable(connectionManager, tableName);
|
|
101
|
+
const { operations } = await collectChanges({
|
|
102
|
+
connectionManager,
|
|
103
|
+
tables: [table],
|
|
104
|
+
startLSN,
|
|
105
|
+
expectedOperationCount: 2
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
expect(operations).toMatchObject([
|
|
109
|
+
{ operation: 'delete', row: initial },
|
|
110
|
+
{ operation: 'insert', row: replacement }
|
|
111
|
+
]);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test('In place updates emit a single UPDATE with the before and after rows', async () => {
|
|
115
|
+
const tableName = 'test_in_place_update';
|
|
116
|
+
await createTestTableWithBasicId(connectionManager, tableName);
|
|
117
|
+
const qualifiedName = toQualifiedTableName(connectionManager.schema, tableName);
|
|
118
|
+
|
|
119
|
+
const beforeInsertLSN = await getLatestLSN(connectionManager);
|
|
120
|
+
const initial = await insertBasicIdTestData(connectionManager, tableName);
|
|
121
|
+
await waitForPendingCDCChanges(beforeInsertLSN, connectionManager);
|
|
122
|
+
|
|
123
|
+
const startLSN = await getLatestReplicatedLSN(connectionManager);
|
|
124
|
+
|
|
125
|
+
const beforeUpdateLSN = await getLatestLSN(connectionManager);
|
|
126
|
+
const updated = { ...initial, description: 'updated' };
|
|
127
|
+
await connectionManager.query(`UPDATE ${qualifiedName} SET description = @description WHERE id = @id`, [
|
|
128
|
+
{ name: 'description', type: sql.NVarChar(sql.MAX), value: updated.description },
|
|
129
|
+
{ name: 'id', type: sql.Int, value: updated.id }
|
|
130
|
+
]);
|
|
131
|
+
await waitForPendingCDCChanges(beforeUpdateLSN, connectionManager);
|
|
132
|
+
|
|
133
|
+
const table = await resolveSourceTable(connectionManager, tableName);
|
|
134
|
+
const { operations } = await collectChanges({
|
|
135
|
+
connectionManager,
|
|
136
|
+
tables: [table],
|
|
137
|
+
startLSN,
|
|
138
|
+
expectedOperationCount: 1
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
expect(operations).toMatchObject([{ operation: 'update', rowBefore: initial, rowAfter: updated }]);
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
test('Committed transaction count covers transactions across all replicated tables', async () => {
|
|
145
|
+
const firstTableName = 'test_transaction_count_first';
|
|
146
|
+
const secondTableName = 'test_transaction_count_second';
|
|
147
|
+
await createTestTableWithBasicId(connectionManager, firstTableName);
|
|
148
|
+
await createTestTableWithBasicId(connectionManager, secondTableName);
|
|
149
|
+
|
|
150
|
+
const startLSN = await getLatestReplicatedLSN(connectionManager);
|
|
151
|
+
|
|
152
|
+
// Two separate transactions, each applying to only one of the tables. Previously this would have been undercounted as 1 transaction.
|
|
153
|
+
const first = await insertBasicIdTestData(connectionManager, firstTableName);
|
|
154
|
+
const second = await insertBasicIdTestData(connectionManager, secondTableName);
|
|
155
|
+
|
|
156
|
+
// CDC captures transactions in commit order, so waiting for a checkpoint written after both
|
|
157
|
+
// inserts guarantees both have been captured before polling starts. That in turn guarantees
|
|
158
|
+
// the poller sees both within a single polling cycle, which is what is being asserted here.
|
|
159
|
+
const beforeCheckpointLSN = await getLatestLSN(connectionManager);
|
|
160
|
+
await createCheckpoint(connectionManager);
|
|
161
|
+
await waitForPendingCDCChanges(beforeCheckpointLSN, connectionManager);
|
|
162
|
+
|
|
163
|
+
const { operations, commits } = await collectChanges({
|
|
164
|
+
connectionManager,
|
|
165
|
+
tables: [
|
|
166
|
+
await resolveSourceTable(connectionManager, firstTableName),
|
|
167
|
+
await resolveSourceTable(connectionManager, secondTableName)
|
|
168
|
+
],
|
|
169
|
+
startLSN,
|
|
170
|
+
expectedOperationCount: 2
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
expect(operations).toMatchObject([
|
|
174
|
+
{ operation: 'insert', row: first },
|
|
175
|
+
{ operation: 'insert', row: second }
|
|
176
|
+
]);
|
|
177
|
+
|
|
178
|
+
// The checkpoint table is not replicated here, so only the two inserts are counted.
|
|
179
|
+
const transactionCount = commits.reduce((total, commit) => total + commit.transactionCount, 0);
|
|
180
|
+
expect(transactionCount).toEqual(2);
|
|
181
|
+
});
|
|
182
|
+
|
|
183
|
+
test('Refreshes the bound capture instance to use its latest metadata', async () => {
|
|
184
|
+
const tableName = 'test_capture_metadata';
|
|
185
|
+
await createTestTableWithBasicId(connectionManager, tableName);
|
|
186
|
+
const table = await resolveSourceTable(connectionManager, tableName);
|
|
187
|
+
const startupInstance = table.captureInstance;
|
|
188
|
+
console.log(startupInstance);
|
|
189
|
+
const beforeInsertLSN = await getLatestLSN(connectionManager);
|
|
190
|
+
const inserted = await insertBasicIdTestData(connectionManager, tableName);
|
|
191
|
+
await waitForPendingCDCChanges(beforeInsertLSN, connectionManager);
|
|
192
|
+
|
|
193
|
+
const captureInstances = await getCaptureInstances({
|
|
194
|
+
connectionManager,
|
|
195
|
+
table: { schema: connectionManager.schema, name: tableName }
|
|
196
|
+
});
|
|
197
|
+
console.log(captureInstances.values());
|
|
198
|
+
const refreshed = captureInstances.get(table.objectId)!.instances[0];
|
|
199
|
+
const eventHandler = new RecordingCDCEventHandler();
|
|
200
|
+
const poller = new CDCPoller({
|
|
201
|
+
connectionManager,
|
|
202
|
+
eventHandler,
|
|
203
|
+
getReplicatedTables: () => [table],
|
|
204
|
+
startLSN: LSN.fromString(LSN.ZERO),
|
|
205
|
+
additionalConfig: {
|
|
206
|
+
pollingBatchSize: 1,
|
|
207
|
+
pollingIntervalMs: 1,
|
|
208
|
+
trustServerCertificate: true
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
(poller as any).captureInstances = captureInstances;
|
|
212
|
+
|
|
213
|
+
const bounds = {
|
|
214
|
+
startLSN: LSN.fromString(LSN.ZERO),
|
|
215
|
+
endLSN: await getLatestReplicatedLSN(connectionManager)
|
|
216
|
+
};
|
|
217
|
+
await (poller as any).pollTable(table, bounds);
|
|
218
|
+
|
|
219
|
+
expect(startupInstance).not.toBe(refreshed);
|
|
220
|
+
expect(bounds.startLSN).toBe(refreshed.minLSN);
|
|
221
|
+
expect(table.captureInstance).toBe(refreshed);
|
|
222
|
+
expect(table.pinnedCaptureObjectId).toBe(refreshed.objectId);
|
|
223
|
+
expect(eventHandler.operations).toMatchObject([{ operation: 'insert', row: inserted }]);
|
|
224
|
+
});
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* A change as it was handed to the CDCEventHandler. The CDC rows are kept as received, so that
|
|
229
|
+
* assertions can match on whichever source columns they care about.
|
|
230
|
+
*/
|
|
231
|
+
type RecordedOperation =
|
|
232
|
+
| { operation: 'insert'; row: any }
|
|
233
|
+
| { operation: 'delete'; row: any }
|
|
234
|
+
| { operation: 'update'; rowBefore: any; rowAfter: any };
|
|
235
|
+
|
|
236
|
+
/**
|
|
237
|
+
* A commit as it was reported to the CDCEventHandler at the end of a polling cycle.
|
|
238
|
+
*/
|
|
239
|
+
interface RecordedCommit {
|
|
240
|
+
lsn: string;
|
|
241
|
+
transactionCount: number;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/**
|
|
245
|
+
* Records the operations emitted by the CDCPoller so that their relative order can be asserted.
|
|
246
|
+
*/
|
|
247
|
+
class RecordingCDCEventHandler implements CDCEventHandler {
|
|
248
|
+
readonly operations: RecordedOperation[] = [];
|
|
249
|
+
readonly commits: RecordedCommit[] = [];
|
|
250
|
+
|
|
251
|
+
async onInsert(row: any): Promise<void> {
|
|
252
|
+
this.operations.push({ operation: 'insert', row });
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
async onUpdate(rowAfter: any, rowBefore: any): Promise<void> {
|
|
256
|
+
this.operations.push({ operation: 'update', rowBefore, rowAfter });
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
async onDelete(row: any): Promise<void> {
|
|
260
|
+
this.operations.push({ operation: 'delete', row });
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
async onCommit(lsn: string, transactionCount: number): Promise<void> {
|
|
264
|
+
this.commits.push({ lsn, transactionCount });
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
async onSchemaChange(): Promise<void> {}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* How long to wait for the expected operations before returning whatever arrived.
|
|
272
|
+
*/
|
|
273
|
+
const COLLECT_CHANGES_TIMEOUT_MS = 20_000;
|
|
274
|
+
|
|
275
|
+
interface CollectChangesOptions {
|
|
276
|
+
connectionManager: MSSQLConnectionManager;
|
|
277
|
+
tables: MSSQLSourceTable[];
|
|
278
|
+
startLSN: LSN;
|
|
279
|
+
expectedOperationCount: number;
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
/**
|
|
283
|
+
* Runs a CDCPoller from startLSN until the expected number of operations has been emitted, and
|
|
284
|
+
* returns the handler holding the operations and the commits in the order they were reported.
|
|
285
|
+
*/
|
|
286
|
+
async function collectChanges(options: CollectChangesOptions): Promise<RecordingCDCEventHandler> {
|
|
287
|
+
const { connectionManager, tables, startLSN, expectedOperationCount } = options;
|
|
288
|
+
const eventHandler = new RecordingCDCEventHandler();
|
|
289
|
+
|
|
290
|
+
const poller = new CDCPoller({
|
|
291
|
+
connectionManager,
|
|
292
|
+
eventHandler,
|
|
293
|
+
getReplicatedTables: () => tables,
|
|
294
|
+
startLSN,
|
|
295
|
+
additionalConfig: {
|
|
296
|
+
pollingBatchSize: 100,
|
|
297
|
+
pollingIntervalMs: 50,
|
|
298
|
+
trustServerCertificate: true
|
|
299
|
+
},
|
|
300
|
+
schemaCheckIntervalMs: 60_000
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
const pollerPromise = poller.replicateUntilStopped();
|
|
304
|
+
try {
|
|
305
|
+
const deadline = Date.now() + COLLECT_CHANGES_TIMEOUT_MS;
|
|
306
|
+
while (eventHandler.operations.length < expectedOperationCount && Date.now() < deadline) {
|
|
307
|
+
// Rethrows if the poller fails instead of waiting out the full timeout.
|
|
308
|
+
await Promise.race([timers.setTimeout(50), pollerPromise]);
|
|
309
|
+
}
|
|
310
|
+
} finally {
|
|
311
|
+
await poller.stop();
|
|
312
|
+
await pollerPromise;
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
return eventHandler;
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
/**
|
|
319
|
+
* Creates a table with a clustered primary key and a unique index over a separate column.
|
|
320
|
+
* A unique index is one of the prerequisites for triggering deferred updates in SQL Server.
|
|
321
|
+
*/
|
|
322
|
+
async function createDeferredUpdateTestTable(
|
|
323
|
+
connectionManager: MSSQLConnectionManager,
|
|
324
|
+
tableName: string
|
|
325
|
+
): Promise<void> {
|
|
326
|
+
const qualifiedName = toQualifiedTableName(connectionManager.schema, tableName);
|
|
327
|
+
await connectionManager.query(`
|
|
328
|
+
CREATE TABLE ${qualifiedName} (
|
|
329
|
+
id INT NOT NULL PRIMARY KEY,
|
|
330
|
+
code VARCHAR(100) NOT NULL
|
|
331
|
+
)
|
|
332
|
+
`);
|
|
333
|
+
await connectionManager.query(
|
|
334
|
+
`CREATE UNIQUE NONCLUSTERED INDEX ${escapeIdentifier(`UX_${tableName}_code`)} ON ${qualifiedName}(code)`
|
|
335
|
+
);
|
|
336
|
+
await enableCDCForTable({ connectionManager, table: tableName });
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* Builds the MSSQLSourceTable that the CDCPoller needs, including the persisted capture-instance binding
|
|
341
|
+
* that would normally be populated by source-table reconciliation.
|
|
342
|
+
*/
|
|
343
|
+
async function resolveSourceTable(
|
|
344
|
+
connectionManager: MSSQLConnectionManager,
|
|
345
|
+
tableName: string
|
|
346
|
+
): Promise<MSSQLSourceTable> {
|
|
347
|
+
const captureInstances = await getCaptureInstances({
|
|
348
|
+
connectionManager,
|
|
349
|
+
table: { schema: connectionManager.schema, name: tableName }
|
|
350
|
+
});
|
|
351
|
+
const details = [...captureInstances.values()][0];
|
|
352
|
+
if (details == null) {
|
|
353
|
+
throw new Error(`No CDC capture instance found for table ${tableName}`);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
const replicaIdColumnsResult = await getReplicationIdentityColumns({
|
|
357
|
+
connectionManager,
|
|
358
|
+
tableName,
|
|
359
|
+
schema: connectionManager.schema
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
const ref = {
|
|
363
|
+
connectionTag: connectionManager.connectionTag,
|
|
364
|
+
objectId: details.sourceTable.objectId,
|
|
365
|
+
schema: details.sourceTable.schema,
|
|
366
|
+
name: details.sourceTable.name,
|
|
367
|
+
replicaIdColumns: replicaIdColumnsResult.columns
|
|
368
|
+
};
|
|
369
|
+
const sourceTable = new SourceTable({
|
|
370
|
+
id: `${details.sourceTable.objectId}`,
|
|
371
|
+
ref,
|
|
372
|
+
objectId: details.sourceTable.objectId,
|
|
373
|
+
replicaIdColumns: replicaIdColumnsResult.columns,
|
|
374
|
+
snapshotComplete: true,
|
|
375
|
+
bucketDataSources: [],
|
|
376
|
+
parameterLookupSources: [],
|
|
377
|
+
sourceMetadata: { captureTableObjectId: details.instances[0].objectId }
|
|
378
|
+
});
|
|
379
|
+
const table = new MSSQLSourceTable(ref, [sourceTable]);
|
|
380
|
+
table.setCaptureInstance(details.instances);
|
|
381
|
+
return table;
|
|
382
|
+
}
|
|
@@ -110,14 +110,16 @@ function defineCDCStreamTests(config: storage.TestStorageConfig) {
|
|
|
110
110
|
expect(data).toMatchObject([putOp('test_data', testData), removeOp('test_data', testData.id)]);
|
|
111
111
|
});
|
|
112
112
|
|
|
113
|
-
test('Replicate
|
|
113
|
+
test('Replicate multiple tables in sync rules', async () => {
|
|
114
114
|
await using context = await CDCStreamTestContext.open(factory);
|
|
115
115
|
const { connectionManager } = context;
|
|
116
|
+
// Table wildcards are not supported - each table is listed explicitly.
|
|
116
117
|
await context.updateSyncRules(`
|
|
117
118
|
bucket_definitions:
|
|
118
119
|
global:
|
|
119
120
|
data:
|
|
120
|
-
- SELECT id, description FROM "
|
|
121
|
+
- SELECT id, description FROM "test_data_1"
|
|
122
|
+
- SELECT id, description FROM "test_data_2"`);
|
|
121
123
|
|
|
122
124
|
await createTestTable(connectionManager, 'test_data_1');
|
|
123
125
|
await createTestTable(connectionManager, 'test_data_2');
|
|
@@ -148,6 +150,8 @@ function defineCDCStreamTests(config: storage.TestStorageConfig) {
|
|
|
148
150
|
const { connectionManager } = context;
|
|
149
151
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
150
152
|
|
|
153
|
+
// test_data is in the sync config and must exist for replication to start; test_donotsync is not.
|
|
154
|
+
await createTestTable(connectionManager, 'test_data');
|
|
151
155
|
await createTestTable(connectionManager, 'test_donotsync');
|
|
152
156
|
|
|
153
157
|
await context.replicateSnapshot();
|
|
@@ -74,6 +74,13 @@ export class CDCStreamTestContext implements AsyncDisposable {
|
|
|
74
74
|
return this.connectionManager.connectionTag;
|
|
75
75
|
}
|
|
76
76
|
|
|
77
|
+
/**
|
|
78
|
+
* Resolves when streaming stops, or rejects with the failure that stopped it.
|
|
79
|
+
*/
|
|
80
|
+
get streamingPromise() {
|
|
81
|
+
return this.streamPromise;
|
|
82
|
+
}
|
|
83
|
+
|
|
77
84
|
async updateSyncRules(content: string) {
|
|
78
85
|
const replicationStream = await this.factory.updateSyncRules(
|
|
79
86
|
updateSyncRulesFromYaml(content, { validate: true, storageVersion: LEGACY_STORAGE_VERSION })
|
|
@@ -203,7 +210,7 @@ export class CDCStreamTestContext implements AsyncDisposable {
|
|
|
203
210
|
while (true) {
|
|
204
211
|
const batch = this.storage!.getBucketDataBatch(checkpoint, map);
|
|
205
212
|
|
|
206
|
-
const batches = await test_utils.
|
|
213
|
+
const batches = await test_utils.getBatchArray(batch);
|
|
207
214
|
data = data.concat(batches[0]?.chunkData.data ?? []);
|
|
208
215
|
if (batches.length == 0 || !batches[0]!.chunkData.has_more) {
|
|
209
216
|
break;
|
|
@@ -230,7 +237,7 @@ export class CDCStreamTestContext implements AsyncDisposable {
|
|
|
230
237
|
const syncConfigContent = this.getSyncConfigContent();
|
|
231
238
|
const map = [bucketRequest(syncConfigContent, bucket, start)];
|
|
232
239
|
const batch = this.storage!.getBucketDataBatch(checkpoint, map);
|
|
233
|
-
const batches = await test_utils.
|
|
240
|
+
const batches = await test_utils.getBatchArray(batch);
|
|
234
241
|
return batches[0]?.chunkData.data ?? [];
|
|
235
242
|
}
|
|
236
243
|
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { CaptureInstance } from '@module/common/CaptureInstance.js';
|
|
2
|
+
import { createCaptureReconciler, readCaptureMetadata } from '@module/replication/CaptureReconciler.js';
|
|
3
|
+
import type { MSSQLTableReconciliationContext } from '@module/replication/MSSQLTableReconciliationContext.js';
|
|
4
|
+
import { MSSQLTableReconciliationState } from '@module/replication/MSSQLTableReconciliationContext.js';
|
|
5
|
+
import { SourceEntityDescriptor, SourceTable } from '@powersync/service-core';
|
|
6
|
+
import { describe, expect, it } from 'vitest';
|
|
7
|
+
import { createCaptureInstance, createSourceDescriptor, createSourceTableCandidate } from './util.js';
|
|
8
|
+
|
|
9
|
+
function reconcile(context: MSSQLTableReconciliationContext, candidates: SourceTable[]) {
|
|
10
|
+
return createCaptureReconciler(context)({ source: context.source, candidates });
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
describe('readCaptureMetadata', () => {
|
|
14
|
+
it('parses a valid metadata object', () => {
|
|
15
|
+
expect(readCaptureMetadata({ captureTableObjectId: 7 })).toEqual({ captureTableObjectId: 7 });
|
|
16
|
+
});
|
|
17
|
+
|
|
18
|
+
it('returns null for legacy metadata', () => {
|
|
19
|
+
expect(readCaptureMetadata(null)).toBeNull();
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
it('rejects malformed metadata', () => {
|
|
23
|
+
expect(() => readCaptureMetadata([1, 2] as any)).toThrow();
|
|
24
|
+
expect(() => readCaptureMetadata({ foo: 1 } as any)).toThrow();
|
|
25
|
+
});
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
describe('createCaptureReconciler', () => {
|
|
29
|
+
it('pins a new binding to the newest available capture instance', () => {
|
|
30
|
+
const resolution = reconcile(readyContext([createCaptureInstance(50), createCaptureInstance(40)]), []);
|
|
31
|
+
expect(resolution.compatibleTables).toEqual([]);
|
|
32
|
+
expect(resolution.incompatibleTables).toEqual([]);
|
|
33
|
+
expect(resolution.newTableValues).toEqual({ sourceMetadata: { captureTableObjectId: 50 } });
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it('reports a missing table with no persisted binding as not ready', () => {
|
|
37
|
+
expect(() =>
|
|
38
|
+
reconcile(
|
|
39
|
+
unavailableContext(
|
|
40
|
+
MSSQLTableReconciliationState.TABLE_MISSING,
|
|
41
|
+
createSourceDescriptor({ objectId: undefined, replicaIdColumns: [] })
|
|
42
|
+
),
|
|
43
|
+
[]
|
|
44
|
+
)
|
|
45
|
+
).toThrow(/does not exist/);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it('reports a missing table with a persisted binding as unavailable', () => {
|
|
49
|
+
expect(() =>
|
|
50
|
+
reconcile(
|
|
51
|
+
unavailableContext(
|
|
52
|
+
MSSQLTableReconciliationState.TABLE_MISSING,
|
|
53
|
+
createSourceDescriptor({ objectId: undefined, replicaIdColumns: [] })
|
|
54
|
+
),
|
|
55
|
+
[createSourceTableCandidate('old', { captureTableObjectId: 40 })]
|
|
56
|
+
)
|
|
57
|
+
).toThrow(/no longer matches the source table binding/);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it('fails a new binding when CDC is disabled', () => {
|
|
61
|
+
expect(() => reconcile(unavailableContext(MSSQLTableReconciliationState.CDC_DISABLED), [])).toThrow(
|
|
62
|
+
/CDC is not enabled/
|
|
63
|
+
);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('updates legacy metadata-free candidates to the newest capture instance', () => {
|
|
67
|
+
const resolution = reconcile(readyContext([createCaptureInstance(50)]), [
|
|
68
|
+
createSourceTableCandidate('a'),
|
|
69
|
+
createSourceTableCandidate('b')
|
|
70
|
+
]);
|
|
71
|
+
expect(
|
|
72
|
+
resolution.compatibleTables.map((table) => ({ id: table.id, sourceMetadata: table.sourceMetadata }))
|
|
73
|
+
).toEqual([
|
|
74
|
+
{ id: 'a', sourceMetadata: { captureTableObjectId: 50 } },
|
|
75
|
+
{ id: 'b', sourceMetadata: { captureTableObjectId: 50 } }
|
|
76
|
+
]);
|
|
77
|
+
expect(resolution.incompatibleTables).toEqual([]);
|
|
78
|
+
expect(resolution.newTableValues).toEqual({ sourceMetadata: { captureTableObjectId: 50 } });
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
it('fails an existing binding when CDC is disabled', () => {
|
|
82
|
+
expect(() =>
|
|
83
|
+
reconcile(unavailableContext(MSSQLTableReconciliationState.CDC_DISABLED), [createSourceTableCandidate('a')])
|
|
84
|
+
).toThrow(/CDC is no longer enabled/);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
it('reports a changed source identity as unavailable when CDC is disabled', () => {
|
|
88
|
+
const changedSource = createSourceDescriptor({ objectId: 200 });
|
|
89
|
+
expect(() =>
|
|
90
|
+
reconcile(unavailableContext(MSSQLTableReconciliationState.CDC_DISABLED, changedSource), [
|
|
91
|
+
createSourceTableCandidate('old')
|
|
92
|
+
])
|
|
93
|
+
).toThrow(/no longer matches the source table binding/);
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it('preserves a pinned capture identity that is still available', () => {
|
|
97
|
+
const resolution = reconcile(readyContext([createCaptureInstance(50), createCaptureInstance(40)]), [
|
|
98
|
+
createSourceTableCandidate('a', { captureTableObjectId: 40 })
|
|
99
|
+
]);
|
|
100
|
+
expect(resolution.compatibleTables.map((table) => table.id)).toEqual(['a']);
|
|
101
|
+
expect(resolution.compatibleTables[0].sourceMetadata).toEqual({ captureTableObjectId: 40 });
|
|
102
|
+
expect(resolution.incompatibleTables).toEqual([]);
|
|
103
|
+
expect(resolution.newTableValues).toEqual({ sourceMetadata: { captureTableObjectId: 40 } });
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
it('fails when the pinned capture instance was dropped, even with a replacement available', () => {
|
|
107
|
+
expect(() =>
|
|
108
|
+
reconcile(readyContext([createCaptureInstance(50)]), [
|
|
109
|
+
createSourceTableCandidate('a', { captureTableObjectId: 40 })
|
|
110
|
+
])
|
|
111
|
+
).toThrow(/no longer available/);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it('fails on a mixture of metadata-free and pinned candidates', () => {
|
|
115
|
+
expect(() =>
|
|
116
|
+
reconcile(readyContext([createCaptureInstance(40)]), [
|
|
117
|
+
createSourceTableCandidate('a'),
|
|
118
|
+
createSourceTableCandidate('b', { captureTableObjectId: 40 })
|
|
119
|
+
])
|
|
120
|
+
).toThrow(/mixture/);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it('fails on multiple distinct pinned identities', () => {
|
|
124
|
+
expect(() =>
|
|
125
|
+
reconcile(readyContext([createCaptureInstance(40), createCaptureInstance(41)]), [
|
|
126
|
+
createSourceTableCandidate('a', { captureTableObjectId: 40 }),
|
|
127
|
+
createSourceTableCandidate('b', { captureTableObjectId: 41 })
|
|
128
|
+
])
|
|
129
|
+
).toThrow(/multiple persisted capture identities/);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
it('does not replace an existing binding when the source identity changed', () => {
|
|
133
|
+
const changedSource = createSourceDescriptor({ objectId: 200 });
|
|
134
|
+
expect(() =>
|
|
135
|
+
reconcile(readyContext([createCaptureInstance(50)], changedSource), [
|
|
136
|
+
createSourceTableCandidate('old', { captureTableObjectId: 40 })
|
|
137
|
+
])
|
|
138
|
+
).toThrow(/Table \[dbo]\.\[users] no longer matches the source table binding.*already-replicated data is retained/);
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
it('drops stale incompatible candidates when a compatible candidate anchors the binding', () => {
|
|
142
|
+
const resolution = reconcile(readyContext([createCaptureInstance(50)]), [
|
|
143
|
+
createSourceTableCandidate('a'),
|
|
144
|
+
createSourceTableCandidate('mismatch', undefined, {
|
|
145
|
+
replicaIdColumns: [{ name: 'id', type: 'bigint', typeId: 127 }]
|
|
146
|
+
})
|
|
147
|
+
]);
|
|
148
|
+
expect(
|
|
149
|
+
resolution.compatibleTables.map((table) => ({ id: table.id, sourceMetadata: table.sourceMetadata }))
|
|
150
|
+
).toEqual([{ id: 'a', sourceMetadata: { captureTableObjectId: 50 } }]);
|
|
151
|
+
expect(resolution.incompatibleTables.map((table) => table.id)).toEqual(['mismatch']);
|
|
152
|
+
});
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
function readyContext(
|
|
156
|
+
captureInstances: CaptureInstance[],
|
|
157
|
+
sourceDescriptor: SourceEntityDescriptor = createSourceDescriptor()
|
|
158
|
+
): MSSQLTableReconciliationContext {
|
|
159
|
+
return {
|
|
160
|
+
state: MSSQLTableReconciliationState.READY,
|
|
161
|
+
source: sourceDescriptor,
|
|
162
|
+
captureInstances
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function unavailableContext(
|
|
167
|
+
state: MSSQLTableReconciliationState.TABLE_MISSING | MSSQLTableReconciliationState.CDC_DISABLED,
|
|
168
|
+
sourceDescriptor: SourceEntityDescriptor = createSourceDescriptor()
|
|
169
|
+
): MSSQLTableReconciliationContext {
|
|
170
|
+
return { state, source: sourceDescriptor };
|
|
171
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { MSSQLSourceTable } from '@module/common/MSSQLSourceTable.js';
|
|
2
|
+
import { describe, expect, it } from 'vitest';
|
|
3
|
+
import { createCaptureInstance, createSourceDescriptor, createSourceTableCandidate } from './util.js';
|
|
4
|
+
|
|
5
|
+
describe('MSSQLSourceTable.setCaptureInstance', () => {
|
|
6
|
+
it('sets the instance matching the persisted capture-table object id', () => {
|
|
7
|
+
const table = new MSSQLSourceTable(createSourceDescriptor(), [
|
|
8
|
+
createSourceTableCandidate('a', { captureTableObjectId: 40 })
|
|
9
|
+
]);
|
|
10
|
+
const expected = createCaptureInstance(40);
|
|
11
|
+
|
|
12
|
+
table.setCaptureInstance([createCaptureInstance(50), expected]);
|
|
13
|
+
|
|
14
|
+
expect(table.captureInstance).toBe(expected);
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
it('sets null when the binding is legacy or the pinned instance is unavailable', () => {
|
|
18
|
+
const legacy = new MSSQLSourceTable(createSourceDescriptor(), [createSourceTableCandidate('legacy')]);
|
|
19
|
+
const pinned = new MSSQLSourceTable(createSourceDescriptor(), [
|
|
20
|
+
createSourceTableCandidate('pinned', { captureTableObjectId: 40 })
|
|
21
|
+
]);
|
|
22
|
+
|
|
23
|
+
legacy.setCaptureInstance([createCaptureInstance(40)]);
|
|
24
|
+
pinned.setCaptureInstance([createCaptureInstance(40)]);
|
|
25
|
+
pinned.setCaptureInstance([createCaptureInstance(50)]);
|
|
26
|
+
|
|
27
|
+
expect(legacy.captureInstance).toBeNull();
|
|
28
|
+
expect(pinned.captureInstance).toBeNull();
|
|
29
|
+
});
|
|
30
|
+
});
|