@powersync/service-module-postgres 0.0.0-dev-20260203155513 → 0.0.0-dev-20260223080959
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -9
- package/dist/replication/replication-utils.d.ts +2 -2
- package/package.json +11 -11
- package/src/replication/replication-utils.ts +2 -2
- package/test/src/checkpoints.test.ts +4 -4
- package/test/src/chunked_snapshots.test.ts +8 -4
- package/test/src/large_batch.test.ts +16 -22
- package/test/src/resuming_snapshots.test.ts +14 -7
- package/test/src/route_api_adapter.test.ts +3 -1
- package/test/src/schema_changes.test.ts +87 -37
- package/test/src/slow_tests.test.ts +12 -6
- package/test/src/util.ts +35 -9
- package/test/src/validation.test.ts +4 -3
- package/test/src/wal_stream.test.ts +26 -24
- package/test/src/wal_stream_utils.ts +56 -10
- package/test/tsconfig.json +3 -7
- package/tsconfig.tsbuildinfo +1 -1
- package/test/src/__snapshots__/schema_changes.test.ts.snap +0 -5
|
@@ -2,8 +2,7 @@ import { compareIds, putOp, reduceBucket, removeOp, test_utils } from '@powersyn
|
|
|
2
2
|
import * as timers from 'timers/promises';
|
|
3
3
|
import { describe, expect, test } from 'vitest';
|
|
4
4
|
|
|
5
|
-
import {
|
|
6
|
-
import { describeWithStorage } from './util.js';
|
|
5
|
+
import { describeWithStorage, StorageVersionTestContext } from './util.js';
|
|
7
6
|
import { WalStreamTestContext } from './wal_stream_utils.js';
|
|
8
7
|
|
|
9
8
|
describe('schema changes', { timeout: 20_000 }, function () {
|
|
@@ -24,9 +23,12 @@ const PUT_T3 = test_utils.putOp('test_data', { id: 't3', description: 'test3' })
|
|
|
24
23
|
const REMOVE_T1 = test_utils.removeOp('test_data', 't1');
|
|
25
24
|
const REMOVE_T2 = test_utils.removeOp('test_data', 't2');
|
|
26
25
|
|
|
27
|
-
function defineTests(factory:
|
|
26
|
+
function defineTests({ factory, storageVersion }: StorageVersionTestContext) {
|
|
27
|
+
const openContext = (options?: Parameters<typeof WalStreamTestContext.open>[1]) => {
|
|
28
|
+
return WalStreamTestContext.open(factory, { ...options, storageVersion });
|
|
29
|
+
};
|
|
28
30
|
test('re-create table', async () => {
|
|
29
|
-
await using context = await
|
|
31
|
+
await using context = await openContext();
|
|
30
32
|
|
|
31
33
|
// Drop a table and re-create it.
|
|
32
34
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
@@ -49,6 +51,11 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
49
51
|
|
|
50
52
|
const data = await context.getBucketData('global[]');
|
|
51
53
|
|
|
54
|
+
// "Reduce" the bucket to get a stable output to test.
|
|
55
|
+
// slice(1) to skip the CLEAR op.
|
|
56
|
+
const reduced = reduceBucket(data).slice(1);
|
|
57
|
+
expect(reduced.sort(compareIds)).toMatchObject([PUT_T3]);
|
|
58
|
+
|
|
52
59
|
// Initial inserts
|
|
53
60
|
expect(data.slice(0, 2)).toMatchObject([PUT_T1, PUT_T2]);
|
|
54
61
|
|
|
@@ -65,7 +72,7 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
65
72
|
});
|
|
66
73
|
|
|
67
74
|
test('add table', async () => {
|
|
68
|
-
await using context = await
|
|
75
|
+
await using context = await openContext();
|
|
69
76
|
// Add table after initial replication
|
|
70
77
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
71
78
|
const { pool } = context;
|
|
@@ -78,6 +85,11 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
78
85
|
|
|
79
86
|
const data = await context.getBucketData('global[]');
|
|
80
87
|
|
|
88
|
+
// "Reduce" the bucket to get a stable output to test.
|
|
89
|
+
// slice(1) to skip the CLEAR op.
|
|
90
|
+
const reduced = reduceBucket(data).slice(1);
|
|
91
|
+
expect(reduced.sort(compareIds)).toMatchObject([PUT_T1]);
|
|
92
|
+
|
|
81
93
|
expect(data).toMatchObject([
|
|
82
94
|
// Snapshot insert
|
|
83
95
|
PUT_T1,
|
|
@@ -88,7 +100,7 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
88
100
|
});
|
|
89
101
|
|
|
90
102
|
test('rename table (1)', async () => {
|
|
91
|
-
await using context = await
|
|
103
|
+
await using context = await openContext();
|
|
92
104
|
const { pool } = context;
|
|
93
105
|
|
|
94
106
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
@@ -108,6 +120,11 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
108
120
|
|
|
109
121
|
const data = await context.getBucketData('global[]');
|
|
110
122
|
|
|
123
|
+
// "Reduce" the bucket to get a stable output to test.
|
|
124
|
+
// slice(1) to skip the CLEAR op.
|
|
125
|
+
const reduced = reduceBucket(data).slice(1);
|
|
126
|
+
expect(reduced.sort(compareIds)).toMatchObject([PUT_T1, PUT_T2]);
|
|
127
|
+
|
|
111
128
|
expect(data.slice(0, 2).sort(compareIds)).toMatchObject([
|
|
112
129
|
// Snapshot insert
|
|
113
130
|
PUT_T1,
|
|
@@ -121,7 +138,7 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
121
138
|
});
|
|
122
139
|
|
|
123
140
|
test('rename table (2)', async () => {
|
|
124
|
-
await using context = await
|
|
141
|
+
await using context = await openContext();
|
|
125
142
|
// Rename table in sync rules -> in sync rules
|
|
126
143
|
const { pool } = context;
|
|
127
144
|
|
|
@@ -146,6 +163,14 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
146
163
|
|
|
147
164
|
const data = await context.getBucketData('global[]');
|
|
148
165
|
|
|
166
|
+
// "Reduce" the bucket to get a stable output to test.
|
|
167
|
+
// slice(1) to skip the CLEAR op.
|
|
168
|
+
const reduced = reduceBucket(data).slice(1);
|
|
169
|
+
expect(reduced.sort(compareIds)).toMatchObject([
|
|
170
|
+
putOp('test_data2', { id: 't1', description: 'test1' }),
|
|
171
|
+
putOp('test_data2', { id: 't2', description: 'test2' })
|
|
172
|
+
]);
|
|
173
|
+
|
|
149
174
|
expect(data.slice(0, 2)).toMatchObject([
|
|
150
175
|
// Initial replication
|
|
151
176
|
putOp('test_data1', { id: 't1', description: 'test1' }),
|
|
@@ -166,7 +191,7 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
166
191
|
});
|
|
167
192
|
|
|
168
193
|
test('rename table (3)', async () => {
|
|
169
|
-
await using context = await
|
|
194
|
+
await using context = await openContext();
|
|
170
195
|
// Rename table in sync rules -> not in sync rules
|
|
171
196
|
|
|
172
197
|
const { pool } = context;
|
|
@@ -193,10 +218,15 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
193
218
|
// Truncate
|
|
194
219
|
REMOVE_T1
|
|
195
220
|
]);
|
|
221
|
+
|
|
222
|
+
// "Reduce" the bucket to get a stable output to test.
|
|
223
|
+
// slice(1) to skip the CLEAR op.
|
|
224
|
+
const reduced = reduceBucket(data).slice(1);
|
|
225
|
+
expect(reduced.sort(compareIds)).toMatchObject([]);
|
|
196
226
|
});
|
|
197
227
|
|
|
198
228
|
test('change replica id', async () => {
|
|
199
|
-
await using context = await
|
|
229
|
+
await using context = await openContext();
|
|
200
230
|
// Change replica id from default to full
|
|
201
231
|
// Causes a re-import of the table.
|
|
202
232
|
|
|
@@ -217,6 +247,11 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
217
247
|
|
|
218
248
|
const data = await context.getBucketData('global[]');
|
|
219
249
|
|
|
250
|
+
// "Reduce" the bucket to get a stable output to test.
|
|
251
|
+
// slice(1) to skip the CLEAR op.
|
|
252
|
+
const reduced = reduceBucket(data).slice(1);
|
|
253
|
+
expect(reduced.sort(compareIds)).toMatchObject([PUT_T1, PUT_T2]);
|
|
254
|
+
|
|
220
255
|
expect(data.slice(0, 2)).toMatchObject([
|
|
221
256
|
// Initial inserts
|
|
222
257
|
PUT_T1,
|
|
@@ -235,7 +270,7 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
235
270
|
});
|
|
236
271
|
|
|
237
272
|
test('change full replica id by adding column', async () => {
|
|
238
|
-
await using context = await
|
|
273
|
+
await using context = await openContext();
|
|
239
274
|
// Change replica id from full by adding column
|
|
240
275
|
// Causes a re-import of the table.
|
|
241
276
|
// Other changes such as renaming column would have the same effect
|
|
@@ -278,7 +313,7 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
278
313
|
});
|
|
279
314
|
|
|
280
315
|
test('change default replica id by changing column type', async () => {
|
|
281
|
-
await using context = await
|
|
316
|
+
await using context = await openContext();
|
|
282
317
|
// Change default replica id by changing column type
|
|
283
318
|
// Causes a re-import of the table.
|
|
284
319
|
const { pool } = context;
|
|
@@ -315,7 +350,7 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
315
350
|
});
|
|
316
351
|
|
|
317
352
|
test('change index id by changing column type', async () => {
|
|
318
|
-
await using context = await
|
|
353
|
+
await using context = await openContext();
|
|
319
354
|
// Change index replica id by changing column type
|
|
320
355
|
// Causes a re-import of the table.
|
|
321
356
|
// Secondary functionality tested here is that replica id column order stays
|
|
@@ -348,24 +383,30 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
348
383
|
PUT_T2
|
|
349
384
|
]);
|
|
350
385
|
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
//
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
]);
|
|
386
|
+
// "Reduce" the bucket to get a stable output to test.
|
|
387
|
+
// slice(1) to skip the CLEAR op.
|
|
388
|
+
const reduced = reduceBucket(data).slice(1);
|
|
389
|
+
expect(reduced.sort(compareIds)).toMatchObject([PUT_T1, PUT_T2, PUT_T3]);
|
|
390
|
+
|
|
391
|
+
// Previously had more specific tests, but this varies too much based on timing:
|
|
392
|
+
// expect(data.slice(2, 4).sort(compareIds)).toMatchObject([
|
|
393
|
+
// // Truncate - any order
|
|
394
|
+
// REMOVE_T1,
|
|
395
|
+
// REMOVE_T2
|
|
396
|
+
// ]);
|
|
397
|
+
|
|
398
|
+
// // Snapshot - order doesn't matter
|
|
399
|
+
// expect(data.slice(4, 7).sort(compareIds)).toMatchObject([PUT_T1, PUT_T2, PUT_T3]);
|
|
400
|
+
|
|
401
|
+
// expect(data.slice(7).sort(compareIds)).toMatchObject([
|
|
402
|
+
// // Replicated insert
|
|
403
|
+
// // We may eventually be able to de-duplicate this
|
|
404
|
+
// PUT_T3
|
|
405
|
+
// ]);
|
|
365
406
|
});
|
|
366
407
|
|
|
367
408
|
test('add to publication', async () => {
|
|
368
|
-
await using context = await
|
|
409
|
+
await using context = await openContext();
|
|
369
410
|
// Add table to publication after initial replication
|
|
370
411
|
const { pool } = context;
|
|
371
412
|
|
|
@@ -401,12 +442,14 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
401
442
|
PUT_T3
|
|
402
443
|
]);
|
|
403
444
|
|
|
404
|
-
|
|
405
|
-
|
|
445
|
+
// "Reduce" the bucket to get a stable output to test.
|
|
446
|
+
// slice(1) to skip the CLEAR op.
|
|
447
|
+
const reduced = reduceBucket(data).slice(1);
|
|
448
|
+
expect(reduced.sort(compareIds)).toMatchObject([PUT_T1, PUT_T2, PUT_T3]);
|
|
406
449
|
});
|
|
407
450
|
|
|
408
451
|
test('add to publication (not in sync rules)', async () => {
|
|
409
|
-
await using context = await
|
|
452
|
+
await using context = await openContext();
|
|
410
453
|
// Add table to publication after initial replication
|
|
411
454
|
// Since the table is not in sync rules, it should not be replicated.
|
|
412
455
|
const { pool } = context;
|
|
@@ -430,13 +473,10 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
430
473
|
|
|
431
474
|
const data = await context.getBucketData('global[]');
|
|
432
475
|
expect(data).toMatchObject([]);
|
|
433
|
-
|
|
434
|
-
const metrics = await storage.factory.getStorageMetrics();
|
|
435
|
-
expect(metrics.replication_size_bytes).toMatchSnapshot();
|
|
436
476
|
});
|
|
437
477
|
|
|
438
478
|
test('replica identity nothing', async () => {
|
|
439
|
-
await using context = await
|
|
479
|
+
await using context = await openContext();
|
|
440
480
|
// Technically not a schema change, but fits here.
|
|
441
481
|
// Replica ID works a little differently here - the table doesn't have
|
|
442
482
|
// one defined, but we generate a unique one for each replicated row.
|
|
@@ -475,10 +515,15 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
475
515
|
REMOVE_T1,
|
|
476
516
|
REMOVE_T2
|
|
477
517
|
]);
|
|
518
|
+
|
|
519
|
+
// "Reduce" the bucket to get a stable output to test.
|
|
520
|
+
// slice(1) to skip the CLEAR op.
|
|
521
|
+
const reduced = reduceBucket(data).slice(1);
|
|
522
|
+
expect(reduced.sort(compareIds)).toMatchObject([]);
|
|
478
523
|
});
|
|
479
524
|
|
|
480
525
|
test('replica identity default without PK', async () => {
|
|
481
|
-
await using context = await
|
|
526
|
+
await using context = await openContext();
|
|
482
527
|
// Same as no replica identity
|
|
483
528
|
const { pool } = context;
|
|
484
529
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
@@ -513,6 +558,11 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
513
558
|
REMOVE_T1,
|
|
514
559
|
REMOVE_T2
|
|
515
560
|
]);
|
|
561
|
+
|
|
562
|
+
// "Reduce" the bucket to get a stable output to test.
|
|
563
|
+
// slice(1) to skip the CLEAR op.
|
|
564
|
+
const reduced = reduceBucket(data).slice(1);
|
|
565
|
+
expect(reduced.sort(compareIds)).toMatchObject([]);
|
|
516
566
|
});
|
|
517
567
|
|
|
518
568
|
// Test consistency of table snapshots.
|
|
@@ -525,7 +575,7 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
525
575
|
// await new Promise((resolve) => setTimeout(resolve, 100));
|
|
526
576
|
// await this.snapshotTable(batch, db, result.table);
|
|
527
577
|
test('table snapshot consistency', async () => {
|
|
528
|
-
await using context = await
|
|
578
|
+
await using context = await openContext();
|
|
529
579
|
const { pool } = context;
|
|
530
580
|
|
|
531
581
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
@@ -592,7 +642,7 @@ function defineTests(factory: storage.TestStorageFactory) {
|
|
|
592
642
|
});
|
|
593
643
|
|
|
594
644
|
test('custom types', async () => {
|
|
595
|
-
await using context = await
|
|
645
|
+
await using context = await openContext();
|
|
596
646
|
|
|
597
647
|
await context.updateSyncRules(`
|
|
598
648
|
streams:
|
|
@@ -7,6 +7,7 @@ import {
|
|
|
7
7
|
connectPgPool,
|
|
8
8
|
describeWithStorage,
|
|
9
9
|
getClientCheckpoint,
|
|
10
|
+
StorageVersionTestContext,
|
|
10
11
|
TEST_CONNECTION_OPTIONS
|
|
11
12
|
} from './util.js';
|
|
12
13
|
|
|
@@ -14,7 +15,11 @@ import * as pgwire from '@powersync/service-jpgwire';
|
|
|
14
15
|
import { SqliteRow } from '@powersync/service-sync-rules';
|
|
15
16
|
|
|
16
17
|
import { PgManager } from '@module/replication/PgManager.js';
|
|
17
|
-
import {
|
|
18
|
+
import {
|
|
19
|
+
createCoreReplicationMetrics,
|
|
20
|
+
initializeCoreReplicationMetrics,
|
|
21
|
+
updateSyncRulesFromYaml
|
|
22
|
+
} from '@powersync/service-core';
|
|
18
23
|
import { METRICS_HELPER, test_utils } from '@powersync/service-core-tests';
|
|
19
24
|
import * as mongo_storage from '@powersync/service-module-mongodb-storage';
|
|
20
25
|
import * as postgres_storage from '@powersync/service-module-postgres-storage';
|
|
@@ -22,12 +27,12 @@ import * as timers from 'node:timers/promises';
|
|
|
22
27
|
import { CustomTypeRegistry } from '@module/types/registry.js';
|
|
23
28
|
|
|
24
29
|
describe.skipIf(!(env.CI || env.SLOW_TESTS))('slow tests', function () {
|
|
25
|
-
describeWithStorage({ timeout: 120_000 }, function (factory) {
|
|
26
|
-
defineSlowTests(factory);
|
|
30
|
+
describeWithStorage({ timeout: 120_000 }, function ({ factory, storageVersion }) {
|
|
31
|
+
defineSlowTests({ factory, storageVersion });
|
|
27
32
|
});
|
|
28
33
|
});
|
|
29
34
|
|
|
30
|
-
function defineSlowTests(factory:
|
|
35
|
+
function defineSlowTests({ factory, storageVersion }: StorageVersionTestContext) {
|
|
31
36
|
let walStream: WalStream | undefined;
|
|
32
37
|
let connections: PgManager | undefined;
|
|
33
38
|
let abortController: AbortController | undefined;
|
|
@@ -81,7 +86,7 @@ bucket_definitions:
|
|
|
81
86
|
data:
|
|
82
87
|
- SELECT * FROM "test_data"
|
|
83
88
|
`;
|
|
84
|
-
const syncRules = await f.updateSyncRules({
|
|
89
|
+
const syncRules = await f.updateSyncRules(updateSyncRulesFromYaml(syncRuleContent, { storageVersion }));
|
|
85
90
|
const storage = f.getInstance(syncRules);
|
|
86
91
|
abortController = new AbortController();
|
|
87
92
|
const options: WalStreamOptions = {
|
|
@@ -306,7 +311,8 @@ bucket_definitions:
|
|
|
306
311
|
data:
|
|
307
312
|
- SELECT id, description FROM "test_data"
|
|
308
313
|
`;
|
|
309
|
-
|
|
314
|
+
|
|
315
|
+
const syncRules = await f.updateSyncRules(updateSyncRulesFromYaml(syncRuleContent, { storageVersion }));
|
|
310
316
|
const storage = f.getInstance(syncRules);
|
|
311
317
|
|
|
312
318
|
// 1. Setup some base data that will be replicated in initial replication
|
package/test/src/util.ts
CHANGED
|
@@ -2,12 +2,18 @@ import { PostgresRouteAPIAdapter } from '@module/api/PostgresRouteAPIAdapter.js'
|
|
|
2
2
|
import * as types from '@module/types/types.js';
|
|
3
3
|
import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
4
4
|
import { logger } from '@powersync/lib-services-framework';
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
BucketStorageFactory,
|
|
7
|
+
CURRENT_STORAGE_VERSION,
|
|
8
|
+
InternalOpId,
|
|
9
|
+
LEGACY_STORAGE_VERSION,
|
|
10
|
+
TestStorageFactory
|
|
11
|
+
} from '@powersync/service-core';
|
|
6
12
|
import * as pgwire from '@powersync/service-jpgwire';
|
|
7
13
|
import * as mongo_storage from '@powersync/service-module-mongodb-storage';
|
|
8
14
|
import * as postgres_storage from '@powersync/service-module-postgres-storage';
|
|
9
|
-
import { env } from './env.js';
|
|
10
15
|
import { describe, TestOptions } from 'vitest';
|
|
16
|
+
import { env } from './env.js';
|
|
11
17
|
|
|
12
18
|
export const TEST_URI = env.PG_TEST_URL;
|
|
13
19
|
|
|
@@ -20,14 +26,34 @@ export const INITIALIZED_POSTGRES_STORAGE_FACTORY = postgres_storage.test_utils.
|
|
|
20
26
|
url: env.PG_STORAGE_TEST_URL
|
|
21
27
|
});
|
|
22
28
|
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
29
|
+
const TEST_STORAGE_VERSIONS = [LEGACY_STORAGE_VERSION, CURRENT_STORAGE_VERSION];
|
|
30
|
+
|
|
31
|
+
export interface StorageVersionTestContext {
|
|
32
|
+
factory: TestStorageFactory;
|
|
33
|
+
storageVersion: number;
|
|
34
|
+
}
|
|
27
35
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
36
|
+
export function describeWithStorage(options: TestOptions, fn: (context: StorageVersionTestContext) => void) {
|
|
37
|
+
const describeFactory = (storageName: string, factory: TestStorageFactory) => {
|
|
38
|
+
describe(`${storageName} storage`, options, function () {
|
|
39
|
+
for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
40
|
+
describe(`storage v${storageVersion}`, function () {
|
|
41
|
+
fn({
|
|
42
|
+
factory,
|
|
43
|
+
storageVersion
|
|
44
|
+
});
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
});
|
|
48
|
+
};
|
|
49
|
+
|
|
50
|
+
if (env.TEST_MONGO_STORAGE) {
|
|
51
|
+
describeFactory('mongodb', INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
if (env.TEST_POSTGRES_STORAGE) {
|
|
55
|
+
describeFactory('postgres', INITIALIZED_POSTGRES_STORAGE_FACTORY);
|
|
56
|
+
}
|
|
31
57
|
}
|
|
32
58
|
|
|
33
59
|
export const TEST_CONNECTION_OPTIONS = types.normalizeConnectionConfig({
|
|
@@ -3,6 +3,7 @@ import { expect, test } from 'vitest';
|
|
|
3
3
|
|
|
4
4
|
import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
|
|
5
5
|
import { WalStreamTestContext } from './wal_stream_utils.js';
|
|
6
|
+
import { updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
6
7
|
|
|
7
8
|
test('validate tables', async () => {
|
|
8
9
|
await using context = await WalStreamTestContext.open(INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
@@ -19,15 +20,15 @@ bucket_definitions:
|
|
|
19
20
|
- SELECT * FROM "other%"
|
|
20
21
|
`;
|
|
21
22
|
|
|
22
|
-
const syncRules = await context.factory.updateSyncRules(
|
|
23
|
+
const syncRules = await context.factory.updateSyncRules(updateSyncRulesFromYaml(syncRuleContent));
|
|
23
24
|
|
|
24
|
-
const tablePatterns = syncRules.parsed({ defaultSchema: 'public' }).sync_rules.getSourceTables();
|
|
25
|
+
const tablePatterns = syncRules.parsed({ defaultSchema: 'public' }).sync_rules.config.getSourceTables();
|
|
25
26
|
const tableInfo = await getDebugTablesInfo({
|
|
26
27
|
db: pool,
|
|
27
28
|
publicationName: context.publicationName,
|
|
28
29
|
connectionTag: context.connectionTag,
|
|
29
30
|
tablePatterns: tablePatterns,
|
|
30
|
-
syncRules: syncRules.parsed({ defaultSchema: 'public' }).sync_rules
|
|
31
|
+
syncRules: syncRules.parsed({ defaultSchema: 'public' }).sync_rules.config
|
|
31
32
|
});
|
|
32
33
|
expect(tableInfo).toEqual([
|
|
33
34
|
{
|
|
@@ -1,11 +1,10 @@
|
|
|
1
1
|
import { MissingReplicationSlotError } from '@module/replication/WalStream.js';
|
|
2
|
-
import { storage } from '@powersync/service-core';
|
|
3
2
|
import { METRICS_HELPER, putOp, removeOp } from '@powersync/service-core-tests';
|
|
4
3
|
import { pgwireRows } from '@powersync/service-jpgwire';
|
|
5
4
|
import { ReplicationMetric } from '@powersync/service-types';
|
|
6
5
|
import * as crypto from 'crypto';
|
|
7
|
-
import {
|
|
8
|
-
import { describeWithStorage } from './util.js';
|
|
6
|
+
import { describe, expect, test } from 'vitest';
|
|
7
|
+
import { describeWithStorage, StorageVersionTestContext } from './util.js';
|
|
9
8
|
import { WalStreamTestContext, withMaxWalSize } from './wal_stream_utils.js';
|
|
10
9
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
11
10
|
|
|
@@ -20,9 +19,12 @@ describe('wal stream', () => {
|
|
|
20
19
|
describeWithStorage({ timeout: 20_000 }, defineWalStreamTests);
|
|
21
20
|
});
|
|
22
21
|
|
|
23
|
-
function defineWalStreamTests(factory:
|
|
22
|
+
function defineWalStreamTests({ factory, storageVersion }: StorageVersionTestContext) {
|
|
23
|
+
const openContext = (options?: Parameters<typeof WalStreamTestContext.open>[1]) => {
|
|
24
|
+
return WalStreamTestContext.open(factory, { ...options, storageVersion });
|
|
25
|
+
};
|
|
24
26
|
test('replicating basic values', async () => {
|
|
25
|
-
await using context = await
|
|
27
|
+
await using context = await openContext();
|
|
26
28
|
const { pool } = context;
|
|
27
29
|
await context.updateSyncRules(`
|
|
28
30
|
bucket_definitions:
|
|
@@ -57,7 +59,7 @@ bucket_definitions:
|
|
|
57
59
|
});
|
|
58
60
|
|
|
59
61
|
test('replicating case sensitive table', async () => {
|
|
60
|
-
await using context = await
|
|
62
|
+
await using context = await openContext();
|
|
61
63
|
const { pool } = context;
|
|
62
64
|
await context.updateSyncRules(`
|
|
63
65
|
bucket_definitions:
|
|
@@ -88,7 +90,7 @@ bucket_definitions:
|
|
|
88
90
|
});
|
|
89
91
|
|
|
90
92
|
test('replicating TOAST values', async () => {
|
|
91
|
-
await using context = await
|
|
93
|
+
await using context = await openContext();
|
|
92
94
|
const { pool } = context;
|
|
93
95
|
await context.updateSyncRules(`
|
|
94
96
|
bucket_definitions:
|
|
@@ -126,7 +128,7 @@ bucket_definitions:
|
|
|
126
128
|
});
|
|
127
129
|
|
|
128
130
|
test('replicating TRUNCATE', async () => {
|
|
129
|
-
await using context = await
|
|
131
|
+
await using context = await openContext();
|
|
130
132
|
const { pool } = context;
|
|
131
133
|
const syncRuleContent = `
|
|
132
134
|
bucket_definitions:
|
|
@@ -157,7 +159,7 @@ bucket_definitions:
|
|
|
157
159
|
});
|
|
158
160
|
|
|
159
161
|
test('replicating changing primary key', async () => {
|
|
160
|
-
await using context = await
|
|
162
|
+
await using context = await openContext();
|
|
161
163
|
const { pool } = context;
|
|
162
164
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
163
165
|
await pool.query(`DROP TABLE IF EXISTS test_data`);
|
|
@@ -198,7 +200,7 @@ bucket_definitions:
|
|
|
198
200
|
});
|
|
199
201
|
|
|
200
202
|
test('initial sync', async () => {
|
|
201
|
-
await using context = await
|
|
203
|
+
await using context = await openContext();
|
|
202
204
|
const { pool } = context;
|
|
203
205
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
204
206
|
|
|
@@ -217,7 +219,7 @@ bucket_definitions:
|
|
|
217
219
|
});
|
|
218
220
|
|
|
219
221
|
test('record too large', async () => {
|
|
220
|
-
await using context = await
|
|
222
|
+
await using context = await openContext();
|
|
221
223
|
await context.updateSyncRules(`bucket_definitions:
|
|
222
224
|
global:
|
|
223
225
|
data:
|
|
@@ -254,7 +256,7 @@ bucket_definitions:
|
|
|
254
256
|
});
|
|
255
257
|
|
|
256
258
|
test('table not in sync rules', async () => {
|
|
257
|
-
await using context = await
|
|
259
|
+
await using context = await openContext();
|
|
258
260
|
const { pool } = context;
|
|
259
261
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
260
262
|
|
|
@@ -280,7 +282,7 @@ bucket_definitions:
|
|
|
280
282
|
|
|
281
283
|
test('reporting slot issues', async () => {
|
|
282
284
|
{
|
|
283
|
-
await using context = await
|
|
285
|
+
await using context = await openContext();
|
|
284
286
|
const { pool } = context;
|
|
285
287
|
await context.updateSyncRules(`
|
|
286
288
|
bucket_definitions:
|
|
@@ -310,7 +312,7 @@ bucket_definitions:
|
|
|
310
312
|
}
|
|
311
313
|
|
|
312
314
|
{
|
|
313
|
-
await using context = await
|
|
315
|
+
await using context = await openContext({ doNotClear: true });
|
|
314
316
|
const { pool } = context;
|
|
315
317
|
await pool.query('DROP PUBLICATION powersync');
|
|
316
318
|
await pool.query(`UPDATE test_data SET description = 'updated'`);
|
|
@@ -345,7 +347,7 @@ bucket_definitions:
|
|
|
345
347
|
|
|
346
348
|
test('dropped replication slot', async () => {
|
|
347
349
|
{
|
|
348
|
-
await using context = await
|
|
350
|
+
await using context = await openContext();
|
|
349
351
|
const { pool } = context;
|
|
350
352
|
await context.updateSyncRules(`
|
|
351
353
|
bucket_definitions:
|
|
@@ -375,7 +377,7 @@ bucket_definitions:
|
|
|
375
377
|
}
|
|
376
378
|
|
|
377
379
|
{
|
|
378
|
-
await using context = await
|
|
380
|
+
await using context = await openContext({ doNotClear: true });
|
|
379
381
|
const { pool } = context;
|
|
380
382
|
const storage = await context.factory.getActiveStorage();
|
|
381
383
|
|
|
@@ -396,7 +398,7 @@ bucket_definitions:
|
|
|
396
398
|
});
|
|
397
399
|
|
|
398
400
|
test('replication slot lost', async () => {
|
|
399
|
-
await using baseContext = await
|
|
401
|
+
await using baseContext = await openContext({ doNotClear: true });
|
|
400
402
|
|
|
401
403
|
const serverVersion = await baseContext.connectionManager.getServerVersion();
|
|
402
404
|
if (serverVersion!.compareMain('13.0.0') < 0) {
|
|
@@ -408,7 +410,7 @@ bucket_definitions:
|
|
|
408
410
|
await using s = await withMaxWalSize(baseContext.pool, '100MB');
|
|
409
411
|
|
|
410
412
|
{
|
|
411
|
-
await using context = await
|
|
413
|
+
await using context = await openContext();
|
|
412
414
|
const { pool } = context;
|
|
413
415
|
await context.updateSyncRules(`
|
|
414
416
|
bucket_definitions:
|
|
@@ -438,7 +440,7 @@ bucket_definitions:
|
|
|
438
440
|
}
|
|
439
441
|
|
|
440
442
|
{
|
|
441
|
-
await using context = await
|
|
443
|
+
await using context = await openContext({ doNotClear: true });
|
|
442
444
|
const { pool } = context;
|
|
443
445
|
const storage = await context.factory.getActiveStorage();
|
|
444
446
|
const slotName = storage?.slot_name!;
|
|
@@ -479,7 +481,7 @@ bucket_definitions:
|
|
|
479
481
|
});
|
|
480
482
|
|
|
481
483
|
test('old date format', async () => {
|
|
482
|
-
await using context = await
|
|
484
|
+
await using context = await openContext();
|
|
483
485
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
484
486
|
|
|
485
487
|
const { pool } = context;
|
|
@@ -494,7 +496,7 @@ bucket_definitions:
|
|
|
494
496
|
});
|
|
495
497
|
|
|
496
498
|
test('new date format', async () => {
|
|
497
|
-
await using context = await
|
|
499
|
+
await using context = await openContext();
|
|
498
500
|
await context.updateSyncRules(`
|
|
499
501
|
streams:
|
|
500
502
|
stream:
|
|
@@ -515,7 +517,7 @@ config:
|
|
|
515
517
|
});
|
|
516
518
|
|
|
517
519
|
test('custom types', async () => {
|
|
518
|
-
await using context = await
|
|
520
|
+
await using context = await openContext();
|
|
519
521
|
|
|
520
522
|
await context.updateSyncRules(`
|
|
521
523
|
streams:
|
|
@@ -550,7 +552,7 @@ config:
|
|
|
550
552
|
});
|
|
551
553
|
|
|
552
554
|
test('custom types in primary key', async () => {
|
|
553
|
-
await using context = await
|
|
555
|
+
await using context = await openContext();
|
|
554
556
|
|
|
555
557
|
await context.updateSyncRules(`
|
|
556
558
|
streams:
|
|
@@ -576,7 +578,7 @@ config:
|
|
|
576
578
|
test('replica identity handling', async () => {
|
|
577
579
|
// This specifically test a case of timestamps being used as part of the replica identity.
|
|
578
580
|
// There was a regression in versions 1.15.0-1.15.5, which this tests for.
|
|
579
|
-
await using context = await
|
|
581
|
+
await using context = await openContext();
|
|
580
582
|
const { pool } = context;
|
|
581
583
|
await context.updateSyncRules(BASIC_SYNC_RULES);
|
|
582
584
|
|