@powersync/service-module-postgres-storage 0.12.0 → 0.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +45 -0
  2. package/dist/.tsbuildinfo +1 -1
  3. package/dist/@types/migrations/scripts/1771424826685-current-data-pending-deletes.d.ts +3 -0
  4. package/dist/@types/storage/PostgresBucketStorageFactory.d.ts +4 -0
  5. package/dist/@types/storage/PostgresCompactor.d.ts +8 -2
  6. package/dist/@types/storage/PostgresSyncRulesStorage.d.ts +10 -4
  7. package/dist/@types/storage/batch/OperationBatch.d.ts +2 -2
  8. package/dist/@types/storage/batch/PostgresBucketBatch.d.ts +13 -9
  9. package/dist/@types/storage/batch/PostgresPersistedBatch.d.ts +17 -5
  10. package/dist/@types/storage/current-data-store.d.ts +85 -0
  11. package/dist/@types/storage/current-data-table.d.ts +9 -0
  12. package/dist/@types/storage/table-id.d.ts +2 -0
  13. package/dist/@types/types/models/CurrentData.d.ts +18 -3
  14. package/dist/@types/utils/bson.d.ts +1 -1
  15. package/dist/@types/utils/test-utils.d.ts +1 -1
  16. package/dist/migrations/scripts/1771424826685-current-data-pending-deletes.js +8 -0
  17. package/dist/migrations/scripts/1771424826685-current-data-pending-deletes.js.map +1 -0
  18. package/dist/storage/PostgresBucketStorageFactory.js +41 -4
  19. package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
  20. package/dist/storage/PostgresCompactor.js +14 -6
  21. package/dist/storage/PostgresCompactor.js.map +1 -1
  22. package/dist/storage/PostgresSyncRulesStorage.js +98 -24
  23. package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
  24. package/dist/storage/batch/OperationBatch.js +2 -1
  25. package/dist/storage/batch/OperationBatch.js.map +1 -1
  26. package/dist/storage/batch/PostgresBucketBatch.js +295 -213
  27. package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
  28. package/dist/storage/batch/PostgresPersistedBatch.js +86 -81
  29. package/dist/storage/batch/PostgresPersistedBatch.js.map +1 -1
  30. package/dist/storage/current-data-store.js +270 -0
  31. package/dist/storage/current-data-store.js.map +1 -0
  32. package/dist/storage/current-data-table.js +22 -0
  33. package/dist/storage/current-data-table.js.map +1 -0
  34. package/dist/storage/table-id.js +8 -0
  35. package/dist/storage/table-id.js.map +1 -0
  36. package/dist/types/models/CurrentData.js +11 -2
  37. package/dist/types/models/CurrentData.js.map +1 -1
  38. package/dist/utils/bson.js.map +1 -1
  39. package/dist/utils/db.js +9 -0
  40. package/dist/utils/db.js.map +1 -1
  41. package/dist/utils/test-utils.js +13 -6
  42. package/dist/utils/test-utils.js.map +1 -1
  43. package/package.json +8 -8
  44. package/src/migrations/scripts/1771424826685-current-data-pending-deletes.ts +10 -0
  45. package/src/storage/PostgresBucketStorageFactory.ts +53 -5
  46. package/src/storage/PostgresCompactor.ts +17 -8
  47. package/src/storage/PostgresSyncRulesStorage.ts +47 -31
  48. package/src/storage/batch/OperationBatch.ts +4 -3
  49. package/src/storage/batch/PostgresBucketBatch.ts +316 -238
  50. package/src/storage/batch/PostgresPersistedBatch.ts +92 -84
  51. package/src/storage/current-data-store.ts +326 -0
  52. package/src/storage/current-data-table.ts +26 -0
  53. package/src/storage/table-id.ts +9 -0
  54. package/src/types/models/CurrentData.ts +17 -4
  55. package/src/utils/bson.ts +1 -1
  56. package/src/utils/db.ts +10 -0
  57. package/src/utils/test-utils.ts +14 -7
  58. package/test/src/__snapshots__/storage.test.ts.snap +151 -0
  59. package/test/src/__snapshots__/storage_compacting.test.ts.snap +17 -0
  60. package/test/src/__snapshots__/storage_sync.test.ts.snap +1111 -16
  61. package/test/src/env.ts +1 -1
  62. package/test/src/migrations.test.ts +1 -1
  63. package/test/src/storage.test.ts +138 -131
  64. package/test/src/storage_compacting.test.ts +80 -11
  65. package/test/src/storage_sync.test.ts +57 -54
  66. package/test/src/util.ts +4 -4
package/test/src/env.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { utils } from '@powersync/lib-services-framework';
2
2
 
3
3
  export const env = utils.collectEnvironmentVariables({
4
- PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5431/powersync_storage_test'),
4
+ PG_STORAGE_TEST_URL: utils.type.string.default('postgres://postgres:postgres@localhost:5432/powersync_storage_test'),
5
5
  CI: utils.type.boolean.default('false')
6
6
  });
@@ -28,7 +28,7 @@ describe('Migrations', () => {
28
28
  register.registerMigrationTests(MIGRATION_AGENT_FACTORY);
29
29
 
30
30
  it('Should have tables declared', async () => {
31
- const { db } = await POSTGRES_STORAGE_FACTORY();
31
+ const { db } = await POSTGRES_STORAGE_FACTORY.factory();
32
32
 
33
33
  const tables = await db.sql`
34
34
  SELECT
@@ -1,149 +1,156 @@
1
1
  import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
2
- import { bucketRequestMap, register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
2
+ import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
3
3
  import { describe, expect, test } from 'vitest';
4
- import { POSTGRES_STORAGE_FACTORY } from './util.js';
5
-
6
- describe('Postgres Sync Bucket Storage - Parameters', () =>
7
- register.registerDataStorageParameterTests(POSTGRES_STORAGE_FACTORY));
8
-
9
- describe('Postgres Sync Bucket Storage - Data', () => register.registerDataStorageDataTests(POSTGRES_STORAGE_FACTORY));
10
-
11
- describe('Postgres Sync Bucket Storage - Checkpoints', () =>
12
- register.registerDataStorageCheckpointTests(POSTGRES_STORAGE_FACTORY));
4
+ import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
13
5
 
14
6
  describe('Sync Bucket Validation', register.registerBucketValidationTests);
15
7
 
16
- describe('Postgres Sync Bucket Storage - pg-specific', () => {
17
- /**
18
- * The split of returned results can vary depending on storage drivers.
19
- * The large rows here are 2MB large while the default chunk limit is 1mb.
20
- * The Postgres storage driver will detect if the next row will increase the batch
21
- * over the limit and separate that row into a new batch (or single row batch) if applicable.
22
- */
23
- test('large batch (2)', async () => {
24
- // Test syncing a batch of data that is small in count,
25
- // but large enough in size to be split over multiple returned chunks.
26
- // Similar to the above test, but splits over 1MB chunks.
27
- await using factory = await POSTGRES_STORAGE_FACTORY();
28
- const syncRules = await factory.updateSyncRules(
29
- updateSyncRulesFromYaml(`
8
+ for (let storageVersion of TEST_STORAGE_VERSIONS) {
9
+ describe(`Postgres Sync Bucket Storage - Parameters - v${storageVersion}`, () =>
10
+ register.registerDataStorageParameterTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
11
+
12
+ describe(`Postgres Sync Bucket Storage - Data - v${storageVersion}`, () =>
13
+ register.registerDataStorageDataTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
14
+
15
+ describe(`Postgres Sync Bucket Storage - Checkpoints - v${storageVersion}`, () =>
16
+ register.registerDataStorageCheckpointTests({ ...POSTGRES_STORAGE_FACTORY, storageVersion }));
17
+
18
+ describe(`Postgres Sync Bucket Storage - pg-specific - v${storageVersion}`, () => {
19
+ /**
20
+ * The split of returned results can vary depending on storage drivers.
21
+ * The large rows here are 2MB large while the default chunk limit is 1mb.
22
+ * The Postgres storage driver will detect if the next row will increase the batch
23
+ * over the limit and separate that row into a new batch (or single row batch) if applicable.
24
+ */
25
+ test('large batch (2)', async () => {
26
+ // Test syncing a batch of data that is small in count,
27
+ // but large enough in size to be split over multiple returned chunks.
28
+ // Similar to the above test, but splits over 1MB chunks.
29
+ await using factory = await POSTGRES_STORAGE_FACTORY.factory();
30
+ const syncRules = await factory.updateSyncRules(
31
+ updateSyncRulesFromYaml(
32
+ `
30
33
  bucket_definitions:
31
34
  global:
32
35
  data:
33
36
  - SELECT id, description FROM "%"
34
- `)
35
- );
36
- const bucketStorage = factory.getInstance(syncRules);
37
-
38
- const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
39
- const sourceTable = TEST_TABLE;
40
-
41
- const largeDescription = '0123456789'.repeat(2_000_00);
42
-
43
- await batch.save({
44
- sourceTable,
45
- tag: storage.SaveOperationTag.INSERT,
46
- after: {
47
- id: 'test1',
48
- description: 'test1'
49
- },
50
- afterReplicaId: test_utils.rid('test1')
37
+ `,
38
+ { storageVersion }
39
+ )
40
+ );
41
+ const bucketStorage = factory.getInstance(syncRules);
42
+ const globalBucket = bucketRequest(syncRules, 'global[]');
43
+
44
+ const result = await (async () => {
45
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
46
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
47
+
48
+ const largeDescription = '0123456789'.repeat(2_000_00);
49
+
50
+ await writer.save({
51
+ sourceTable,
52
+ tag: storage.SaveOperationTag.INSERT,
53
+ after: {
54
+ id: 'test1',
55
+ description: 'test1'
56
+ },
57
+ afterReplicaId: test_utils.rid('test1')
58
+ });
59
+
60
+ await writer.save({
61
+ sourceTable,
62
+ tag: storage.SaveOperationTag.INSERT,
63
+ after: {
64
+ id: 'large1',
65
+ description: largeDescription
66
+ },
67
+ afterReplicaId: test_utils.rid('large1')
68
+ });
69
+
70
+ // Large enough to split the returned batch
71
+ await writer.save({
72
+ sourceTable,
73
+ tag: storage.SaveOperationTag.INSERT,
74
+ after: {
75
+ id: 'large2',
76
+ description: largeDescription
77
+ },
78
+ afterReplicaId: test_utils.rid('large2')
79
+ });
80
+
81
+ await writer.save({
82
+ sourceTable,
83
+ tag: storage.SaveOperationTag.INSERT,
84
+ after: {
85
+ id: 'test3',
86
+ description: 'test3'
87
+ },
88
+ afterReplicaId: test_utils.rid('test3')
89
+ });
90
+ return writer.flush();
91
+ })();
92
+
93
+ const checkpoint = result!.flushed_op;
94
+
95
+ const options: storage.BucketDataBatchOptions = {};
96
+
97
+ const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [globalBucket], options));
98
+ expect(test_utils.getBatchData(batch1)).toEqual([
99
+ { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
100
+ ]);
101
+ expect(test_utils.getBatchMeta(batch1)).toEqual({
102
+ after: '0',
103
+ has_more: true,
104
+ next_after: '1'
51
105
  });
52
106
 
53
- await batch.save({
54
- sourceTable,
55
- tag: storage.SaveOperationTag.INSERT,
56
- after: {
57
- id: 'large1',
58
- description: largeDescription
59
- },
60
- afterReplicaId: test_utils.rid('large1')
107
+ const batch2 = await test_utils.fromAsync(
108
+ bucketStorage.getBucketDataBatch(
109
+ checkpoint,
110
+ [{ ...globalBucket, start: BigInt(batch1[0].chunkData.next_after) }],
111
+ options
112
+ )
113
+ );
114
+ expect(test_utils.getBatchData(batch2)).toEqual([
115
+ { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 }
116
+ ]);
117
+ expect(test_utils.getBatchMeta(batch2)).toEqual({
118
+ after: '1',
119
+ has_more: true,
120
+ next_after: '2'
61
121
  });
62
122
 
63
- // Large enough to split the returned batch
64
- await batch.save({
65
- sourceTable,
66
- tag: storage.SaveOperationTag.INSERT,
67
- after: {
68
- id: 'large2',
69
- description: largeDescription
70
- },
71
- afterReplicaId: test_utils.rid('large2')
123
+ const batch3 = await test_utils.fromAsync(
124
+ bucketStorage.getBucketDataBatch(
125
+ checkpoint,
126
+ [{ ...globalBucket, start: BigInt(batch2[0].chunkData.next_after) }],
127
+ options
128
+ )
129
+ );
130
+ expect(test_utils.getBatchData(batch3)).toEqual([
131
+ { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }
132
+ ]);
133
+ expect(test_utils.getBatchMeta(batch3)).toEqual({
134
+ after: '2',
135
+ has_more: true,
136
+ next_after: '3'
72
137
  });
73
138
 
74
- await batch.save({
75
- sourceTable,
76
- tag: storage.SaveOperationTag.INSERT,
77
- after: {
78
- id: 'test3',
79
- description: 'test3'
80
- },
81
- afterReplicaId: test_utils.rid('test3')
139
+ const batch4 = await test_utils.fromAsync(
140
+ bucketStorage.getBucketDataBatch(
141
+ checkpoint,
142
+ [{ ...globalBucket, start: BigInt(batch3[0].chunkData.next_after) }],
143
+ options
144
+ )
145
+ );
146
+ expect(test_utils.getBatchData(batch4)).toEqual([
147
+ { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
148
+ ]);
149
+ expect(test_utils.getBatchMeta(batch4)).toEqual({
150
+ after: '3',
151
+ has_more: false,
152
+ next_after: '4'
82
153
  });
83
154
  });
84
-
85
- const checkpoint = result!.flushed_op;
86
-
87
- const options: storage.BucketDataBatchOptions = {};
88
-
89
- const batch1 = await test_utils.fromAsync(
90
- bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]), options)
91
- );
92
- expect(test_utils.getBatchData(batch1)).toEqual([
93
- { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
94
- ]);
95
- expect(test_utils.getBatchMeta(batch1)).toEqual({
96
- after: '0',
97
- has_more: true,
98
- next_after: '1'
99
- });
100
-
101
- const batch2 = await test_utils.fromAsync(
102
- bucketStorage.getBucketDataBatch(
103
- checkpoint,
104
- bucketRequestMap(syncRules, [['global[]', BigInt(batch1[0].chunkData.next_after)]]),
105
- options
106
- )
107
- );
108
- expect(test_utils.getBatchData(batch2)).toEqual([
109
- { op_id: '2', op: 'PUT', object_id: 'large1', checksum: 1178768505 }
110
- ]);
111
- expect(test_utils.getBatchMeta(batch2)).toEqual({
112
- after: '1',
113
- has_more: true,
114
- next_after: '2'
115
- });
116
-
117
- const batch3 = await test_utils.fromAsync(
118
- bucketStorage.getBucketDataBatch(
119
- checkpoint,
120
- bucketRequestMap(syncRules, [['global[]', BigInt(batch2[0].chunkData.next_after)]]),
121
- options
122
- )
123
- );
124
- expect(test_utils.getBatchData(batch3)).toEqual([
125
- { op_id: '3', op: 'PUT', object_id: 'large2', checksum: 1607205872 }
126
- ]);
127
- expect(test_utils.getBatchMeta(batch3)).toEqual({
128
- after: '2',
129
- has_more: true,
130
- next_after: '3'
131
- });
132
-
133
- const batch4 = await test_utils.fromAsync(
134
- bucketStorage.getBucketDataBatch(
135
- checkpoint,
136
- bucketRequestMap(syncRules, [['global[]', BigInt(batch3[0].chunkData.next_after)]]),
137
- options
138
- )
139
- );
140
- expect(test_utils.getBatchData(batch4)).toEqual([
141
- { op_id: '4', op: 'PUT', object_id: 'test3', checksum: 1359888332 }
142
- ]);
143
- expect(test_utils.getBatchMeta(batch4)).toEqual({
144
- after: '3',
145
- has_more: false,
146
- next_after: '4'
147
- });
148
155
  });
149
- });
156
+ }
@@ -1,13 +1,14 @@
1
1
  import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
2
- import { bucketRequest, bucketRequestMap, register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
2
+ import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
3
3
  import { describe, expect, test } from 'vitest';
4
+ import { PostgresCompactor } from '../../src/storage/PostgresCompactor.js';
4
5
  import { POSTGRES_STORAGE_FACTORY } from './util.js';
5
6
 
6
7
  describe('Postgres Sync Bucket Storage Compact', () => register.registerCompactTests(POSTGRES_STORAGE_FACTORY));
7
8
 
8
9
  describe('Postgres Compact - explicit bucket name', () => {
9
10
  test('compacts a specific bucket by exact name', async () => {
10
- await using factory = await POSTGRES_STORAGE_FACTORY();
11
+ await using factory = await POSTGRES_STORAGE_FACTORY.factory();
11
12
  const syncRules = await factory.updateSyncRules(
12
13
  updateSyncRulesFromYaml(`
13
14
  bucket_definitions:
@@ -17,33 +18,38 @@ bucket_definitions:
17
18
  );
18
19
  const bucketStorage = factory.getInstance(syncRules);
19
20
 
20
- const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
21
- await batch.save({
22
- sourceTable: TEST_TABLE,
21
+ const result = await (async () => {
22
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
23
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
24
+ await writer.save({
25
+ sourceTable: testTable,
23
26
  tag: storage.SaveOperationTag.INSERT,
24
27
  after: { id: 't1' },
25
28
  afterReplicaId: test_utils.rid('t1')
26
29
  });
27
- await batch.save({
28
- sourceTable: TEST_TABLE,
30
+ await writer.save({
31
+ sourceTable: testTable,
29
32
  tag: storage.SaveOperationTag.UPDATE,
30
33
  after: { id: 't1' },
31
34
  afterReplicaId: test_utils.rid('t1')
32
35
  });
33
- await batch.commit('1/1');
34
- });
36
+ await writer.markAllSnapshotDone('1/1');
37
+ const flushed = await writer.flush();
38
+ await writer.commit('1/1');
39
+ return flushed;
40
+ })();
35
41
 
36
42
  const checkpoint = result!.flushed_op;
37
43
 
38
44
  // Compact with an explicit bucket name — exercises the this.buckets
39
45
  // iteration path, NOT the compactAllBuckets discovery path.
40
46
  await bucketStorage.compact({
41
- compactBuckets: [bucketRequest(syncRules, 'global[]')],
47
+ compactBuckets: [bucketRequest(syncRules, 'global[]').bucket],
42
48
  minBucketChanges: 1
43
49
  });
44
50
 
45
51
  const batch = await test_utils.oneFromAsync(
46
- bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
52
+ bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)])
47
53
  );
48
54
 
49
55
  expect(batch.chunkData.data).toMatchObject([
@@ -51,4 +57,67 @@ bucket_definitions:
51
57
  { op_id: '2', op: 'PUT', object_id: 't1' }
52
58
  ]);
53
59
  });
60
+
61
+ test('clearBucket fails fast when prefix includes PUT', async () => {
62
+ // This tests the specific implementation, to check that our operation type guard is working
63
+ // for CLEAR compacting.
64
+ await using factory = await POSTGRES_STORAGE_FACTORY.factory();
65
+ const syncRules = await factory.updateSyncRules(
66
+ updateSyncRulesFromYaml(`
67
+ bucket_definitions:
68
+ global:
69
+ data: [select * from test]
70
+ `)
71
+ );
72
+ const bucketStorage = factory.getInstance(syncRules);
73
+ const request = bucketRequest(syncRules, 'global[]');
74
+
75
+ const result = await (async () => {
76
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
77
+ const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
78
+ await writer.markAllSnapshotDone('1/1');
79
+ await writer.save({
80
+ sourceTable: testTable,
81
+ tag: storage.SaveOperationTag.INSERT,
82
+ after: { id: 't1' },
83
+ afterReplicaId: test_utils.rid('t1')
84
+ });
85
+ await writer.save({
86
+ sourceTable: testTable,
87
+ tag: storage.SaveOperationTag.DELETE,
88
+ before: { id: 't1' },
89
+ beforeReplicaId: test_utils.rid('t1')
90
+ });
91
+ await writer.save({
92
+ sourceTable: testTable,
93
+ tag: storage.SaveOperationTag.INSERT,
94
+ after: { id: 't2' },
95
+ afterReplicaId: test_utils.rid('t2')
96
+ });
97
+ await writer.save({
98
+ sourceTable: testTable,
99
+ tag: storage.SaveOperationTag.DELETE,
100
+ before: { id: 't2' },
101
+ beforeReplicaId: test_utils.rid('t2')
102
+ });
103
+ const flushed = await writer.flush();
104
+ await writer.commit('1/1');
105
+ return flushed;
106
+ })();
107
+
108
+ const checkpoint = result!.flushed_op;
109
+ const rowsBefore = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
110
+ const dataBefore = test_utils.getBatchData(rowsBefore);
111
+ const clearToOpId = BigInt(dataBefore[2].op_id);
112
+
113
+ const compactor = new PostgresCompactor(factory.db, bucketStorage.group_id, {});
114
+ // Trigger the private method directly
115
+ await expect(compactor.clearBucketForTests(request.bucket, clearToOpId)).rejects.toThrow(
116
+ /Unexpected PUT operation/
117
+ );
118
+
119
+ // The method wraps in a transaction; on assertion error the bucket must remain unchanged.
120
+ const rowsAfter = await test_utils.oneFromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
121
+ expect(test_utils.getBatchData(rowsAfter)).toEqual(dataBefore);
122
+ });
54
123
  });
@@ -1,5 +1,5 @@
1
1
  import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
2
- import { bucketRequest, register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
2
+ import { bucketRequest, register, resolveTestTable, test_utils } from '@powersync/service-core-tests';
3
3
  import { describe, expect, test } from 'vitest';
4
4
  import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
5
5
 
@@ -12,13 +12,16 @@ function registerStorageVersionTests(storageVersion: number) {
12
12
  describe(`storage v${storageVersion}`, () => {
13
13
  const storageFactory = POSTGRES_STORAGE_FACTORY;
14
14
 
15
- register.registerSyncTests(storageFactory, { storageVersion });
15
+ register.registerSyncTests(storageFactory.factory, {
16
+ storageVersion,
17
+ tableIdStrings: storageFactory.tableIdStrings
18
+ });
16
19
 
17
20
  test('large batch (2)', async () => {
18
21
  // Test syncing a batch of data that is small in count,
19
22
  // but large enough in size to be split over multiple returned chunks.
20
23
  // Similar to the above test, but splits over 1MB chunks.
21
- await using factory = await storageFactory();
24
+ await using factory = await storageFactory.factory();
22
25
  const syncRules = await factory.updateSyncRules(
23
26
  updateSyncRulesFromYaml(
24
27
  `
@@ -33,60 +36,60 @@ function registerStorageVersionTests(storageVersion: number) {
33
36
  const bucketStorage = factory.getInstance(syncRules);
34
37
  const globalBucket = bucketRequest(syncRules, 'global[]');
35
38
 
36
- const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
37
- const sourceTable = TEST_TABLE;
38
-
39
- const largeDescription = '0123456789'.repeat(2_000_00);
40
-
41
- await batch.save({
42
- sourceTable,
43
- tag: storage.SaveOperationTag.INSERT,
44
- after: {
45
- id: 'test1',
46
- description: 'test1'
47
- },
48
- afterReplicaId: test_utils.rid('test1')
49
- });
50
-
51
- await batch.save({
52
- sourceTable,
53
- tag: storage.SaveOperationTag.INSERT,
54
- after: {
55
- id: 'large1',
56
- description: largeDescription
57
- },
58
- afterReplicaId: test_utils.rid('large1')
59
- });
60
-
61
- // Large enough to split the returned batch
62
- await batch.save({
63
- sourceTable,
64
- tag: storage.SaveOperationTag.INSERT,
65
- after: {
66
- id: 'large2',
67
- description: largeDescription
68
- },
69
- afterReplicaId: test_utils.rid('large2')
70
- });
71
-
72
- await batch.save({
73
- sourceTable,
74
- tag: storage.SaveOperationTag.INSERT,
75
- after: {
76
- id: 'test3',
77
- description: 'test3'
78
- },
79
- afterReplicaId: test_utils.rid('test3')
80
- });
39
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
40
+
41
+ const sourceTable = await resolveTestTable(writer, 'test', ['id'], storageFactory);
42
+
43
+ const largeDescription = '0123456789'.repeat(2_000_00);
44
+
45
+ await writer.save({
46
+ sourceTable,
47
+ tag: storage.SaveOperationTag.INSERT,
48
+ after: {
49
+ id: 'test1',
50
+ description: 'test1'
51
+ },
52
+ afterReplicaId: test_utils.rid('test1')
53
+ });
54
+
55
+ await writer.save({
56
+ sourceTable,
57
+ tag: storage.SaveOperationTag.INSERT,
58
+ after: {
59
+ id: 'large1',
60
+ description: largeDescription
61
+ },
62
+ afterReplicaId: test_utils.rid('large1')
81
63
  });
82
64
 
65
+ // Large enough to split the returned batch
66
+ await writer.save({
67
+ sourceTable,
68
+ tag: storage.SaveOperationTag.INSERT,
69
+ after: {
70
+ id: 'large2',
71
+ description: largeDescription
72
+ },
73
+ afterReplicaId: test_utils.rid('large2')
74
+ });
75
+
76
+ await writer.save({
77
+ sourceTable,
78
+ tag: storage.SaveOperationTag.INSERT,
79
+ after: {
80
+ id: 'test3',
81
+ description: 'test3'
82
+ },
83
+ afterReplicaId: test_utils.rid('test3')
84
+ });
85
+
86
+ const result = await writer.flush();
87
+
83
88
  const checkpoint = result!.flushed_op;
84
89
 
85
90
  const options: storage.BucketDataBatchOptions = {};
86
91
 
87
- const batch1 = await test_utils.fromAsync(
88
- bucketStorage.getBucketDataBatch(checkpoint, new Map([[globalBucket, 0n]]), options)
89
- );
92
+ const batch1 = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [globalBucket], options));
90
93
  expect(test_utils.getBatchData(batch1)).toEqual([
91
94
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 }
92
95
  ]);
@@ -99,7 +102,7 @@ function registerStorageVersionTests(storageVersion: number) {
99
102
  const batch2 = await test_utils.fromAsync(
100
103
  bucketStorage.getBucketDataBatch(
101
104
  checkpoint,
102
- new Map([[globalBucket, BigInt(batch1[0].chunkData.next_after)]]),
105
+ [{ ...globalBucket, start: BigInt(batch1[0].chunkData.next_after) }],
103
106
  options
104
107
  )
105
108
  );
@@ -115,7 +118,7 @@ function registerStorageVersionTests(storageVersion: number) {
115
118
  const batch3 = await test_utils.fromAsync(
116
119
  bucketStorage.getBucketDataBatch(
117
120
  checkpoint,
118
- new Map([[globalBucket, BigInt(batch2[0].chunkData.next_after)]]),
121
+ [{ ...globalBucket, start: BigInt(batch2[0].chunkData.next_after) }],
119
122
  options
120
123
  )
121
124
  );
@@ -131,7 +134,7 @@ function registerStorageVersionTests(storageVersion: number) {
131
134
  const batch4 = await test_utils.fromAsync(
132
135
  bucketStorage.getBucketDataBatch(
133
136
  checkpoint,
134
- new Map([[globalBucket, BigInt(batch3[0].chunkData.next_after)]]),
137
+ [{ ...globalBucket, start: BigInt(batch3[0].chunkData.next_after) }],
135
138
  options
136
139
  )
137
140
  );
package/test/src/util.ts CHANGED
@@ -1,9 +1,9 @@
1
+ import { SUPPORTED_STORAGE_VERSIONS } from '@powersync/service-core';
1
2
  import path from 'path';
2
3
  import { fileURLToPath } from 'url';
3
4
  import { normalizePostgresStorageConfig, PostgresMigrationAgent } from '../../src/index.js';
4
- import { env } from './env.js';
5
5
  import { postgresTestSetup } from '../../src/utils/test-utils.js';
6
- import { CURRENT_STORAGE_VERSION, LEGACY_STORAGE_VERSION } from '@powersync/service-core';
6
+ import { env } from './env.js';
7
7
 
8
8
  const __filename = fileURLToPath(import.meta.url);
9
9
  const __dirname = path.dirname(__filename);
@@ -33,7 +33,7 @@ export const POSTGRES_STORAGE_SETUP = postgresTestSetup({
33
33
  migrationAgent: (config) => new TestPostgresMigrationAgent(config)
34
34
  });
35
35
 
36
- export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.factory;
36
+ export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP;
37
37
  export const POSTGRES_REPORT_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.reportFactory;
38
38
 
39
- export const TEST_STORAGE_VERSIONS = [LEGACY_STORAGE_VERSION, CURRENT_STORAGE_VERSION];
39
+ export const TEST_STORAGE_VERSIONS = SUPPORTED_STORAGE_VERSIONS;