@powersync/service-module-postgres-storage 0.19.5 → 0.19.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/storage/batch/PostgresBucketBatch.js +37 -34
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/storage/batch/PostgresPersistedBatch.d.ts +68 -25
- package/dist/storage/batch/PostgresPersistedBatch.js +179 -147
- package/dist/storage/batch/PostgresPersistedBatch.js.map +1 -1
- package/dist/storage/current-data-store.d.ts +9 -7
- package/dist/storage/current-data-store.js +8 -2
- package/dist/storage/current-data-store.js.map +1 -1
- package/dist/types/types.d.ts +9 -8
- package/dist/types/types.js +9 -8
- package/dist/types/types.js.map +1 -1
- package/package.json +2 -2
- package/src/storage/batch/PostgresBucketBatch.ts +38 -35
- package/src/storage/batch/PostgresPersistedBatch.ts +238 -192
- package/src/storage/current-data-store.ts +10 -4
- package/src/types/types.ts +9 -8
- package/test/src/storage_flush.test.ts +162 -41
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -2,7 +2,6 @@ import * as lib_postgres from '@powersync/lib-service-postgres';
|
|
|
2
2
|
import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
3
3
|
import { test_utils } from '@powersync/service-core-tests';
|
|
4
4
|
import { afterEach, describe, expect, test, vi } from 'vitest';
|
|
5
|
-
import { chunkRowsBySize } from '../../src/storage/batch/PostgresPersistedBatch.js';
|
|
6
5
|
import { createPostgresTestSetup, POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
|
|
7
6
|
|
|
8
7
|
/**
|
|
@@ -29,26 +28,68 @@ streams:
|
|
|
29
28
|
query: SELECT * FROM test WHERE owner_id IN (SELECT id FROM users WHERE name = subscription.parameter('name'))
|
|
30
29
|
`;
|
|
31
30
|
|
|
31
|
+
type BucketDataRow = { bucket_name: string; data: string | null; source_key: string };
|
|
32
|
+
type ParameterDataRow = { lookup: string; bucket_parameters: string; source_key: string };
|
|
33
|
+
|
|
32
34
|
/**
|
|
33
|
-
*
|
|
35
|
+
* A statement as passed to `query()`: either a script string or a statement with parameters.
|
|
34
36
|
*/
|
|
35
|
-
|
|
37
|
+
type QueryCall = string | { statement: string; params: { value: unknown }[] };
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Records the statements executed on any connection.
|
|
41
|
+
*/
|
|
42
|
+
function spyOnStatements() {
|
|
36
43
|
const spy = vi.spyOn(lib_postgres.AbstractPostgresConnection.prototype, 'query');
|
|
44
|
+
const calls = () => spy.mock.calls.map(([statement]) => statement as QueryCall);
|
|
45
|
+
const inserts = (table: string) => {
|
|
46
|
+
const pattern = new RegExp(`INSERT INTO\\s+${table}\\b`);
|
|
47
|
+
return calls().filter((call): call is Exclude<QueryCall, string> => {
|
|
48
|
+
return typeof call == 'object' && pattern.test(call.statement);
|
|
49
|
+
});
|
|
50
|
+
};
|
|
37
51
|
return {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
52
|
+
insertCount(table: string) {
|
|
53
|
+
return inserts(table).length;
|
|
54
|
+
},
|
|
55
|
+
/**
|
|
56
|
+
* The rows passed to each INSERT statement on the table, in execution order.
|
|
57
|
+
*/
|
|
58
|
+
insertedRows<T>(table: string): T[][] {
|
|
59
|
+
return inserts(table).map((call) => call.params[0].value as T[]);
|
|
60
|
+
},
|
|
61
|
+
/**
|
|
62
|
+
* The number of replication transactions started.
|
|
63
|
+
*/
|
|
64
|
+
transactionCount() {
|
|
65
|
+
return calls().filter((call) => {
|
|
66
|
+
return typeof call == 'string' && call.includes('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ');
|
|
42
67
|
}).length;
|
|
43
68
|
}
|
|
44
69
|
};
|
|
45
70
|
}
|
|
46
71
|
|
|
72
|
+
/**
|
|
73
|
+
* The estimates used by PostgresPersistedBatch for the size of a statement.
|
|
74
|
+
*/
|
|
75
|
+
function estimateBucketDataSize(rows: BucketDataRow[]) {
|
|
76
|
+
return rows.reduce((total, row) => {
|
|
77
|
+
return total + row.bucket_name.length + (row.data?.length ?? 0) + row.source_key.length + 100;
|
|
78
|
+
}, 0);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function estimateParameterDataSize(rows: ParameterDataRow[]) {
|
|
82
|
+
return rows.reduce((total, row) => {
|
|
83
|
+
return total + row.lookup.length + row.bucket_parameters.length + row.source_key.length + 100;
|
|
84
|
+
}, 0);
|
|
85
|
+
}
|
|
86
|
+
|
|
47
87
|
async function readBucketData(db: lib_postgres.DatabaseClient, groupId: number) {
|
|
48
88
|
return await db.sql`
|
|
49
89
|
SELECT
|
|
50
90
|
bucket_name,
|
|
51
91
|
op_id,
|
|
92
|
+
op,
|
|
52
93
|
data
|
|
53
94
|
FROM
|
|
54
95
|
bucket_data
|
|
@@ -56,36 +97,16 @@ async function readBucketData(db: lib_postgres.DatabaseClient, groupId: number)
|
|
|
56
97
|
group_id = ${{ type: 'int4', value: groupId }}
|
|
57
98
|
ORDER BY
|
|
58
99
|
op_id
|
|
59
|
-
`.rows<{ bucket_name: string; op_id: bigint; data: string }>();
|
|
100
|
+
`.rows<{ bucket_name: string; op_id: bigint; op: string; data: string | null }>();
|
|
60
101
|
}
|
|
61
102
|
|
|
62
|
-
describe('chunkRowsBySize', () => {
|
|
63
|
-
const size = (row: string) => row.length;
|
|
64
|
-
|
|
65
|
-
test('splits rows into consecutive chunks by estimated size', () => {
|
|
66
|
-
expect(chunkRowsBySize(['aa', 'bb', 'cc', 'dd', 'e'], size, 4)).toEqual([['aa', 'bb'], ['cc', 'dd'], ['e']]);
|
|
67
|
-
});
|
|
68
|
-
|
|
69
|
-
test('keeps all rows in one chunk within the limit', () => {
|
|
70
|
-
expect(chunkRowsBySize(['aa', 'bb'], size, 4)).toEqual([['aa', 'bb']]);
|
|
71
|
-
});
|
|
72
|
-
|
|
73
|
-
test('places a single row larger than the limit in its own chunk', () => {
|
|
74
|
-
expect(chunkRowsBySize(['aa', 'bbbbbbbb', 'cc', 'dd'], size, 4)).toEqual([['aa'], ['bbbbbbbb'], ['cc', 'dd']]);
|
|
75
|
-
});
|
|
76
|
-
|
|
77
|
-
test('returns no chunks for no rows', () => {
|
|
78
|
-
expect(chunkRowsBySize([], size, 4)).toEqual([]);
|
|
79
|
-
});
|
|
80
|
-
});
|
|
81
|
-
|
|
82
103
|
for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
83
|
-
describe(`Postgres Sync Bucket Storage - flush
|
|
104
|
+
describe(`Postgres Sync Bucket Storage - flush statements - v${storageVersion}`, () => {
|
|
84
105
|
afterEach(() => {
|
|
85
106
|
vi.restoreAllMocks();
|
|
86
107
|
});
|
|
87
108
|
|
|
88
|
-
test('
|
|
109
|
+
test('inserts bucket_data in statements bounded by max_statement_size', async () => {
|
|
89
110
|
const setup = createPostgresTestSetup({ batchLimits: { max_statement_size: 2_000 } });
|
|
90
111
|
await using factory = await setup.factory();
|
|
91
112
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(BY_TAG_SYNC_CONFIG, { storageVersion }));
|
|
@@ -93,10 +114,10 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
|
93
114
|
|
|
94
115
|
const tags = Array.from({ length: 50 }, (_, i) => `tag${i}`);
|
|
95
116
|
const description = 'x'.repeat(500);
|
|
96
|
-
const inserts = spyOnInserts();
|
|
97
117
|
|
|
98
118
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
99
119
|
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
|
|
120
|
+
const statements = spyOnStatements();
|
|
100
121
|
await writer.save({
|
|
101
122
|
sourceTable,
|
|
102
123
|
tag: storage.SaveOperationTag.INSERT,
|
|
@@ -109,8 +130,13 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
|
109
130
|
});
|
|
110
131
|
await writer.flush();
|
|
111
132
|
|
|
112
|
-
// Each row is roughly
|
|
113
|
-
|
|
133
|
+
// Each row is roughly 700 bytes, so this needs multiple statements, all in one transaction.
|
|
134
|
+
const inserted = statements.insertedRows<BucketDataRow>('bucket_data');
|
|
135
|
+
expect(inserted.length).toBeGreaterThan(1);
|
|
136
|
+
expect(statements.transactionCount()).toEqual(1);
|
|
137
|
+
for (const statementRows of inserted) {
|
|
138
|
+
expect(estimateBucketDataSize(statementRows)).toBeLessThanOrEqual(2_000);
|
|
139
|
+
}
|
|
114
140
|
|
|
115
141
|
const rows = await readBucketData(factory.db, bucketStorage.replicationStreamId);
|
|
116
142
|
expect(rows.length).toEqual(tags.length);
|
|
@@ -120,11 +146,12 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
|
120
146
|
expect(rows[i].op_id).toBeGreaterThan(rows[i - 1].op_id);
|
|
121
147
|
}
|
|
122
148
|
for (const row of rows) {
|
|
123
|
-
expect(
|
|
149
|
+
expect(row.op).toEqual('PUT');
|
|
150
|
+
expect(JSON.parse(row.data!)).toEqual({ id: 'test1', tags: JSON.stringify(tags), description });
|
|
124
151
|
}
|
|
125
152
|
});
|
|
126
153
|
|
|
127
|
-
test('
|
|
154
|
+
test('inserts bucket_parameters in statements bounded by max_statement_size', async () => {
|
|
128
155
|
const setup = createPostgresTestSetup({ batchLimits: { max_statement_size: 2_000 } });
|
|
129
156
|
await using factory = await setup.factory();
|
|
130
157
|
const syncRules = await factory.updateSyncRules(
|
|
@@ -133,10 +160,10 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
|
133
160
|
const bucketStorage = factory.getInstance(syncRules);
|
|
134
161
|
|
|
135
162
|
const userCount = 30;
|
|
136
|
-
const inserts = spyOnInserts();
|
|
137
163
|
|
|
138
164
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
139
165
|
const sourceTable = await test_utils.resolveTestTable(writer, 'users', ['id'], POSTGRES_STORAGE_FACTORY, 2);
|
|
166
|
+
const statements = spyOnStatements();
|
|
140
167
|
for (let i = 0; i < userCount; i++) {
|
|
141
168
|
await writer.save({
|
|
142
169
|
sourceTable,
|
|
@@ -150,7 +177,12 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
|
150
177
|
}
|
|
151
178
|
await writer.flush();
|
|
152
179
|
|
|
153
|
-
|
|
180
|
+
const inserted = statements.insertedRows<ParameterDataRow>('bucket_parameters');
|
|
181
|
+
expect(inserted.length).toBeGreaterThan(1);
|
|
182
|
+
expect(statements.transactionCount()).toEqual(1);
|
|
183
|
+
for (const statementRows of inserted) {
|
|
184
|
+
expect(estimateParameterDataSize(statementRows)).toBeLessThanOrEqual(2_000);
|
|
185
|
+
}
|
|
154
186
|
|
|
155
187
|
const rows = await factory.db.sql`
|
|
156
188
|
SELECT
|
|
@@ -177,10 +209,10 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
|
177
209
|
|
|
178
210
|
const tags = Array.from({ length: 300 }, (_, i) => `tag${i}`);
|
|
179
211
|
const description = '0123456789'.repeat(30_000);
|
|
180
|
-
const inserts = spyOnInserts();
|
|
181
212
|
|
|
182
213
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
183
214
|
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
|
|
215
|
+
const statements = spyOnStatements();
|
|
184
216
|
await writer.save({
|
|
185
217
|
sourceTable,
|
|
186
218
|
tag: storage.SaveOperationTag.INSERT,
|
|
@@ -193,14 +225,103 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
|
|
|
193
225
|
});
|
|
194
226
|
await writer.flush();
|
|
195
227
|
|
|
196
|
-
// 300 buckets x 300kB exceeds
|
|
197
|
-
|
|
228
|
+
// 300 buckets x 300kB exceeds the default max_estimated_size (5MB), so the rows are inserted
|
|
229
|
+
// in multiple statements, all in the transaction of this batch.
|
|
230
|
+
expect(statements.insertCount('bucket_data')).toBeGreaterThan(1);
|
|
231
|
+
expect(statements.transactionCount()).toEqual(1);
|
|
198
232
|
|
|
199
233
|
const rows = await readBucketData(factory.db, bucketStorage.replicationStreamId);
|
|
200
234
|
expect(rows.length).toEqual(tags.length);
|
|
201
235
|
expect(new Set(rows.map((row) => row.bucket_name)).size).toEqual(tags.length);
|
|
202
236
|
for (const row of rows) {
|
|
203
|
-
expect(
|
|
237
|
+
expect(row.op).toEqual('PUT');
|
|
238
|
+
expect(JSON.parse(row.data!).description).toEqual(description);
|
|
239
|
+
}
|
|
240
|
+
});
|
|
241
|
+
|
|
242
|
+
test('keeps the transaction size limit while inserting rows early', async () => {
|
|
243
|
+
// Rows inserted before the end of a batch still count towards max_estimated_size,
|
|
244
|
+
// so inserting early must not grow the transactions.
|
|
245
|
+
const setup = createPostgresTestSetup({ batchLimits: { max_estimated_size: 5_000, max_statement_size: 2_000 } });
|
|
246
|
+
await using factory = await setup.factory();
|
|
247
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(BY_TAG_SYNC_CONFIG, { storageVersion }));
|
|
248
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
249
|
+
|
|
250
|
+
const rowCount = 30;
|
|
251
|
+
const description = 'x'.repeat(500);
|
|
252
|
+
|
|
253
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
254
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
|
|
255
|
+
const statements = spyOnStatements();
|
|
256
|
+
for (let i = 0; i < rowCount; i++) {
|
|
257
|
+
await writer.save({
|
|
258
|
+
sourceTable,
|
|
259
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
260
|
+
after: {
|
|
261
|
+
id: `test${i}`,
|
|
262
|
+
tags: JSON.stringify([`tag${i}`]),
|
|
263
|
+
description
|
|
264
|
+
},
|
|
265
|
+
afterReplicaId: test_utils.rid(`test${i}`)
|
|
266
|
+
});
|
|
267
|
+
}
|
|
268
|
+
await writer.flush();
|
|
269
|
+
|
|
270
|
+
// Roughly 700 bytes per row against a 5000 byte transaction limit.
|
|
271
|
+
expect(statements.transactionCount()).toBeGreaterThanOrEqual(3);
|
|
272
|
+
|
|
273
|
+
const rows = await readBucketData(factory.db, bucketStorage.replicationStreamId);
|
|
274
|
+
expect(rows.length).toEqual(rowCount);
|
|
275
|
+
for (let i = 1; i < rows.length; i++) {
|
|
276
|
+
expect(rows[i].op_id).toBeGreaterThan(rows[i - 1].op_id);
|
|
277
|
+
}
|
|
278
|
+
});
|
|
279
|
+
|
|
280
|
+
test('inserts the REMOVE operations of a truncate in statements bounded by max_statement_size', async () => {
|
|
281
|
+
// The REMOVE operations are inserted on the truncate transaction while its current_data rows
|
|
282
|
+
// are processed, on the connection those rows were read from.
|
|
283
|
+
const setup = createPostgresTestSetup({ batchLimits: { max_statement_size: 2_000 } });
|
|
284
|
+
await using factory = await setup.factory();
|
|
285
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(BY_TAG_SYNC_CONFIG, { storageVersion }));
|
|
286
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
287
|
+
|
|
288
|
+
const rowCount = 20;
|
|
289
|
+
const tags = Array.from({ length: 20 }, (_, i) => `tag${i}`);
|
|
290
|
+
|
|
291
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
292
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
|
|
293
|
+
for (let i = 0; i < rowCount; i++) {
|
|
294
|
+
await writer.save({
|
|
295
|
+
sourceTable,
|
|
296
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
297
|
+
after: {
|
|
298
|
+
id: `test${i}`,
|
|
299
|
+
tags: JSON.stringify(tags)
|
|
300
|
+
},
|
|
301
|
+
afterReplicaId: test_utils.rid(`test${i}`)
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
await writer.flush();
|
|
305
|
+
|
|
306
|
+
const statements = spyOnStatements();
|
|
307
|
+
await writer.truncate([sourceTable]);
|
|
308
|
+
|
|
309
|
+
// 400 REMOVE operations of roughly 150 bytes each, all in the transaction of the truncate.
|
|
310
|
+
const inserted = statements.insertedRows<BucketDataRow>('bucket_data');
|
|
311
|
+
expect(inserted.length).toBeGreaterThan(1);
|
|
312
|
+
expect(statements.transactionCount()).toEqual(1);
|
|
313
|
+
for (const statementRows of inserted) {
|
|
314
|
+
expect(estimateBucketDataSize(statementRows)).toBeLessThanOrEqual(2_000);
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
const rows = await readBucketData(factory.db, bucketStorage.replicationStreamId);
|
|
318
|
+
const operationCount = rowCount * tags.length;
|
|
319
|
+
expect(rows.length).toEqual(operationCount * 2);
|
|
320
|
+
// The REMOVE operations follow the PUT operations, with op_ids in insertion order.
|
|
321
|
+
expect(rows.slice(0, operationCount).every((row) => row.op == 'PUT')).toBe(true);
|
|
322
|
+
expect(rows.slice(operationCount).every((row) => row.op == 'REMOVE')).toBe(true);
|
|
323
|
+
for (let i = 1; i < rows.length; i++) {
|
|
324
|
+
expect(rows[i].op_id).toBeGreaterThan(rows[i - 1].op_id);
|
|
204
325
|
}
|
|
205
326
|
});
|
|
206
327
|
});
|