@powersync/service-module-postgres-storage 0.19.6 → 0.19.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,7 +2,6 @@ import * as lib_postgres from '@powersync/lib-service-postgres';
2
2
  import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
3
3
  import { test_utils } from '@powersync/service-core-tests';
4
4
  import { afterEach, describe, expect, test, vi } from 'vitest';
5
- import { chunkRowsBySize } from '../../src/storage/batch/PostgresPersistedBatch.js';
6
5
  import { createPostgresTestSetup, POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
7
6
 
8
7
  /**
@@ -29,26 +28,68 @@ streams:
29
28
  query: SELECT * FROM test WHERE owner_id IN (SELECT id FROM users WHERE name = subscription.parameter('name'))
30
29
  `;
31
30
 
31
+ type BucketDataRow = { bucket_name: string; data: string | null; source_key: string };
32
+ type ParameterDataRow = { lookup: string; bucket_parameters: string; source_key: string };
33
+
32
34
  /**
33
- * Records the INSERT statements executed on any connection, by target table.
35
+ * A statement as passed to `query()`: either a script string or a statement with parameters.
34
36
  */
35
- function spyOnInserts() {
37
+ type QueryCall = string | { statement: string; params: { value: unknown }[] };
38
+
39
+ /**
40
+ * Records the statements executed on any connection.
41
+ */
42
+ function spyOnStatements() {
36
43
  const spy = vi.spyOn(lib_postgres.AbstractPostgresConnection.prototype, 'query');
44
+ const calls = () => spy.mock.calls.map(([statement]) => statement as QueryCall);
45
+ const inserts = (table: string) => {
46
+ const pattern = new RegExp(`INSERT INTO\\s+${table}\\b`);
47
+ return calls().filter((call): call is Exclude<QueryCall, string> => {
48
+ return typeof call == 'object' && pattern.test(call.statement);
49
+ });
50
+ };
37
51
  return {
38
- count(table: string) {
39
- const pattern = new RegExp(`INSERT INTO\\s+${table}\\b`);
40
- return spy.mock.calls.filter(([statement]) => {
41
- return typeof statement == 'object' && pattern.test(statement.statement);
52
+ insertCount(table: string) {
53
+ return inserts(table).length;
54
+ },
55
+ /**
56
+ * The rows passed to each INSERT statement on the table, in execution order.
57
+ */
58
+ insertedRows<T>(table: string): T[][] {
59
+ return inserts(table).map((call) => call.params[0].value as T[]);
60
+ },
61
+ /**
62
+ * The number of replication transactions started.
63
+ */
64
+ transactionCount() {
65
+ return calls().filter((call) => {
66
+ return typeof call == 'string' && call.includes('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ');
42
67
  }).length;
43
68
  }
44
69
  };
45
70
  }
46
71
 
72
+ /**
73
+ * The estimates used by PostgresPersistedBatch for the size of a statement.
74
+ */
75
+ function estimateBucketDataSize(rows: BucketDataRow[]) {
76
+ return rows.reduce((total, row) => {
77
+ return total + row.bucket_name.length + (row.data?.length ?? 0) + row.source_key.length + 100;
78
+ }, 0);
79
+ }
80
+
81
+ function estimateParameterDataSize(rows: ParameterDataRow[]) {
82
+ return rows.reduce((total, row) => {
83
+ return total + row.lookup.length + row.bucket_parameters.length + row.source_key.length + 100;
84
+ }, 0);
85
+ }
86
+
47
87
  async function readBucketData(db: lib_postgres.DatabaseClient, groupId: number) {
48
88
  return await db.sql`
49
89
  SELECT
50
90
  bucket_name,
51
91
  op_id,
92
+ op,
52
93
  data
53
94
  FROM
54
95
  bucket_data
@@ -56,36 +97,16 @@ async function readBucketData(db: lib_postgres.DatabaseClient, groupId: number)
56
97
  group_id = ${{ type: 'int4', value: groupId }}
57
98
  ORDER BY
58
99
  op_id
59
- `.rows<{ bucket_name: string; op_id: bigint; data: string }>();
100
+ `.rows<{ bucket_name: string; op_id: bigint; op: string; data: string | null }>();
60
101
  }
61
102
 
62
- describe('chunkRowsBySize', () => {
63
- const size = (row: string) => row.length;
64
-
65
- test('splits rows into consecutive chunks by estimated size', () => {
66
- expect(chunkRowsBySize(['aa', 'bb', 'cc', 'dd', 'e'], size, 4)).toEqual([['aa', 'bb'], ['cc', 'dd'], ['e']]);
67
- });
68
-
69
- test('keeps all rows in one chunk within the limit', () => {
70
- expect(chunkRowsBySize(['aa', 'bb'], size, 4)).toEqual([['aa', 'bb']]);
71
- });
72
-
73
- test('places a single row larger than the limit in its own chunk', () => {
74
- expect(chunkRowsBySize(['aa', 'bbbbbbbb', 'cc', 'dd'], size, 4)).toEqual([['aa'], ['bbbbbbbb'], ['cc', 'dd']]);
75
- });
76
-
77
- test('returns no chunks for no rows', () => {
78
- expect(chunkRowsBySize([], size, 4)).toEqual([]);
79
- });
80
- });
81
-
82
103
  for (const storageVersion of TEST_STORAGE_VERSIONS) {
83
- describe(`Postgres Sync Bucket Storage - flush chunking - v${storageVersion}`, () => {
104
+ describe(`Postgres Sync Bucket Storage - flush statements - v${storageVersion}`, () => {
84
105
  afterEach(() => {
85
106
  vi.restoreAllMocks();
86
107
  });
87
108
 
88
- test('splits bucket_data inserts by max_statement_size', async () => {
109
+ test('inserts bucket_data in statements bounded by max_statement_size', async () => {
89
110
  const setup = createPostgresTestSetup({ batchLimits: { max_statement_size: 2_000 } });
90
111
  await using factory = await setup.factory();
91
112
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(BY_TAG_SYNC_CONFIG, { storageVersion }));
@@ -93,10 +114,10 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
93
114
 
94
115
  const tags = Array.from({ length: 50 }, (_, i) => `tag${i}`);
95
116
  const description = 'x'.repeat(500);
96
- const inserts = spyOnInserts();
97
117
 
98
118
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
99
119
  const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
120
+ const statements = spyOnStatements();
100
121
  await writer.save({
101
122
  sourceTable,
102
123
  tag: storage.SaveOperationTag.INSERT,
@@ -109,8 +130,13 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
109
130
  });
110
131
  await writer.flush();
111
132
 
112
- // Each row is roughly 600 bytes, so this needs multiple statements.
113
- expect(inserts.count('bucket_data')).toBeGreaterThan(1);
133
+ // Each row is roughly 700 bytes, so this needs multiple statements, all in one transaction.
134
+ const inserted = statements.insertedRows<BucketDataRow>('bucket_data');
135
+ expect(inserted.length).toBeGreaterThan(1);
136
+ expect(statements.transactionCount()).toEqual(1);
137
+ for (const statementRows of inserted) {
138
+ expect(estimateBucketDataSize(statementRows)).toBeLessThanOrEqual(2_000);
139
+ }
114
140
 
115
141
  const rows = await readBucketData(factory.db, bucketStorage.replicationStreamId);
116
142
  expect(rows.length).toEqual(tags.length);
@@ -120,11 +146,12 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
120
146
  expect(rows[i].op_id).toBeGreaterThan(rows[i - 1].op_id);
121
147
  }
122
148
  for (const row of rows) {
123
- expect(JSON.parse(row.data)).toEqual({ id: 'test1', tags: JSON.stringify(tags), description });
149
+ expect(row.op).toEqual('PUT');
150
+ expect(JSON.parse(row.data!)).toEqual({ id: 'test1', tags: JSON.stringify(tags), description });
124
151
  }
125
152
  });
126
153
 
127
- test('splits bucket_parameters inserts by max_statement_size', async () => {
154
+ test('inserts bucket_parameters in statements bounded by max_statement_size', async () => {
128
155
  const setup = createPostgresTestSetup({ batchLimits: { max_statement_size: 2_000 } });
129
156
  await using factory = await setup.factory();
130
157
  const syncRules = await factory.updateSyncRules(
@@ -133,10 +160,10 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
133
160
  const bucketStorage = factory.getInstance(syncRules);
134
161
 
135
162
  const userCount = 30;
136
- const inserts = spyOnInserts();
137
163
 
138
164
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
139
165
  const sourceTable = await test_utils.resolveTestTable(writer, 'users', ['id'], POSTGRES_STORAGE_FACTORY, 2);
166
+ const statements = spyOnStatements();
140
167
  for (let i = 0; i < userCount; i++) {
141
168
  await writer.save({
142
169
  sourceTable,
@@ -150,7 +177,12 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
150
177
  }
151
178
  await writer.flush();
152
179
 
153
- expect(inserts.count('bucket_parameters')).toBeGreaterThan(1);
180
+ const inserted = statements.insertedRows<ParameterDataRow>('bucket_parameters');
181
+ expect(inserted.length).toBeGreaterThan(1);
182
+ expect(statements.transactionCount()).toEqual(1);
183
+ for (const statementRows of inserted) {
184
+ expect(estimateParameterDataSize(statementRows)).toBeLessThanOrEqual(2_000);
185
+ }
154
186
 
155
187
  const rows = await factory.db.sql`
156
188
  SELECT
@@ -177,10 +209,10 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
177
209
 
178
210
  const tags = Array.from({ length: 300 }, (_, i) => `tag${i}`);
179
211
  const description = '0123456789'.repeat(30_000);
180
- const inserts = spyOnInserts();
181
212
 
182
213
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
183
214
  const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
215
+ const statements = spyOnStatements();
184
216
  await writer.save({
185
217
  sourceTable,
186
218
  tag: storage.SaveOperationTag.INSERT,
@@ -193,14 +225,103 @@ for (const storageVersion of TEST_STORAGE_VERSIONS) {
193
225
  });
194
226
  await writer.flush();
195
227
 
196
- // 300 buckets x 300kB exceeds MAX_FLUSH_STATEMENT_SIZE (64MB).
197
- expect(inserts.count('bucket_data')).toBeGreaterThan(1);
228
+ // 300 buckets x 300kB exceeds the default max_estimated_size (5MB), so the rows are inserted
229
+ // in multiple statements, all in the transaction of this batch.
230
+ expect(statements.insertCount('bucket_data')).toBeGreaterThan(1);
231
+ expect(statements.transactionCount()).toEqual(1);
198
232
 
199
233
  const rows = await readBucketData(factory.db, bucketStorage.replicationStreamId);
200
234
  expect(rows.length).toEqual(tags.length);
201
235
  expect(new Set(rows.map((row) => row.bucket_name)).size).toEqual(tags.length);
202
236
  for (const row of rows) {
203
- expect(JSON.parse(row.data).description).toEqual(description);
237
+ expect(row.op).toEqual('PUT');
238
+ expect(JSON.parse(row.data!).description).toEqual(description);
239
+ }
240
+ });
241
+
242
+ test('keeps the transaction size limit while inserting rows early', async () => {
243
+ // Rows inserted before the end of a batch still count towards max_estimated_size,
244
+ // so inserting early must not grow the transactions.
245
+ const setup = createPostgresTestSetup({ batchLimits: { max_estimated_size: 5_000, max_statement_size: 2_000 } });
246
+ await using factory = await setup.factory();
247
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(BY_TAG_SYNC_CONFIG, { storageVersion }));
248
+ const bucketStorage = factory.getInstance(syncRules);
249
+
250
+ const rowCount = 30;
251
+ const description = 'x'.repeat(500);
252
+
253
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
254
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
255
+ const statements = spyOnStatements();
256
+ for (let i = 0; i < rowCount; i++) {
257
+ await writer.save({
258
+ sourceTable,
259
+ tag: storage.SaveOperationTag.INSERT,
260
+ after: {
261
+ id: `test${i}`,
262
+ tags: JSON.stringify([`tag${i}`]),
263
+ description
264
+ },
265
+ afterReplicaId: test_utils.rid(`test${i}`)
266
+ });
267
+ }
268
+ await writer.flush();
269
+
270
+ // Roughly 700 bytes per row against a 5000 byte transaction limit.
271
+ expect(statements.transactionCount()).toBeGreaterThanOrEqual(3);
272
+
273
+ const rows = await readBucketData(factory.db, bucketStorage.replicationStreamId);
274
+ expect(rows.length).toEqual(rowCount);
275
+ for (let i = 1; i < rows.length; i++) {
276
+ expect(rows[i].op_id).toBeGreaterThan(rows[i - 1].op_id);
277
+ }
278
+ });
279
+
280
+ test('inserts the REMOVE operations of a truncate in statements bounded by max_statement_size', async () => {
281
+ // The REMOVE operations are inserted on the truncate transaction while its current_data rows
282
+ // are processed, on the connection those rows were read from.
283
+ const setup = createPostgresTestSetup({ batchLimits: { max_statement_size: 2_000 } });
284
+ await using factory = await setup.factory();
285
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(BY_TAG_SYNC_CONFIG, { storageVersion }));
286
+ const bucketStorage = factory.getInstance(syncRules);
287
+
288
+ const rowCount = 20;
289
+ const tags = Array.from({ length: 20 }, (_, i) => `tag${i}`);
290
+
291
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
292
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
293
+ for (let i = 0; i < rowCount; i++) {
294
+ await writer.save({
295
+ sourceTable,
296
+ tag: storage.SaveOperationTag.INSERT,
297
+ after: {
298
+ id: `test${i}`,
299
+ tags: JSON.stringify(tags)
300
+ },
301
+ afterReplicaId: test_utils.rid(`test${i}`)
302
+ });
303
+ }
304
+ await writer.flush();
305
+
306
+ const statements = spyOnStatements();
307
+ await writer.truncate([sourceTable]);
308
+
309
+ // 400 REMOVE operations of roughly 150 bytes each, all in the transaction of the truncate.
310
+ const inserted = statements.insertedRows<BucketDataRow>('bucket_data');
311
+ expect(inserted.length).toBeGreaterThan(1);
312
+ expect(statements.transactionCount()).toEqual(1);
313
+ for (const statementRows of inserted) {
314
+ expect(estimateBucketDataSize(statementRows)).toBeLessThanOrEqual(2_000);
315
+ }
316
+
317
+ const rows = await readBucketData(factory.db, bucketStorage.replicationStreamId);
318
+ const operationCount = rowCount * tags.length;
319
+ expect(rows.length).toEqual(operationCount * 2);
320
+ // The REMOVE operations follow the PUT operations, with op_ids in insertion order.
321
+ expect(rows.slice(0, operationCount).every((row) => row.op == 'PUT')).toBe(true);
322
+ expect(rows.slice(operationCount).every((row) => row.op == 'REMOVE')).toBe(true);
323
+ for (let i = 1; i < rows.length; i++) {
324
+ expect(rows[i].op_id).toBeGreaterThan(rows[i - 1].op_id);
204
325
  }
205
326
  });
206
327
  });