@powersync/service-module-postgres-storage 0.19.4 → 0.19.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/CHANGELOG.md +41 -0
  2. package/LICENSE +201 -67
  3. package/dist/storage/PostgresBucketStorageFactory.d.ts +2 -0
  4. package/dist/storage/PostgresBucketStorageFactory.js +105 -95
  5. package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
  6. package/dist/storage/PostgresReportStorage.js +1 -2
  7. package/dist/storage/PostgresReportStorage.js.map +1 -1
  8. package/dist/storage/PostgresSyncRulesStorage.js +1 -2
  9. package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
  10. package/dist/storage/batch/PostgresBucketBatch.js +14 -4
  11. package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
  12. package/dist/storage/batch/PostgresPersistedBatch.d.ts +15 -0
  13. package/dist/storage/batch/PostgresPersistedBatch.js +58 -4
  14. package/dist/storage/batch/PostgresPersistedBatch.js.map +1 -1
  15. package/dist/types/types.d.ts +22 -1
  16. package/dist/types/types.js +18 -1
  17. package/dist/types/types.js.map +1 -1
  18. package/dist/utils/test-utils.d.ts +6 -1
  19. package/dist/utils/test-utils.js +5 -1
  20. package/dist/utils/test-utils.js.map +1 -1
  21. package/package.json +10 -11
  22. package/src/storage/PostgresBucketStorageFactory.ts +114 -96
  23. package/src/storage/PostgresReportStorage.ts +1 -2
  24. package/src/storage/PostgresSyncRulesStorage.ts +1 -2
  25. package/src/storage/batch/PostgresBucketBatch.ts +16 -6
  26. package/src/storage/batch/PostgresPersistedBatch.ts +69 -4
  27. package/src/types/types.ts +25 -2
  28. package/src/utils/test-utils.ts +16 -2
  29. package/test/src/invalid_sync_config.test.ts +100 -0
  30. package/test/src/storage_flush.test.ts +207 -0
  31. package/test/src/util.ts +10 -5
  32. package/tsconfig.tsbuildinfo +1 -1
@@ -3,7 +3,11 @@ import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOption
3
3
  import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js';
4
4
  import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
5
5
  import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
6
- import { normalizePostgresStorageConfig, PostgresStorageConfigDecoded } from '../types/types.js';
6
+ import {
7
+ normalizePostgresStorageConfig,
8
+ PostgresStorageConfigDecoded,
9
+ RequiredOperationBatchLimits
10
+ } from '../types/types.js';
7
11
  import { truncateTables } from './db.js';
8
12
 
9
13
  export type PostgresTestStorageOptions = {
@@ -14,6 +18,12 @@ export type PostgresTestStorageOptions = {
14
18
  * This allows for providing a custom PostgresMigrationAgent.
15
19
  */
16
20
  migrationAgent?: (config: PostgresStorageConfigDecoded) => PostgresMigrationAgent;
21
+
22
+ /**
23
+ * Override the default batch limits for the storage factory, including the internal limits
24
+ * that are not part of the storage config.
25
+ */
26
+ batchLimits?: Partial<RequiredOperationBatchLimits>;
17
27
  };
18
28
 
19
29
  export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
@@ -23,7 +33,11 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
23
33
  sslmode: 'disable' as const
24
34
  };
25
35
 
26
- const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG);
36
+ const normalizedConfig = normalizePostgresStorageConfig(BASE_CONFIG);
37
+ const TEST_CONNECTION_OPTIONS = {
38
+ ...normalizedConfig,
39
+ batch_limits: { ...normalizedConfig.batch_limits, ...factoryOptions.batchLimits }
40
+ };
27
41
 
28
42
  const runMigrations = async (options: { down: boolean; up: boolean }) => {
29
43
  await using migrationManager: PowerSyncMigrationManager = new framework.MigrationManager();
@@ -0,0 +1,100 @@
1
+ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
2
+ import { register, test_utils } from '@powersync/service-core-tests';
3
+ import { describe, expect, test, vi } from 'vitest';
4
+ import { POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
5
+
6
+ for (const storageVersion of TEST_STORAGE_VERSIONS) {
7
+ describe(`storage v${storageVersion}`, () => {
8
+ register.registerInvalidSyncConfigTests(POSTGRES_STORAGE_FACTORY, storageVersion);
9
+
10
+ test.each([{ deploymentTiming: 'before recovery reads' }, { deploymentTiming: 'after recovery reads' }] as const)(
11
+ 'preserves a deployment submitted after cancellation: $deploymentTiming',
12
+ async ({ deploymentTiming }) => {
13
+ await using backend = await POSTGRES_STORAGE_FACTORY.factory();
14
+ await using otherWorker = await POSTGRES_STORAGE_FACTORY.factory({ doNotClear: true });
15
+ const content = /* yaml */ `
16
+ # Existing active deployment.
17
+ config:
18
+ edition: 3
19
+ streams:
20
+ todos:
21
+ query: SELECT id FROM todos
22
+ `;
23
+ const original = await backend.updateSyncRules(
24
+ updateSyncRulesFromYaml(content, { storageVersion, version_label: 'active' })
25
+ );
26
+ const instance = await test_utils.getTestStorage(backend, original);
27
+ {
28
+ await using writer = await instance.createWriter(test_utils.BATCH_OPTIONS);
29
+ await writer.markAllSnapshotDone('1/1');
30
+ await writer.commit('1/1');
31
+ }
32
+ await backend.restartReplication(original.replicationStreamId);
33
+ // Model rejected startup cancelling recovery while A remains errored and readable.
34
+ await backend.stopProcessingSyncConfigs();
35
+ expect((await backend.getActiveSyncConfig())?.replicationStream.state).toBe(storage.SyncRuleState.ERRORED);
36
+
37
+ const replacement = updateSyncRulesFromYaml(content + '\n# New valid deployment C', {
38
+ storageVersion,
39
+ version_label: 'newer'
40
+ });
41
+ const transaction = backend.db.transaction.bind(backend.db);
42
+ const recoveryRead = Promise.withResolvers<void>();
43
+ const resumeRecovery = Promise.withResolvers<void>();
44
+ using race = vi.spyOn(backend.db, 'transaction').mockImplementationOnce((callback) =>
45
+ transaction(async (db) => {
46
+ if (deploymentTiming === 'before recovery reads') {
47
+ // Commit C at the former gap between recovery's read and replacement transaction.
48
+ await otherWorker.updateSyncRules(replacement);
49
+ } else {
50
+ const queryRows = db.queryRows.bind(db);
51
+ // Pause after the recovery snapshot is read, keeping its transaction open.
52
+ vi.spyOn(db, 'queryRows').mockImplementationOnce(async function <T>(
53
+ ...args: Parameters<typeof db.queryRows>
54
+ ): Promise<T[]> {
55
+ const rows = await queryRows<T>(...args);
56
+ recoveryRead.resolve();
57
+ await resumeRecovery.promise;
58
+ return rows;
59
+ });
60
+ }
61
+ return callback(db);
62
+ })
63
+ );
64
+ const recovering = backend.restartReplication(original.replicationStreamId);
65
+ if (deploymentTiming === 'before recovery reads') {
66
+ await recovering;
67
+ } else {
68
+ await recoveryRead.promise;
69
+ let deploymentFinished = false;
70
+ const deploying = otherWorker.updateSyncRules(replacement).finally(() => {
71
+ deploymentFinished = true;
72
+ });
73
+ try {
74
+ // Wait until C either commits or is blocked by recovery's metadata lock. Without the
75
+ // lock it commits first, and recovery then incorrectly cancels it using its stale read.
76
+ await expect
77
+ .poll(async () => {
78
+ const [waiting] = await backend.db.sql`
79
+ SELECT EXISTS (
80
+ SELECT 1 FROM pg_locks
81
+ WHERE relation = 'powersync.sync_rules'::regclass AND NOT granted
82
+ ) AS blocked
83
+ `.rows<{ blocked: boolean }>();
84
+ return deploymentFinished || waiting.blocked;
85
+ })
86
+ .toBe(true);
87
+ } finally {
88
+ resumeRecovery.resolve();
89
+ await Promise.all([recovering, deploying]);
90
+ }
91
+ }
92
+
93
+ const pending = await backend.getDeployingSyncConfig();
94
+ expect(pending?.content.sync_rules_content).toBe(replacement.config.yaml);
95
+ expect(pending?.content.version_label).toBe('newer');
96
+ expect((await backend.getActiveSyncConfig())?.content.sync_rules_content).toBe(content);
97
+ }
98
+ );
99
+ });
100
+ }
@@ -0,0 +1,207 @@
1
+ import * as lib_postgres from '@powersync/lib-service-postgres';
2
+ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
3
+ import { test_utils } from '@powersync/service-core-tests';
4
+ import { afterEach, describe, expect, test, vi } from 'vitest';
5
+ import { chunkRowsBySize } from '../../src/storage/batch/PostgresPersistedBatch.js';
6
+ import { createPostgresTestSetup, POSTGRES_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
7
+
8
+ /**
9
+ * Evaluates each row of "test" into one bucket per entry in its "tags" JSON array.
10
+ */
11
+ const BY_TAG_SYNC_CONFIG = `
12
+ config:
13
+ edition: 3
14
+
15
+ streams:
16
+ by_tag:
17
+ query: SELECT * FROM test WHERE subscription.parameter('tag') IN (SELECT value FROM json_each(test.tags))
18
+ `;
19
+
20
+ /**
21
+ * Stores a parameter lookup row for each row of "users".
22
+ */
23
+ const BY_OWNER_SYNC_CONFIG = `
24
+ config:
25
+ edition: 3
26
+
27
+ streams:
28
+ by_owner:
29
+ query: SELECT * FROM test WHERE owner_id IN (SELECT id FROM users WHERE name = subscription.parameter('name'))
30
+ `;
31
+
32
+ /**
33
+ * Records the INSERT statements executed on any connection, by target table.
34
+ */
35
+ function spyOnInserts() {
36
+ const spy = vi.spyOn(lib_postgres.AbstractPostgresConnection.prototype, 'query');
37
+ return {
38
+ count(table: string) {
39
+ const pattern = new RegExp(`INSERT INTO\\s+${table}\\b`);
40
+ return spy.mock.calls.filter(([statement]) => {
41
+ return typeof statement == 'object' && pattern.test(statement.statement);
42
+ }).length;
43
+ }
44
+ };
45
+ }
46
+
47
+ async function readBucketData(db: lib_postgres.DatabaseClient, groupId: number) {
48
+ return await db.sql`
49
+ SELECT
50
+ bucket_name,
51
+ op_id,
52
+ data
53
+ FROM
54
+ bucket_data
55
+ WHERE
56
+ group_id = ${{ type: 'int4', value: groupId }}
57
+ ORDER BY
58
+ op_id
59
+ `.rows<{ bucket_name: string; op_id: bigint; data: string }>();
60
+ }
61
+
62
+ describe('chunkRowsBySize', () => {
63
+ const size = (row: string) => row.length;
64
+
65
+ test('splits rows into consecutive chunks by estimated size', () => {
66
+ expect(chunkRowsBySize(['aa', 'bb', 'cc', 'dd', 'e'], size, 4)).toEqual([['aa', 'bb'], ['cc', 'dd'], ['e']]);
67
+ });
68
+
69
+ test('keeps all rows in one chunk within the limit', () => {
70
+ expect(chunkRowsBySize(['aa', 'bb'], size, 4)).toEqual([['aa', 'bb']]);
71
+ });
72
+
73
+ test('places a single row larger than the limit in its own chunk', () => {
74
+ expect(chunkRowsBySize(['aa', 'bbbbbbbb', 'cc', 'dd'], size, 4)).toEqual([['aa'], ['bbbbbbbb'], ['cc', 'dd']]);
75
+ });
76
+
77
+ test('returns no chunks for no rows', () => {
78
+ expect(chunkRowsBySize([], size, 4)).toEqual([]);
79
+ });
80
+ });
81
+
82
+ for (const storageVersion of TEST_STORAGE_VERSIONS) {
83
+ describe(`Postgres Sync Bucket Storage - flush chunking - v${storageVersion}`, () => {
84
+ afterEach(() => {
85
+ vi.restoreAllMocks();
86
+ });
87
+
88
+ test('splits bucket_data inserts by max_statement_size', async () => {
89
+ const setup = createPostgresTestSetup({ batchLimits: { max_statement_size: 2_000 } });
90
+ await using factory = await setup.factory();
91
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(BY_TAG_SYNC_CONFIG, { storageVersion }));
92
+ const bucketStorage = factory.getInstance(syncRules);
93
+
94
+ const tags = Array.from({ length: 50 }, (_, i) => `tag${i}`);
95
+ const description = 'x'.repeat(500);
96
+ const inserts = spyOnInserts();
97
+
98
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
99
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
100
+ await writer.save({
101
+ sourceTable,
102
+ tag: storage.SaveOperationTag.INSERT,
103
+ after: {
104
+ id: 'test1',
105
+ tags: JSON.stringify(tags),
106
+ description
107
+ },
108
+ afterReplicaId: test_utils.rid('test1')
109
+ });
110
+ await writer.flush();
111
+
112
+ // Each row is roughly 600 bytes, so this needs multiple statements.
113
+ expect(inserts.count('bucket_data')).toBeGreaterThan(1);
114
+
115
+ const rows = await readBucketData(factory.db, bucketStorage.replicationStreamId);
116
+ expect(rows.length).toEqual(tags.length);
117
+ // One PUT operation per bucket, with op_ids in insertion order.
118
+ expect(new Set(rows.map((row) => row.bucket_name)).size).toEqual(tags.length);
119
+ for (let i = 1; i < rows.length; i++) {
120
+ expect(rows[i].op_id).toBeGreaterThan(rows[i - 1].op_id);
121
+ }
122
+ for (const row of rows) {
123
+ expect(JSON.parse(row.data)).toEqual({ id: 'test1', tags: JSON.stringify(tags), description });
124
+ }
125
+ });
126
+
127
+ test('splits bucket_parameters inserts by max_statement_size', async () => {
128
+ const setup = createPostgresTestSetup({ batchLimits: { max_statement_size: 2_000 } });
129
+ await using factory = await setup.factory();
130
+ const syncRules = await factory.updateSyncRules(
131
+ updateSyncRulesFromYaml(BY_OWNER_SYNC_CONFIG, { storageVersion })
132
+ );
133
+ const bucketStorage = factory.getInstance(syncRules);
134
+
135
+ const userCount = 30;
136
+ const inserts = spyOnInserts();
137
+
138
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
139
+ const sourceTable = await test_utils.resolveTestTable(writer, 'users', ['id'], POSTGRES_STORAGE_FACTORY, 2);
140
+ for (let i = 0; i < userCount; i++) {
141
+ await writer.save({
142
+ sourceTable,
143
+ tag: storage.SaveOperationTag.INSERT,
144
+ after: {
145
+ id: `user${i}`,
146
+ name: `name${i}`
147
+ },
148
+ afterReplicaId: test_utils.rid(`user${i}`)
149
+ });
150
+ }
151
+ await writer.flush();
152
+
153
+ expect(inserts.count('bucket_parameters')).toBeGreaterThan(1);
154
+
155
+ const rows = await factory.db.sql`
156
+ SELECT
157
+ bucket_parameters
158
+ FROM
159
+ bucket_parameters
160
+ WHERE
161
+ group_id = ${{ type: 'int4', value: bucketStorage.replicationStreamId }}
162
+ ORDER BY
163
+ id
164
+ `.rows<{ bucket_parameters: string }>();
165
+ // The looked-up "id" column is stored by position, in insertion order.
166
+ expect(rows.map((row) => JSON.parse(row.bucket_parameters))).toEqual(
167
+ Array.from({ length: userCount }, (_, i) => [{ 0: `user${i}` }])
168
+ );
169
+ });
170
+
171
+ test('flushes a single row evaluated into many buckets with large data', { timeout: 30_000 }, async () => {
172
+ // Regression test: a single row with large data evaluated into many buckets could
173
+ // previously exceed the maximum string length when serializing the INSERT parameter.
174
+ await using factory = await POSTGRES_STORAGE_FACTORY.factory();
175
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(BY_TAG_SYNC_CONFIG, { storageVersion }));
176
+ const bucketStorage = factory.getInstance(syncRules);
177
+
178
+ const tags = Array.from({ length: 300 }, (_, i) => `tag${i}`);
179
+ const description = '0123456789'.repeat(30_000);
180
+ const inserts = spyOnInserts();
181
+
182
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
183
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], POSTGRES_STORAGE_FACTORY);
184
+ await writer.save({
185
+ sourceTable,
186
+ tag: storage.SaveOperationTag.INSERT,
187
+ after: {
188
+ id: 'large1',
189
+ tags: JSON.stringify(tags),
190
+ description
191
+ },
192
+ afterReplicaId: test_utils.rid('large1')
193
+ });
194
+ await writer.flush();
195
+
196
+ // 300 buckets x 300kB exceeds MAX_FLUSH_STATEMENT_SIZE (64MB).
197
+ expect(inserts.count('bucket_data')).toBeGreaterThan(1);
198
+
199
+ const rows = await readBucketData(factory.db, bucketStorage.replicationStreamId);
200
+ expect(rows.length).toEqual(tags.length);
201
+ expect(new Set(rows.map((row) => row.bucket_name)).size).toEqual(tags.length);
202
+ for (const row of rows) {
203
+ expect(JSON.parse(row.data).description).toEqual(description);
204
+ }
205
+ });
206
+ });
207
+ }
package/test/src/util.ts CHANGED
@@ -2,7 +2,7 @@ import { SUPPORTED_STORAGE_VERSIONS } from '@powersync/service-core';
2
2
  import path from 'path';
3
3
  import { fileURLToPath } from 'url';
4
4
  import { normalizePostgresStorageConfig, PostgresMigrationAgent } from '../../src/index.js';
5
- import { postgresTestSetup } from '../../src/utils/test-utils.js';
5
+ import { postgresTestSetup, PostgresTestStorageOptions } from '../../src/utils/test-utils.js';
6
6
  import { env } from './env.js';
7
7
 
8
8
  const __filename = fileURLToPath(import.meta.url);
@@ -28,10 +28,15 @@ class TestPostgresMigrationAgent extends PostgresMigrationAgent {
28
28
  }
29
29
  }
30
30
 
31
- export const POSTGRES_STORAGE_SETUP = postgresTestSetup({
32
- url: env.PG_STORAGE_TEST_URL,
33
- migrationAgent: (config) => new TestPostgresMigrationAgent(config)
34
- });
31
+ export function createPostgresTestSetup(options?: Pick<PostgresTestStorageOptions, 'batchLimits'>) {
32
+ return postgresTestSetup({
33
+ url: env.PG_STORAGE_TEST_URL,
34
+ migrationAgent: (config) => new TestPostgresMigrationAgent(config),
35
+ ...options
36
+ });
37
+ }
38
+
39
+ export const POSTGRES_STORAGE_SETUP = createPostgresTestSetup();
35
40
 
36
41
  export const POSTGRES_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP;
37
42
  export const POSTGRES_REPORT_STORAGE_FACTORY = POSTGRES_STORAGE_SETUP.reportFactory;