@powersync/service-module-postgres-storage 0.19.4 → 0.19.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +34 -0
- package/LICENSE +201 -67
- package/dist/storage/PostgresBucketStorageFactory.d.ts +2 -0
- package/dist/storage/PostgresBucketStorageFactory.js +105 -95
- package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
- package/dist/storage/PostgresReportStorage.js +1 -2
- package/dist/storage/PostgresReportStorage.js.map +1 -1
- package/dist/storage/PostgresSyncRulesStorage.js +1 -2
- package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
- package/dist/storage/batch/PostgresBucketBatch.js +14 -4
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/storage/batch/PostgresPersistedBatch.d.ts +15 -0
- package/dist/storage/batch/PostgresPersistedBatch.js +58 -4
- package/dist/storage/batch/PostgresPersistedBatch.js.map +1 -1
- package/dist/types/types.d.ts +22 -1
- package/dist/types/types.js +18 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +6 -1
- package/dist/utils/test-utils.js +5 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/package.json +10 -11
- package/src/storage/PostgresBucketStorageFactory.ts +114 -96
- package/src/storage/PostgresReportStorage.ts +1 -2
- package/src/storage/PostgresSyncRulesStorage.ts +1 -2
- package/src/storage/batch/PostgresBucketBatch.ts +16 -6
- package/src/storage/batch/PostgresPersistedBatch.ts +69 -4
- package/src/types/types.ts +25 -2
- package/src/utils/test-utils.ts +16 -2
- package/test/src/invalid_sync_config.test.ts +100 -0
- package/test/src/storage_flush.test.ts +207 -0
- package/test/src/util.ts +10 -5
- package/tsconfig.tsbuildinfo +1 -1
package/dist/types/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAEhE,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,KAAK,MAAM,MAAM,0BAA0B,CAAC;AAEnD,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAE3C,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAElD,MAAM,CAAC,MAAM,2BAA2B,GAAG,UAAU,CAAC;AAEtD,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC;;OAEG;IACH,kBAAkB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACvC;;OAEG;IACH,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACtC,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,oBAAoB,GAAG,WAAW,CAAC,GAAG,CACjD,CAAC,CAAC,MAAM,CAAC;IACP;;OAEG;IACH,2BAA2B,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACjD,CAAC,CACH,CAAC;AAIF,MAAM,CAAC,MAAM,qBAAqB,GAAG,UAAU,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,4BAA4B,CAAC,CAAC,GAAG,CAClH,CAAC,CAAC,MAAM,CAAC;IACP;;;;;OAKG;IACH,YAAY,EAAE,oBAAoB,CAAC,QAAQ,EAAE;CAC9C,CAAC,CACH,CAAC;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAEhE,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,KAAK,MAAM,MAAM,0BAA0B,CAAC;AAEnD,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAE3C,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAElD,MAAM,CAAC,MAAM,2BAA2B,GAAG,UAAU,CAAC;AAEtD;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,UAAU,CAAC;AAEnD,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC;;OAEG;IACH,kBAAkB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACvC;;OAEG;IACH,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACtC,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,oBAAoB,GAAG,WAAW,CAAC,GAAG,CACjD,CAAC,CAAC,MAAM,CAAC;IACP;;OAEG;IACH,2BAA2B,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACjD,CAAC,CACH,CAAC;AAIF,MAAM,CAAC,MAAM,qBAAqB,GAAG,UAAU,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,4BAA4B,CAAC,CAAC,GAAG,CAClH,CAAC,CAAC,MAAM,CAAC;IACP;;;;;OAKG;IACH,YAAY,EAAE,oBAAoB,CAAC,QAAQ,EAAE;CAC9C,CAAC,CACH,CAAC;AAiBF,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAC5C,UAAwC,EACP,EAAE;IACnC,OAAO;QACL,GAAG,YAAY,CAAC,yBAAyB,CAAC,UAAU,CAAC;QACrD,YAAY,EAAE;YACZ,2BAA2B,EAAE,UAAU,CAAC,YAAY,EAAE,2BAA2B,IAAI,2BAA2B;YAChH,kBAAkB,EAAE,UAAU,CAAC,YAAY,EAAE,kBAAkB,IAAI,wBAAwB;YAC3F,gBAAgB,EAAE,UAAU,CAAC,YAAY,EAAE,gBAAgB,IAAI,sBAAsB;YACrF,kBAAkB,EAAE,wBAAwB;SAC7C;KACF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,MAAoC,EAAmC,EAAE;IAC/G,OAAO,MAAM,CAAC,IAAI,IAAI,YAAY,CAAC,wBAAwB,CAAC;AAC9D,CAAC,CAAC"}
|
|
@@ -2,7 +2,7 @@ import { framework, TestStorageOptions } from '@powersync/service-core';
|
|
|
2
2
|
import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js';
|
|
3
3
|
import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
|
|
4
4
|
import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
|
|
5
|
-
import { PostgresStorageConfigDecoded } from '../types/types.js';
|
|
5
|
+
import { PostgresStorageConfigDecoded, RequiredOperationBatchLimits } from '../types/types.js';
|
|
6
6
|
export type PostgresTestStorageOptions = {
|
|
7
7
|
url: string;
|
|
8
8
|
/**
|
|
@@ -10,6 +10,11 @@ export type PostgresTestStorageOptions = {
|
|
|
10
10
|
* This allows for providing a custom PostgresMigrationAgent.
|
|
11
11
|
*/
|
|
12
12
|
migrationAgent?: (config: PostgresStorageConfigDecoded) => PostgresMigrationAgent;
|
|
13
|
+
/**
|
|
14
|
+
* Override the default batch limits for the storage factory, including the internal limits
|
|
15
|
+
* that are not part of the storage config.
|
|
16
|
+
*/
|
|
17
|
+
batchLimits?: Partial<RequiredOperationBatchLimits>;
|
|
13
18
|
};
|
|
14
19
|
export declare function postgresTestSetup(factoryOptions: PostgresTestStorageOptions): {
|
|
15
20
|
reportFactory: (options?: TestStorageOptions) => Promise<PostgresReportStorage>;
|
package/dist/utils/test-utils.js
CHANGED
|
@@ -11,7 +11,11 @@ export function postgresTestSetup(factoryOptions) {
|
|
|
11
11
|
uri: factoryOptions.url,
|
|
12
12
|
sslmode: 'disable'
|
|
13
13
|
};
|
|
14
|
-
const
|
|
14
|
+
const normalizedConfig = normalizePostgresStorageConfig(BASE_CONFIG);
|
|
15
|
+
const TEST_CONNECTION_OPTIONS = {
|
|
16
|
+
...normalizedConfig,
|
|
17
|
+
batch_limits: { ...normalizedConfig.batch_limits, ...factoryOptions.batchLimits }
|
|
18
|
+
};
|
|
15
19
|
const runMigrations = async (options) => {
|
|
16
20
|
await using migrationManager = new framework.MigrationManager();
|
|
17
21
|
await using migrationAgent = factoryOptions.migrationAgent
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,aAAa,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AACtG,OAAO,EAAE,SAAS,EAAiE,MAAM,yBAAyB,CAAC;AACnH,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACjF,OAAO,EAAE,4BAA4B,EAAE,MAAM,4CAA4C,CAAC;AAC1F,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,OAAO,
|
|
1
|
+
{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,aAAa,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AACtG,OAAO,EAAE,SAAS,EAAiE,MAAM,yBAAyB,CAAC;AACnH,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACjF,OAAO,EAAE,4BAA4B,EAAE,MAAM,4CAA4C,CAAC;AAC1F,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,OAAO,EACL,8BAA8B,EAG/B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAkBzC,MAAM,UAAU,iBAAiB,CAAC,cAA0C;IAC1E,MAAM,WAAW,GAAG;QAClB,IAAI,EAAE,YAAqB;QAC3B,GAAG,EAAE,cAAc,CAAC,GAAG;QACvB,OAAO,EAAE,SAAkB;KAC5B,CAAC;IAEF,MAAM,gBAAgB,GAAG,8BAA8B,CAAC,WAAW,CAAC,CAAC;IACrE,MAAM,uBAAuB,GAAG;QAC9B,GAAG,gBAAgB;QACnB,YAAY,EAAE,EAAE,GAAG,gBAAgB,CAAC,YAAY,EAAE,GAAG,cAAc,CAAC,WAAW,EAAE;KAClF,CAAC;IAEF,MAAM,aAAa,GAAG,KAAK,EAAE,OAAuC,EAAE,EAAE;QACtE,YAAY,gBAAgB,GAA8B,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;QAC3F,YAAY,cAAc,GAAG,cAAc,CAAC,cAAc;YACxD,CAAC,CAAC,cAAc,CAAC,cAAc,CAAC,WAAW,CAAC;YAC5C,CAAC,CAAC,IAAI,sBAAsB,CAAC,WAAW,CAAC,CAAC;QAC5C,gBAAgB,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;QAExD,MAAM,kBAAkB,GAAG,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAA+B,CAAC;QAEpG,+EAA+E;QAC/E,MAAM,MAAM,GAAG,YAAY,CAAC;YAC1B,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,UAAU,EAAE,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;SACvC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI;gBAC9C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC5C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,EAAE,SAAyC,EAAE,EAAE;QAClE,MAAM,aAAa,CAAC;YAClB,IAAI,EAAE,IAAI;YACV,EAAE,EAAE,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;SACnD,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;QAC9B,MAAM,aAAa,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,YAAY,cAAc,GAAG,IAAI,4BAA4B,CAAC;YAC5D,MAAM,EAAE,uBAAuB;YAC/B,2BAA2B,EAAE,OAAO;SACrC,CAAC,CAAC;QACH,MAAM,cAAc,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC,CAAC;IAEF,OAAO;QACL,aAAa,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YACpD,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,qBAAqB,CAAC;oBAC/B,MAAM,EAAE,uBAAuB;iBAChC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,4BAA4B,CAAC;oBACtC,MAAM,EAAE,uBAAuB;oBAC/B,2BAA2B,EAAE,OAAO;iBACrC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO;QACP,cAAc,EAAE,IAAI;KACrB,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@powersync/service-module-postgres-storage",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
|
-
"version": "0.19.
|
|
5
|
-
"license": "
|
|
4
|
+
"version": "0.19.5",
|
|
5
|
+
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"publishConfig": {
|
|
8
8
|
"access": "public"
|
|
@@ -12,21 +12,20 @@
|
|
|
12
12
|
"./types": "./dist/types/types.js"
|
|
13
13
|
},
|
|
14
14
|
"dependencies": {
|
|
15
|
-
"ix": "^5.0.0",
|
|
16
15
|
"lru-cache": "^10.2.2",
|
|
17
16
|
"ts-codec": "^1.3.0",
|
|
18
17
|
"uuid": "^14.0.0",
|
|
19
|
-
"@powersync/lib-service-postgres": "0.5.
|
|
20
|
-
"@powersync/lib-services-framework": "0.10.
|
|
21
|
-
"@powersync/service-
|
|
22
|
-
"@powersync/service-
|
|
23
|
-
"@powersync/service-
|
|
24
|
-
"@powersync/service-jsonbig": "0.17.
|
|
25
|
-
"@powersync/service-sync-rules": "0.
|
|
18
|
+
"@powersync/lib-service-postgres": "0.5.8",
|
|
19
|
+
"@powersync/lib-services-framework": "0.10.5",
|
|
20
|
+
"@powersync/service-types": "0.21.0",
|
|
21
|
+
"@powersync/service-jpgwire": "0.21.27",
|
|
22
|
+
"@powersync/service-core": "1.28.0",
|
|
23
|
+
"@powersync/service-jsonbig": "0.17.14",
|
|
24
|
+
"@powersync/service-sync-rules": "0.45.0"
|
|
26
25
|
},
|
|
27
26
|
"devDependencies": {
|
|
28
27
|
"typescript": "~6.0.3",
|
|
29
|
-
"@powersync/service-core-tests": "0.18.
|
|
28
|
+
"@powersync/service-core-tests": "0.18.5"
|
|
30
29
|
},
|
|
31
30
|
"scripts": {
|
|
32
31
|
"build": "tsc -b",
|
|
@@ -161,7 +161,24 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
161
161
|
};
|
|
162
162
|
}
|
|
163
163
|
|
|
164
|
+
async stopProcessingSyncConfigs(): Promise<void> {
|
|
165
|
+
await this.db.sql`
|
|
166
|
+
UPDATE sync_rules
|
|
167
|
+
SET
|
|
168
|
+
state = ${{ type: 'varchar', value: storage.SyncRuleState.STOP }}
|
|
169
|
+
WHERE
|
|
170
|
+
state = ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }}
|
|
171
|
+
`.execute();
|
|
172
|
+
}
|
|
173
|
+
|
|
164
174
|
async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<PostgresPersistedReplicationStream> {
|
|
175
|
+
return this.db.transaction((db) => this.updateSyncRulesInTransaction(options, db));
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
private async updateSyncRulesInTransaction(
|
|
179
|
+
options: storage.UpdateSyncRulesOptions,
|
|
180
|
+
db: lib_postgres.WrappedConnection
|
|
181
|
+
): Promise<PostgresPersistedReplicationStream> {
|
|
165
182
|
const storageVersion =
|
|
166
183
|
options.storageVersion ??
|
|
167
184
|
options.config.parsed.config.storageVersion ??
|
|
@@ -174,77 +191,78 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
174
191
|
`Unsupported storage version ${storageVersion} for PostgreSQL storage`
|
|
175
192
|
);
|
|
176
193
|
}
|
|
177
|
-
await this.initializeStorageVersion(storageConfig);
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
`.execute();
|
|
194
|
+
await this.initializeStorageVersion(storageConfig, db);
|
|
195
|
+
await db.sql`
|
|
196
|
+
UPDATE sync_rules
|
|
197
|
+
SET
|
|
198
|
+
state = ${{ type: 'varchar', value: storage.SyncRuleState.STOP }}
|
|
199
|
+
WHERE
|
|
200
|
+
state = ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }}
|
|
201
|
+
`.execute();
|
|
186
202
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
203
|
+
// Uses the legacy sequence name "sync_rules_id_sequence" for backwards-compatibility. See AGENTS.md.
|
|
204
|
+
const newSyncRulesRow = await db.sql`
|
|
205
|
+
WITH
|
|
206
|
+
next_id AS (
|
|
207
|
+
SELECT
|
|
208
|
+
nextval('sync_rules_id_sequence') AS id
|
|
209
|
+
)
|
|
210
|
+
INSERT INTO
|
|
211
|
+
sync_rules (
|
|
212
|
+
id,
|
|
213
|
+
content,
|
|
214
|
+
sync_plan,
|
|
215
|
+
state,
|
|
216
|
+
slot_name,
|
|
217
|
+
storage_version,
|
|
218
|
+
version_label
|
|
219
|
+
)
|
|
220
|
+
VALUES
|
|
221
|
+
(
|
|
205
222
|
(
|
|
223
|
+
SELECT
|
|
224
|
+
id
|
|
225
|
+
FROM
|
|
226
|
+
next_id
|
|
227
|
+
),
|
|
228
|
+
${{ type: 'varchar', value: options.config.yaml }},
|
|
229
|
+
${{ type: 'json', value: options.config.plan }},
|
|
230
|
+
${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }},
|
|
231
|
+
CONCAT(
|
|
232
|
+
${{ type: 'varchar', value: this.replicationStreamNamePrefix }},
|
|
206
233
|
(
|
|
207
234
|
SELECT
|
|
208
235
|
id
|
|
209
236
|
FROM
|
|
210
237
|
next_id
|
|
211
238
|
),
|
|
212
|
-
|
|
213
|
-
${{ type: '
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
'_',
|
|
224
|
-
${{ type: 'varchar', value: crypto.randomBytes(2).toString('hex') }}
|
|
225
|
-
),
|
|
226
|
-
${{ type: 'int4', value: storageVersion }},
|
|
227
|
-
${{ type: 'varchar', value: options.version_label ?? null }}
|
|
228
|
-
)
|
|
229
|
-
RETURNING
|
|
230
|
-
*
|
|
231
|
-
`
|
|
232
|
-
.decoded(models.SyncRules)
|
|
233
|
-
.first();
|
|
239
|
+
'_',
|
|
240
|
+
${{ type: 'varchar', value: crypto.randomBytes(2).toString('hex') }}
|
|
241
|
+
),
|
|
242
|
+
${{ type: 'int4', value: storageVersion }},
|
|
243
|
+
${{ type: 'varchar', value: options.version_label ?? null }}
|
|
244
|
+
)
|
|
245
|
+
RETURNING
|
|
246
|
+
*
|
|
247
|
+
`
|
|
248
|
+
.decoded(models.SyncRules)
|
|
249
|
+
.first();
|
|
234
250
|
|
|
235
|
-
|
|
236
|
-
});
|
|
251
|
+
return new PostgresPersistedReplicationStream(this.db, newSyncRulesRow!);
|
|
237
252
|
}
|
|
238
253
|
|
|
239
254
|
/**
|
|
240
255
|
* Lazy-initializes storage-version-specific structures, if needed.
|
|
241
256
|
*/
|
|
242
|
-
private async initializeStorageVersion(
|
|
257
|
+
private async initializeStorageVersion(
|
|
258
|
+
storageConfig: storage.StorageVersionConfig,
|
|
259
|
+
db: lib_postgres.WrappedConnection
|
|
260
|
+
) {
|
|
243
261
|
if (!storageConfig.softDeleteCurrentData) {
|
|
244
262
|
return;
|
|
245
263
|
}
|
|
246
264
|
|
|
247
|
-
await
|
|
265
|
+
await db.sql`
|
|
248
266
|
CREATE TABLE IF NOT EXISTS v3_current_data (
|
|
249
267
|
group_id integer NOT NULL,
|
|
250
268
|
source_table TEXT NOT NULL,
|
|
@@ -257,7 +275,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
257
275
|
)
|
|
258
276
|
`.execute();
|
|
259
277
|
|
|
260
|
-
await
|
|
278
|
+
await db.sql`
|
|
261
279
|
CREATE INDEX IF NOT EXISTS v3_current_data_pending_deletes ON v3_current_data (group_id, pending_delete)
|
|
262
280
|
WHERE
|
|
263
281
|
pending_delete IS NOT NULL
|
|
@@ -265,50 +283,50 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
|
|
|
265
283
|
}
|
|
266
284
|
|
|
267
285
|
async restartReplication(replicationStreamId: number): Promise<void> {
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
// Pro-actively stop replicating
|
|
278
|
-
await this.db.sql`
|
|
279
|
-
UPDATE sync_rules
|
|
280
|
-
SET
|
|
281
|
-
state = ${{ value: storage.SyncRuleState.STOP, type: 'varchar' }}
|
|
282
|
-
WHERE
|
|
283
|
-
id = ${{ value: next.content.replicationStreamId, type: 'int4' }}
|
|
284
|
-
AND state = ${{ value: storage.SyncRuleState.PROCESSING, type: 'varchar' }}
|
|
285
|
-
`.execute();
|
|
286
|
-
} else if (next == null && active?.content.replicationStreamId == replicationStreamId) {
|
|
287
|
-
// Slot removed for "active" replication stream, while there is no "next" one.
|
|
288
|
-
await this.updateSyncRules(active.content.asUpdateOptions());
|
|
289
|
-
|
|
290
|
-
// Pro-actively stop replicating, but still serve clients with existing data
|
|
291
|
-
await this.db.sql`
|
|
292
|
-
UPDATE sync_rules
|
|
293
|
-
SET
|
|
294
|
-
state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
|
|
286
|
+
await this.db.transaction(async (db) => {
|
|
287
|
+
// Protect the absence of pending work as well as existing rows. Deployments and activation
|
|
288
|
+
// take ROW EXCLUSIVE locks, so neither can change the decision before recovery commits.
|
|
289
|
+
await db.sql`LOCK TABLE sync_rules IN SHARE ROW EXCLUSIVE MODE`.execute();
|
|
290
|
+
const rows = await db.sql`
|
|
291
|
+
SELECT
|
|
292
|
+
*
|
|
293
|
+
FROM
|
|
294
|
+
sync_rules
|
|
295
295
|
WHERE
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
296
|
+
state IN (
|
|
297
|
+
${{ value: storage.SyncRuleState.PROCESSING, type: 'varchar' }},
|
|
298
|
+
${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }},
|
|
299
|
+
${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
|
|
300
|
+
)
|
|
301
|
+
ORDER BY
|
|
302
|
+
id DESC
|
|
303
|
+
`
|
|
304
|
+
.decoded(models.SyncRules)
|
|
305
|
+
.rows();
|
|
306
|
+
const next = rows.find((row) => row.state === storage.SyncRuleState.PROCESSING);
|
|
307
|
+
const active = rows.find(
|
|
308
|
+
(row) => row.state === storage.SyncRuleState.ACTIVE || row.state === storage.SyncRuleState.ERRORED
|
|
309
|
+
);
|
|
301
310
|
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
311
|
+
if (next?.id === replicationStreamId) {
|
|
312
|
+
// Replace the failed pending stream only while it is still the current deployment.
|
|
313
|
+
await this.updateSyncRulesInTransaction(this.resolvedSyncConfigFromRow(next).content.asUpdateOptions(), db);
|
|
314
|
+
} else if (active?.id === replicationStreamId) {
|
|
315
|
+
if (next == null) {
|
|
316
|
+
await this.updateSyncRulesInTransaction(this.resolvedSyncConfigFromRow(active).content.asUpdateOptions(), db);
|
|
317
|
+
}
|
|
318
|
+
// Keep serving the old data, but stop replication. An existing newer deployment takes
|
|
319
|
+
// precedence over recovery and must remain eligible to activate.
|
|
320
|
+
await db.sql`
|
|
321
|
+
UPDATE sync_rules
|
|
322
|
+
SET
|
|
323
|
+
state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
|
|
324
|
+
WHERE
|
|
325
|
+
id = ${{ value: active.id, type: 'int4' }}
|
|
326
|
+
AND state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
|
|
327
|
+
`.execute();
|
|
328
|
+
}
|
|
329
|
+
});
|
|
312
330
|
}
|
|
313
331
|
|
|
314
332
|
async getActiveSyncConfig(): Promise<storage.ResolvedSyncConfig | null> {
|
|
@@ -3,7 +3,6 @@ import { logger } from '@powersync/lib-services-framework';
|
|
|
3
3
|
import { storage } from '@powersync/service-core';
|
|
4
4
|
import * as pg_wire from '@powersync/service-jpgwire';
|
|
5
5
|
import { event_types } from '@powersync/service-types';
|
|
6
|
-
import { toInteger } from 'ix/util/tointeger.js';
|
|
7
6
|
import { v4 } from 'uuid';
|
|
8
7
|
import { SdkReporting, SdkReportingDecoded } from '../types/models/SdkReporting.js';
|
|
9
8
|
import { NormalizedPostgresStorageConfig } from '../types/types.js';
|
|
@@ -329,7 +328,7 @@ export class PostgresReportStorage implements storage.ReportStorage {
|
|
|
329
328
|
)
|
|
330
329
|
);
|
|
331
330
|
`.execute();
|
|
332
|
-
const deletedRows =
|
|
331
|
+
const deletedRows = Number(result.results[1].status.split(' ')[1] || '0');
|
|
333
332
|
if (deletedRows > 0) {
|
|
334
333
|
logger.info(
|
|
335
334
|
`TTL from ${date.toISOString()}: ${deletedRows} PostgresSQL rows have been removed from connection_report_events.`
|
|
@@ -30,7 +30,6 @@ import { mapOpEntry } from '../utils/bucket-data.js';
|
|
|
30
30
|
|
|
31
31
|
import * as framework from '@powersync/lib-services-framework';
|
|
32
32
|
import type { Statement } from '@powersync/service-jpgwire';
|
|
33
|
-
import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
|
|
34
33
|
import * as t from 'ts-codec';
|
|
35
34
|
import { ActiveCheckpointDecoded } from '../types/models/ActiveCheckpoint.js';
|
|
36
35
|
import * as checkpointUtils from '../utils/checkpoints.js';
|
|
@@ -716,7 +715,7 @@ export class PostgresSyncRulesStorage
|
|
|
716
715
|
|
|
717
716
|
const { signal, user_id } = options;
|
|
718
717
|
|
|
719
|
-
const iter =
|
|
718
|
+
const iter = this.sharedIterator[Symbol.asyncIterator](signal);
|
|
720
719
|
for await (const cp of iter) {
|
|
721
720
|
const { checkpoint, lsn } = cp;
|
|
722
721
|
|
|
@@ -202,6 +202,19 @@ export class PostgresBucketBatch
|
|
|
202
202
|
const resolution = await reconcile({ source, candidates });
|
|
203
203
|
storage.validateSourceTableCandidateResolution(candidates, resolution);
|
|
204
204
|
|
|
205
|
+
const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
|
|
206
|
+
const compatibleTables = materializedResolution.compatibleTables;
|
|
207
|
+
|
|
208
|
+
if (!syncRules.tableSyncsAny(source)) {
|
|
209
|
+
// Still reconcile old mappings so renames out of scope remove previously replicated data.
|
|
210
|
+
// Skip updating source_metadata below: these records are dropped by the caller in this
|
|
211
|
+
// same transaction, so persisting new metadata into them first is wasted work.
|
|
212
|
+
return {
|
|
213
|
+
tables: [],
|
|
214
|
+
dropTables: [...materializedResolution.incompatibleTables, ...compatibleTables]
|
|
215
|
+
};
|
|
216
|
+
}
|
|
217
|
+
|
|
205
218
|
for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
|
|
206
219
|
await db.sql`
|
|
207
220
|
UPDATE source_tables
|
|
@@ -212,9 +225,6 @@ export class PostgresBucketBatch
|
|
|
212
225
|
`.execute();
|
|
213
226
|
}
|
|
214
227
|
|
|
215
|
-
const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
|
|
216
|
-
const compatibleTables = materializedResolution.compatibleTables;
|
|
217
|
-
|
|
218
228
|
// Keep one record, preferring one that has already been snapshotted.
|
|
219
229
|
const reuse = compatibleTables.find((candidate) => candidate.snapshotComplete) ?? compatibleTables[0] ?? null;
|
|
220
230
|
|
|
@@ -1060,9 +1070,9 @@ export class PostgresBucketBatch
|
|
|
1060
1070
|
// Not an error if we re-apply a transaction
|
|
1061
1071
|
existingBuckets = [];
|
|
1062
1072
|
existingLookups = [];
|
|
1063
|
-
//
|
|
1064
|
-
|
|
1065
|
-
if (storeCurrentData) {
|
|
1073
|
+
// A complete row is evaluated and stored in current_data below, so a resnapshot is only
|
|
1074
|
+
// needed when TOAST values are missing.
|
|
1075
|
+
if (!utils.isCompleteRow(storeCurrentData, after!)) {
|
|
1066
1076
|
if (this.markRecordUnavailable != null) {
|
|
1067
1077
|
// This will trigger a "resnapshot" of the record.
|
|
1068
1078
|
// This is not relevant if storeCurrentData is false, since we'll get the full row
|
|
@@ -62,6 +62,7 @@ export class PostgresPersistedBatch {
|
|
|
62
62
|
|
|
63
63
|
readonly maxTransactionBatchSize: number;
|
|
64
64
|
readonly maxTransactionDocCount: number;
|
|
65
|
+
readonly maxStatementSize: number;
|
|
65
66
|
|
|
66
67
|
/**
|
|
67
68
|
* Ordered set of bucket_data insert operation parameters
|
|
@@ -83,6 +84,7 @@ export class PostgresPersistedBatch {
|
|
|
83
84
|
|
|
84
85
|
this.maxTransactionBatchSize = options.max_estimated_size;
|
|
85
86
|
this.maxTransactionDocCount = options.max_record_count;
|
|
87
|
+
this.maxStatementSize = options.max_statement_size;
|
|
86
88
|
|
|
87
89
|
this.bucketDataInserts = [];
|
|
88
90
|
this.parameterDataInserts = [];
|
|
@@ -304,7 +306,20 @@ export class PostgresPersistedBatch {
|
|
|
304
306
|
}
|
|
305
307
|
|
|
306
308
|
protected async flushBucketData(db: lib_postgres.WrappedConnection) {
|
|
307
|
-
if (this.bucketDataInserts.length
|
|
309
|
+
if (this.bucketDataInserts.length == 0) {
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
const chunks = chunkRowsBySize(this.bucketDataInserts, estimateBucketDataRowSize, this.maxStatementSize);
|
|
314
|
+
if (chunks.length > 1) {
|
|
315
|
+
logger.debug(
|
|
316
|
+
`powersync_${this.group_id} Flushing ${this.bucketDataInserts.length} bucket_data rows in ${chunks.length} statements.`
|
|
317
|
+
);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
// Sequential statements in the same transaction preserve the order of the operations,
|
|
321
|
+
// since op_id is assigned from the sequence per row.
|
|
322
|
+
for (const chunk of chunks) {
|
|
308
323
|
await db.sql`
|
|
309
324
|
INSERT INTO
|
|
310
325
|
bucket_data (
|
|
@@ -333,7 +348,7 @@ export class PostgresPersistedBatch {
|
|
|
333
348
|
data,
|
|
334
349
|
target_op
|
|
335
350
|
FROM
|
|
336
|
-
json_to_recordset(${{ type: 'json', value:
|
|
351
|
+
json_to_recordset(${{ type: 'json', value: chunk }}::json) AS t (
|
|
337
352
|
group_id integer,
|
|
338
353
|
bucket_name text,
|
|
339
354
|
source_table text,
|
|
@@ -350,7 +365,18 @@ export class PostgresPersistedBatch {
|
|
|
350
365
|
}
|
|
351
366
|
|
|
352
367
|
protected async flushParameterData(db: lib_postgres.WrappedConnection) {
|
|
353
|
-
if (this.parameterDataInserts.length
|
|
368
|
+
if (this.parameterDataInserts.length == 0) {
|
|
369
|
+
return;
|
|
370
|
+
}
|
|
371
|
+
|
|
372
|
+
const chunks = chunkRowsBySize(this.parameterDataInserts, estimateParameterDataRowSize, this.maxStatementSize);
|
|
373
|
+
if (chunks.length > 1) {
|
|
374
|
+
logger.debug(
|
|
375
|
+
`powersync_${this.group_id} Flushing ${this.parameterDataInserts.length} bucket_parameters rows in ${chunks.length} statements.`
|
|
376
|
+
);
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
for (const chunk of chunks) {
|
|
354
380
|
await db.sql`
|
|
355
381
|
INSERT INTO
|
|
356
382
|
bucket_parameters (
|
|
@@ -367,7 +393,7 @@ export class PostgresPersistedBatch {
|
|
|
367
393
|
decode(lookup, 'hex') AS lookup, -- Decode hex to bytea
|
|
368
394
|
bucket_parameters
|
|
369
395
|
FROM
|
|
370
|
-
json_to_recordset(${{ type: 'json', value:
|
|
396
|
+
json_to_recordset(${{ type: 'json', value: chunk }}::json) AS t (
|
|
371
397
|
group_id integer,
|
|
372
398
|
source_table text,
|
|
373
399
|
source_key text, -- Input as hex string
|
|
@@ -417,3 +443,42 @@ export class PostgresPersistedBatch {
|
|
|
417
443
|
export function currentBucketKey(b: models.CurrentBucket) {
|
|
418
444
|
return `${b.bucket}/${b.table}/${b.id}`;
|
|
419
445
|
}
|
|
446
|
+
|
|
447
|
+
/**
|
|
448
|
+
* Rough estimate of the serialized size of a bucket_data row, matching the `currentSize` accounting.
|
|
449
|
+
*/
|
|
450
|
+
export function estimateBucketDataRowSize(row: models.BucketData) {
|
|
451
|
+
return row.bucket_name.length + (row.data?.length ?? 0) + (row.source_key?.length ?? 0) + 100;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* Rough estimate of the serialized size of a bucket_parameters row, matching the `currentSize` accounting.
|
|
456
|
+
*/
|
|
457
|
+
export function estimateParameterDataRowSize(row: models.BucketParameters) {
|
|
458
|
+
return row.lookup.length + row.bucket_parameters.length + row.source_key.length + 100;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* Splits rows into consecutive chunks, each with a total estimated size of at most `maxSize`.
|
|
463
|
+
*
|
|
464
|
+
* The order of the rows is preserved. A single row larger than `maxSize` is placed in its own chunk.
|
|
465
|
+
*/
|
|
466
|
+
export function chunkRowsBySize<T>(rows: T[], estimateSize: (row: T) => number, maxSize: number): T[][] {
|
|
467
|
+
const chunks: T[][] = [];
|
|
468
|
+
let currentChunk: T[] = [];
|
|
469
|
+
let currentChunkSize = 0;
|
|
470
|
+
for (const row of rows) {
|
|
471
|
+
const size = estimateSize(row);
|
|
472
|
+
if (currentChunk.length > 0 && currentChunkSize + size > maxSize) {
|
|
473
|
+
chunks.push(currentChunk);
|
|
474
|
+
currentChunk = [];
|
|
475
|
+
currentChunkSize = 0;
|
|
476
|
+
}
|
|
477
|
+
currentChunk.push(row);
|
|
478
|
+
currentChunkSize += size;
|
|
479
|
+
}
|
|
480
|
+
if (currentChunk.length > 0) {
|
|
481
|
+
chunks.push(currentChunk);
|
|
482
|
+
}
|
|
483
|
+
return chunks;
|
|
484
|
+
}
|
package/src/types/types.ts
CHANGED
|
@@ -10,6 +10,23 @@ export const MAX_BATCH_ESTIMATED_SIZE = 5_000_000;
|
|
|
10
10
|
|
|
11
11
|
export const MAX_BATCH_CURRENT_DATA_SIZE = 50_000_000;
|
|
12
12
|
|
|
13
|
+
/**
|
|
14
|
+
* Maximum estimated size of the rows written by a single INSERT statement when flushing
|
|
15
|
+
* bucket_data and bucket_parameters.
|
|
16
|
+
*
|
|
17
|
+
* A single source row can produce many operations when it is evaluated into many buckets,
|
|
18
|
+
* which is not limited by `max_estimated_size`. Flushes larger than this limit are split
|
|
19
|
+
* into multiple statements in the same transaction.
|
|
20
|
+
*
|
|
21
|
+
* The rows are passed to Postgres as a single JSON parameter, which is serialized to a string
|
|
22
|
+
* before being sent. Node limits a string to roughly 512MB. The estimate used for chunking does
|
|
23
|
+
* not account for the JSON escaping of the already-serialized row data, so the actual parameter
|
|
24
|
+
* can be close to double the estimate. 64MB keeps a wide margin for that.
|
|
25
|
+
*
|
|
26
|
+
* This is not part of `batch_limits`: the limit comes from the runtime, not from the workload.
|
|
27
|
+
*/
|
|
28
|
+
export const MAX_FLUSH_STATEMENT_SIZE = 64_000_000;
|
|
29
|
+
|
|
13
30
|
export const BatchLimits = t.object({
|
|
14
31
|
/**
|
|
15
32
|
* Maximum size of operations we write in a single transaction.
|
|
@@ -49,7 +66,12 @@ export const PostgresStorageConfig = configFile.BaseStorageConfig.and(lib_postgr
|
|
|
49
66
|
export type PostgresStorageConfig = t.Encoded<typeof PostgresStorageConfig>;
|
|
50
67
|
export type PostgresStorageConfigDecoded = t.Decoded<typeof PostgresStorageConfig>;
|
|
51
68
|
|
|
52
|
-
export type RequiredOperationBatchLimits = Required<OperationBatchLimits
|
|
69
|
+
export type RequiredOperationBatchLimits = Required<OperationBatchLimits> & {
|
|
70
|
+
/**
|
|
71
|
+
* Internal limit for the size of a single flush statement. See {@link MAX_FLUSH_STATEMENT_SIZE}.
|
|
72
|
+
*/
|
|
73
|
+
max_statement_size: number;
|
|
74
|
+
};
|
|
53
75
|
|
|
54
76
|
export type NormalizedPostgresStorageConfig = pg_wire.NormalizedConnectionConfig & {
|
|
55
77
|
batch_limits: RequiredOperationBatchLimits;
|
|
@@ -64,7 +86,8 @@ export const normalizePostgresStorageConfig = (
|
|
|
64
86
|
batch_limits: {
|
|
65
87
|
max_current_data_batch_size: baseConfig.batch_limits?.max_current_data_batch_size ?? MAX_BATCH_CURRENT_DATA_SIZE,
|
|
66
88
|
max_estimated_size: baseConfig.batch_limits?.max_estimated_size ?? MAX_BATCH_ESTIMATED_SIZE,
|
|
67
|
-
max_record_count: baseConfig.batch_limits?.max_record_count ?? MAX_BATCH_RECORD_COUNT
|
|
89
|
+
max_record_count: baseConfig.batch_limits?.max_record_count ?? MAX_BATCH_RECORD_COUNT,
|
|
90
|
+
max_statement_size: MAX_FLUSH_STATEMENT_SIZE
|
|
68
91
|
}
|
|
69
92
|
};
|
|
70
93
|
};
|
package/src/utils/test-utils.ts
CHANGED
|
@@ -3,7 +3,11 @@ import { framework, PowerSyncMigrationManager, ServiceContext, TestStorageOption
|
|
|
3
3
|
import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js';
|
|
4
4
|
import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
|
|
5
5
|
import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
|
|
6
|
-
import {
|
|
6
|
+
import {
|
|
7
|
+
normalizePostgresStorageConfig,
|
|
8
|
+
PostgresStorageConfigDecoded,
|
|
9
|
+
RequiredOperationBatchLimits
|
|
10
|
+
} from '../types/types.js';
|
|
7
11
|
import { truncateTables } from './db.js';
|
|
8
12
|
|
|
9
13
|
export type PostgresTestStorageOptions = {
|
|
@@ -14,6 +18,12 @@ export type PostgresTestStorageOptions = {
|
|
|
14
18
|
* This allows for providing a custom PostgresMigrationAgent.
|
|
15
19
|
*/
|
|
16
20
|
migrationAgent?: (config: PostgresStorageConfigDecoded) => PostgresMigrationAgent;
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Override the default batch limits for the storage factory, including the internal limits
|
|
24
|
+
* that are not part of the storage config.
|
|
25
|
+
*/
|
|
26
|
+
batchLimits?: Partial<RequiredOperationBatchLimits>;
|
|
17
27
|
};
|
|
18
28
|
|
|
19
29
|
export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
|
|
@@ -23,7 +33,11 @@ export function postgresTestSetup(factoryOptions: PostgresTestStorageOptions) {
|
|
|
23
33
|
sslmode: 'disable' as const
|
|
24
34
|
};
|
|
25
35
|
|
|
26
|
-
const
|
|
36
|
+
const normalizedConfig = normalizePostgresStorageConfig(BASE_CONFIG);
|
|
37
|
+
const TEST_CONNECTION_OPTIONS = {
|
|
38
|
+
...normalizedConfig,
|
|
39
|
+
batch_limits: { ...normalizedConfig.batch_limits, ...factoryOptions.batchLimits }
|
|
40
|
+
};
|
|
27
41
|
|
|
28
42
|
const runMigrations = async (options: { down: boolean; up: boolean }) => {
|
|
29
43
|
await using migrationManager: PowerSyncMigrationManager = new framework.MigrationManager();
|