@powersync/service-module-postgres-storage 0.19.3 → 0.19.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/CHANGELOG.md +40 -0
  2. package/LICENSE +201 -67
  3. package/dist/storage/PostgresBucketStorageFactory.d.ts +2 -0
  4. package/dist/storage/PostgresBucketStorageFactory.js +105 -95
  5. package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
  6. package/dist/storage/PostgresReportStorage.js +1 -2
  7. package/dist/storage/PostgresReportStorage.js.map +1 -1
  8. package/dist/storage/PostgresSyncRulesStorage.js +1 -2
  9. package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
  10. package/dist/storage/batch/PostgresBucketBatch.js +14 -4
  11. package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
  12. package/dist/storage/batch/PostgresPersistedBatch.d.ts +15 -0
  13. package/dist/storage/batch/PostgresPersistedBatch.js +58 -4
  14. package/dist/storage/batch/PostgresPersistedBatch.js.map +1 -1
  15. package/dist/types/types.d.ts +22 -1
  16. package/dist/types/types.js +18 -1
  17. package/dist/types/types.js.map +1 -1
  18. package/dist/utils/test-utils.d.ts +6 -1
  19. package/dist/utils/test-utils.js +5 -1
  20. package/dist/utils/test-utils.js.map +1 -1
  21. package/package.json +10 -11
  22. package/src/storage/PostgresBucketStorageFactory.ts +114 -96
  23. package/src/storage/PostgresReportStorage.ts +1 -2
  24. package/src/storage/PostgresSyncRulesStorage.ts +1 -2
  25. package/src/storage/batch/PostgresBucketBatch.ts +16 -6
  26. package/src/storage/batch/PostgresPersistedBatch.ts +69 -4
  27. package/src/types/types.ts +25 -2
  28. package/src/utils/test-utils.ts +16 -2
  29. package/test/src/invalid_sync_config.test.ts +100 -0
  30. package/test/src/storage_flush.test.ts +207 -0
  31. package/test/src/util.ts +10 -5
  32. package/tsconfig.tsbuildinfo +1 -1
@@ -5,6 +5,22 @@ export * as models from './models/models-index.js';
5
5
  export const MAX_BATCH_RECORD_COUNT = 2000;
6
6
  export const MAX_BATCH_ESTIMATED_SIZE = 5_000_000;
7
7
  export const MAX_BATCH_CURRENT_DATA_SIZE = 50_000_000;
8
+ /**
9
+ * Maximum estimated size of the rows written by a single INSERT statement when flushing
10
+ * bucket_data and bucket_parameters.
11
+ *
12
+ * A single source row can produce many operations when it is evaluated into many buckets,
13
+ * which is not limited by `max_estimated_size`. Flushes larger than this limit are split
14
+ * into multiple statements in the same transaction.
15
+ *
16
+ * The rows are passed to Postgres as a single JSON parameter, which is serialized to a string
17
+ * before being sent. Node limits a string to roughly 512MB. The estimate used for chunking does
18
+ * not account for the JSON escaping of the already-serialized row data, so the actual parameter
19
+ * can be close to double the estimate. 64MB keeps a wide margin for that.
20
+ *
21
+ * This is not part of `batch_limits`: the limit comes from the runtime, not from the workload.
22
+ */
23
+ export const MAX_FLUSH_STATEMENT_SIZE = 64_000_000;
8
24
  export const BatchLimits = t.object({
9
25
  /**
10
26
  * Maximum size of operations we write in a single transaction.
@@ -36,7 +52,8 @@ export const normalizePostgresStorageConfig = (baseConfig) => {
36
52
  batch_limits: {
37
53
  max_current_data_batch_size: baseConfig.batch_limits?.max_current_data_batch_size ?? MAX_BATCH_CURRENT_DATA_SIZE,
38
54
  max_estimated_size: baseConfig.batch_limits?.max_estimated_size ?? MAX_BATCH_ESTIMATED_SIZE,
39
- max_record_count: baseConfig.batch_limits?.max_record_count ?? MAX_BATCH_RECORD_COUNT
55
+ max_record_count: baseConfig.batch_limits?.max_record_count ?? MAX_BATCH_RECORD_COUNT,
56
+ max_statement_size: MAX_FLUSH_STATEMENT_SIZE
40
57
  }
41
58
  };
42
59
  };
@@ -1 +1 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAEhE,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,KAAK,MAAM,MAAM,0BAA0B,CAAC;AAEnD,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAE3C,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAElD,MAAM,CAAC,MAAM,2BAA2B,GAAG,UAAU,CAAC;AAEtD,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC;;OAEG;IACH,kBAAkB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACvC;;OAEG;IACH,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACtC,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,oBAAoB,GAAG,WAAW,CAAC,GAAG,CACjD,CAAC,CAAC,MAAM,CAAC;IACP;;OAEG;IACH,2BAA2B,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACjD,CAAC,CACH,CAAC;AAIF,MAAM,CAAC,MAAM,qBAAqB,GAAG,UAAU,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,4BAA4B,CAAC,CAAC,GAAG,CAClH,CAAC,CAAC,MAAM,CAAC;IACP;;;;;OAKG;IACH,YAAY,EAAE,oBAAoB,CAAC,QAAQ,EAAE;CAC9C,CAAC,CACH,CAAC;AAYF,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAC5C,UAAwC,EACP,EAAE;IACnC,OAAO;QACL,GAAG,YAAY,CAAC,yBAAyB,CAAC,UAAU,CAAC;QACrD,YAAY,EAAE;YACZ,2BAA2B,EAAE,UAAU,CAAC,YAAY,EAAE,2BAA2B,IAAI,2BAA2B;YAChH,kBAAkB,EAAE,UAAU,CAAC,YAAY,EAAE,kBAAkB,IAAI,wBAAwB;YAC3F,gBAAgB,EAAE,UAAU,CAAC,YAAY,EAAE,gBAAgB,IAAI,sBAAsB;SACtF;KACF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,MAAoC,EAAmC,EAAE;IAC/G,OAAO,MAAM,CAAC,IAAI,IAAI,YAAY,CAAC,wBAAwB,CAAC;AAC9D,CAAC,CAAC"}
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAEhE,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,KAAK,MAAM,MAAM,0BAA0B,CAAC;AAEnD,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAE3C,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAElD,MAAM,CAAC,MAAM,2BAA2B,GAAG,UAAU,CAAC;AAEtD;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,UAAU,CAAC;AAEnD,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC;;OAEG;IACH,kBAAkB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACvC;;OAEG;IACH,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACtC,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,oBAAoB,GAAG,WAAW,CAAC,GAAG,CACjD,CAAC,CAAC,MAAM,CAAC;IACP;;OAEG;IACH,2BAA2B,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACjD,CAAC,CACH,CAAC;AAIF,MAAM,CAAC,MAAM,qBAAqB,GAAG,UAAU,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,4BAA4B,CAAC,CAAC,GAAG,CAClH,CAAC,CAAC,MAAM,CAAC;IACP;;;;;OAKG;IACH,YAAY,EAAE,oBAAoB,CAAC,QAAQ,EAAE;CAC9C,CAAC,CACH,CAAC;AAiBF,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAC5C,UAAwC,EACP,EAAE;IACnC,OAAO;QACL,GAAG,YAAY,CAAC,yBAAyB,CAAC,UAAU,CAAC;QACrD,YAAY,EAAE;YACZ,2BAA2B,EAAE,UAAU,CAAC,YAAY,EAAE,2BAA2B,IAAI,2BAA2B;YAChH,kBAAkB,EAAE,UAAU,CAAC,YAAY,EAAE,kBAAkB,IAAI,wBAAwB;YAC3F,gBAAgB,EAAE,UAAU,CAAC,YAAY,EAAE,gBAAgB,IAAI,sBAAsB;YACrF,kBAAkB,EAAE,wBAAwB;SAC7C;KACF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,MAAoC,EAAmC,EAAE;IAC/G,OAAO,MAAM,CAAC,IAAI,IAAI,YAAY,CAAC,wBAAwB,CAAC;AAC9D,CAAC,CAAC"}
@@ -2,7 +2,7 @@ import { framework, TestStorageOptions } from '@powersync/service-core';
2
2
  import { PostgresMigrationAgent } from '../migrations/PostgresMigrationAgent.js';
3
3
  import { PostgresBucketStorageFactory } from '../storage/PostgresBucketStorageFactory.js';
4
4
  import { PostgresReportStorage } from '../storage/PostgresReportStorage.js';
5
- import { PostgresStorageConfigDecoded } from '../types/types.js';
5
+ import { PostgresStorageConfigDecoded, RequiredOperationBatchLimits } from '../types/types.js';
6
6
  export type PostgresTestStorageOptions = {
7
7
  url: string;
8
8
  /**
@@ -10,6 +10,11 @@ export type PostgresTestStorageOptions = {
10
10
  * This allows for providing a custom PostgresMigrationAgent.
11
11
  */
12
12
  migrationAgent?: (config: PostgresStorageConfigDecoded) => PostgresMigrationAgent;
13
+ /**
14
+ * Override the default batch limits for the storage factory, including the internal limits
15
+ * that are not part of the storage config.
16
+ */
17
+ batchLimits?: Partial<RequiredOperationBatchLimits>;
13
18
  };
14
19
  export declare function postgresTestSetup(factoryOptions: PostgresTestStorageOptions): {
15
20
  reportFactory: (options?: TestStorageOptions) => Promise<PostgresReportStorage>;
@@ -11,7 +11,11 @@ export function postgresTestSetup(factoryOptions) {
11
11
  uri: factoryOptions.url,
12
12
  sslmode: 'disable'
13
13
  };
14
- const TEST_CONNECTION_OPTIONS = normalizePostgresStorageConfig(BASE_CONFIG);
14
+ const normalizedConfig = normalizePostgresStorageConfig(BASE_CONFIG);
15
+ const TEST_CONNECTION_OPTIONS = {
16
+ ...normalizedConfig,
17
+ batch_limits: { ...normalizedConfig.batch_limits, ...factoryOptions.batchLimits }
18
+ };
15
19
  const runMigrations = async (options) => {
16
20
  await using migrationManager = new framework.MigrationManager();
17
21
  await using migrationAgent = factoryOptions.migrationAgent
@@ -1 +1 @@
1
- {"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,aAAa,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AACtG,OAAO,EAAE,SAAS,EAAiE,MAAM,yBAAyB,CAAC;AACnH,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACjF,OAAO,EAAE,4BAA4B,EAAE,MAAM,4CAA4C,CAAC;AAC1F,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,OAAO,EAAE,8BAA8B,EAAgC,MAAM,mBAAmB,CAAC;AACjG,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAYzC,MAAM,UAAU,iBAAiB,CAAC,cAA0C;IAC1E,MAAM,WAAW,GAAG;QAClB,IAAI,EAAE,YAAqB;QAC3B,GAAG,EAAE,cAAc,CAAC,GAAG;QACvB,OAAO,EAAE,SAAkB;KAC5B,CAAC;IAEF,MAAM,uBAAuB,GAAG,8BAA8B,CAAC,WAAW,CAAC,CAAC;IAE5E,MAAM,aAAa,GAAG,KAAK,EAAE,OAAuC,EAAE,EAAE;QACtE,YAAY,gBAAgB,GAA8B,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;QAC3F,YAAY,cAAc,GAAG,cAAc,CAAC,cAAc;YACxD,CAAC,CAAC,cAAc,CAAC,cAAc,CAAC,WAAW,CAAC;YAC5C,CAAC,CAAC,IAAI,sBAAsB,CAAC,WAAW,CAAC,CAAC;QAC5C,gBAAgB,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;QAExD,MAAM,kBAAkB,GAAG,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAA+B,CAAC;QAEpG,+EAA+E;QAC/E,MAAM,MAAM,GAAG,YAAY,CAAC;YAC1B,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,UAAU,EAAE,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;SACvC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI;gBAC9C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC5C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,EAAE,SAAyC,EAAE,EAAE;QAClE,MAAM,aAAa,CAAC;YAClB,IAAI,EAAE,IAAI;YACV,EAAE,EAAE,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;SACnD,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;QAC9B,MAAM,aAAa,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,YAAY,cAAc,GAAG,IAAI,4BAA4B,CAAC;YAC5D,MAAM,EAAE,uBAAuB;YAC/B,2BAA2B,EAAE,OAAO;SACrC,CAAC,CAAC;QACH,MAAM,cAAc,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC,CAAC;IAEF,OAAO;QACL,aAAa,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YACpD,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,qBAAqB,CAAC;oBAC/B,MAAM,EAAE,uBAAuB;iBAChC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,4BAA4B,CAAC;oBACtC,MAAM,EAAE,uBAAuB;oBAC/B,2BAA2B,EAAE,OAAO;iBACrC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO;QACP,cAAc,EAAE,IAAI;KACrB,CAAC;AACJ,CAAC"}
1
+ {"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,IAAI,aAAa,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AACtG,OAAO,EAAE,SAAS,EAAiE,MAAM,yBAAyB,CAAC;AACnH,OAAO,EAAE,sBAAsB,EAAE,MAAM,yCAAyC,CAAC;AACjF,OAAO,EAAE,4BAA4B,EAAE,MAAM,4CAA4C,CAAC;AAC1F,OAAO,EAAE,qBAAqB,EAAE,MAAM,qCAAqC,CAAC;AAC5E,OAAO,EACL,8BAA8B,EAG/B,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAkBzC,MAAM,UAAU,iBAAiB,CAAC,cAA0C;IAC1E,MAAM,WAAW,GAAG;QAClB,IAAI,EAAE,YAAqB;QAC3B,GAAG,EAAE,cAAc,CAAC,GAAG;QACvB,OAAO,EAAE,SAAkB;KAC5B,CAAC;IAEF,MAAM,gBAAgB,GAAG,8BAA8B,CAAC,WAAW,CAAC,CAAC;IACrE,MAAM,uBAAuB,GAAG;QAC9B,GAAG,gBAAgB;QACnB,YAAY,EAAE,EAAE,GAAG,gBAAgB,CAAC,YAAY,EAAE,GAAG,cAAc,CAAC,WAAW,EAAE;KAClF,CAAC;IAEF,MAAM,aAAa,GAAG,KAAK,EAAE,OAAuC,EAAE,EAAE;QACtE,YAAY,gBAAgB,GAA8B,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;QAC3F,YAAY,cAAc,GAAG,cAAc,CAAC,cAAc;YACxD,CAAC,CAAC,cAAc,CAAC,cAAc,CAAC,WAAW,CAAC;YAC5C,CAAC,CAAC,IAAI,sBAAsB,CAAC,WAAW,CAAC,CAAC;QAC5C,gBAAgB,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;QAExD,MAAM,kBAAkB,GAAG,EAAE,aAAa,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,EAA+B,CAAC;QAEpG,+EAA+E;QAC/E,MAAM,MAAM,GAAG,YAAY,CAAC;YAC1B,KAAK,EAAE,MAAM;YACb,MAAM,EAAE,aAAa,CAAC,MAAM;YAC5B,UAAU,EAAE,CAAC,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;SACvC,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YACjB,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI;gBAC9C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAO,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,gBAAgB,CAAC,OAAO,CAAC;gBAC7B,SAAS,EAAE,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC5C,gBAAgB,EAAE;oBAChB,eAAe,EAAE,kBAAkB;iBACpC;gBACD,MAAM;aACP,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,KAAK,EAAE,SAAyC,EAAE,EAAE;QAClE,MAAM,aAAa,CAAC;YAClB,IAAI,EAAE,IAAI;YACV,EAAE,EAAE,SAAS,IAAI,SAAS,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;SACnD,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;QAC9B,MAAM,aAAa,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/C,YAAY,cAAc,GAAG,IAAI,4BAA4B,CAAC;YAC5D,MAAM,EAAE,uBAAuB;YAC/B,2BAA2B,EAAE,OAAO;SACrC,CAAC,CAAC;QACH,MAAM,cAAc,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IAC1C,CAAC,CAAC;IAEF,OAAO;QACL,aAAa,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YACpD,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,qBAAqB,CAAC;oBAC/B,MAAM,EAAE,uBAAuB;iBAChC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,IAAI,CAAC;gBACH,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;oBACzB,MAAM,YAAY,EAAE,CAAC;gBACvB,CAAC;gBAED,OAAO,IAAI,4BAA4B,CAAC;oBACtC,MAAM,EAAE,uBAAuB;oBAC/B,2BAA2B,EAAE,OAAO;iBACrC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,EAAE,EAAE,CAAC;gBACZ,iFAAiF;gBACjF,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC;gBAC5B,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC;QACD,OAAO;QACP,cAAc,EAAE,IAAI;KACrB,CAAC;AACJ,CAAC"}
package/package.json CHANGED
@@ -1,8 +1,8 @@
1
1
  {
2
2
  "name": "@powersync/service-module-postgres-storage",
3
3
  "repository": "https://github.com/powersync-ja/powersync-service",
4
- "version": "0.19.3",
5
- "license": "FSL-1.1-ALv2",
4
+ "version": "0.19.5",
5
+ "license": "Apache-2.0",
6
6
  "type": "module",
7
7
  "publishConfig": {
8
8
  "access": "public"
@@ -12,21 +12,20 @@
12
12
  "./types": "./dist/types/types.js"
13
13
  },
14
14
  "dependencies": {
15
- "ix": "^5.0.0",
16
15
  "lru-cache": "^10.2.2",
17
16
  "ts-codec": "^1.3.0",
18
17
  "uuid": "^14.0.0",
19
- "@powersync/lib-service-postgres": "0.5.7",
20
- "@powersync/lib-services-framework": "0.10.4",
21
- "@powersync/service-core": "1.27.1",
22
- "@powersync/service-types": "0.20.0",
23
- "@powersync/service-jpgwire": "0.21.26",
24
- "@powersync/service-jsonbig": "0.17.13",
25
- "@powersync/service-sync-rules": "0.44.0"
18
+ "@powersync/lib-service-postgres": "0.5.8",
19
+ "@powersync/lib-services-framework": "0.10.5",
20
+ "@powersync/service-types": "0.21.0",
21
+ "@powersync/service-jpgwire": "0.21.27",
22
+ "@powersync/service-core": "1.28.0",
23
+ "@powersync/service-jsonbig": "0.17.14",
24
+ "@powersync/service-sync-rules": "0.45.0"
26
25
  },
27
26
  "devDependencies": {
28
27
  "typescript": "~6.0.3",
29
- "@powersync/service-core-tests": "0.18.4"
28
+ "@powersync/service-core-tests": "0.18.5"
30
29
  },
31
30
  "scripts": {
32
31
  "build": "tsc -b",
@@ -161,7 +161,24 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
161
161
  };
162
162
  }
163
163
 
164
+ async stopProcessingSyncConfigs(): Promise<void> {
165
+ await this.db.sql`
166
+ UPDATE sync_rules
167
+ SET
168
+ state = ${{ type: 'varchar', value: storage.SyncRuleState.STOP }}
169
+ WHERE
170
+ state = ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }}
171
+ `.execute();
172
+ }
173
+
164
174
  async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<PostgresPersistedReplicationStream> {
175
+ return this.db.transaction((db) => this.updateSyncRulesInTransaction(options, db));
176
+ }
177
+
178
+ private async updateSyncRulesInTransaction(
179
+ options: storage.UpdateSyncRulesOptions,
180
+ db: lib_postgres.WrappedConnection
181
+ ): Promise<PostgresPersistedReplicationStream> {
165
182
  const storageVersion =
166
183
  options.storageVersion ??
167
184
  options.config.parsed.config.storageVersion ??
@@ -174,77 +191,78 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
174
191
  `Unsupported storage version ${storageVersion} for PostgreSQL storage`
175
192
  );
176
193
  }
177
- await this.initializeStorageVersion(storageConfig);
178
- return this.db.transaction(async (db) => {
179
- await db.sql`
180
- UPDATE sync_rules
181
- SET
182
- state = ${{ type: 'varchar', value: storage.SyncRuleState.STOP }}
183
- WHERE
184
- state = ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }}
185
- `.execute();
194
+ await this.initializeStorageVersion(storageConfig, db);
195
+ await db.sql`
196
+ UPDATE sync_rules
197
+ SET
198
+ state = ${{ type: 'varchar', value: storage.SyncRuleState.STOP }}
199
+ WHERE
200
+ state = ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }}
201
+ `.execute();
186
202
 
187
- // Uses the legacy sequence name "sync_rules_id_sequence" for backwards-compatibility. See AGENTS.md.
188
- const newSyncRulesRow = await db.sql`
189
- WITH
190
- next_id AS (
191
- SELECT
192
- nextval('sync_rules_id_sequence') AS id
193
- )
194
- INSERT INTO
195
- sync_rules (
196
- id,
197
- content,
198
- sync_plan,
199
- state,
200
- slot_name,
201
- storage_version,
202
- version_label
203
- )
204
- VALUES
203
+ // Uses the legacy sequence name "sync_rules_id_sequence" for backwards-compatibility. See AGENTS.md.
204
+ const newSyncRulesRow = await db.sql`
205
+ WITH
206
+ next_id AS (
207
+ SELECT
208
+ nextval('sync_rules_id_sequence') AS id
209
+ )
210
+ INSERT INTO
211
+ sync_rules (
212
+ id,
213
+ content,
214
+ sync_plan,
215
+ state,
216
+ slot_name,
217
+ storage_version,
218
+ version_label
219
+ )
220
+ VALUES
221
+ (
205
222
  (
223
+ SELECT
224
+ id
225
+ FROM
226
+ next_id
227
+ ),
228
+ ${{ type: 'varchar', value: options.config.yaml }},
229
+ ${{ type: 'json', value: options.config.plan }},
230
+ ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }},
231
+ CONCAT(
232
+ ${{ type: 'varchar', value: this.replicationStreamNamePrefix }},
206
233
  (
207
234
  SELECT
208
235
  id
209
236
  FROM
210
237
  next_id
211
238
  ),
212
- ${{ type: 'varchar', value: options.config.yaml }},
213
- ${{ type: 'json', value: options.config.plan }},
214
- ${{ type: 'varchar', value: storage.SyncRuleState.PROCESSING }},
215
- CONCAT(
216
- ${{ type: 'varchar', value: this.replicationStreamNamePrefix }},
217
- (
218
- SELECT
219
- id
220
- FROM
221
- next_id
222
- ),
223
- '_',
224
- ${{ type: 'varchar', value: crypto.randomBytes(2).toString('hex') }}
225
- ),
226
- ${{ type: 'int4', value: storageVersion }},
227
- ${{ type: 'varchar', value: options.version_label ?? null }}
228
- )
229
- RETURNING
230
- *
231
- `
232
- .decoded(models.SyncRules)
233
- .first();
239
+ '_',
240
+ ${{ type: 'varchar', value: crypto.randomBytes(2).toString('hex') }}
241
+ ),
242
+ ${{ type: 'int4', value: storageVersion }},
243
+ ${{ type: 'varchar', value: options.version_label ?? null }}
244
+ )
245
+ RETURNING
246
+ *
247
+ `
248
+ .decoded(models.SyncRules)
249
+ .first();
234
250
 
235
- return new PostgresPersistedReplicationStream(this.db, newSyncRulesRow!);
236
- });
251
+ return new PostgresPersistedReplicationStream(this.db, newSyncRulesRow!);
237
252
  }
238
253
 
239
254
  /**
240
255
  * Lazy-initializes storage-version-specific structures, if needed.
241
256
  */
242
- private async initializeStorageVersion(storageConfig: storage.StorageVersionConfig) {
257
+ private async initializeStorageVersion(
258
+ storageConfig: storage.StorageVersionConfig,
259
+ db: lib_postgres.WrappedConnection
260
+ ) {
243
261
  if (!storageConfig.softDeleteCurrentData) {
244
262
  return;
245
263
  }
246
264
 
247
- await this.db.sql`
265
+ await db.sql`
248
266
  CREATE TABLE IF NOT EXISTS v3_current_data (
249
267
  group_id integer NOT NULL,
250
268
  source_table TEXT NOT NULL,
@@ -257,7 +275,7 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
257
275
  )
258
276
  `.execute();
259
277
 
260
- await this.db.sql`
278
+ await db.sql`
261
279
  CREATE INDEX IF NOT EXISTS v3_current_data_pending_deletes ON v3_current_data (group_id, pending_delete)
262
280
  WHERE
263
281
  pending_delete IS NOT NULL
@@ -265,50 +283,50 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory {
265
283
  }
266
284
 
267
285
  async restartReplication(replicationStreamId: number): Promise<void> {
268
- const next = await this.getDeployingSyncConfig();
269
- const active = await this.getActiveSyncConfig();
270
-
271
- // In both the below cases, we create a new replication stream.
272
- // The current one will continue serving sync requests until the next one has finished processing.
273
- if (next != null && next.content.replicationStreamId == replicationStreamId) {
274
- // We need to redo the "next" replication stream
275
-
276
- await this.updateSyncRules(next.content.asUpdateOptions());
277
- // Pro-actively stop replicating
278
- await this.db.sql`
279
- UPDATE sync_rules
280
- SET
281
- state = ${{ value: storage.SyncRuleState.STOP, type: 'varchar' }}
282
- WHERE
283
- id = ${{ value: next.content.replicationStreamId, type: 'int4' }}
284
- AND state = ${{ value: storage.SyncRuleState.PROCESSING, type: 'varchar' }}
285
- `.execute();
286
- } else if (next == null && active?.content.replicationStreamId == replicationStreamId) {
287
- // Slot removed for "active" replication stream, while there is no "next" one.
288
- await this.updateSyncRules(active.content.asUpdateOptions());
289
-
290
- // Pro-actively stop replicating, but still serve clients with existing data
291
- await this.db.sql`
292
- UPDATE sync_rules
293
- SET
294
- state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
286
+ await this.db.transaction(async (db) => {
287
+ // Protect the absence of pending work as well as existing rows. Deployments and activation
288
+ // take ROW EXCLUSIVE locks, so neither can change the decision before recovery commits.
289
+ await db.sql`LOCK TABLE sync_rules IN SHARE ROW EXCLUSIVE MODE`.execute();
290
+ const rows = await db.sql`
291
+ SELECT
292
+ *
293
+ FROM
294
+ sync_rules
295
295
  WHERE
296
- id = ${{ value: active.content.replicationStreamId, type: 'int4' }}
297
- AND state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
298
- `.execute();
299
- } else if (next != null && active?.content.replicationStreamId == replicationStreamId) {
300
- // Already have "next" replication stream - don't update any.
296
+ state IN (
297
+ ${{ value: storage.SyncRuleState.PROCESSING, type: 'varchar' }},
298
+ ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }},
299
+ ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
300
+ )
301
+ ORDER BY
302
+ id DESC
303
+ `
304
+ .decoded(models.SyncRules)
305
+ .rows();
306
+ const next = rows.find((row) => row.state === storage.SyncRuleState.PROCESSING);
307
+ const active = rows.find(
308
+ (row) => row.state === storage.SyncRuleState.ACTIVE || row.state === storage.SyncRuleState.ERRORED
309
+ );
301
310
 
302
- // Pro-actively stop replicating, but still serve clients with existing data
303
- await this.db.sql`
304
- UPDATE sync_rules
305
- SET
306
- state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
307
- WHERE
308
- id = ${{ value: active.content.replicationStreamId, type: 'int4' }}
309
- AND state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
310
- `.execute();
311
- }
311
+ if (next?.id === replicationStreamId) {
312
+ // Replace the failed pending stream only while it is still the current deployment.
313
+ await this.updateSyncRulesInTransaction(this.resolvedSyncConfigFromRow(next).content.asUpdateOptions(), db);
314
+ } else if (active?.id === replicationStreamId) {
315
+ if (next == null) {
316
+ await this.updateSyncRulesInTransaction(this.resolvedSyncConfigFromRow(active).content.asUpdateOptions(), db);
317
+ }
318
+ // Keep serving the old data, but stop replication. An existing newer deployment takes
319
+ // precedence over recovery and must remain eligible to activate.
320
+ await db.sql`
321
+ UPDATE sync_rules
322
+ SET
323
+ state = ${{ value: storage.SyncRuleState.ERRORED, type: 'varchar' }}
324
+ WHERE
325
+ id = ${{ value: active.id, type: 'int4' }}
326
+ AND state = ${{ value: storage.SyncRuleState.ACTIVE, type: 'varchar' }}
327
+ `.execute();
328
+ }
329
+ });
312
330
  }
313
331
 
314
332
  async getActiveSyncConfig(): Promise<storage.ResolvedSyncConfig | null> {
@@ -3,7 +3,6 @@ import { logger } from '@powersync/lib-services-framework';
3
3
  import { storage } from '@powersync/service-core';
4
4
  import * as pg_wire from '@powersync/service-jpgwire';
5
5
  import { event_types } from '@powersync/service-types';
6
- import { toInteger } from 'ix/util/tointeger.js';
7
6
  import { v4 } from 'uuid';
8
7
  import { SdkReporting, SdkReportingDecoded } from '../types/models/SdkReporting.js';
9
8
  import { NormalizedPostgresStorageConfig } from '../types/types.js';
@@ -329,7 +328,7 @@ export class PostgresReportStorage implements storage.ReportStorage {
329
328
  )
330
329
  );
331
330
  `.execute();
332
- const deletedRows = toInteger(result.results[1].status.split(' ')[1] || '0');
331
+ const deletedRows = Number(result.results[1].status.split(' ')[1] || '0');
333
332
  if (deletedRows > 0) {
334
333
  logger.info(
335
334
  `TTL from ${date.toISOString()}: ${deletedRows} PostgresSQL rows have been removed from connection_report_events.`
@@ -30,7 +30,6 @@ import { mapOpEntry } from '../utils/bucket-data.js';
30
30
 
31
31
  import * as framework from '@powersync/lib-services-framework';
32
32
  import type { Statement } from '@powersync/service-jpgwire';
33
- import { wrapWithAbort } from 'ix/asynciterable/operators/withabort.js';
34
33
  import * as t from 'ts-codec';
35
34
  import { ActiveCheckpointDecoded } from '../types/models/ActiveCheckpoint.js';
36
35
  import * as checkpointUtils from '../utils/checkpoints.js';
@@ -716,7 +715,7 @@ export class PostgresSyncRulesStorage
716
715
 
717
716
  const { signal, user_id } = options;
718
717
 
719
- const iter = wrapWithAbort(this.sharedIterator, signal);
718
+ const iter = this.sharedIterator[Symbol.asyncIterator](signal);
720
719
  for await (const cp of iter) {
721
720
  const { checkpoint, lsn } = cp;
722
721
 
@@ -202,6 +202,19 @@ export class PostgresBucketBatch
202
202
  const resolution = await reconcile({ source, candidates });
203
203
  storage.validateSourceTableCandidateResolution(candidates, resolution);
204
204
 
205
+ const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
206
+ const compatibleTables = materializedResolution.compatibleTables;
207
+
208
+ if (!syncRules.tableSyncsAny(source)) {
209
+ // Still reconcile old mappings so renames out of scope remove previously replicated data.
210
+ // Skip updating source_metadata below: these records are dropped by the caller in this
211
+ // same transaction, so persisting new metadata into them first is wasted work.
212
+ return {
213
+ tables: [],
214
+ dropTables: [...materializedResolution.incompatibleTables, ...compatibleTables]
215
+ };
216
+ }
217
+
205
218
  for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
206
219
  await db.sql`
207
220
  UPDATE source_tables
@@ -212,9 +225,6 @@ export class PostgresBucketBatch
212
225
  `.execute();
213
226
  }
214
227
 
215
- const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
216
- const compatibleTables = materializedResolution.compatibleTables;
217
-
218
228
  // Keep one record, preferring one that has already been snapshotted.
219
229
  const reuse = compatibleTables.find((candidate) => candidate.snapshotComplete) ?? compatibleTables[0] ?? null;
220
230
 
@@ -1060,9 +1070,9 @@ export class PostgresBucketBatch
1060
1070
  // Not an error if we re-apply a transaction
1061
1071
  existingBuckets = [];
1062
1072
  existingLookups = [];
1063
- // Log to help with debugging if there was a consistency issue
1064
-
1065
- if (storeCurrentData) {
1073
+ // A complete row is evaluated and stored in current_data below, so a resnapshot is only
1074
+ // needed when TOAST values are missing.
1075
+ if (!utils.isCompleteRow(storeCurrentData, after!)) {
1066
1076
  if (this.markRecordUnavailable != null) {
1067
1077
  // This will trigger a "resnapshot" of the record.
1068
1078
  // This is not relevant if storeCurrentData is false, since we'll get the full row
@@ -62,6 +62,7 @@ export class PostgresPersistedBatch {
62
62
 
63
63
  readonly maxTransactionBatchSize: number;
64
64
  readonly maxTransactionDocCount: number;
65
+ readonly maxStatementSize: number;
65
66
 
66
67
  /**
67
68
  * Ordered set of bucket_data insert operation parameters
@@ -83,6 +84,7 @@ export class PostgresPersistedBatch {
83
84
 
84
85
  this.maxTransactionBatchSize = options.max_estimated_size;
85
86
  this.maxTransactionDocCount = options.max_record_count;
87
+ this.maxStatementSize = options.max_statement_size;
86
88
 
87
89
  this.bucketDataInserts = [];
88
90
  this.parameterDataInserts = [];
@@ -304,7 +306,20 @@ export class PostgresPersistedBatch {
304
306
  }
305
307
 
306
308
  protected async flushBucketData(db: lib_postgres.WrappedConnection) {
307
- if (this.bucketDataInserts.length > 0) {
309
+ if (this.bucketDataInserts.length == 0) {
310
+ return;
311
+ }
312
+
313
+ const chunks = chunkRowsBySize(this.bucketDataInserts, estimateBucketDataRowSize, this.maxStatementSize);
314
+ if (chunks.length > 1) {
315
+ logger.debug(
316
+ `powersync_${this.group_id} Flushing ${this.bucketDataInserts.length} bucket_data rows in ${chunks.length} statements.`
317
+ );
318
+ }
319
+
320
+ // Sequential statements in the same transaction preserve the order of the operations,
321
+ // since op_id is assigned from the sequence per row.
322
+ for (const chunk of chunks) {
308
323
  await db.sql`
309
324
  INSERT INTO
310
325
  bucket_data (
@@ -333,7 +348,7 @@ export class PostgresPersistedBatch {
333
348
  data,
334
349
  target_op
335
350
  FROM
336
- json_to_recordset(${{ type: 'json', value: this.bucketDataInserts }}::json) AS t (
351
+ json_to_recordset(${{ type: 'json', value: chunk }}::json) AS t (
337
352
  group_id integer,
338
353
  bucket_name text,
339
354
  source_table text,
@@ -350,7 +365,18 @@ export class PostgresPersistedBatch {
350
365
  }
351
366
 
352
367
  protected async flushParameterData(db: lib_postgres.WrappedConnection) {
353
- if (this.parameterDataInserts.length > 0) {
368
+ if (this.parameterDataInserts.length == 0) {
369
+ return;
370
+ }
371
+
372
+ const chunks = chunkRowsBySize(this.parameterDataInserts, estimateParameterDataRowSize, this.maxStatementSize);
373
+ if (chunks.length > 1) {
374
+ logger.debug(
375
+ `powersync_${this.group_id} Flushing ${this.parameterDataInserts.length} bucket_parameters rows in ${chunks.length} statements.`
376
+ );
377
+ }
378
+
379
+ for (const chunk of chunks) {
354
380
  await db.sql`
355
381
  INSERT INTO
356
382
  bucket_parameters (
@@ -367,7 +393,7 @@ export class PostgresPersistedBatch {
367
393
  decode(lookup, 'hex') AS lookup, -- Decode hex to bytea
368
394
  bucket_parameters
369
395
  FROM
370
- json_to_recordset(${{ type: 'json', value: this.parameterDataInserts }}::json) AS t (
396
+ json_to_recordset(${{ type: 'json', value: chunk }}::json) AS t (
371
397
  group_id integer,
372
398
  source_table text,
373
399
  source_key text, -- Input as hex string
@@ -417,3 +443,42 @@ export class PostgresPersistedBatch {
417
443
  export function currentBucketKey(b: models.CurrentBucket) {
418
444
  return `${b.bucket}/${b.table}/${b.id}`;
419
445
  }
446
+
447
+ /**
448
+ * Rough estimate of the serialized size of a bucket_data row, matching the `currentSize` accounting.
449
+ */
450
+ export function estimateBucketDataRowSize(row: models.BucketData) {
451
+ return row.bucket_name.length + (row.data?.length ?? 0) + (row.source_key?.length ?? 0) + 100;
452
+ }
453
+
454
+ /**
455
+ * Rough estimate of the serialized size of a bucket_parameters row, matching the `currentSize` accounting.
456
+ */
457
+ export function estimateParameterDataRowSize(row: models.BucketParameters) {
458
+ return row.lookup.length + row.bucket_parameters.length + row.source_key.length + 100;
459
+ }
460
+
461
+ /**
462
+ * Splits rows into consecutive chunks, each with a total estimated size of at most `maxSize`.
463
+ *
464
+ * The order of the rows is preserved. A single row larger than `maxSize` is placed in its own chunk.
465
+ */
466
+ export function chunkRowsBySize<T>(rows: T[], estimateSize: (row: T) => number, maxSize: number): T[][] {
467
+ const chunks: T[][] = [];
468
+ let currentChunk: T[] = [];
469
+ let currentChunkSize = 0;
470
+ for (const row of rows) {
471
+ const size = estimateSize(row);
472
+ if (currentChunk.length > 0 && currentChunkSize + size > maxSize) {
473
+ chunks.push(currentChunk);
474
+ currentChunk = [];
475
+ currentChunkSize = 0;
476
+ }
477
+ currentChunk.push(row);
478
+ currentChunkSize += size;
479
+ }
480
+ if (currentChunk.length > 0) {
481
+ chunks.push(currentChunk);
482
+ }
483
+ return chunks;
484
+ }
@@ -10,6 +10,23 @@ export const MAX_BATCH_ESTIMATED_SIZE = 5_000_000;
10
10
 
11
11
  export const MAX_BATCH_CURRENT_DATA_SIZE = 50_000_000;
12
12
 
13
+ /**
14
+ * Maximum estimated size of the rows written by a single INSERT statement when flushing
15
+ * bucket_data and bucket_parameters.
16
+ *
17
+ * A single source row can produce many operations when it is evaluated into many buckets,
18
+ * which is not limited by `max_estimated_size`. Flushes larger than this limit are split
19
+ * into multiple statements in the same transaction.
20
+ *
21
+ * The rows are passed to Postgres as a single JSON parameter, which is serialized to a string
22
+ * before being sent. Node limits a string to roughly 512MB. The estimate used for chunking does
23
+ * not account for the JSON escaping of the already-serialized row data, so the actual parameter
24
+ * can be close to double the estimate. 64MB keeps a wide margin for that.
25
+ *
26
+ * This is not part of `batch_limits`: the limit comes from the runtime, not from the workload.
27
+ */
28
+ export const MAX_FLUSH_STATEMENT_SIZE = 64_000_000;
29
+
13
30
  export const BatchLimits = t.object({
14
31
  /**
15
32
  * Maximum size of operations we write in a single transaction.
@@ -49,7 +66,12 @@ export const PostgresStorageConfig = configFile.BaseStorageConfig.and(lib_postgr
49
66
  export type PostgresStorageConfig = t.Encoded<typeof PostgresStorageConfig>;
50
67
  export type PostgresStorageConfigDecoded = t.Decoded<typeof PostgresStorageConfig>;
51
68
 
52
- export type RequiredOperationBatchLimits = Required<OperationBatchLimits>;
69
+ export type RequiredOperationBatchLimits = Required<OperationBatchLimits> & {
70
+ /**
71
+ * Internal limit for the size of a single flush statement. See {@link MAX_FLUSH_STATEMENT_SIZE}.
72
+ */
73
+ max_statement_size: number;
74
+ };
53
75
 
54
76
  export type NormalizedPostgresStorageConfig = pg_wire.NormalizedConnectionConfig & {
55
77
  batch_limits: RequiredOperationBatchLimits;
@@ -64,7 +86,8 @@ export const normalizePostgresStorageConfig = (
64
86
  batch_limits: {
65
87
  max_current_data_batch_size: baseConfig.batch_limits?.max_current_data_batch_size ?? MAX_BATCH_CURRENT_DATA_SIZE,
66
88
  max_estimated_size: baseConfig.batch_limits?.max_estimated_size ?? MAX_BATCH_ESTIMATED_SIZE,
67
- max_record_count: baseConfig.batch_limits?.max_record_count ?? MAX_BATCH_RECORD_COUNT
89
+ max_record_count: baseConfig.batch_limits?.max_record_count ?? MAX_BATCH_RECORD_COUNT,
90
+ max_statement_size: MAX_FLUSH_STATEMENT_SIZE
68
91
  }
69
92
  };
70
93
  };