@powersync/service-module-postgres-storage 0.19.6 → 0.19.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -6,17 +6,18 @@ export const MAX_BATCH_RECORD_COUNT = 2000;
6
6
  export const MAX_BATCH_ESTIMATED_SIZE = 5_000_000;
7
7
  export const MAX_BATCH_CURRENT_DATA_SIZE = 50_000_000;
8
8
  /**
9
- * Maximum estimated size of the rows written by a single INSERT statement when flushing
10
- * bucket_data and bucket_parameters.
9
+ * Upper bound for the estimated size of the rows written by a single INSERT statement when
10
+ * flushing bucket_data and bucket_parameters.
11
11
  *
12
- * A single source row can produce many operations when it is evaluated into many buckets,
13
- * which is not limited by `max_estimated_size`. Flushes larger than this limit are split
14
- * into multiple statements in the same transaction.
12
+ * Rows are inserted while they accumulate, once the pending rows would exceed the smaller of
13
+ * `max_estimated_size` and this limit. A single source row can produce many operations when it
14
+ * is evaluated into many buckets, which `max_estimated_size` alone does not bound since it is
15
+ * only checked between rows.
15
16
  *
16
17
  * The rows are passed to Postgres as a single JSON parameter, which is serialized to a string
17
- * before being sent. Node limits a string to roughly 512MB. The estimate used for chunking does
18
- * not account for the JSON escaping of the already-serialized row data, so the actual parameter
19
- * can be close to double the estimate. 64MB keeps a wide margin for that.
18
+ * before being sent. Node limits a string to roughly 512MB. The estimate does not account for
19
+ * the JSON escaping of the already-serialized row data, so the actual parameter can be close to
20
+ * double the estimate. 64MB keeps a wide margin for that.
20
21
  *
21
22
  * This is not part of `batch_limits`: the limit comes from the runtime, not from the workload.
22
23
  */
@@ -1 +1 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAEhE,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,KAAK,MAAM,MAAM,0BAA0B,CAAC;AAEnD,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAE3C,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAElD,MAAM,CAAC,MAAM,2BAA2B,GAAG,UAAU,CAAC;AAEtD;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,UAAU,CAAC;AAEnD,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC;;OAEG;IACH,kBAAkB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACvC;;OAEG;IACH,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACtC,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,oBAAoB,GAAG,WAAW,CAAC,GAAG,CACjD,CAAC,CAAC,MAAM,CAAC;IACP;;OAEG;IACH,2BAA2B,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACjD,CAAC,CACH,CAAC;AAIF,MAAM,CAAC,MAAM,qBAAqB,GAAG,UAAU,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,4BAA4B,CAAC,CAAC,GAAG,CAClH,CAAC,CAAC,MAAM,CAAC;IACP;;;;;OAKG;IACH,YAAY,EAAE,oBAAoB,CAAC,QAAQ,EAAE;CAC9C,CAAC,CACH,CAAC;AAiBF,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAC5C,UAAwC,EACP,EAAE;IACnC,OAAO;QACL,GAAG,YAAY,CAAC,yBAAyB,CAAC,UAAU,CAAC;QACrD,YAAY,EAAE;YACZ,2BAA2B,EAAE,UAAU,CAAC,YAAY,EAAE,2BAA2B,IAAI,2BAA2B;YAChH,kBAAkB,EAAE,UAAU,CAAC,YAAY,EAAE,kBAAkB,IAAI,wBAAwB;YAC3F,gBAAgB,EAAE,UAAU,CAAC,YAAY,EAAE,gBAAgB,IAAI,sBAAsB;YACrF,kBAAkB,EAAE,wBAAwB;SAC7C;KACF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,MAAoC,EAAmC,EAAE;IAC/G,OAAO,MAAM,CAAC,IAAI,IAAI,YAAY,CAAC,wBAAwB,CAAC;AAC9D,CAAC,CAAC"}
1
+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAEhE,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,KAAK,MAAM,MAAM,0BAA0B,CAAC;AAEnD,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAE3C,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAElD,MAAM,CAAC,MAAM,2BAA2B,GAAG,UAAU,CAAC;AAEtD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,UAAU,CAAC;AAEnD,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC;;OAEG;IACH,kBAAkB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACvC;;OAEG;IACH,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACtC,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,oBAAoB,GAAG,WAAW,CAAC,GAAG,CACjD,CAAC,CAAC,MAAM,CAAC;IACP;;OAEG;IACH,2BAA2B,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACjD,CAAC,CACH,CAAC;AAIF,MAAM,CAAC,MAAM,qBAAqB,GAAG,UAAU,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,4BAA4B,CAAC,CAAC,GAAG,CAClH,CAAC,CAAC,MAAM,CAAC;IACP;;;;;OAKG;IACH,YAAY,EAAE,oBAAoB,CAAC,QAAQ,EAAE;CAC9C,CAAC,CACH,CAAC;AAiBF,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAC5C,UAAwC,EACP,EAAE;IACnC,OAAO;QACL,GAAG,YAAY,CAAC,yBAAyB,CAAC,UAAU,CAAC;QACrD,YAAY,EAAE;YACZ,2BAA2B,EAAE,UAAU,CAAC,YAAY,EAAE,2BAA2B,IAAI,2BAA2B;YAChH,kBAAkB,EAAE,UAAU,CAAC,YAAY,EAAE,kBAAkB,IAAI,wBAAwB;YAC3F,gBAAgB,EAAE,UAAU,CAAC,YAAY,EAAE,gBAAgB,IAAI,sBAAsB;YACrF,kBAAkB,EAAE,wBAAwB;SAC7C;KACF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,MAAoC,EAAmC,EAAE;IAC/G,OAAO,MAAM,CAAC,IAAI,IAAI,YAAY,CAAC,wBAAwB,CAAC;AAC9D,CAAC,CAAC"}
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@powersync/service-module-postgres-storage",
3
3
  "repository": "https://github.com/powersync-ja/powersync-service",
4
- "version": "0.19.6",
4
+ "version": "0.19.8",
5
5
  "license": "Apache-2.0",
6
6
  "type": "module",
7
7
  "publishConfig": {
@@ -17,7 +17,7 @@
17
17
  "uuid": "^14.0.0",
18
18
  "@powersync/lib-service-postgres": "0.5.8",
19
19
  "@powersync/lib-services-framework": "0.10.5",
20
- "@powersync/service-core": "1.28.1",
20
+ "@powersync/service-core": "1.28.3",
21
21
  "@powersync/service-types": "0.21.0",
22
22
  "@powersync/service-jpgwire": "0.21.27",
23
23
  "@powersync/service-jsonbig": "0.17.14",
@@ -406,42 +406,44 @@ export class PostgresBucketBatch
406
406
  const persistedBatch = new PostgresPersistedBatch({
407
407
  group_id: this.group_id,
408
408
  storageConfig: this.storageConfig,
409
+ db,
409
410
  ...this.options.batch_limits
410
411
  });
411
412
 
412
- for await (const rows of this.currentDataStore.streamTruncateRows(db, {
413
+ // Read in full rather than streamed, since the batch may write on this connection while
414
+ // the rows are processed (see PostgresPersistedBatch). BATCH_LIMIT bounds the rows in memory.
415
+ const rows = await this.currentDataStore.queryTruncateRows(db, {
413
416
  groupId: this.group_id,
414
417
  sourceTableId: postgresTableId(sourceTable.id),
415
418
  limit: BATCH_LIMIT
416
- })) {
417
- lastBatchCount += rows.length;
418
- processedCount += rows.length;
419
-
420
- const decodedRows = rows.map((row) => this.currentDataStore.decodeTruncateRow(row));
421
- for (const value of decodedRows) {
422
- const source_key = deserializeReplicaId(value.source_key);
423
- persistedBatch.saveBucketData({
424
- before_buckets: value.buckets,
425
- evaluated: [],
426
- table: sourceTable,
427
- source_key
428
- });
429
- persistedBatch.saveParameterData({
430
- existing_lookups: value.lookups,
431
- evaluated: [],
432
- table: sourceTable,
433
- source_key
434
- });
435
- persistedBatch.deleteCurrentData({
436
- // This is serialized since we got it from a DB query
437
- serialized_source_key: value.source_key,
438
- source_table_id: postgresTableId(sourceTable.id),
439
- // No need for soft delete, since this is not streaming replication
440
- soft: false
441
- });
442
- }
419
+ });
420
+ lastBatchCount = rows.length;
421
+ processedCount += rows.length;
422
+
423
+ for (const row of rows) {
424
+ const value = this.currentDataStore.decodeTruncateRow(row);
425
+ const source_key = deserializeReplicaId(value.source_key);
426
+ await persistedBatch.saveBucketData({
427
+ before_buckets: value.buckets,
428
+ evaluated: [],
429
+ table: sourceTable,
430
+ source_key
431
+ });
432
+ await persistedBatch.saveParameterData({
433
+ existing_lookups: value.lookups,
434
+ evaluated: [],
435
+ table: sourceTable,
436
+ source_key
437
+ });
438
+ persistedBatch.deleteCurrentData({
439
+ // This is serialized since we got it from a DB query
440
+ serialized_source_key: value.source_key,
441
+ source_table_id: postgresTableId(sourceTable.id),
442
+ // No need for soft delete, since this is not streaming replication
443
+ soft: false
444
+ });
443
445
  }
444
- const { flushedAny } = await persistedBatch.flush(db);
446
+ const { flushedAny } = await persistedBatch.flush();
445
447
  clearedError = flushedAny && !this.clearedError;
446
448
  });
447
449
  if (clearedError) {
@@ -978,6 +980,7 @@ export class PostgresBucketBatch
978
980
  let persistedBatch: PostgresPersistedBatch | null = new PostgresPersistedBatch({
979
981
  group_id: this.group_id,
980
982
  storageConfig: this.storageConfig,
983
+ db,
981
984
  ...this.options.batch_limits
982
985
  });
983
986
 
@@ -1001,7 +1004,7 @@ export class PostgresBucketBatch
1001
1004
  }
1002
1005
 
1003
1006
  if (persistedBatch!.shouldFlushTransaction()) {
1004
- const { flushedAny } = await persistedBatch!.flush(db);
1007
+ const { flushedAny } = await persistedBatch!.flush();
1005
1008
  didFlush ||= flushedAny;
1006
1009
  // The operations stored in this batch will be processed in the `resumeBatch`
1007
1010
  persistedBatch = null;
@@ -1015,7 +1018,7 @@ export class PostgresBucketBatch
1015
1018
  * The operations were less than the max size if here. Flush now.
1016
1019
  * `persistedBatch` will be `null` if the operations should be flushed in a new transaction.
1017
1020
  */
1018
- const { flushedAny } = await persistedBatch.flush(db);
1021
+ const { flushedAny } = await persistedBatch.flush();
1019
1022
  didFlush ||= flushedAny;
1020
1023
  }
1021
1024
  }
@@ -1149,7 +1152,7 @@ export class PostgresBucketBatch
1149
1152
  // Source ID updated
1150
1153
  if (sourceTable.syncData) {
1151
1154
  // Delete old record
1152
- persistedBatch.saveBucketData({
1155
+ await persistedBatch.saveBucketData({
1153
1156
  source_key: beforeId,
1154
1157
  table: sourceTable,
1155
1158
  before_buckets: existingBuckets,
@@ -1161,7 +1164,7 @@ export class PostgresBucketBatch
1161
1164
 
1162
1165
  if (sourceTable.syncParameters) {
1163
1166
  // Delete old parameters
1164
- persistedBatch.saveParameterData({
1167
+ await persistedBatch.saveParameterData({
1165
1168
  source_key: beforeId,
1166
1169
  table: sourceTable,
1167
1170
  evaluated: [],
@@ -1203,7 +1206,7 @@ export class PostgresBucketBatch
1203
1206
  }
1204
1207
 
1205
1208
  // Save new one
1206
- persistedBatch.saveBucketData({
1209
+ await persistedBatch.saveBucketData({
1207
1210
  source_key: afterId,
1208
1211
  evaluated,
1209
1212
  table: sourceTable,
@@ -1243,7 +1246,7 @@ export class PostgresBucketBatch
1243
1246
  );
1244
1247
  }
1245
1248
 
1246
- persistedBatch.saveParameterData({
1249
+ await persistedBatch.saveParameterData({
1247
1250
  source_key: afterId,
1248
1251
  table: sourceTable,
1249
1252
  evaluated: paramEvaluated,