@powersync/service-module-postgres-storage 0.19.6 → 0.19.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/dist/storage/batch/PostgresBucketBatch.js +37 -34
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/storage/batch/PostgresPersistedBatch.d.ts +68 -25
- package/dist/storage/batch/PostgresPersistedBatch.js +179 -147
- package/dist/storage/batch/PostgresPersistedBatch.js.map +1 -1
- package/dist/storage/current-data-store.d.ts +9 -7
- package/dist/storage/current-data-store.js +8 -2
- package/dist/storage/current-data-store.js.map +1 -1
- package/dist/types/types.d.ts +9 -8
- package/dist/types/types.js +9 -8
- package/dist/types/types.js.map +1 -1
- package/package.json +2 -2
- package/src/storage/batch/PostgresBucketBatch.ts +38 -35
- package/src/storage/batch/PostgresPersistedBatch.ts +238 -192
- package/src/storage/current-data-store.ts +10 -4
- package/src/types/types.ts +9 -8
- package/test/src/storage_flush.test.ts +162 -41
- package/tsconfig.tsbuildinfo +1 -1
package/dist/types/types.js
CHANGED
|
@@ -6,17 +6,18 @@ export const MAX_BATCH_RECORD_COUNT = 2000;
|
|
|
6
6
|
export const MAX_BATCH_ESTIMATED_SIZE = 5_000_000;
|
|
7
7
|
export const MAX_BATCH_CURRENT_DATA_SIZE = 50_000_000;
|
|
8
8
|
/**
|
|
9
|
-
*
|
|
10
|
-
* bucket_data and bucket_parameters.
|
|
9
|
+
* Upper bound for the estimated size of the rows written by a single INSERT statement when
|
|
10
|
+
* flushing bucket_data and bucket_parameters.
|
|
11
11
|
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* into
|
|
12
|
+
* Rows are inserted while they accumulate, once the pending rows would exceed the smaller of
|
|
13
|
+
* `max_estimated_size` and this limit. A single source row can produce many operations when it
|
|
14
|
+
* is evaluated into many buckets, which `max_estimated_size` alone does not bound since it is
|
|
15
|
+
* only checked between rows.
|
|
15
16
|
*
|
|
16
17
|
* The rows are passed to Postgres as a single JSON parameter, which is serialized to a string
|
|
17
|
-
* before being sent. Node limits a string to roughly 512MB. The estimate
|
|
18
|
-
*
|
|
19
|
-
*
|
|
18
|
+
* before being sent. Node limits a string to roughly 512MB. The estimate does not account for
|
|
19
|
+
* the JSON escaping of the already-serialized row data, so the actual parameter can be close to
|
|
20
|
+
* double the estimate. 64MB keeps a wide margin for that.
|
|
20
21
|
*
|
|
21
22
|
* This is not part of `batch_limits`: the limit comes from the runtime, not from the workload.
|
|
22
23
|
*/
|
package/dist/types/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAEhE,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,KAAK,MAAM,MAAM,0BAA0B,CAAC;AAEnD,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAE3C,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAElD,MAAM,CAAC,MAAM,2BAA2B,GAAG,UAAU,CAAC;AAEtD
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,YAAY,MAAM,iCAAiC,CAAC;AAEhE,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAC9B,OAAO,KAAK,MAAM,MAAM,0BAA0B,CAAC;AAEnD,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,CAAC;AAE3C,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAElD,MAAM,CAAC,MAAM,2BAA2B,GAAG,UAAU,CAAC;AAEtD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAG,UAAU,CAAC;AAEnD,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC;;OAEG;IACH,kBAAkB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACvC;;OAEG;IACH,gBAAgB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACtC,CAAC,CAAC;AAIH,MAAM,CAAC,MAAM,oBAAoB,GAAG,WAAW,CAAC,GAAG,CACjD,CAAC,CAAC,MAAM,CAAC;IACP;;OAEG;IACH,2BAA2B,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CACjD,CAAC,CACH,CAAC;AAIF,MAAM,CAAC,MAAM,qBAAqB,GAAG,UAAU,CAAC,iBAAiB,CAAC,GAAG,CAAC,YAAY,CAAC,4BAA4B,CAAC,CAAC,GAAG,CAClH,CAAC,CAAC,MAAM,CAAC;IACP;;;;;OAKG;IACH,YAAY,EAAE,oBAAoB,CAAC,QAAQ,EAAE;CAC9C,CAAC,CACH,CAAC;AAiBF,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAC5C,UAAwC,EACP,EAAE;IACnC,OAAO;QACL,GAAG,YAAY,CAAC,yBAAyB,CAAC,UAAU,CAAC;QACrD,YAAY,EAAE;YACZ,2BAA2B,EAAE,UAAU,CAAC,YAAY,EAAE,2BAA2B,IAAI,2BAA2B;YAChH,kBAAkB,EAAE,UAAU,CAAC,YAAY,EAAE,kBAAkB,IAAI,wBAAwB;YAC3F,gBAAgB,EAAE,UAAU,CAAC,YAAY,EAAE,gBAAgB,IAAI,sBAAsB;YACrF,kBAAkB,EAAE,wBAAwB;SAC7C;KACF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,MAAoC,EAAmC,EAAE;IAC/G,OAAO,MAAM,CAAC,IAAI,IAAI,YAAY,CAAC,wBAAwB,CAAC;AAC9D,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@powersync/service-module-postgres-storage",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
|
-
"version": "0.19.
|
|
4
|
+
"version": "0.19.8",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"publishConfig": {
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
"uuid": "^14.0.0",
|
|
18
18
|
"@powersync/lib-service-postgres": "0.5.8",
|
|
19
19
|
"@powersync/lib-services-framework": "0.10.5",
|
|
20
|
-
"@powersync/service-core": "1.28.
|
|
20
|
+
"@powersync/service-core": "1.28.3",
|
|
21
21
|
"@powersync/service-types": "0.21.0",
|
|
22
22
|
"@powersync/service-jpgwire": "0.21.27",
|
|
23
23
|
"@powersync/service-jsonbig": "0.17.14",
|
|
@@ -406,42 +406,44 @@ export class PostgresBucketBatch
|
|
|
406
406
|
const persistedBatch = new PostgresPersistedBatch({
|
|
407
407
|
group_id: this.group_id,
|
|
408
408
|
storageConfig: this.storageConfig,
|
|
409
|
+
db,
|
|
409
410
|
...this.options.batch_limits
|
|
410
411
|
});
|
|
411
412
|
|
|
412
|
-
|
|
413
|
+
// Read in full rather than streamed, since the batch may write on this connection while
|
|
414
|
+
// the rows are processed (see PostgresPersistedBatch). BATCH_LIMIT bounds the rows in memory.
|
|
415
|
+
const rows = await this.currentDataStore.queryTruncateRows(db, {
|
|
413
416
|
groupId: this.group_id,
|
|
414
417
|
sourceTableId: postgresTableId(sourceTable.id),
|
|
415
418
|
limit: BATCH_LIMIT
|
|
416
|
-
})
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
}
|
|
419
|
+
});
|
|
420
|
+
lastBatchCount = rows.length;
|
|
421
|
+
processedCount += rows.length;
|
|
422
|
+
|
|
423
|
+
for (const row of rows) {
|
|
424
|
+
const value = this.currentDataStore.decodeTruncateRow(row);
|
|
425
|
+
const source_key = deserializeReplicaId(value.source_key);
|
|
426
|
+
await persistedBatch.saveBucketData({
|
|
427
|
+
before_buckets: value.buckets,
|
|
428
|
+
evaluated: [],
|
|
429
|
+
table: sourceTable,
|
|
430
|
+
source_key
|
|
431
|
+
});
|
|
432
|
+
await persistedBatch.saveParameterData({
|
|
433
|
+
existing_lookups: value.lookups,
|
|
434
|
+
evaluated: [],
|
|
435
|
+
table: sourceTable,
|
|
436
|
+
source_key
|
|
437
|
+
});
|
|
438
|
+
persistedBatch.deleteCurrentData({
|
|
439
|
+
// This is serialized since we got it from a DB query
|
|
440
|
+
serialized_source_key: value.source_key,
|
|
441
|
+
source_table_id: postgresTableId(sourceTable.id),
|
|
442
|
+
// No need for soft delete, since this is not streaming replication
|
|
443
|
+
soft: false
|
|
444
|
+
});
|
|
443
445
|
}
|
|
444
|
-
const { flushedAny } = await persistedBatch.flush(
|
|
446
|
+
const { flushedAny } = await persistedBatch.flush();
|
|
445
447
|
clearedError = flushedAny && !this.clearedError;
|
|
446
448
|
});
|
|
447
449
|
if (clearedError) {
|
|
@@ -978,6 +980,7 @@ export class PostgresBucketBatch
|
|
|
978
980
|
let persistedBatch: PostgresPersistedBatch | null = new PostgresPersistedBatch({
|
|
979
981
|
group_id: this.group_id,
|
|
980
982
|
storageConfig: this.storageConfig,
|
|
983
|
+
db,
|
|
981
984
|
...this.options.batch_limits
|
|
982
985
|
});
|
|
983
986
|
|
|
@@ -1001,7 +1004,7 @@ export class PostgresBucketBatch
|
|
|
1001
1004
|
}
|
|
1002
1005
|
|
|
1003
1006
|
if (persistedBatch!.shouldFlushTransaction()) {
|
|
1004
|
-
const { flushedAny } = await persistedBatch!.flush(
|
|
1007
|
+
const { flushedAny } = await persistedBatch!.flush();
|
|
1005
1008
|
didFlush ||= flushedAny;
|
|
1006
1009
|
// The operations stored in this batch will be processed in the `resumeBatch`
|
|
1007
1010
|
persistedBatch = null;
|
|
@@ -1015,7 +1018,7 @@ export class PostgresBucketBatch
|
|
|
1015
1018
|
* The operations were less than the max size if here. Flush now.
|
|
1016
1019
|
* `persistedBatch` will be `null` if the operations should be flushed in a new transaction.
|
|
1017
1020
|
*/
|
|
1018
|
-
const { flushedAny } = await persistedBatch.flush(
|
|
1021
|
+
const { flushedAny } = await persistedBatch.flush();
|
|
1019
1022
|
didFlush ||= flushedAny;
|
|
1020
1023
|
}
|
|
1021
1024
|
}
|
|
@@ -1149,7 +1152,7 @@ export class PostgresBucketBatch
|
|
|
1149
1152
|
// Source ID updated
|
|
1150
1153
|
if (sourceTable.syncData) {
|
|
1151
1154
|
// Delete old record
|
|
1152
|
-
persistedBatch.saveBucketData({
|
|
1155
|
+
await persistedBatch.saveBucketData({
|
|
1153
1156
|
source_key: beforeId,
|
|
1154
1157
|
table: sourceTable,
|
|
1155
1158
|
before_buckets: existingBuckets,
|
|
@@ -1161,7 +1164,7 @@ export class PostgresBucketBatch
|
|
|
1161
1164
|
|
|
1162
1165
|
if (sourceTable.syncParameters) {
|
|
1163
1166
|
// Delete old parameters
|
|
1164
|
-
persistedBatch.saveParameterData({
|
|
1167
|
+
await persistedBatch.saveParameterData({
|
|
1165
1168
|
source_key: beforeId,
|
|
1166
1169
|
table: sourceTable,
|
|
1167
1170
|
evaluated: [],
|
|
@@ -1203,7 +1206,7 @@ export class PostgresBucketBatch
|
|
|
1203
1206
|
}
|
|
1204
1207
|
|
|
1205
1208
|
// Save new one
|
|
1206
|
-
persistedBatch.saveBucketData({
|
|
1209
|
+
await persistedBatch.saveBucketData({
|
|
1207
1210
|
source_key: afterId,
|
|
1208
1211
|
evaluated,
|
|
1209
1212
|
table: sourceTable,
|
|
@@ -1243,7 +1246,7 @@ export class PostgresBucketBatch
|
|
|
1243
1246
|
);
|
|
1244
1247
|
}
|
|
1245
1248
|
|
|
1246
|
-
persistedBatch.saveParameterData({
|
|
1249
|
+
await persistedBatch.saveParameterData({
|
|
1247
1250
|
source_key: afterId,
|
|
1248
1251
|
table: sourceTable,
|
|
1249
1252
|
evaluated: paramEvaluated,
|