@powersync/service-module-postgres-storage 0.0.0-dev-20260223080959 → 0.0.0-dev-20260224151854
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -7
- package/dist/.tsbuildinfo +1 -1
- package/dist/@types/storage/PostgresCompactor.d.ts +2 -1
- package/dist/@types/types/types.d.ts +2 -0
- package/dist/storage/PostgresCompactor.js +41 -60
- package/dist/storage/PostgresCompactor.js.map +1 -1
- package/package.json +8 -8
- package/src/storage/PostgresCompactor.ts +46 -64
- package/test/src/storage_compacting.test.ts +51 -2
|
@@ -22,7 +22,8 @@ export declare class PostgresCompactor {
|
|
|
22
22
|
* See /docs/compacting-operations.md for details.
|
|
23
23
|
*/
|
|
24
24
|
compact(): Promise<void>;
|
|
25
|
-
|
|
25
|
+
private compactAllBuckets;
|
|
26
|
+
private compactSingleBucket;
|
|
26
27
|
private flush;
|
|
27
28
|
/**
|
|
28
29
|
* Perform a CLEAR compact for a bucket.
|
|
@@ -50,6 +50,7 @@ export declare const PostgresStorageConfig: t.Intersection<t.Codec<{
|
|
|
50
50
|
reject_ip_ranges?: string[] | undefined;
|
|
51
51
|
slot_name_prefix?: string | undefined;
|
|
52
52
|
max_pool_size?: number | undefined;
|
|
53
|
+
connect_timeout?: number | undefined;
|
|
53
54
|
}, {
|
|
54
55
|
type: string;
|
|
55
56
|
max_pool_size?: number | undefined;
|
|
@@ -71,6 +72,7 @@ export declare const PostgresStorageConfig: t.Intersection<t.Codec<{
|
|
|
71
72
|
reject_ip_ranges?: string[] | undefined;
|
|
72
73
|
slot_name_prefix?: string | undefined;
|
|
73
74
|
max_pool_size?: number | undefined;
|
|
75
|
+
connect_timeout?: number | undefined;
|
|
74
76
|
}, string, t.CodecProps>, t.ObjectCodec<{
|
|
75
77
|
/**
|
|
76
78
|
* Allow batch operation limits to be configurable.
|
|
@@ -38,36 +38,48 @@ export class PostgresCompactor {
|
|
|
38
38
|
async compact() {
|
|
39
39
|
if (this.buckets) {
|
|
40
40
|
for (let bucket of this.buckets) {
|
|
41
|
-
|
|
42
|
-
// through the buckets in a single query.
|
|
43
|
-
// That makes batching more tricky, so we leave for later.
|
|
44
|
-
await this.compactInternal(bucket);
|
|
41
|
+
await this.compactSingleBucket(bucket);
|
|
45
42
|
}
|
|
46
43
|
}
|
|
47
44
|
else {
|
|
48
|
-
await this.
|
|
45
|
+
await this.compactAllBuckets();
|
|
49
46
|
}
|
|
50
47
|
}
|
|
51
|
-
async
|
|
52
|
-
const
|
|
53
|
-
let
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
48
|
+
async compactAllBuckets() {
|
|
49
|
+
const DISCOVERY_BATCH_SIZE = 200;
|
|
50
|
+
let lastBucket = '';
|
|
51
|
+
while (true) {
|
|
52
|
+
const bucketRows = (await this.db.sql `
|
|
53
|
+
SELECT DISTINCT
|
|
54
|
+
bucket_name
|
|
55
|
+
FROM
|
|
56
|
+
bucket_data
|
|
57
|
+
WHERE
|
|
58
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
59
|
+
AND bucket_name > ${{ type: 'varchar', value: lastBucket }}
|
|
60
|
+
ORDER BY
|
|
61
|
+
bucket_name ASC
|
|
62
|
+
LIMIT
|
|
63
|
+
${{ type: 'int4', value: DISCOVERY_BATCH_SIZE }}
|
|
64
|
+
`.rows());
|
|
65
|
+
if (bucketRows.length === 0) {
|
|
66
|
+
break;
|
|
67
|
+
}
|
|
68
|
+
for (const row of bucketRows) {
|
|
69
|
+
await this.compactSingleBucket(row.bucket_name);
|
|
70
|
+
}
|
|
71
|
+
lastBucket = bucketRows[bucketRows.length - 1].bucket_name;
|
|
70
72
|
}
|
|
73
|
+
}
|
|
74
|
+
async compactSingleBucket(bucket) {
|
|
75
|
+
const idLimitBytes = this.idLimitBytes;
|
|
76
|
+
let currentState = {
|
|
77
|
+
bucket: bucket,
|
|
78
|
+
seen: new Map(),
|
|
79
|
+
trackingSize: 0,
|
|
80
|
+
lastNotPut: null,
|
|
81
|
+
opsSincePut: 0
|
|
82
|
+
};
|
|
71
83
|
let upperOpIdLimit = BIGINT_MAX;
|
|
72
84
|
while (true) {
|
|
73
85
|
const batch = await this.db.sql `
|
|
@@ -83,16 +95,9 @@ export class PostgresCompactor {
|
|
|
83
95
|
bucket_data
|
|
84
96
|
WHERE
|
|
85
97
|
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
86
|
-
AND bucket_name
|
|
87
|
-
AND
|
|
88
|
-
(
|
|
89
|
-
bucket_name = ${{ type: 'varchar', value: bucketUpper }}
|
|
90
|
-
AND op_id < ${{ type: 'int8', value: upperOpIdLimit }}
|
|
91
|
-
)
|
|
92
|
-
OR bucket_name < ${{ type: 'varchar', value: bucketUpper }} COLLATE "C" -- Use binary comparison
|
|
93
|
-
)
|
|
98
|
+
AND bucket_name = ${{ type: 'varchar', value: bucket }}
|
|
99
|
+
AND op_id < ${{ type: 'int8', value: upperOpIdLimit }}
|
|
94
100
|
ORDER BY
|
|
95
|
-
bucket_name DESC,
|
|
96
101
|
op_id DESC
|
|
97
102
|
LIMIT
|
|
98
103
|
${{ type: 'int4', value: this.moveBatchQueryLimit }}
|
|
@@ -106,27 +111,7 @@ export class PostgresCompactor {
|
|
|
106
111
|
// Set upperBound for the next batch
|
|
107
112
|
const lastBatchItem = batch[batch.length - 1];
|
|
108
113
|
upperOpIdLimit = lastBatchItem.op_id;
|
|
109
|
-
bucketUpper = lastBatchItem.bucket_name;
|
|
110
114
|
for (const doc of batch) {
|
|
111
|
-
if (currentState == null || doc.bucket_name != currentState.bucket) {
|
|
112
|
-
if (currentState != null && currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
|
|
113
|
-
// Important to flush before clearBucket()
|
|
114
|
-
await this.flush();
|
|
115
|
-
logger.info(`Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
|
|
116
|
-
const bucket = currentState.bucket;
|
|
117
|
-
const clearOp = currentState.lastNotPut;
|
|
118
|
-
// Free memory before clearing bucket
|
|
119
|
-
currentState = null;
|
|
120
|
-
await this.clearBucket(bucket, clearOp);
|
|
121
|
-
}
|
|
122
|
-
currentState = {
|
|
123
|
-
bucket: doc.bucket_name,
|
|
124
|
-
seen: new Map(),
|
|
125
|
-
trackingSize: 0,
|
|
126
|
-
lastNotPut: null,
|
|
127
|
-
opsSincePut: 0
|
|
128
|
-
};
|
|
129
|
-
}
|
|
130
115
|
if (this.maxOpId != null && doc.op_id > this.maxOpId) {
|
|
131
116
|
continue;
|
|
132
117
|
}
|
|
@@ -185,14 +170,10 @@ export class PostgresCompactor {
|
|
|
185
170
|
}
|
|
186
171
|
}
|
|
187
172
|
await this.flush();
|
|
188
|
-
currentState
|
|
189
|
-
if (currentState
|
|
173
|
+
currentState.seen.clear();
|
|
174
|
+
if (currentState.lastNotPut != null && currentState.opsSincePut > 1) {
|
|
190
175
|
logger.info(`Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`);
|
|
191
|
-
|
|
192
|
-
const clearOp = currentState.lastNotPut;
|
|
193
|
-
// Free memory before clearing bucket
|
|
194
|
-
currentState = null;
|
|
195
|
-
await this.clearBucket(bucket, clearOp);
|
|
176
|
+
await this.clearBucket(currentState.bucket, currentState.lastNotPut);
|
|
196
177
|
}
|
|
197
178
|
}
|
|
198
179
|
async flush() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PostgresCompactor.js","sourceRoot":"","sources":["../../src/storage/PostgresCompactor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AACtF,OAAO,EAAyB,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGvE,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AA8B5D,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,iBAAiB;IAWlB;IACA;IAXF,OAAO,GAAuB,EAAE,CAAC;IAEjC,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,eAAe,CAAS;IACxB,OAAO,CAA2B;IAClC,OAAO,CAAuB;IAEtC,YACU,EAA+B,EAC/B,QAAgB,EACxB,OAAgC;QAFxB,OAAE,GAAF,EAAE,CAA6B;QAC/B,aAAQ,GAAR,QAAQ,CAAQ;QAGxB,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,
|
|
1
|
+
{"version":3,"file":"PostgresCompactor.js","sourceRoot":"","sources":["../../src/storage/PostgresCompactor.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AACtF,OAAO,EAAyB,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAGvE,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC3C,OAAO,EAAE,GAAG,EAAE,MAAM,gBAAgB,CAAC;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,sBAAsB,CAAC;AAC5C,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AA8B5D,MAAM,yBAAyB,GAAG,IAAI,CAAC;AACvC,MAAM,wBAAwB,GAAG,IAAI,CAAC;AACtC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAE9C,2CAA2C;AAC3C,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAEnC,MAAM,OAAO,iBAAiB;IAWlB;IACA;IAXF,OAAO,GAAuB,EAAE,CAAC;IAEjC,YAAY,CAAS;IACrB,cAAc,CAAS;IACvB,mBAAmB,CAAS;IAC5B,eAAe,CAAS;IACxB,OAAO,CAA2B;IAClC,OAAO,CAAuB;IAEtC,YACU,EAA+B,EAC/B,QAAgB,EACxB,OAAgC;QAFxB,OAAE,GAAF,EAAE,CAA6B;QAC/B,aAAQ,GAAR,QAAQ,CAAQ;QAGxB,IAAI,CAAC,YAAY,GAAG,CAAC,OAAO,EAAE,aAAa,IAAI,uBAAuB,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;QACtF,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,IAAI,wBAAwB,CAAC;QAC1E,IAAI,CAAC,mBAAmB,GAAG,OAAO,EAAE,mBAAmB,IAAI,8BAA8B,CAAC;QAC1F,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,IAAI,yBAAyB,CAAC;QAC7E,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,cAAc,CAAC;IACzC,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,KAAK,IAAI,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBAChC,MAAM,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;YACzC,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACjC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,iBAAiB;QAC7B,MAAM,oBAAoB,GAAG,GAAG,CAAC;QACjC,IAAI,UAAU,GAAG,EAAE,CAAC;QAEpB,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;uBAMpB,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;8BAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE;;;;YAIxD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,oBAAoB,EAAE;OAClD,CAAC,IAAI,EAAE,CAA8B,CAAC;YAEvC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC5B,MAAM;YACR,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC;YAClD,CAAC;YAED,UAAU,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC;QAC7D,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAAC,MAAc;QAC9C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;QAEvC,IAAI,YAAY,GAAuB;YACrC,MAAM,EAAE,MAAM;YACd,IAAI,EAAE,IAAI,GAAG,EAAE;YACf,YAAY,EAAE,CAAC;YACf,UAAU,EAAE,IAAI;YAChB,WAAW,EAAE,CAAC;SACf,CAAC;QAEF,IAAI,cAAc,GAAG,UAAU,CAAC;QAEhC,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;;uBAYd,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;8BAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;wBACxC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE;;;;YAInD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,mBAAmB,EAAE;OACtD;iBACE,OAAO,CACN,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,YAAY,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,CAAC,CAAC,CAC9G;iBACA,IAAI,EAAE,CAAC;YAEV,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;gBACtB,wBAAwB;gBACxB,MAAM;YACR,CAAC;YAED,oCAAoC;YACpC,MAAM,aAAa,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC9C,cAAc,GAAG,aAAa,CAAC,KAAK,CAAC;YAErC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;oBACrD,SAAS;gBACX,CAAC;gBAED,IAAI,eAAe,GAAG,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC;gBAEtC,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,EAAE,CAAC;oBAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,MAAM,IAAI,eAAe,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC,EAAE,CAAC;oBACrG,MAAM,QAAQ,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;oBAC3D,IAAI,QAAQ,EAAE,CAAC;wBACb,8CAA8C;wBAC9C,eAAe,GAAG,KAAK,CAAC;wBAExB,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAA;;;;8BAIH,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;;;;;;;6BAOlC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;oCAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,WAAW,EAAE;8BACjD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE;aACnD,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,IAAI,YAAY,CAAC,YAAY,IAAI,YAAY,EAAE,CAAC;4BAC9C,wBAAwB;4BACxB,6CAA6C;wBAC/C,CAAC;6BAAM,CAAC;4BACN,4DAA4D;4BAC5D,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC;4BACrD,6BAA6B;4BAC7B,oBAAoB;4BACpB,sBAAsB;4BACtB,6BAA6B;4BAC7B,YAAY,CAAC,YAAY,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC;wBAChD,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,eAAe,EAAE,CAAC;oBACpB,YAAY,CAAC,UAAU,GAAG,IAAI,CAAC;oBAC/B,YAAY,CAAC,WAAW,GAAG,CAAC,CAAC;gBAC/B,CAAC;qBAAM,IAAI,GAAG,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;oBAC7B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC;wBACpC,YAAY,CAAC,UAAU,GAAG,GAAG,CAAC,KAAK,CAAC;oBACtC,CAAC;oBACD,YAAY,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChC,CAAC;gBAED,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;oBAC/C,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACnB,YAAY,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,YAAY,CAAC,UAAU,IAAI,IAAI,IAAI,YAAY,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;YACpE,MAAM,CAAC,IAAI,CACT,sBAAsB,IAAI,CAAC,QAAQ,IAAI,YAAY,CAAC,MAAM,IAAI,YAAY,CAAC,UAAU,cAAc,YAAY,CAAC,WAAW,aAAa,CACzI,CAAC;YACF,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,MAAM,EAAE,YAAY,CAAC,UAAU,CAAC,CAAC;QACvE,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC,cAAc,IAAI,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC,CAAC;YACrD,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC;YACrC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACK,KAAK,CAAC,WAAW,CAAC,MAAc,EAAE,EAAgB;QACxD;;;WAGG;QACH,IAAI,IAAI,GAAG,KAAK,CAAC;QACjB,OAAO,CAAC,IAAI,EAAE,CAAC;YACb,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,KAAK,EAAE,EAAE,EAAE,EAAE;gBACxC;;;mBAGG;gBACH,MAAM,EAAE,CAAC,GAAG,CAAA,mDAAmD,CAAC,OAAO,EAAE,CAAC;gBAE1E,IAAI,CAAC;oBACH,IAAI,QAAQ,GAAG,CAAC,CAAC;oBACjB,IAAI,QAAQ,GAAwB,IAAI,CAAC;oBACzC,IAAI,QAAQ,GAAwB,IAAI,CAAC;oBACzC,IAAI,OAAO,GAAG,KAAK,CAAC;oBAEpB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,cAAc,EAAE,YAAY,EAAE,OAAO,EAAE,UAAU,EAAE,WAAW,CAAC,CAAC,CAAC;oBAC9G,IAAI,KAAK,EAAE,MAAM,UAAU,IAAI,EAAE,CAAC,UAAU,CAA0B,GAAG,CAAA;;;;;;;;;;;2BAWxD,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;kCAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;6BACvC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE;;;;gBAIxC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE;WAClD,CAAC,EAAE,CAAC;wBACH,MAAM,UAAU,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;wBAC1D,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;4BAC5B,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;gCACpF,QAAQ,GAAG,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC;gCAC7D,QAAQ,GAAG,EAAE,CAAC,KAAK,CAAC;gCACpB,IAAI,EAAE,CAAC,EAAE,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;oCACjC,OAAO,GAAG,IAAI,CAAC;gCACjB,CAAC;gCACD,IAAI,EAAE,CAAC,SAAS,IAAI,IAAI,EAAE,CAAC;oCACzB,IAAI,QAAQ,IAAI,IAAI,IAAI,EAAE,CAAC,SAAS,GAAG,QAAQ,EAAE,CAAC;wCAChD,QAAQ,GAAG,EAAE,CAAC,SAAS,CAAC;oCAC1B,CAAC;gCACH,CAAC;4BACH,CAAC;iCAAM,CAAC;gCACN,MAAM,IAAI,yBAAyB,CACjC,cAAc,EAAE,CAAC,EAAE,iBAAiB,IAAI,CAAC,QAAQ,IAAI,MAAM,IAAI,EAAE,CAAC,KAAK,EAAE,CAC1E,CAAC;4BACJ,CAAC;wBACH,CAAC;oBACH,CAAC;oBAED,IAAI,CAAC,OAAO,EAAE,CAAC;wBACb,MAAM,EAAE,CAAC,GAAG,CAAA,QAAQ,CAAC,OAAO,EAAE,CAAC;wBAC/B,IAAI,GAAG,IAAI,CAAC;wBACZ,OAAO;oBACT,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,qBAAqB,QAAQ,EAAE,CAAC,CAAC;oBAE7C,MAAM,EAAE,CAAC,GAAG,CAAA;;;2BAGK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;kCAC/B,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;6BACvC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;WACnD,CAAC,OAAO,EAAE,CAAC;oBAEZ,MAAM,EAAE,CAAC,GAAG,CAAA;;;;;;;;;;;;kBAYJ,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE;kBACtC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE;kBAClC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;kBACjC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE;kBAC/C,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;kBACjC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE;;WAExC,CAAC,OAAO,EAAE,CAAC;oBAEZ,MAAM,EAAE,CAAC,GAAG,CAAA,QAAQ,CAAC,OAAO,EAAE,CAAC;gBACjC,CAAC;gBAAC,OAAO,EAAE,EAAE,CAAC;oBACZ,MAAM,EAAE,CAAC,GAAG,CAAA,UAAU,CAAC,OAAO,EAAE,CAAC;oBACjC,MAAM,EAAE,CAAC;gBACX,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}
|
package/package.json
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"name": "@powersync/service-module-postgres-storage",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
4
|
"types": "dist/@types/index.d.ts",
|
|
5
|
-
"version": "0.0.0-dev-
|
|
5
|
+
"version": "0.0.0-dev-20260224151854",
|
|
6
6
|
"main": "dist/index.js",
|
|
7
7
|
"license": "FSL-1.1-ALv2",
|
|
8
8
|
"type": "module",
|
|
@@ -29,17 +29,17 @@
|
|
|
29
29
|
"p-defer": "^4.0.1",
|
|
30
30
|
"ts-codec": "^1.3.0",
|
|
31
31
|
"uuid": "^11.1.0",
|
|
32
|
-
"@powersync/lib-service-postgres": "0.0.0-dev-
|
|
33
|
-
"@powersync/lib-services-framework": "0.0.0-dev-
|
|
34
|
-
"@powersync/service-core": "0.0.0-dev-
|
|
35
|
-
"@powersync/service-types": "0.0.0-dev-
|
|
36
|
-
"@powersync/service-jpgwire": "0.0.0-dev-
|
|
32
|
+
"@powersync/lib-service-postgres": "0.0.0-dev-20260224151854",
|
|
33
|
+
"@powersync/lib-services-framework": "0.0.0-dev-20260224151854",
|
|
34
|
+
"@powersync/service-core": "0.0.0-dev-20260224151854",
|
|
35
|
+
"@powersync/service-types": "0.0.0-dev-20260224151854",
|
|
36
|
+
"@powersync/service-jpgwire": "0.0.0-dev-20260224151854",
|
|
37
37
|
"@powersync/service-jsonbig": "0.17.12",
|
|
38
|
-
"@powersync/service-sync-rules": "0.0.0-dev-
|
|
38
|
+
"@powersync/service-sync-rules": "0.0.0-dev-20260224151854"
|
|
39
39
|
},
|
|
40
40
|
"devDependencies": {
|
|
41
41
|
"typescript": "^5.7.3",
|
|
42
|
-
"@powersync/service-core-tests": "0.0.0-dev-
|
|
42
|
+
"@powersync/service-core-tests": "0.0.0-dev-20260224151854"
|
|
43
43
|
},
|
|
44
44
|
"scripts": {
|
|
45
45
|
"build": "tsc -b",
|
|
@@ -75,37 +75,54 @@ export class PostgresCompactor {
|
|
|
75
75
|
async compact() {
|
|
76
76
|
if (this.buckets) {
|
|
77
77
|
for (let bucket of this.buckets) {
|
|
78
|
-
|
|
79
|
-
// through the buckets in a single query.
|
|
80
|
-
// That makes batching more tricky, so we leave for later.
|
|
81
|
-
await this.compactInternal(bucket);
|
|
78
|
+
await this.compactSingleBucket(bucket);
|
|
82
79
|
}
|
|
83
80
|
} else {
|
|
84
|
-
await this.
|
|
81
|
+
await this.compactAllBuckets();
|
|
85
82
|
}
|
|
86
83
|
}
|
|
87
84
|
|
|
88
|
-
async
|
|
89
|
-
const
|
|
85
|
+
private async compactAllBuckets() {
|
|
86
|
+
const DISCOVERY_BATCH_SIZE = 200;
|
|
87
|
+
let lastBucket = '';
|
|
88
|
+
|
|
89
|
+
while (true) {
|
|
90
|
+
const bucketRows = (await this.db.sql`
|
|
91
|
+
SELECT DISTINCT
|
|
92
|
+
bucket_name
|
|
93
|
+
FROM
|
|
94
|
+
bucket_data
|
|
95
|
+
WHERE
|
|
96
|
+
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
97
|
+
AND bucket_name > ${{ type: 'varchar', value: lastBucket }}
|
|
98
|
+
ORDER BY
|
|
99
|
+
bucket_name ASC
|
|
100
|
+
LIMIT
|
|
101
|
+
${{ type: 'int4', value: DISCOVERY_BATCH_SIZE }}
|
|
102
|
+
`.rows()) as { bucket_name: string }[];
|
|
103
|
+
|
|
104
|
+
if (bucketRows.length === 0) {
|
|
105
|
+
break;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
for (const row of bucketRows) {
|
|
109
|
+
await this.compactSingleBucket(row.bucket_name);
|
|
110
|
+
}
|
|
90
111
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
let bucketLower: string | null = null;
|
|
94
|
-
let bucketUpper: string | null = null;
|
|
95
|
-
const MAX_CHAR = String.fromCodePoint(0xffff);
|
|
96
|
-
|
|
97
|
-
if (bucket == null) {
|
|
98
|
-
bucketLower = '';
|
|
99
|
-
bucketUpper = MAX_CHAR;
|
|
100
|
-
} else if (bucket?.includes('[')) {
|
|
101
|
-
// Exact bucket name
|
|
102
|
-
bucketLower = bucket;
|
|
103
|
-
bucketUpper = bucket;
|
|
104
|
-
} else if (bucket) {
|
|
105
|
-
// Bucket definition name
|
|
106
|
-
bucketLower = `${bucket}[`;
|
|
107
|
-
bucketUpper = `${bucket}[${MAX_CHAR}`;
|
|
112
|
+
lastBucket = bucketRows[bucketRows.length - 1].bucket_name;
|
|
108
113
|
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
private async compactSingleBucket(bucket: string) {
|
|
117
|
+
const idLimitBytes = this.idLimitBytes;
|
|
118
|
+
|
|
119
|
+
let currentState: CurrentBucketState = {
|
|
120
|
+
bucket: bucket,
|
|
121
|
+
seen: new Map(),
|
|
122
|
+
trackingSize: 0,
|
|
123
|
+
lastNotPut: null,
|
|
124
|
+
opsSincePut: 0
|
|
125
|
+
};
|
|
109
126
|
|
|
110
127
|
let upperOpIdLimit = BIGINT_MAX;
|
|
111
128
|
|
|
@@ -123,16 +140,9 @@ export class PostgresCompactor {
|
|
|
123
140
|
bucket_data
|
|
124
141
|
WHERE
|
|
125
142
|
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
126
|
-
AND bucket_name
|
|
127
|
-
AND
|
|
128
|
-
(
|
|
129
|
-
bucket_name = ${{ type: 'varchar', value: bucketUpper }}
|
|
130
|
-
AND op_id < ${{ type: 'int8', value: upperOpIdLimit }}
|
|
131
|
-
)
|
|
132
|
-
OR bucket_name < ${{ type: 'varchar', value: bucketUpper }} COLLATE "C" -- Use binary comparison
|
|
133
|
-
)
|
|
143
|
+
AND bucket_name = ${{ type: 'varchar', value: bucket }}
|
|
144
|
+
AND op_id < ${{ type: 'int8', value: upperOpIdLimit }}
|
|
134
145
|
ORDER BY
|
|
135
|
-
bucket_name DESC,
|
|
136
146
|
op_id DESC
|
|
137
147
|
LIMIT
|
|
138
148
|
${{ type: 'int4', value: this.moveBatchQueryLimit }}
|
|
@@ -150,32 +160,8 @@ export class PostgresCompactor {
|
|
|
150
160
|
// Set upperBound for the next batch
|
|
151
161
|
const lastBatchItem = batch[batch.length - 1];
|
|
152
162
|
upperOpIdLimit = lastBatchItem.op_id;
|
|
153
|
-
bucketUpper = lastBatchItem.bucket_name;
|
|
154
163
|
|
|
155
164
|
for (const doc of batch) {
|
|
156
|
-
if (currentState == null || doc.bucket_name != currentState.bucket) {
|
|
157
|
-
if (currentState != null && currentState.lastNotPut != null && currentState.opsSincePut >= 1) {
|
|
158
|
-
// Important to flush before clearBucket()
|
|
159
|
-
await this.flush();
|
|
160
|
-
logger.info(
|
|
161
|
-
`Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
|
|
162
|
-
);
|
|
163
|
-
|
|
164
|
-
const bucket = currentState.bucket;
|
|
165
|
-
const clearOp = currentState.lastNotPut;
|
|
166
|
-
// Free memory before clearing bucket
|
|
167
|
-
currentState = null;
|
|
168
|
-
await this.clearBucket(bucket, clearOp);
|
|
169
|
-
}
|
|
170
|
-
currentState = {
|
|
171
|
-
bucket: doc.bucket_name,
|
|
172
|
-
seen: new Map(),
|
|
173
|
-
trackingSize: 0,
|
|
174
|
-
lastNotPut: null,
|
|
175
|
-
opsSincePut: 0
|
|
176
|
-
};
|
|
177
|
-
}
|
|
178
|
-
|
|
179
165
|
if (this.maxOpId != null && doc.op_id > this.maxOpId) {
|
|
180
166
|
continue;
|
|
181
167
|
}
|
|
@@ -237,16 +223,12 @@ export class PostgresCompactor {
|
|
|
237
223
|
}
|
|
238
224
|
|
|
239
225
|
await this.flush();
|
|
240
|
-
currentState
|
|
241
|
-
if (currentState
|
|
226
|
+
currentState.seen.clear();
|
|
227
|
+
if (currentState.lastNotPut != null && currentState.opsSincePut > 1) {
|
|
242
228
|
logger.info(
|
|
243
229
|
`Inserting CLEAR at ${this.group_id}:${currentState.bucket}:${currentState.lastNotPut} to remove ${currentState.opsSincePut} operations`
|
|
244
230
|
);
|
|
245
|
-
|
|
246
|
-
const clearOp = currentState.lastNotPut;
|
|
247
|
-
// Free memory before clearing bucket
|
|
248
|
-
currentState = null;
|
|
249
|
-
await this.clearBucket(bucket, clearOp);
|
|
231
|
+
await this.clearBucket(currentState.bucket, currentState.lastNotPut);
|
|
250
232
|
}
|
|
251
233
|
}
|
|
252
234
|
|
|
@@ -1,5 +1,54 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
2
|
+
import { bucketRequest, bucketRequestMap, register, TEST_TABLE, test_utils } from '@powersync/service-core-tests';
|
|
3
|
+
import { describe, expect, test } from 'vitest';
|
|
3
4
|
import { POSTGRES_STORAGE_FACTORY } from './util.js';
|
|
4
5
|
|
|
5
6
|
describe('Postgres Sync Bucket Storage Compact', () => register.registerCompactTests(POSTGRES_STORAGE_FACTORY));
|
|
7
|
+
|
|
8
|
+
describe('Postgres Compact - explicit bucket name', () => {
|
|
9
|
+
test('compacts a specific bucket by exact name', async () => {
|
|
10
|
+
await using factory = await POSTGRES_STORAGE_FACTORY();
|
|
11
|
+
const syncRules = await factory.updateSyncRules(
|
|
12
|
+
updateSyncRulesFromYaml(`
|
|
13
|
+
bucket_definitions:
|
|
14
|
+
global:
|
|
15
|
+
data: [select * from test]
|
|
16
|
+
`)
|
|
17
|
+
);
|
|
18
|
+
const bucketStorage = factory.getInstance(syncRules);
|
|
19
|
+
|
|
20
|
+
const result = await bucketStorage.startBatch(test_utils.BATCH_OPTIONS, async (batch) => {
|
|
21
|
+
await batch.save({
|
|
22
|
+
sourceTable: TEST_TABLE,
|
|
23
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
24
|
+
after: { id: 't1' },
|
|
25
|
+
afterReplicaId: test_utils.rid('t1')
|
|
26
|
+
});
|
|
27
|
+
await batch.save({
|
|
28
|
+
sourceTable: TEST_TABLE,
|
|
29
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
30
|
+
after: { id: 't1' },
|
|
31
|
+
afterReplicaId: test_utils.rid('t1')
|
|
32
|
+
});
|
|
33
|
+
await batch.commit('1/1');
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
const checkpoint = result!.flushed_op;
|
|
37
|
+
|
|
38
|
+
// Compact with an explicit bucket name — exercises the this.buckets
|
|
39
|
+
// iteration path, NOT the compactAllBuckets discovery path.
|
|
40
|
+
await bucketStorage.compact({
|
|
41
|
+
compactBuckets: [bucketRequest(syncRules, 'global[]')],
|
|
42
|
+
minBucketChanges: 1
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
const batch = await test_utils.oneFromAsync(
|
|
46
|
+
bucketStorage.getBucketDataBatch(checkpoint, bucketRequestMap(syncRules, [['global[]', 0n]]))
|
|
47
|
+
);
|
|
48
|
+
|
|
49
|
+
expect(batch.chunkData.data).toMatchObject([
|
|
50
|
+
{ op_id: '1', op: 'MOVE' },
|
|
51
|
+
{ op_id: '2', op: 'PUT', object_id: 't1' }
|
|
52
|
+
]);
|
|
53
|
+
});
|
|
54
|
+
});
|