@powersync/service-module-postgres-storage 0.19.6 → 0.19.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/storage/batch/PostgresBucketBatch.js +37 -34
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/storage/batch/PostgresPersistedBatch.d.ts +68 -25
- package/dist/storage/batch/PostgresPersistedBatch.js +179 -147
- package/dist/storage/batch/PostgresPersistedBatch.js.map +1 -1
- package/dist/storage/current-data-store.d.ts +9 -7
- package/dist/storage/current-data-store.js +8 -2
- package/dist/storage/current-data-store.js.map +1 -1
- package/dist/types/types.d.ts +9 -8
- package/dist/types/types.js +9 -8
- package/dist/types/types.js.map +1 -1
- package/package.json +2 -2
- package/src/storage/batch/PostgresBucketBatch.ts +38 -35
- package/src/storage/batch/PostgresPersistedBatch.ts +238 -192
- package/src/storage/current-data-store.ts +10 -4
- package/src/types/types.ts +9 -8
- package/test/src/storage_flush.test.ts +162 -41
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -46,26 +46,44 @@ export type DeleteCurrentDataOptions = {
|
|
|
46
46
|
export type PostgresPersistedBatchOptions = RequiredOperationBatchLimits & {
|
|
47
47
|
group_id: number;
|
|
48
48
|
storageConfig: storage.StorageVersionConfig;
|
|
49
|
+
/**
|
|
50
|
+
* The replication transaction the batch is written on.
|
|
51
|
+
*/
|
|
52
|
+
db: lib_postgres.WrappedConnection;
|
|
49
53
|
};
|
|
50
54
|
|
|
51
55
|
const EMPTY_DATA = Buffer.from(bson.serialize({}));
|
|
52
56
|
|
|
57
|
+
/**
|
|
58
|
+
* Accumulates the operations of one replication transaction and writes them to Postgres.
|
|
59
|
+
*
|
|
60
|
+
* bucket_data and bucket_parameters rows are written while they accumulate: once the pending rows
|
|
61
|
+
* of a table would exceed `maxStatementSize`, they are inserted on the transaction before the next
|
|
62
|
+
* row is added. This bounds both the memory held for a single source row evaluated into many
|
|
63
|
+
* buckets and the size of the JSON parameter of each statement. current_data is written in `flush()`.
|
|
64
|
+
*
|
|
65
|
+
* Since `saveBucketData()` and `saveParameterData()` may write on the connection, no streamed query
|
|
66
|
+
* may be in progress on it while they are called: the connection processes responses in order, and
|
|
67
|
+
* a streamed query only completes once its rows are consumed.
|
|
68
|
+
*
|
|
69
|
+
* The transaction limits in `shouldFlushTransaction()` count everything written on the transaction,
|
|
70
|
+
* including the rows already inserted.
|
|
71
|
+
*/
|
|
53
72
|
export class PostgresPersistedBatch {
|
|
54
73
|
group_id: number;
|
|
74
|
+
private readonly db: lib_postgres.WrappedConnection;
|
|
55
75
|
private readonly storageConfig: storage.StorageVersionConfig;
|
|
56
76
|
private readonly currentDataStore: PostgresCurrentDataStore;
|
|
57
77
|
|
|
58
|
-
/**
|
|
59
|
-
* Very rough estimate of current operations size in bytes
|
|
60
|
-
*/
|
|
61
|
-
currentSize: number;
|
|
62
|
-
|
|
63
78
|
readonly maxTransactionBatchSize: number;
|
|
64
79
|
readonly maxTransactionDocCount: number;
|
|
80
|
+
/**
|
|
81
|
+
* Pending bucket_data or bucket_parameters rows are inserted before their estimated size exceeds this.
|
|
82
|
+
*/
|
|
65
83
|
readonly maxStatementSize: number;
|
|
66
84
|
|
|
67
85
|
/**
|
|
68
|
-
* Ordered set of bucket_data insert operation parameters
|
|
86
|
+
* Ordered set of bucket_data insert operation parameters, not inserted yet.
|
|
69
87
|
*/
|
|
70
88
|
protected bucketDataInserts: models.BucketData[];
|
|
71
89
|
protected parameterDataInserts: models.BucketParameters[];
|
|
@@ -77,23 +95,49 @@ export class PostgresPersistedBatch {
|
|
|
77
95
|
protected currentDataInserts: Map<string, models.V3CurrentData>;
|
|
78
96
|
protected currentDataDeletes: Map<string, { source_key_hex: string; source_table: string }>;
|
|
79
97
|
|
|
98
|
+
/**
|
|
99
|
+
* Very rough estimates of the size in bytes of the operations not inserted yet.
|
|
100
|
+
*/
|
|
101
|
+
private pendingBucketDataSize = 0;
|
|
102
|
+
private pendingParameterDataSize = 0;
|
|
103
|
+
private pendingCurrentDataSize = 0;
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Rows and estimated size already inserted on the transaction before `flush()`.
|
|
107
|
+
*/
|
|
108
|
+
private insertedBucketDataCount = 0;
|
|
109
|
+
private insertedParameterDataCount = 0;
|
|
110
|
+
private insertedSize = 0;
|
|
111
|
+
/**
|
|
112
|
+
* Statements issued while accumulating, excluding the ones issued by `flush()`.
|
|
113
|
+
*/
|
|
114
|
+
private earlyStatementCount = 0;
|
|
115
|
+
|
|
80
116
|
constructor(options: PostgresPersistedBatchOptions) {
|
|
81
117
|
this.group_id = options.group_id;
|
|
118
|
+
this.db = options.db;
|
|
82
119
|
this.storageConfig = options.storageConfig;
|
|
83
120
|
this.currentDataStore = new PostgresCurrentDataStore(this.storageConfig);
|
|
84
121
|
|
|
85
122
|
this.maxTransactionBatchSize = options.max_estimated_size;
|
|
86
123
|
this.maxTransactionDocCount = options.max_record_count;
|
|
87
|
-
this.maxStatementSize = options.max_statement_size;
|
|
124
|
+
this.maxStatementSize = Math.min(options.max_estimated_size, options.max_statement_size);
|
|
88
125
|
|
|
89
126
|
this.bucketDataInserts = [];
|
|
90
127
|
this.parameterDataInserts = [];
|
|
91
128
|
this.currentDataInserts = new Map();
|
|
92
129
|
this.currentDataDeletes = new Map();
|
|
93
|
-
this.currentSize = 0;
|
|
94
130
|
}
|
|
95
131
|
|
|
96
|
-
|
|
132
|
+
/**
|
|
133
|
+
* Very rough estimate of the size in bytes of all operations in this transaction,
|
|
134
|
+
* including the rows already inserted.
|
|
135
|
+
*/
|
|
136
|
+
get currentSize() {
|
|
137
|
+
return this.insertedSize + this.pendingBucketDataSize + this.pendingParameterDataSize + this.pendingCurrentDataSize;
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
async saveBucketData(options: SaveBucketDataOptions) {
|
|
97
141
|
const remaining_buckets = new Map<string, models.CurrentBucket>();
|
|
98
142
|
for (const b of options.before_buckets) {
|
|
99
143
|
const key = currentBucketKey(b);
|
|
@@ -112,43 +156,58 @@ export class PostgresPersistedBatch {
|
|
|
112
156
|
const data = JSONBig.stringify(k.data);
|
|
113
157
|
const checksum = utils.hashData(k.table, k.id, data);
|
|
114
158
|
|
|
115
|
-
this.
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
159
|
+
await this.addBucketData(
|
|
160
|
+
{
|
|
161
|
+
group_id: this.group_id,
|
|
162
|
+
bucket_name: k.bucket,
|
|
163
|
+
op: models.OpType.PUT,
|
|
164
|
+
source_table: postgresTableId(options.table.id),
|
|
165
|
+
source_key: hexSourceKey,
|
|
166
|
+
table_name: k.table,
|
|
167
|
+
row_id: k.id,
|
|
168
|
+
checksum,
|
|
169
|
+
data,
|
|
170
|
+
op_id: 0, // Will use nextval of sequence
|
|
171
|
+
target_op: null
|
|
172
|
+
},
|
|
173
|
+
k.bucket.length + data.length + hexSourceKey.length + 100
|
|
174
|
+
);
|
|
130
175
|
}
|
|
131
176
|
|
|
132
177
|
for (const bd of remaining_buckets.values()) {
|
|
133
178
|
// REMOVE operation
|
|
134
|
-
this.
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
179
|
+
await this.addBucketData(
|
|
180
|
+
{
|
|
181
|
+
group_id: this.group_id,
|
|
182
|
+
bucket_name: bd.bucket,
|
|
183
|
+
op: models.OpType.REMOVE,
|
|
184
|
+
source_table: postgresTableId(options.table.id),
|
|
185
|
+
source_key: hexSourceKey,
|
|
186
|
+
table_name: bd.table,
|
|
187
|
+
row_id: bd.id,
|
|
188
|
+
checksum: dchecksum,
|
|
189
|
+
op_id: 0, // Will use nextval of sequence
|
|
190
|
+
target_op: null,
|
|
191
|
+
data: null
|
|
192
|
+
},
|
|
193
|
+
bd.bucket.length + hexSourceKey.length + 100
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
/**
|
|
199
|
+
* Inserts the pending bucket_data rows first if adding this row would exceed the statement size.
|
|
200
|
+
*/
|
|
201
|
+
private async addBucketData(row: models.BucketData, estimatedSize: number) {
|
|
202
|
+
if (this.bucketDataInserts.length > 0 && this.pendingBucketDataSize + estimatedSize > this.maxStatementSize) {
|
|
203
|
+
await this.flushBucketData();
|
|
204
|
+
this.earlyStatementCount += 1;
|
|
148
205
|
}
|
|
206
|
+
this.bucketDataInserts.push(row);
|
|
207
|
+
this.pendingBucketDataSize += estimatedSize;
|
|
149
208
|
}
|
|
150
209
|
|
|
151
|
-
saveParameterData(options: SaveParameterDataOptions) {
|
|
210
|
+
async saveParameterData(options: SaveParameterDataOptions) {
|
|
152
211
|
// This is similar to saving bucket data.
|
|
153
212
|
// A key difference is that we don't need to keep the history intact.
|
|
154
213
|
// We do need to keep track of recent history though - enough that we can get consistent data for any specific checkpoint.
|
|
@@ -171,32 +230,48 @@ export class PostgresPersistedBatch {
|
|
|
171
230
|
remaining_lookups.delete(base64);
|
|
172
231
|
const hexLookup = binLookup.toString('hex');
|
|
173
232
|
const serializedBucketParameters = JSONBig.stringify(result.bucketParameters);
|
|
174
|
-
this.
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
233
|
+
await this.addParameterData(
|
|
234
|
+
{
|
|
235
|
+
group_id: this.group_id,
|
|
236
|
+
source_table: postgresTableId(table.id),
|
|
237
|
+
source_key: hexSourceKey,
|
|
238
|
+
bucket_parameters: serializedBucketParameters,
|
|
239
|
+
id: 0, // auto incrementing id
|
|
240
|
+
lookup: hexLookup
|
|
241
|
+
},
|
|
242
|
+
hexLookup.length + serializedBucketParameters.length + hexSourceKey.length + 100
|
|
243
|
+
);
|
|
183
244
|
}
|
|
184
245
|
|
|
185
246
|
// 2. "REMOVE" entries for any lookup not touched.
|
|
186
247
|
for (const lookup of remaining_lookups.values()) {
|
|
187
248
|
const hexLookup = lookup.toString('hex');
|
|
188
|
-
this.
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
249
|
+
await this.addParameterData(
|
|
250
|
+
{
|
|
251
|
+
group_id: this.group_id,
|
|
252
|
+
source_table: postgresTableId(table.id),
|
|
253
|
+
source_key: hexSourceKey,
|
|
254
|
+
bucket_parameters: JSON.stringify([]),
|
|
255
|
+
id: 0, // auto incrementing id
|
|
256
|
+
lookup: hexLookup
|
|
257
|
+
},
|
|
258
|
+
hexLookup.length + hexSourceKey.length + 100
|
|
259
|
+
);
|
|
197
260
|
}
|
|
198
261
|
}
|
|
199
262
|
|
|
263
|
+
/**
|
|
264
|
+
* Inserts the pending bucket_parameters rows first if adding this row would exceed the statement size.
|
|
265
|
+
*/
|
|
266
|
+
private async addParameterData(row: models.BucketParameters, estimatedSize: number) {
|
|
267
|
+
if (this.parameterDataInserts.length > 0 && this.pendingParameterDataSize + estimatedSize > this.maxStatementSize) {
|
|
268
|
+
await this.flushParameterData();
|
|
269
|
+
this.earlyStatementCount += 1;
|
|
270
|
+
}
|
|
271
|
+
this.parameterDataInserts.push(row);
|
|
272
|
+
this.pendingParameterDataSize += estimatedSize;
|
|
273
|
+
}
|
|
274
|
+
|
|
200
275
|
deleteCurrentData(options: DeleteCurrentDataOptions) {
|
|
201
276
|
if (options.soft && this.storageConfig.softDeleteCurrentData) {
|
|
202
277
|
return this.upsertCurrentData(
|
|
@@ -221,7 +296,7 @@ export class PostgresPersistedBatch {
|
|
|
221
296
|
source_key_hex: hexReplicaId,
|
|
222
297
|
source_table: source_table
|
|
223
298
|
});
|
|
224
|
-
this.
|
|
299
|
+
this.pendingCurrentDataSize += serializedReplicaId.byteLength + 100;
|
|
225
300
|
}
|
|
226
301
|
}
|
|
227
302
|
|
|
@@ -253,7 +328,7 @@ export class PostgresPersistedBatch {
|
|
|
253
328
|
pending_delete: options.pending_delete?.toString() ?? null
|
|
254
329
|
});
|
|
255
330
|
|
|
256
|
-
this.
|
|
331
|
+
this.pendingCurrentDataSize +=
|
|
257
332
|
(options.data?.byteLength ?? 0) +
|
|
258
333
|
serializedReplicaId.byteLength +
|
|
259
334
|
buckets.length +
|
|
@@ -266,38 +341,47 @@ export class PostgresPersistedBatch {
|
|
|
266
341
|
shouldFlushTransaction() {
|
|
267
342
|
return (
|
|
268
343
|
this.currentSize >= this.maxTransactionBatchSize ||
|
|
269
|
-
this.bucketDataInserts.length >= this.maxTransactionDocCount ||
|
|
344
|
+
this.insertedBucketDataCount + this.bucketDataInserts.length >= this.maxTransactionDocCount ||
|
|
270
345
|
this.currentDataInserts.size >= this.maxTransactionDocCount ||
|
|
271
346
|
this.currentDataDeletes.size >= this.maxTransactionDocCount ||
|
|
272
|
-
this.parameterDataInserts.length >= this.maxTransactionDocCount
|
|
347
|
+
this.insertedParameterDataCount + this.parameterDataInserts.length >= this.maxTransactionDocCount
|
|
273
348
|
);
|
|
274
349
|
}
|
|
275
350
|
|
|
276
|
-
async flush(
|
|
351
|
+
async flush() {
|
|
277
352
|
const stats = {
|
|
278
|
-
bucketDataCount: this.bucketDataInserts.length,
|
|
279
|
-
parameterDataCount: this.parameterDataInserts.length,
|
|
353
|
+
bucketDataCount: this.insertedBucketDataCount + this.bucketDataInserts.length,
|
|
354
|
+
parameterDataCount: this.insertedParameterDataCount + this.parameterDataInserts.length,
|
|
280
355
|
currentDataCount: this.currentDataInserts.size + this.currentDataDeletes.size
|
|
281
356
|
};
|
|
282
357
|
const flushedAny = stats.bucketDataCount > 0 || stats.parameterDataCount > 0 || stats.currentDataCount > 0;
|
|
283
358
|
|
|
284
359
|
logger.info(
|
|
285
|
-
`powersync_${this.group_id} Flushed ${
|
|
360
|
+
`powersync_${this.group_id} Flushed ${stats.bucketDataCount} + ${stats.parameterDataCount} + ${
|
|
286
361
|
this.currentDataInserts.size
|
|
287
362
|
} updates, ${Math.round(this.currentSize / 1024)}kb.`
|
|
288
363
|
);
|
|
364
|
+
if (this.earlyStatementCount > 0) {
|
|
365
|
+
logger.debug(
|
|
366
|
+
`powersync_${this.group_id} Inserted ${this.insertedBucketDataCount} + ${this.insertedParameterDataCount} rows in ${this.earlyStatementCount} statements while accumulating.`
|
|
367
|
+
);
|
|
368
|
+
}
|
|
289
369
|
|
|
290
|
-
// Flush current_data
|
|
291
|
-
// want to detect those as soon as possible.
|
|
292
|
-
|
|
370
|
+
// Flush current_data before the remaining rows, since this is where lock errors are most likely
|
|
371
|
+
// to occur, and we want to detect those as soon as possible. The rows inserted while accumulating
|
|
372
|
+
// are already on the transaction at this point, so a lock error on a transaction with early
|
|
373
|
+
// inserts also rolls back that work. This only affects transactions with rows of large fan-out.
|
|
374
|
+
await this.flushCurrentData();
|
|
293
375
|
|
|
294
|
-
await this.flushBucketData(
|
|
295
|
-
await this.flushParameterData(
|
|
296
|
-
this.bucketDataInserts = [];
|
|
297
|
-
this.parameterDataInserts = [];
|
|
376
|
+
await this.flushBucketData();
|
|
377
|
+
await this.flushParameterData();
|
|
298
378
|
this.currentDataInserts = new Map();
|
|
299
379
|
this.currentDataDeletes = new Map();
|
|
300
|
-
this.
|
|
380
|
+
this.pendingCurrentDataSize = 0;
|
|
381
|
+
this.insertedBucketDataCount = 0;
|
|
382
|
+
this.insertedParameterDataCount = 0;
|
|
383
|
+
this.insertedSize = 0;
|
|
384
|
+
this.earlyStatementCount = 0;
|
|
301
385
|
|
|
302
386
|
return {
|
|
303
387
|
...stats,
|
|
@@ -305,106 +389,107 @@ export class PostgresPersistedBatch {
|
|
|
305
389
|
};
|
|
306
390
|
}
|
|
307
391
|
|
|
308
|
-
|
|
309
|
-
|
|
392
|
+
/**
|
|
393
|
+
* Inserts the pending bucket_data rows in a single statement.
|
|
394
|
+
*
|
|
395
|
+
* Sequential statements in the same transaction preserve the order of the operations,
|
|
396
|
+
* since op_id is assigned from the sequence per row.
|
|
397
|
+
*/
|
|
398
|
+
protected async flushBucketData() {
|
|
399
|
+
const rows = this.bucketDataInserts;
|
|
400
|
+
if (rows.length == 0) {
|
|
310
401
|
return;
|
|
311
402
|
}
|
|
312
403
|
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
`powersync_${this.group_id} Flushing ${this.bucketDataInserts.length} bucket_data rows in ${chunks.length} statements.`
|
|
317
|
-
);
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
// Sequential statements in the same transaction preserve the order of the operations,
|
|
321
|
-
// since op_id is assigned from the sequence per row.
|
|
322
|
-
for (const chunk of chunks) {
|
|
323
|
-
await db.sql`
|
|
324
|
-
INSERT INTO
|
|
325
|
-
bucket_data (
|
|
326
|
-
group_id,
|
|
327
|
-
bucket_name,
|
|
328
|
-
op_id,
|
|
329
|
-
op,
|
|
330
|
-
source_table,
|
|
331
|
-
source_key,
|
|
332
|
-
table_name,
|
|
333
|
-
row_id,
|
|
334
|
-
checksum,
|
|
335
|
-
data,
|
|
336
|
-
target_op
|
|
337
|
-
)
|
|
338
|
-
SELECT
|
|
404
|
+
await this.db.sql`
|
|
405
|
+
INSERT INTO
|
|
406
|
+
bucket_data (
|
|
339
407
|
group_id,
|
|
340
408
|
bucket_name,
|
|
341
|
-
|
|
409
|
+
op_id,
|
|
342
410
|
op,
|
|
343
411
|
source_table,
|
|
344
|
-
|
|
412
|
+
source_key,
|
|
345
413
|
table_name,
|
|
346
414
|
row_id,
|
|
347
415
|
checksum,
|
|
348
416
|
data,
|
|
349
417
|
target_op
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
418
|
+
)
|
|
419
|
+
SELECT
|
|
420
|
+
group_id,
|
|
421
|
+
bucket_name,
|
|
422
|
+
nextval('op_id_sequence'),
|
|
423
|
+
op,
|
|
424
|
+
source_table,
|
|
425
|
+
decode(source_key, 'hex') AS source_key,
|
|
426
|
+
table_name,
|
|
427
|
+
row_id,
|
|
428
|
+
checksum,
|
|
429
|
+
data,
|
|
430
|
+
target_op
|
|
431
|
+
FROM
|
|
432
|
+
json_to_recordset(${{ type: 'json', value: rows }}::json) AS t (
|
|
433
|
+
group_id integer,
|
|
434
|
+
bucket_name text,
|
|
435
|
+
source_table text,
|
|
436
|
+
source_key text, -- Input as hex string
|
|
437
|
+
table_name text,
|
|
438
|
+
op text,
|
|
439
|
+
row_id text,
|
|
440
|
+
checksum bigint,
|
|
441
|
+
data text,
|
|
442
|
+
target_op bigint
|
|
443
|
+
);
|
|
444
|
+
`.execute();
|
|
445
|
+
|
|
446
|
+
this.insertedBucketDataCount += rows.length;
|
|
447
|
+
this.insertedSize += this.pendingBucketDataSize;
|
|
448
|
+
this.bucketDataInserts = [];
|
|
449
|
+
this.pendingBucketDataSize = 0;
|
|
365
450
|
}
|
|
366
451
|
|
|
367
|
-
|
|
368
|
-
|
|
452
|
+
/**
|
|
453
|
+
* Inserts the pending bucket_parameters rows in a single statement.
|
|
454
|
+
*/
|
|
455
|
+
protected async flushParameterData() {
|
|
456
|
+
const rows = this.parameterDataInserts;
|
|
457
|
+
if (rows.length == 0) {
|
|
369
458
|
return;
|
|
370
459
|
}
|
|
371
460
|
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
`powersync_${this.group_id} Flushing ${this.parameterDataInserts.length} bucket_parameters rows in ${chunks.length} statements.`
|
|
376
|
-
);
|
|
377
|
-
}
|
|
378
|
-
|
|
379
|
-
for (const chunk of chunks) {
|
|
380
|
-
await db.sql`
|
|
381
|
-
INSERT INTO
|
|
382
|
-
bucket_parameters (
|
|
383
|
-
group_id,
|
|
384
|
-
source_table,
|
|
385
|
-
source_key,
|
|
386
|
-
lookup,
|
|
387
|
-
bucket_parameters
|
|
388
|
-
)
|
|
389
|
-
SELECT
|
|
461
|
+
await this.db.sql`
|
|
462
|
+
INSERT INTO
|
|
463
|
+
bucket_parameters (
|
|
390
464
|
group_id,
|
|
391
465
|
source_table,
|
|
392
|
-
|
|
393
|
-
|
|
466
|
+
source_key,
|
|
467
|
+
lookup,
|
|
394
468
|
bucket_parameters
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
469
|
+
)
|
|
470
|
+
SELECT
|
|
471
|
+
group_id,
|
|
472
|
+
source_table,
|
|
473
|
+
decode(source_key, 'hex') AS source_key, -- Decode hex to bytea
|
|
474
|
+
decode(lookup, 'hex') AS lookup, -- Decode hex to bytea
|
|
475
|
+
bucket_parameters
|
|
476
|
+
FROM
|
|
477
|
+
json_to_recordset(${{ type: 'json', value: rows }}::json) AS t (
|
|
478
|
+
group_id integer,
|
|
479
|
+
source_table text,
|
|
480
|
+
source_key text, -- Input as hex string
|
|
481
|
+
lookup text, -- Input as hex string
|
|
482
|
+
bucket_parameters text -- Input as stringified JSON
|
|
483
|
+
)
|
|
484
|
+
`.execute();
|
|
485
|
+
|
|
486
|
+
this.insertedParameterDataCount += rows.length;
|
|
487
|
+
this.insertedSize += this.pendingParameterDataSize;
|
|
488
|
+
this.parameterDataInserts = [];
|
|
489
|
+
this.pendingParameterDataSize = 0;
|
|
405
490
|
}
|
|
406
491
|
|
|
407
|
-
protected async flushCurrentData(
|
|
492
|
+
protected async flushCurrentData() {
|
|
408
493
|
if (this.currentDataInserts.size > 0) {
|
|
409
494
|
const updates = Array.from(this.currentDataInserts.values());
|
|
410
495
|
// Sort by source_table, source_key to ensure consistent order.
|
|
@@ -418,7 +503,7 @@ export class PostgresPersistedBatch {
|
|
|
418
503
|
return 0;
|
|
419
504
|
});
|
|
420
505
|
|
|
421
|
-
await this.currentDataStore.flushUpserts(db, updates);
|
|
506
|
+
await this.currentDataStore.flushUpserts(this.db, updates);
|
|
422
507
|
}
|
|
423
508
|
|
|
424
509
|
if (this.currentDataDeletes.size > 0) {
|
|
@@ -432,7 +517,7 @@ export class PostgresPersistedBatch {
|
|
|
432
517
|
return 0;
|
|
433
518
|
});
|
|
434
519
|
|
|
435
|
-
await this.currentDataStore.flushDeletes(db, {
|
|
520
|
+
await this.currentDataStore.flushDeletes(this.db, {
|
|
436
521
|
groupId: this.group_id,
|
|
437
522
|
deletes
|
|
438
523
|
});
|
|
@@ -443,42 +528,3 @@ export class PostgresPersistedBatch {
|
|
|
443
528
|
export function currentBucketKey(b: models.CurrentBucket) {
|
|
444
529
|
return `${b.bucket}/${b.table}/${b.id}`;
|
|
445
530
|
}
|
|
446
|
-
|
|
447
|
-
/**
|
|
448
|
-
* Rough estimate of the serialized size of a bucket_data row, matching the `currentSize` accounting.
|
|
449
|
-
*/
|
|
450
|
-
export function estimateBucketDataRowSize(row: models.BucketData) {
|
|
451
|
-
return row.bucket_name.length + (row.data?.length ?? 0) + (row.source_key?.length ?? 0) + 100;
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
/**
|
|
455
|
-
* Rough estimate of the serialized size of a bucket_parameters row, matching the `currentSize` accounting.
|
|
456
|
-
*/
|
|
457
|
-
export function estimateParameterDataRowSize(row: models.BucketParameters) {
|
|
458
|
-
return row.lookup.length + row.bucket_parameters.length + row.source_key.length + 100;
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
/**
|
|
462
|
-
* Splits rows into consecutive chunks, each with a total estimated size of at most `maxSize`.
|
|
463
|
-
*
|
|
464
|
-
* The order of the rows is preserved. A single row larger than `maxSize` is placed in its own chunk.
|
|
465
|
-
*/
|
|
466
|
-
export function chunkRowsBySize<T>(rows: T[], estimateSize: (row: T) => number, maxSize: number): T[][] {
|
|
467
|
-
const chunks: T[][] = [];
|
|
468
|
-
let currentChunk: T[] = [];
|
|
469
|
-
let currentChunkSize = 0;
|
|
470
|
-
for (const row of rows) {
|
|
471
|
-
const size = estimateSize(row);
|
|
472
|
-
if (currentChunk.length > 0 && currentChunkSize + size > maxSize) {
|
|
473
|
-
chunks.push(currentChunk);
|
|
474
|
-
currentChunk = [];
|
|
475
|
-
currentChunkSize = 0;
|
|
476
|
-
}
|
|
477
|
-
currentChunk.push(row);
|
|
478
|
-
currentChunkSize += size;
|
|
479
|
-
}
|
|
480
|
-
if (currentChunk.length > 0) {
|
|
481
|
-
chunks.push(currentChunk);
|
|
482
|
-
}
|
|
483
|
-
return chunks;
|
|
484
|
-
}
|
|
@@ -4,7 +4,7 @@ import * as t from 'ts-codec';
|
|
|
4
4
|
import * as models from '../types/models/CurrentData.js';
|
|
5
5
|
import { pick } from '../utils/ts-codec.js';
|
|
6
6
|
|
|
7
|
-
type Queryable = Pick<lib_postgres.DatabaseClient, 'sql' | 'streamRows'>;
|
|
7
|
+
type Queryable = Pick<lib_postgres.DatabaseClient, 'sql' | 'streamRows' | 'queryRows'>;
|
|
8
8
|
|
|
9
9
|
const TruncateCurrentDataCodec = pick(models.V1CurrentData, ['buckets', 'lookups', 'source_key']);
|
|
10
10
|
const LookupKeyCodec = pick(models.V1CurrentData, ['source_key', 'source_table']);
|
|
@@ -27,15 +27,21 @@ export class PostgresCurrentDataStore {
|
|
|
27
27
|
this.table = storageConfig.softDeleteCurrentData ? V3_CURRENT_DATA_TABLE : V1_CURRENT_DATA_TABLE;
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
-
|
|
30
|
+
/**
|
|
31
|
+
* Reads up to `limit` live rows of a source table, locked for the truncate.
|
|
32
|
+
*
|
|
33
|
+
* The rows are read in full rather than streamed, since the caller writes on the same
|
|
34
|
+
* connection while processing them. See `PostgresPersistedBatch`.
|
|
35
|
+
*/
|
|
36
|
+
queryTruncateRows(
|
|
31
37
|
db: Queryable,
|
|
32
38
|
options: {
|
|
33
39
|
groupId: number;
|
|
34
40
|
sourceTableId: string;
|
|
35
41
|
limit: number;
|
|
36
42
|
}
|
|
37
|
-
) {
|
|
38
|
-
return db.
|
|
43
|
+
): Promise<t.Encoded<typeof TruncateCurrentDataCodec>[]> {
|
|
44
|
+
return db.queryRows<t.Encoded<typeof TruncateCurrentDataCodec>>({
|
|
39
45
|
statement: `
|
|
40
46
|
SELECT
|
|
41
47
|
buckets,
|
package/src/types/types.ts
CHANGED
|
@@ -11,17 +11,18 @@ export const MAX_BATCH_ESTIMATED_SIZE = 5_000_000;
|
|
|
11
11
|
export const MAX_BATCH_CURRENT_DATA_SIZE = 50_000_000;
|
|
12
12
|
|
|
13
13
|
/**
|
|
14
|
-
*
|
|
15
|
-
* bucket_data and bucket_parameters.
|
|
14
|
+
* Upper bound for the estimated size of the rows written by a single INSERT statement when
|
|
15
|
+
* flushing bucket_data and bucket_parameters.
|
|
16
16
|
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
* into
|
|
17
|
+
* Rows are inserted while they accumulate, once the pending rows would exceed the smaller of
|
|
18
|
+
* `max_estimated_size` and this limit. A single source row can produce many operations when it
|
|
19
|
+
* is evaluated into many buckets, which `max_estimated_size` alone does not bound since it is
|
|
20
|
+
* only checked between rows.
|
|
20
21
|
*
|
|
21
22
|
* The rows are passed to Postgres as a single JSON parameter, which is serialized to a string
|
|
22
|
-
* before being sent. Node limits a string to roughly 512MB. The estimate
|
|
23
|
-
*
|
|
24
|
-
*
|
|
23
|
+
* before being sent. Node limits a string to roughly 512MB. The estimate does not account for
|
|
24
|
+
* the JSON escaping of the already-serialized row data, so the actual parameter can be close to
|
|
25
|
+
* double the estimate. 64MB keeps a wide margin for that.
|
|
25
26
|
*
|
|
26
27
|
* This is not part of `batch_limits`: the limit comes from the runtime, not from the workload.
|
|
27
28
|
*/
|