@powersync/service-module-postgres-storage 0.19.5 → 0.19.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -46,26 +46,44 @@ export type DeleteCurrentDataOptions = {
46
46
  export type PostgresPersistedBatchOptions = RequiredOperationBatchLimits & {
47
47
  group_id: number;
48
48
  storageConfig: storage.StorageVersionConfig;
49
+ /**
50
+ * The replication transaction the batch is written on.
51
+ */
52
+ db: lib_postgres.WrappedConnection;
49
53
  };
50
54
 
51
55
  const EMPTY_DATA = Buffer.from(bson.serialize({}));
52
56
 
57
+ /**
58
+ * Accumulates the operations of one replication transaction and writes them to Postgres.
59
+ *
60
+ * bucket_data and bucket_parameters rows are written while they accumulate: once the pending rows
61
+ * of a table would exceed `maxStatementSize`, they are inserted on the transaction before the next
62
+ * row is added. This bounds both the memory held for a single source row evaluated into many
63
+ * buckets and the size of the JSON parameter of each statement. current_data is written in `flush()`.
64
+ *
65
+ * Since `saveBucketData()` and `saveParameterData()` may write on the connection, no streamed query
66
+ * may be in progress on it while they are called: the connection processes responses in order, and
67
+ * a streamed query only completes once its rows are consumed.
68
+ *
69
+ * The transaction limits in `shouldFlushTransaction()` count everything written on the transaction,
70
+ * including the rows already inserted.
71
+ */
53
72
  export class PostgresPersistedBatch {
54
73
  group_id: number;
74
+ private readonly db: lib_postgres.WrappedConnection;
55
75
  private readonly storageConfig: storage.StorageVersionConfig;
56
76
  private readonly currentDataStore: PostgresCurrentDataStore;
57
77
 
58
- /**
59
- * Very rough estimate of current operations size in bytes
60
- */
61
- currentSize: number;
62
-
63
78
  readonly maxTransactionBatchSize: number;
64
79
  readonly maxTransactionDocCount: number;
80
+ /**
81
+ * Pending bucket_data or bucket_parameters rows are inserted before their estimated size exceeds this.
82
+ */
65
83
  readonly maxStatementSize: number;
66
84
 
67
85
  /**
68
- * Ordered set of bucket_data insert operation parameters
86
+ * Ordered set of bucket_data insert operation parameters, not inserted yet.
69
87
  */
70
88
  protected bucketDataInserts: models.BucketData[];
71
89
  protected parameterDataInserts: models.BucketParameters[];
@@ -77,23 +95,49 @@ export class PostgresPersistedBatch {
77
95
  protected currentDataInserts: Map<string, models.V3CurrentData>;
78
96
  protected currentDataDeletes: Map<string, { source_key_hex: string; source_table: string }>;
79
97
 
98
+ /**
99
+ * Very rough estimates of the size in bytes of the operations not inserted yet.
100
+ */
101
+ private pendingBucketDataSize = 0;
102
+ private pendingParameterDataSize = 0;
103
+ private pendingCurrentDataSize = 0;
104
+
105
+ /**
106
+ * Rows and estimated size already inserted on the transaction before `flush()`.
107
+ */
108
+ private insertedBucketDataCount = 0;
109
+ private insertedParameterDataCount = 0;
110
+ private insertedSize = 0;
111
+ /**
112
+ * Statements issued while accumulating, excluding the ones issued by `flush()`.
113
+ */
114
+ private earlyStatementCount = 0;
115
+
80
116
  constructor(options: PostgresPersistedBatchOptions) {
81
117
  this.group_id = options.group_id;
118
+ this.db = options.db;
82
119
  this.storageConfig = options.storageConfig;
83
120
  this.currentDataStore = new PostgresCurrentDataStore(this.storageConfig);
84
121
 
85
122
  this.maxTransactionBatchSize = options.max_estimated_size;
86
123
  this.maxTransactionDocCount = options.max_record_count;
87
- this.maxStatementSize = options.max_statement_size;
124
+ this.maxStatementSize = Math.min(options.max_estimated_size, options.max_statement_size);
88
125
 
89
126
  this.bucketDataInserts = [];
90
127
  this.parameterDataInserts = [];
91
128
  this.currentDataInserts = new Map();
92
129
  this.currentDataDeletes = new Map();
93
- this.currentSize = 0;
94
130
  }
95
131
 
96
- saveBucketData(options: SaveBucketDataOptions) {
132
+ /**
133
+ * Very rough estimate of the size in bytes of all operations in this transaction,
134
+ * including the rows already inserted.
135
+ */
136
+ get currentSize() {
137
+ return this.insertedSize + this.pendingBucketDataSize + this.pendingParameterDataSize + this.pendingCurrentDataSize;
138
+ }
139
+
140
+ async saveBucketData(options: SaveBucketDataOptions) {
97
141
  const remaining_buckets = new Map<string, models.CurrentBucket>();
98
142
  for (const b of options.before_buckets) {
99
143
  const key = currentBucketKey(b);
@@ -112,43 +156,58 @@ export class PostgresPersistedBatch {
112
156
  const data = JSONBig.stringify(k.data);
113
157
  const checksum = utils.hashData(k.table, k.id, data);
114
158
 
115
- this.bucketDataInserts.push({
116
- group_id: this.group_id,
117
- bucket_name: k.bucket,
118
- op: models.OpType.PUT,
119
- source_table: postgresTableId(options.table.id),
120
- source_key: hexSourceKey,
121
- table_name: k.table,
122
- row_id: k.id,
123
- checksum,
124
- data,
125
- op_id: 0, // Will use nextval of sequence
126
- target_op: null
127
- });
128
-
129
- this.currentSize += k.bucket.length + data.length + hexSourceKey.length + 100;
159
+ await this.addBucketData(
160
+ {
161
+ group_id: this.group_id,
162
+ bucket_name: k.bucket,
163
+ op: models.OpType.PUT,
164
+ source_table: postgresTableId(options.table.id),
165
+ source_key: hexSourceKey,
166
+ table_name: k.table,
167
+ row_id: k.id,
168
+ checksum,
169
+ data,
170
+ op_id: 0, // Will use nextval of sequence
171
+ target_op: null
172
+ },
173
+ k.bucket.length + data.length + hexSourceKey.length + 100
174
+ );
130
175
  }
131
176
 
132
177
  for (const bd of remaining_buckets.values()) {
133
178
  // REMOVE operation
134
- this.bucketDataInserts.push({
135
- group_id: this.group_id,
136
- bucket_name: bd.bucket,
137
- op: models.OpType.REMOVE,
138
- source_table: postgresTableId(options.table.id),
139
- source_key: hexSourceKey,
140
- table_name: bd.table,
141
- row_id: bd.id,
142
- checksum: dchecksum,
143
- op_id: 0, // Will use nextval of sequence
144
- target_op: null,
145
- data: null
146
- });
147
- this.currentSize += bd.bucket.length + hexSourceKey.length + 100;
179
+ await this.addBucketData(
180
+ {
181
+ group_id: this.group_id,
182
+ bucket_name: bd.bucket,
183
+ op: models.OpType.REMOVE,
184
+ source_table: postgresTableId(options.table.id),
185
+ source_key: hexSourceKey,
186
+ table_name: bd.table,
187
+ row_id: bd.id,
188
+ checksum: dchecksum,
189
+ op_id: 0, // Will use nextval of sequence
190
+ target_op: null,
191
+ data: null
192
+ },
193
+ bd.bucket.length + hexSourceKey.length + 100
194
+ );
195
+ }
196
+ }
197
+
198
+ /**
199
+ * Inserts the pending bucket_data rows first if adding this row would exceed the statement size.
200
+ */
201
+ private async addBucketData(row: models.BucketData, estimatedSize: number) {
202
+ if (this.bucketDataInserts.length > 0 && this.pendingBucketDataSize + estimatedSize > this.maxStatementSize) {
203
+ await this.flushBucketData();
204
+ this.earlyStatementCount += 1;
148
205
  }
206
+ this.bucketDataInserts.push(row);
207
+ this.pendingBucketDataSize += estimatedSize;
149
208
  }
150
209
 
151
- saveParameterData(options: SaveParameterDataOptions) {
210
+ async saveParameterData(options: SaveParameterDataOptions) {
152
211
  // This is similar to saving bucket data.
153
212
  // A key difference is that we don't need to keep the history intact.
154
213
  // We do need to keep track of recent history though - enough that we can get consistent data for any specific checkpoint.
@@ -171,32 +230,48 @@ export class PostgresPersistedBatch {
171
230
  remaining_lookups.delete(base64);
172
231
  const hexLookup = binLookup.toString('hex');
173
232
  const serializedBucketParameters = JSONBig.stringify(result.bucketParameters);
174
- this.parameterDataInserts.push({
175
- group_id: this.group_id,
176
- source_table: postgresTableId(table.id),
177
- source_key: hexSourceKey,
178
- bucket_parameters: serializedBucketParameters,
179
- id: 0, // auto incrementing id
180
- lookup: hexLookup
181
- });
182
- this.currentSize += hexLookup.length + serializedBucketParameters.length + hexSourceKey.length + 100;
233
+ await this.addParameterData(
234
+ {
235
+ group_id: this.group_id,
236
+ source_table: postgresTableId(table.id),
237
+ source_key: hexSourceKey,
238
+ bucket_parameters: serializedBucketParameters,
239
+ id: 0, // auto incrementing id
240
+ lookup: hexLookup
241
+ },
242
+ hexLookup.length + serializedBucketParameters.length + hexSourceKey.length + 100
243
+ );
183
244
  }
184
245
 
185
246
  // 2. "REMOVE" entries for any lookup not touched.
186
247
  for (const lookup of remaining_lookups.values()) {
187
248
  const hexLookup = lookup.toString('hex');
188
- this.parameterDataInserts.push({
189
- group_id: this.group_id,
190
- source_table: postgresTableId(table.id),
191
- source_key: hexSourceKey,
192
- bucket_parameters: JSON.stringify([]),
193
- id: 0, // auto incrementing id
194
- lookup: hexLookup
195
- });
196
- this.currentSize += hexLookup.length + hexSourceKey.length + 100;
249
+ await this.addParameterData(
250
+ {
251
+ group_id: this.group_id,
252
+ source_table: postgresTableId(table.id),
253
+ source_key: hexSourceKey,
254
+ bucket_parameters: JSON.stringify([]),
255
+ id: 0, // auto incrementing id
256
+ lookup: hexLookup
257
+ },
258
+ hexLookup.length + hexSourceKey.length + 100
259
+ );
197
260
  }
198
261
  }
199
262
 
263
+ /**
264
+ * Inserts the pending bucket_parameters rows first if adding this row would exceed the statement size.
265
+ */
266
+ private async addParameterData(row: models.BucketParameters, estimatedSize: number) {
267
+ if (this.parameterDataInserts.length > 0 && this.pendingParameterDataSize + estimatedSize > this.maxStatementSize) {
268
+ await this.flushParameterData();
269
+ this.earlyStatementCount += 1;
270
+ }
271
+ this.parameterDataInserts.push(row);
272
+ this.pendingParameterDataSize += estimatedSize;
273
+ }
274
+
200
275
  deleteCurrentData(options: DeleteCurrentDataOptions) {
201
276
  if (options.soft && this.storageConfig.softDeleteCurrentData) {
202
277
  return this.upsertCurrentData(
@@ -221,7 +296,7 @@ export class PostgresPersistedBatch {
221
296
  source_key_hex: hexReplicaId,
222
297
  source_table: source_table
223
298
  });
224
- this.currentSize += serializedReplicaId.byteLength + 100;
299
+ this.pendingCurrentDataSize += serializedReplicaId.byteLength + 100;
225
300
  }
226
301
  }
227
302
 
@@ -253,7 +328,7 @@ export class PostgresPersistedBatch {
253
328
  pending_delete: options.pending_delete?.toString() ?? null
254
329
  });
255
330
 
256
- this.currentSize +=
331
+ this.pendingCurrentDataSize +=
257
332
  (options.data?.byteLength ?? 0) +
258
333
  serializedReplicaId.byteLength +
259
334
  buckets.length +
@@ -266,38 +341,47 @@ export class PostgresPersistedBatch {
266
341
  shouldFlushTransaction() {
267
342
  return (
268
343
  this.currentSize >= this.maxTransactionBatchSize ||
269
- this.bucketDataInserts.length >= this.maxTransactionDocCount ||
344
+ this.insertedBucketDataCount + this.bucketDataInserts.length >= this.maxTransactionDocCount ||
270
345
  this.currentDataInserts.size >= this.maxTransactionDocCount ||
271
346
  this.currentDataDeletes.size >= this.maxTransactionDocCount ||
272
- this.parameterDataInserts.length >= this.maxTransactionDocCount
347
+ this.insertedParameterDataCount + this.parameterDataInserts.length >= this.maxTransactionDocCount
273
348
  );
274
349
  }
275
350
 
276
- async flush(db: lib_postgres.WrappedConnection) {
351
+ async flush() {
277
352
  const stats = {
278
- bucketDataCount: this.bucketDataInserts.length,
279
- parameterDataCount: this.parameterDataInserts.length,
353
+ bucketDataCount: this.insertedBucketDataCount + this.bucketDataInserts.length,
354
+ parameterDataCount: this.insertedParameterDataCount + this.parameterDataInserts.length,
280
355
  currentDataCount: this.currentDataInserts.size + this.currentDataDeletes.size
281
356
  };
282
357
  const flushedAny = stats.bucketDataCount > 0 || stats.parameterDataCount > 0 || stats.currentDataCount > 0;
283
358
 
284
359
  logger.info(
285
- `powersync_${this.group_id} Flushed ${this.bucketDataInserts.length} + ${this.parameterDataInserts.length} + ${
360
+ `powersync_${this.group_id} Flushed ${stats.bucketDataCount} + ${stats.parameterDataCount} + ${
286
361
  this.currentDataInserts.size
287
362
  } updates, ${Math.round(this.currentSize / 1024)}kb.`
288
363
  );
364
+ if (this.earlyStatementCount > 0) {
365
+ logger.debug(
366
+ `powersync_${this.group_id} Inserted ${this.insertedBucketDataCount} + ${this.insertedParameterDataCount} rows in ${this.earlyStatementCount} statements while accumulating.`
367
+ );
368
+ }
289
369
 
290
- // Flush current_data first, since this is where lock errors are most likely to occur, and we
291
- // want to detect those as soon as possible.
292
- await this.flushCurrentData(db);
370
+ // Flush current_data before the remaining rows, since this is where lock errors are most likely
371
+ // to occur, and we want to detect those as soon as possible. The rows inserted while accumulating
372
+ // are already on the transaction at this point, so a lock error on a transaction with early
373
+ // inserts also rolls back that work. This only affects transactions with rows of large fan-out.
374
+ await this.flushCurrentData();
293
375
 
294
- await this.flushBucketData(db);
295
- await this.flushParameterData(db);
296
- this.bucketDataInserts = [];
297
- this.parameterDataInserts = [];
376
+ await this.flushBucketData();
377
+ await this.flushParameterData();
298
378
  this.currentDataInserts = new Map();
299
379
  this.currentDataDeletes = new Map();
300
- this.currentSize = 0;
380
+ this.pendingCurrentDataSize = 0;
381
+ this.insertedBucketDataCount = 0;
382
+ this.insertedParameterDataCount = 0;
383
+ this.insertedSize = 0;
384
+ this.earlyStatementCount = 0;
301
385
 
302
386
  return {
303
387
  ...stats,
@@ -305,106 +389,107 @@ export class PostgresPersistedBatch {
305
389
  };
306
390
  }
307
391
 
308
- protected async flushBucketData(db: lib_postgres.WrappedConnection) {
309
- if (this.bucketDataInserts.length == 0) {
392
+ /**
393
+ * Inserts the pending bucket_data rows in a single statement.
394
+ *
395
+ * Sequential statements in the same transaction preserve the order of the operations,
396
+ * since op_id is assigned from the sequence per row.
397
+ */
398
+ protected async flushBucketData() {
399
+ const rows = this.bucketDataInserts;
400
+ if (rows.length == 0) {
310
401
  return;
311
402
  }
312
403
 
313
- const chunks = chunkRowsBySize(this.bucketDataInserts, estimateBucketDataRowSize, this.maxStatementSize);
314
- if (chunks.length > 1) {
315
- logger.debug(
316
- `powersync_${this.group_id} Flushing ${this.bucketDataInserts.length} bucket_data rows in ${chunks.length} statements.`
317
- );
318
- }
319
-
320
- // Sequential statements in the same transaction preserve the order of the operations,
321
- // since op_id is assigned from the sequence per row.
322
- for (const chunk of chunks) {
323
- await db.sql`
324
- INSERT INTO
325
- bucket_data (
326
- group_id,
327
- bucket_name,
328
- op_id,
329
- op,
330
- source_table,
331
- source_key,
332
- table_name,
333
- row_id,
334
- checksum,
335
- data,
336
- target_op
337
- )
338
- SELECT
404
+ await this.db.sql`
405
+ INSERT INTO
406
+ bucket_data (
339
407
  group_id,
340
408
  bucket_name,
341
- nextval('op_id_sequence'),
409
+ op_id,
342
410
  op,
343
411
  source_table,
344
- decode(source_key, 'hex') AS source_key,
412
+ source_key,
345
413
  table_name,
346
414
  row_id,
347
415
  checksum,
348
416
  data,
349
417
  target_op
350
- FROM
351
- json_to_recordset(${{ type: 'json', value: chunk }}::json) AS t (
352
- group_id integer,
353
- bucket_name text,
354
- source_table text,
355
- source_key text, -- Input as hex string
356
- table_name text,
357
- op text,
358
- row_id text,
359
- checksum bigint,
360
- data text,
361
- target_op bigint
362
- );
363
- `.execute();
364
- }
418
+ )
419
+ SELECT
420
+ group_id,
421
+ bucket_name,
422
+ nextval('op_id_sequence'),
423
+ op,
424
+ source_table,
425
+ decode(source_key, 'hex') AS source_key,
426
+ table_name,
427
+ row_id,
428
+ checksum,
429
+ data,
430
+ target_op
431
+ FROM
432
+ json_to_recordset(${{ type: 'json', value: rows }}::json) AS t (
433
+ group_id integer,
434
+ bucket_name text,
435
+ source_table text,
436
+ source_key text, -- Input as hex string
437
+ table_name text,
438
+ op text,
439
+ row_id text,
440
+ checksum bigint,
441
+ data text,
442
+ target_op bigint
443
+ );
444
+ `.execute();
445
+
446
+ this.insertedBucketDataCount += rows.length;
447
+ this.insertedSize += this.pendingBucketDataSize;
448
+ this.bucketDataInserts = [];
449
+ this.pendingBucketDataSize = 0;
365
450
  }
366
451
 
367
- protected async flushParameterData(db: lib_postgres.WrappedConnection) {
368
- if (this.parameterDataInserts.length == 0) {
452
+ /**
453
+ * Inserts the pending bucket_parameters rows in a single statement.
454
+ */
455
+ protected async flushParameterData() {
456
+ const rows = this.parameterDataInserts;
457
+ if (rows.length == 0) {
369
458
  return;
370
459
  }
371
460
 
372
- const chunks = chunkRowsBySize(this.parameterDataInserts, estimateParameterDataRowSize, this.maxStatementSize);
373
- if (chunks.length > 1) {
374
- logger.debug(
375
- `powersync_${this.group_id} Flushing ${this.parameterDataInserts.length} bucket_parameters rows in ${chunks.length} statements.`
376
- );
377
- }
378
-
379
- for (const chunk of chunks) {
380
- await db.sql`
381
- INSERT INTO
382
- bucket_parameters (
383
- group_id,
384
- source_table,
385
- source_key,
386
- lookup,
387
- bucket_parameters
388
- )
389
- SELECT
461
+ await this.db.sql`
462
+ INSERT INTO
463
+ bucket_parameters (
390
464
  group_id,
391
465
  source_table,
392
- decode(source_key, 'hex') AS source_key, -- Decode hex to bytea
393
- decode(lookup, 'hex') AS lookup, -- Decode hex to bytea
466
+ source_key,
467
+ lookup,
394
468
  bucket_parameters
395
- FROM
396
- json_to_recordset(${{ type: 'json', value: chunk }}::json) AS t (
397
- group_id integer,
398
- source_table text,
399
- source_key text, -- Input as hex string
400
- lookup text, -- Input as hex string
401
- bucket_parameters text -- Input as stringified JSON
402
- )
403
- `.execute();
404
- }
469
+ )
470
+ SELECT
471
+ group_id,
472
+ source_table,
473
+ decode(source_key, 'hex') AS source_key, -- Decode hex to bytea
474
+ decode(lookup, 'hex') AS lookup, -- Decode hex to bytea
475
+ bucket_parameters
476
+ FROM
477
+ json_to_recordset(${{ type: 'json', value: rows }}::json) AS t (
478
+ group_id integer,
479
+ source_table text,
480
+ source_key text, -- Input as hex string
481
+ lookup text, -- Input as hex string
482
+ bucket_parameters text -- Input as stringified JSON
483
+ )
484
+ `.execute();
485
+
486
+ this.insertedParameterDataCount += rows.length;
487
+ this.insertedSize += this.pendingParameterDataSize;
488
+ this.parameterDataInserts = [];
489
+ this.pendingParameterDataSize = 0;
405
490
  }
406
491
 
407
- protected async flushCurrentData(db: lib_postgres.WrappedConnection) {
492
+ protected async flushCurrentData() {
408
493
  if (this.currentDataInserts.size > 0) {
409
494
  const updates = Array.from(this.currentDataInserts.values());
410
495
  // Sort by source_table, source_key to ensure consistent order.
@@ -418,7 +503,7 @@ export class PostgresPersistedBatch {
418
503
  return 0;
419
504
  });
420
505
 
421
- await this.currentDataStore.flushUpserts(db, updates);
506
+ await this.currentDataStore.flushUpserts(this.db, updates);
422
507
  }
423
508
 
424
509
  if (this.currentDataDeletes.size > 0) {
@@ -432,7 +517,7 @@ export class PostgresPersistedBatch {
432
517
  return 0;
433
518
  });
434
519
 
435
- await this.currentDataStore.flushDeletes(db, {
520
+ await this.currentDataStore.flushDeletes(this.db, {
436
521
  groupId: this.group_id,
437
522
  deletes
438
523
  });
@@ -443,42 +528,3 @@ export class PostgresPersistedBatch {
443
528
  export function currentBucketKey(b: models.CurrentBucket) {
444
529
  return `${b.bucket}/${b.table}/${b.id}`;
445
530
  }
446
-
447
- /**
448
- * Rough estimate of the serialized size of a bucket_data row, matching the `currentSize` accounting.
449
- */
450
- export function estimateBucketDataRowSize(row: models.BucketData) {
451
- return row.bucket_name.length + (row.data?.length ?? 0) + (row.source_key?.length ?? 0) + 100;
452
- }
453
-
454
- /**
455
- * Rough estimate of the serialized size of a bucket_parameters row, matching the `currentSize` accounting.
456
- */
457
- export function estimateParameterDataRowSize(row: models.BucketParameters) {
458
- return row.lookup.length + row.bucket_parameters.length + row.source_key.length + 100;
459
- }
460
-
461
- /**
462
- * Splits rows into consecutive chunks, each with a total estimated size of at most `maxSize`.
463
- *
464
- * The order of the rows is preserved. A single row larger than `maxSize` is placed in its own chunk.
465
- */
466
- export function chunkRowsBySize<T>(rows: T[], estimateSize: (row: T) => number, maxSize: number): T[][] {
467
- const chunks: T[][] = [];
468
- let currentChunk: T[] = [];
469
- let currentChunkSize = 0;
470
- for (const row of rows) {
471
- const size = estimateSize(row);
472
- if (currentChunk.length > 0 && currentChunkSize + size > maxSize) {
473
- chunks.push(currentChunk);
474
- currentChunk = [];
475
- currentChunkSize = 0;
476
- }
477
- currentChunk.push(row);
478
- currentChunkSize += size;
479
- }
480
- if (currentChunk.length > 0) {
481
- chunks.push(currentChunk);
482
- }
483
- return chunks;
484
- }
@@ -4,7 +4,7 @@ import * as t from 'ts-codec';
4
4
  import * as models from '../types/models/CurrentData.js';
5
5
  import { pick } from '../utils/ts-codec.js';
6
6
 
7
- type Queryable = Pick<lib_postgres.DatabaseClient, 'sql' | 'streamRows'>;
7
+ type Queryable = Pick<lib_postgres.DatabaseClient, 'sql' | 'streamRows' | 'queryRows'>;
8
8
 
9
9
  const TruncateCurrentDataCodec = pick(models.V1CurrentData, ['buckets', 'lookups', 'source_key']);
10
10
  const LookupKeyCodec = pick(models.V1CurrentData, ['source_key', 'source_table']);
@@ -27,15 +27,21 @@ export class PostgresCurrentDataStore {
27
27
  this.table = storageConfig.softDeleteCurrentData ? V3_CURRENT_DATA_TABLE : V1_CURRENT_DATA_TABLE;
28
28
  }
29
29
 
30
- streamTruncateRows(
30
+ /**
31
+ * Reads up to `limit` live rows of a source table, locked for the truncate.
32
+ *
33
+ * The rows are read in full rather than streamed, since the caller writes on the same
34
+ * connection while processing them. See `PostgresPersistedBatch`.
35
+ */
36
+ queryTruncateRows(
31
37
  db: Queryable,
32
38
  options: {
33
39
  groupId: number;
34
40
  sourceTableId: string;
35
41
  limit: number;
36
42
  }
37
- ) {
38
- return db.streamRows<t.Encoded<typeof TruncateCurrentDataCodec>>({
43
+ ): Promise<t.Encoded<typeof TruncateCurrentDataCodec>[]> {
44
+ return db.queryRows<t.Encoded<typeof TruncateCurrentDataCodec>>({
39
45
  statement: `
40
46
  SELECT
41
47
  buckets,
@@ -11,17 +11,18 @@ export const MAX_BATCH_ESTIMATED_SIZE = 5_000_000;
11
11
  export const MAX_BATCH_CURRENT_DATA_SIZE = 50_000_000;
12
12
 
13
13
  /**
14
- * Maximum estimated size of the rows written by a single INSERT statement when flushing
15
- * bucket_data and bucket_parameters.
14
+ * Upper bound for the estimated size of the rows written by a single INSERT statement when
15
+ * flushing bucket_data and bucket_parameters.
16
16
  *
17
- * A single source row can produce many operations when it is evaluated into many buckets,
18
- * which is not limited by `max_estimated_size`. Flushes larger than this limit are split
19
- * into multiple statements in the same transaction.
17
+ * Rows are inserted while they accumulate, once the pending rows would exceed the smaller of
18
+ * `max_estimated_size` and this limit. A single source row can produce many operations when it
19
+ * is evaluated into many buckets, which `max_estimated_size` alone does not bound since it is
20
+ * only checked between rows.
20
21
  *
21
22
  * The rows are passed to Postgres as a single JSON parameter, which is serialized to a string
22
- * before being sent. Node limits a string to roughly 512MB. The estimate used for chunking does
23
- * not account for the JSON escaping of the already-serialized row data, so the actual parameter
24
- * can be close to double the estimate. 64MB keeps a wide margin for that.
23
+ * before being sent. Node limits a string to roughly 512MB. The estimate does not account for
24
+ * the JSON escaping of the already-serialized row data, so the actual parameter can be close to
25
+ * double the estimate. 64MB keeps a wide margin for that.
25
26
  *
26
27
  * This is not part of `batch_limits`: the limit comes from the runtime, not from the workload.
27
28
  */