@powersync/service-module-postgres-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -6
- package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.d.ts +3 -0
- package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js +42 -0
- package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js.map +1 -0
- package/dist/{@types/storage → storage}/PostgresBucketStorageFactory.d.ts +13 -12
- package/dist/storage/PostgresBucketStorageFactory.js +59 -55
- package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
- package/dist/{@types/storage → storage}/PostgresCompactor.d.ts +3 -1
- package/dist/storage/PostgresCompactor.js +21 -1
- package/dist/storage/PostgresCompactor.js.map +1 -1
- package/dist/storage/PostgresReportStorage.js.map +1 -1
- package/dist/storage/PostgresStorageProvider.js +2 -1
- package/dist/storage/PostgresStorageProvider.js.map +1 -1
- package/dist/{@types/storage → storage}/PostgresSyncRulesStorage.d.ts +36 -11
- package/dist/storage/PostgresSyncRulesStorage.js +250 -177
- package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
- package/dist/{@types/storage → storage}/batch/PostgresBucketBatch.d.ts +8 -24
- package/dist/storage/batch/PostgresBucketBatch.js +229 -194
- package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
- package/dist/{@types/storage → storage}/checkpoints/PostgresWriteCheckpointAPI.d.ts +1 -1
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js +161 -24
- package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map +1 -1
- package/dist/{@types/storage → storage}/current-data-store.d.ts +8 -2
- package/dist/storage/current-data-store.js +66 -11
- package/dist/storage/current-data-store.js.map +1 -1
- package/dist/{@types/storage → storage}/storage-index.d.ts +1 -0
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.d.ts +17 -0
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js +90 -0
- package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js.map +1 -0
- package/dist/{@types/types → types}/models/WriteCheckpoint.d.ts +2 -0
- package/dist/types/models/WriteCheckpoint.js +5 -2
- package/dist/types/models/WriteCheckpoint.js.map +1 -1
- package/dist/utils/checkpoints.d.ts +9 -0
- package/dist/utils/checkpoints.js +26 -0
- package/dist/utils/checkpoints.js.map +1 -0
- package/dist/utils/test-utils.js +2 -2
- package/dist/utils/test-utils.js.map +1 -1
- package/package.json +10 -22
- package/src/migrations/scripts/1782950400000-checkpoint-requested-at.ts +51 -0
- package/src/storage/PostgresBucketStorageFactory.ts +67 -58
- package/src/storage/PostgresCompactor.ts +25 -1
- package/src/storage/PostgresReportStorage.ts +1 -2
- package/src/storage/PostgresStorageProvider.ts +2 -1
- package/src/storage/PostgresSyncRulesStorage.ts +275 -191
- package/src/storage/batch/PostgresBucketBatch.ts +256 -214
- package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +174 -24
- package/src/storage/current-data-store.ts +66 -11
- package/src/storage/storage-index.ts +1 -0
- package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts +98 -0
- package/src/types/models/WriteCheckpoint.ts +5 -2
- package/src/utils/checkpoints.ts +31 -0
- package/src/utils/test-utils.ts +2 -2
- package/test/src/__snapshots__/storage_sync.test.ts.snap +0 -582
- package/test/src/checkpoint_notifications.test.ts +522 -0
- package/test/src/migrations.test.ts +1 -1
- package/test/src/storage.test.ts +226 -10
- package/test/src/storage_compacting.test.ts +15 -7
- package/test/src/storage_sync.test.ts +11 -8
- package/test/tsconfig.json +1 -3
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -0
- package/dist/.tsbuildinfo +0 -1
- package/dist/@types/storage/sync-rules/PostgresPersistedSyncRulesContent.d.ts +0 -9
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js +0 -49
- package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js.map +0 -1
- package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +0 -54
- /package/dist/{@types/index.d.ts → index.d.ts} +0 -0
- /package/dist/{@types/migrations → migrations}/PostgresMigrationAgent.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/PostgresMigrationStore.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/migration-utils.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1684951997326-init.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1749024804042-snapshot-progress.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1756282360128-connection-reporting.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771232439485-storage-version.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771424826685-current-data-pending-deletes.d.ts +0 -0
- /package/dist/{@types/migrations → migrations}/scripts/1771491856000-sync-plan.d.ts +0 -0
- /package/dist/{@types/module → module}/PostgresStorageModule.d.ts +0 -0
- /package/dist/{@types/storage → storage}/PostgresReportStorage.d.ts +0 -0
- /package/dist/{@types/storage → storage}/PostgresStorageProvider.d.ts +0 -0
- /package/dist/{@types/storage → storage}/batch/OperationBatch.d.ts +0 -0
- /package/dist/{@types/storage → storage}/batch/PostgresPersistedBatch.d.ts +0 -0
- /package/dist/{@types/storage → storage}/table-id.d.ts +0 -0
- /package/dist/{@types/types → types}/codecs.d.ts +0 -0
- /package/dist/{@types/types → types}/models/ActiveCheckpoint.d.ts +0 -0
- /package/dist/{@types/types → types}/models/ActiveCheckpointNotification.d.ts +0 -0
- /package/dist/{@types/types → types}/models/BucketData.d.ts +0 -0
- /package/dist/{@types/types → types}/models/BucketParameters.d.ts +0 -0
- /package/dist/{@types/types → types}/models/CurrentData.d.ts +0 -0
- /package/dist/{@types/types → types}/models/Instance.d.ts +0 -0
- /package/dist/{@types/types → types}/models/Migration.d.ts +0 -0
- /package/dist/{@types/types → types}/models/SdkReporting.d.ts +0 -0
- /package/dist/{@types/types → types}/models/SourceTable.d.ts +0 -0
- /package/dist/{@types/types → types}/models/SyncRules.d.ts +0 -0
- /package/dist/{@types/types → types}/models/json.d.ts +0 -0
- /package/dist/{@types/types → types}/models/models-index.d.ts +0 -0
- /package/dist/{@types/types → types}/types.d.ts +0 -0
- /package/dist/{@types/utils → utils}/application-name.d.ts +0 -0
- /package/dist/{@types/utils → utils}/bson.d.ts +0 -0
- /package/dist/{@types/utils → utils}/bucket-data.d.ts +0 -0
- /package/dist/{@types/utils → utils}/db.d.ts +0 -0
- /package/dist/{@types/utils → utils}/test-utils.d.ts +0 -0
- /package/dist/{@types/utils → utils}/ts-codec.d.ts +0 -0
- /package/dist/{@types/utils → utils}/utils-index.d.ts +0 -0
|
@@ -38,8 +38,8 @@ export interface PostgresBucketBatchOptions {
|
|
|
38
38
|
logger: Logger;
|
|
39
39
|
db: lib_postgres.DatabaseClient;
|
|
40
40
|
sync_rules: sync_rules.HydratedSyncConfig;
|
|
41
|
-
|
|
42
|
-
|
|
41
|
+
replicationStreamId: number;
|
|
42
|
+
replicationStreamName: string;
|
|
43
43
|
last_checkpoint_lsn: string | null;
|
|
44
44
|
store_current_data: boolean;
|
|
45
45
|
keep_alive_op?: InternalOpId | null;
|
|
@@ -60,7 +60,6 @@ export interface PostgresBucketBatchOptions {
|
|
|
60
60
|
* via the Postgres NOTIFY protocol.
|
|
61
61
|
*/
|
|
62
62
|
const StatefulCheckpoint = models.ActiveCheckpoint.and(t.object({ state: t.Enum(storage.SyncRuleState) }));
|
|
63
|
-
type StatefulCheckpointDecoded = t.Decoded<typeof StatefulCheckpoint>;
|
|
64
63
|
|
|
65
64
|
const CheckpointWithStatus = StatefulCheckpoint.and(
|
|
66
65
|
t.object({
|
|
@@ -112,7 +111,7 @@ export class PostgresBucketBatch
|
|
|
112
111
|
super();
|
|
113
112
|
this.logger = options.logger;
|
|
114
113
|
this.db = options.db;
|
|
115
|
-
this.group_id = options.
|
|
114
|
+
this.group_id = options.replicationStreamId;
|
|
116
115
|
this.last_checkpoint_lsn = options.last_checkpoint_lsn;
|
|
117
116
|
this.resumeFromLsn = options.resumeFromLsn;
|
|
118
117
|
this.skipExistingRows = options.skip_existing_rows;
|
|
@@ -129,10 +128,6 @@ export class PostgresBucketBatch
|
|
|
129
128
|
}
|
|
130
129
|
}
|
|
131
130
|
|
|
132
|
-
get lastCheckpointLsn() {
|
|
133
|
-
return this.last_checkpoint_lsn;
|
|
134
|
-
}
|
|
135
|
-
|
|
136
131
|
async [Symbol.asyncDispose]() {
|
|
137
132
|
if (this.batch != null || this.write_checkpoint_batch.length > 0) {
|
|
138
133
|
// We don't error here, since:
|
|
@@ -148,15 +143,16 @@ export class PostgresBucketBatch
|
|
|
148
143
|
}
|
|
149
144
|
|
|
150
145
|
async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
|
|
151
|
-
const syncRules = options.
|
|
146
|
+
const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
|
|
152
147
|
const { connection_id, source } = options;
|
|
153
|
-
const { schema, name: table, objectId, replicaIdColumns, connectionTag } = source;
|
|
148
|
+
const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
|
|
154
149
|
|
|
155
150
|
const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
|
|
156
151
|
name: column.name,
|
|
157
152
|
type: column.type,
|
|
158
153
|
type_oid: typeof column.typeId !== 'undefined' ? Number(column.typeId) : column.typeId
|
|
159
154
|
}));
|
|
155
|
+
|
|
160
156
|
return this.db.transaction(async (db) => {
|
|
161
157
|
let sourceTableRow: SourceTableDecoded | null;
|
|
162
158
|
if (objectId != null) {
|
|
@@ -240,6 +236,7 @@ export class PostgresBucketBatch
|
|
|
240
236
|
sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
|
|
241
237
|
sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
|
|
242
238
|
sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
|
|
239
|
+
sourceTable.storeCurrentData = sendsCompleteRows !== true;
|
|
243
240
|
|
|
244
241
|
let truncatedTables: SourceTableDecoded[] = [];
|
|
245
242
|
if (objectId != null) {
|
|
@@ -359,19 +356,23 @@ export class PostgresBucketBatch
|
|
|
359
356
|
async save(record: storage.SaveOptions): Promise<storage.FlushedResult | null> {
|
|
360
357
|
// TODO maybe share with abstract class
|
|
361
358
|
const { after, before, sourceTable, tag } = record;
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
359
|
+
const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
|
|
360
|
+
// Only the table designated as event carrier by resolveTables may fire events.
|
|
361
|
+
if (sourceTable.syncEvent) {
|
|
362
|
+
for (const event of this.getTableEvents(sourceTable)) {
|
|
363
|
+
this.iterateListeners((cb) =>
|
|
364
|
+
cb.replicationEvent?.({
|
|
365
|
+
batch: this,
|
|
366
|
+
table: sourceTable,
|
|
367
|
+
data: {
|
|
368
|
+
op: tag,
|
|
369
|
+
after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined,
|
|
370
|
+
before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined
|
|
371
|
+
},
|
|
372
|
+
event
|
|
373
|
+
})
|
|
374
|
+
);
|
|
375
|
+
}
|
|
375
376
|
}
|
|
376
377
|
/**
|
|
377
378
|
* Return if the table is just an event table
|
|
@@ -420,7 +421,9 @@ export class PostgresBucketBatch
|
|
|
420
421
|
let processedCount = 0;
|
|
421
422
|
while (lastBatchCount == BATCH_LIMIT) {
|
|
422
423
|
lastBatchCount = 0;
|
|
424
|
+
let clearedError = false;
|
|
423
425
|
await this.withReplicationTransaction(async (db) => {
|
|
426
|
+
clearedError = false;
|
|
424
427
|
const persistedBatch = new PostgresPersistedBatch({
|
|
425
428
|
group_id: this.group_id,
|
|
426
429
|
storageConfig: this.storageConfig,
|
|
@@ -460,10 +463,14 @@ export class PostgresBucketBatch
|
|
|
460
463
|
}
|
|
461
464
|
}
|
|
462
465
|
const { flushedAny } = await persistedBatch.flush(db);
|
|
463
|
-
|
|
464
|
-
await this.clearError(db);
|
|
465
|
-
}
|
|
466
|
+
clearedError = flushedAny && !this.clearedError;
|
|
466
467
|
});
|
|
468
|
+
if (clearedError) {
|
|
469
|
+
// No need to clear an error more than once per batch, since an error would always result in restarting the batch.
|
|
470
|
+
// Cleared outside the replication transaction - see flushInner.
|
|
471
|
+
await this.clearError();
|
|
472
|
+
this.clearedError = true;
|
|
473
|
+
}
|
|
467
474
|
}
|
|
468
475
|
if (processedCount == 0) {
|
|
469
476
|
// The op sequence should not have progressed
|
|
@@ -516,13 +523,25 @@ export class PostgresBucketBatch
|
|
|
516
523
|
await this.hooks?.beforeBatchFlush?.(this);
|
|
517
524
|
|
|
518
525
|
let resumeBatch: OperationBatch | null = null;
|
|
526
|
+
let clearedError = false;
|
|
519
527
|
|
|
520
528
|
const lastOp = await this.withReplicationTransaction(async (db) => {
|
|
521
|
-
|
|
529
|
+
clearedError = false;
|
|
530
|
+
const result = await this.replicateBatch(db, batch);
|
|
531
|
+
resumeBatch = result.resumeBatch;
|
|
532
|
+
clearedError ||= result.clearedError;
|
|
522
533
|
|
|
523
534
|
return this.getLastOpIdSequence(db);
|
|
524
535
|
});
|
|
525
536
|
|
|
537
|
+
if (clearedError) {
|
|
538
|
+
// Clear the error outside the replication transaction (plain autocommit update,
|
|
539
|
+
// like the keepalive), to avoid serialization conflicts on the sync_rules row
|
|
540
|
+
// when multiple writers flush concurrently.
|
|
541
|
+
await this.clearError();
|
|
542
|
+
this.clearedError = true;
|
|
543
|
+
}
|
|
544
|
+
|
|
526
545
|
// null if done, set if we need another flush
|
|
527
546
|
this.batch = resumeBatch;
|
|
528
547
|
|
|
@@ -545,136 +564,144 @@ export class PostgresBucketBatch
|
|
|
545
564
|
|
|
546
565
|
const persisted_op = this.persisted_op ?? null;
|
|
547
566
|
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
567
|
+
// Transaction failures restart replication from the last durable checkpoint.
|
|
568
|
+
const result = await this.db.transaction(async (db) => {
|
|
569
|
+
const checkpoint = await db.sql`
|
|
570
|
+
WITH
|
|
571
|
+
selected AS (
|
|
572
|
+
SELECT
|
|
573
|
+
id,
|
|
574
|
+
state,
|
|
575
|
+
last_checkpoint,
|
|
576
|
+
last_checkpoint_lsn,
|
|
577
|
+
snapshot_done,
|
|
578
|
+
no_checkpoint_before,
|
|
579
|
+
keepalive_op,
|
|
580
|
+
(
|
|
581
|
+
snapshot_done = TRUE
|
|
582
|
+
AND (
|
|
583
|
+
last_checkpoint_lsn IS NULL
|
|
584
|
+
OR last_checkpoint_lsn <= ${{ type: 'varchar', value: lsn }}
|
|
585
|
+
)
|
|
586
|
+
AND (
|
|
587
|
+
no_checkpoint_before IS NULL
|
|
588
|
+
OR no_checkpoint_before <= ${{ type: 'varchar', value: lsn }}
|
|
589
|
+
)
|
|
590
|
+
) AS can_checkpoint
|
|
591
|
+
FROM
|
|
592
|
+
sync_rules
|
|
593
|
+
WHERE
|
|
594
|
+
id = ${{ type: 'int4', value: this.group_id }}
|
|
595
|
+
FOR UPDATE
|
|
596
|
+
),
|
|
597
|
+
computed AS (
|
|
598
|
+
SELECT
|
|
599
|
+
selected.*,
|
|
600
|
+
CASE
|
|
601
|
+
WHEN selected.can_checkpoint THEN GREATEST(
|
|
602
|
+
selected.last_checkpoint,
|
|
603
|
+
${{ type: 'int8', value: persisted_op }},
|
|
604
|
+
selected.keepalive_op,
|
|
605
|
+
0
|
|
606
|
+
)
|
|
607
|
+
ELSE selected.last_checkpoint
|
|
608
|
+
END AS new_last_checkpoint,
|
|
609
|
+
CASE
|
|
610
|
+
WHEN selected.can_checkpoint THEN NULL
|
|
611
|
+
ELSE GREATEST(
|
|
612
|
+
selected.keepalive_op,
|
|
613
|
+
${{ type: 'int8', value: persisted_op }},
|
|
614
|
+
0
|
|
615
|
+
)
|
|
616
|
+
END AS new_keepalive_op
|
|
617
|
+
FROM
|
|
618
|
+
selected
|
|
619
|
+
),
|
|
620
|
+
updated AS (
|
|
621
|
+
UPDATE sync_rules AS sr
|
|
622
|
+
SET
|
|
623
|
+
last_checkpoint_lsn = CASE
|
|
624
|
+
WHEN computed.can_checkpoint THEN ${{ type: 'varchar', value: lsn }}
|
|
625
|
+
ELSE sr.last_checkpoint_lsn
|
|
626
|
+
END,
|
|
627
|
+
last_checkpoint_ts = CASE
|
|
628
|
+
WHEN computed.can_checkpoint THEN ${{ type: 1184, value: now }}
|
|
629
|
+
ELSE sr.last_checkpoint_ts
|
|
630
|
+
END,
|
|
631
|
+
last_keepalive_ts = ${{ type: 1184, value: now }},
|
|
632
|
+
last_fatal_error = CASE
|
|
633
|
+
WHEN computed.can_checkpoint THEN NULL
|
|
634
|
+
ELSE sr.last_fatal_error
|
|
635
|
+
END,
|
|
636
|
+
keepalive_op = computed.new_keepalive_op,
|
|
637
|
+
last_checkpoint = computed.new_last_checkpoint,
|
|
638
|
+
snapshot_lsn = CASE
|
|
639
|
+
WHEN computed.can_checkpoint THEN NULL
|
|
640
|
+
ELSE sr.snapshot_lsn
|
|
641
|
+
END
|
|
642
|
+
FROM
|
|
643
|
+
computed
|
|
644
|
+
WHERE
|
|
645
|
+
sr.id = computed.id
|
|
565
646
|
AND (
|
|
566
|
-
|
|
567
|
-
OR
|
|
568
|
-
|
|
569
|
-
) AS can_checkpoint
|
|
570
|
-
FROM
|
|
571
|
-
sync_rules
|
|
572
|
-
WHERE
|
|
573
|
-
id = ${{ type: 'int4', value: this.group_id }}
|
|
574
|
-
FOR UPDATE
|
|
575
|
-
),
|
|
576
|
-
computed AS (
|
|
577
|
-
SELECT
|
|
578
|
-
selected.*,
|
|
579
|
-
CASE
|
|
580
|
-
WHEN selected.can_checkpoint THEN GREATEST(
|
|
581
|
-
selected.last_checkpoint,
|
|
582
|
-
${{ type: 'int8', value: persisted_op }},
|
|
583
|
-
selected.keepalive_op,
|
|
584
|
-
0
|
|
585
|
-
)
|
|
586
|
-
ELSE selected.last_checkpoint
|
|
587
|
-
END AS new_last_checkpoint,
|
|
588
|
-
CASE
|
|
589
|
-
WHEN selected.can_checkpoint THEN NULL
|
|
590
|
-
ELSE GREATEST(
|
|
591
|
-
selected.keepalive_op,
|
|
592
|
-
${{ type: 'int8', value: persisted_op }},
|
|
593
|
-
0
|
|
647
|
+
sr.keepalive_op IS DISTINCT FROM computed.new_keepalive_op
|
|
648
|
+
OR sr.last_checkpoint IS DISTINCT FROM computed.new_last_checkpoint
|
|
649
|
+
OR ${{ type: 'bool', value: createEmptyCheckpoints }}
|
|
594
650
|
)
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
keepalive_op,
|
|
650
|
-
new_last_checkpoint,
|
|
651
|
-
TRUE AS created_checkpoint
|
|
652
|
-
FROM
|
|
653
|
-
updated
|
|
654
|
-
UNION ALL
|
|
655
|
-
SELECT
|
|
656
|
-
id,
|
|
657
|
-
state,
|
|
658
|
-
new_last_checkpoint AS last_checkpoint,
|
|
659
|
-
last_checkpoint_lsn,
|
|
660
|
-
snapshot_done,
|
|
661
|
-
no_checkpoint_before,
|
|
662
|
-
can_checkpoint,
|
|
663
|
-
keepalive_op,
|
|
664
|
-
new_last_checkpoint,
|
|
665
|
-
FALSE AS created_checkpoint
|
|
666
|
-
FROM
|
|
667
|
-
computed
|
|
668
|
-
WHERE
|
|
669
|
-
NOT EXISTS (
|
|
670
|
-
SELECT
|
|
671
|
-
1
|
|
672
|
-
FROM
|
|
673
|
-
updated
|
|
674
|
-
)
|
|
675
|
-
`
|
|
676
|
-
.decoded(CheckpointWithStatus)
|
|
677
|
-
.first();
|
|
651
|
+
RETURNING
|
|
652
|
+
sr.id,
|
|
653
|
+
sr.state,
|
|
654
|
+
sr.last_checkpoint,
|
|
655
|
+
sr.last_checkpoint_lsn,
|
|
656
|
+
sr.snapshot_done,
|
|
657
|
+
sr.no_checkpoint_before,
|
|
658
|
+
computed.can_checkpoint,
|
|
659
|
+
computed.keepalive_op,
|
|
660
|
+
computed.new_last_checkpoint
|
|
661
|
+
)
|
|
662
|
+
SELECT
|
|
663
|
+
id,
|
|
664
|
+
state,
|
|
665
|
+
last_checkpoint,
|
|
666
|
+
last_checkpoint_lsn,
|
|
667
|
+
snapshot_done,
|
|
668
|
+
no_checkpoint_before,
|
|
669
|
+
can_checkpoint,
|
|
670
|
+
keepalive_op,
|
|
671
|
+
new_last_checkpoint,
|
|
672
|
+
TRUE AS created_checkpoint
|
|
673
|
+
FROM
|
|
674
|
+
updated
|
|
675
|
+
UNION ALL
|
|
676
|
+
SELECT
|
|
677
|
+
id,
|
|
678
|
+
state,
|
|
679
|
+
new_last_checkpoint AS last_checkpoint,
|
|
680
|
+
last_checkpoint_lsn,
|
|
681
|
+
snapshot_done,
|
|
682
|
+
no_checkpoint_before,
|
|
683
|
+
can_checkpoint,
|
|
684
|
+
keepalive_op,
|
|
685
|
+
new_last_checkpoint,
|
|
686
|
+
FALSE AS created_checkpoint
|
|
687
|
+
FROM
|
|
688
|
+
computed
|
|
689
|
+
WHERE
|
|
690
|
+
NOT EXISTS (
|
|
691
|
+
SELECT
|
|
692
|
+
1
|
|
693
|
+
FROM
|
|
694
|
+
updated
|
|
695
|
+
)
|
|
696
|
+
`
|
|
697
|
+
.decoded(CheckpointWithStatus)
|
|
698
|
+
.first();
|
|
699
|
+
|
|
700
|
+
if (checkpoint?.can_checkpoint && checkpoint.state == storage.SyncRuleState.ACTIVE) {
|
|
701
|
+
await notifySyncRulesUpdate(db, checkpoint);
|
|
702
|
+
}
|
|
703
|
+
return checkpoint;
|
|
704
|
+
});
|
|
678
705
|
|
|
679
706
|
if (result == null) {
|
|
680
707
|
throw new ReplicationAssertionError('Failed to update sync_rules during checkpoint');
|
|
@@ -706,10 +733,8 @@ export class PostgresBucketBatch
|
|
|
706
733
|
});
|
|
707
734
|
}
|
|
708
735
|
}
|
|
709
|
-
await this.autoActivate(lsn
|
|
710
|
-
await notifySyncRulesUpdate(this.db, {
|
|
736
|
+
await this.autoActivate(lsn, {
|
|
711
737
|
id: result.id,
|
|
712
|
-
state: result.state,
|
|
713
738
|
last_checkpoint: result.last_checkpoint,
|
|
714
739
|
last_checkpoint_lsn: result.last_checkpoint_lsn
|
|
715
740
|
});
|
|
@@ -758,23 +783,22 @@ export class PostgresBucketBatch
|
|
|
758
783
|
|
|
759
784
|
async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void> {
|
|
760
785
|
await this.db.transaction(async (db) => {
|
|
761
|
-
const
|
|
786
|
+
const blockingTables = await db.sql`
|
|
762
787
|
SELECT
|
|
763
|
-
|
|
788
|
+
schema_name,
|
|
789
|
+
table_name
|
|
764
790
|
FROM
|
|
765
791
|
source_tables
|
|
766
792
|
WHERE
|
|
767
793
|
group_id = ${{ type: 'int4', value: this.group_id }}
|
|
768
794
|
AND snapshot_done = FALSE
|
|
769
795
|
`
|
|
770
|
-
.decoded(t.object({
|
|
771
|
-
.
|
|
772
|
-
if (
|
|
796
|
+
.decoded(t.object({ schema_name: t.string, table_name: t.string }))
|
|
797
|
+
.rows();
|
|
798
|
+
if (blockingTables.length > 0) {
|
|
773
799
|
if (options?.throwOnConflict ?? true) {
|
|
774
800
|
throw new ReplicationAssertionError(
|
|
775
|
-
`Cannot mark snapshot done while ${
|
|
776
|
-
snapshotRequiredCount?.count == 1n ? '' : 's'
|
|
777
|
-
} still require snapshotting`
|
|
801
|
+
`Cannot mark snapshot done while source tables still require snapshotting. Tables: ${blockingTables.map((t) => `${t.schema_name}.${t.table_name}`).join(', ')}`
|
|
778
802
|
);
|
|
779
803
|
} else {
|
|
780
804
|
return;
|
|
@@ -890,10 +914,17 @@ export class PostgresBucketBatch
|
|
|
890
914
|
});
|
|
891
915
|
}
|
|
892
916
|
|
|
893
|
-
protected async replicateBatch(
|
|
917
|
+
protected async replicateBatch(
|
|
918
|
+
db: lib_postgres.WrappedConnection,
|
|
919
|
+
batch: OperationBatch
|
|
920
|
+
): Promise<{ resumeBatch: OperationBatch | null; clearedError: boolean }> {
|
|
894
921
|
let sizes: Map<string, number> | undefined = undefined;
|
|
895
|
-
if
|
|
896
|
-
|
|
922
|
+
// Check if any table in this batch needs to store current_data
|
|
923
|
+
const anyTableStoresCurrentData =
|
|
924
|
+
this.options.store_current_data && batch.batch.some((r) => r.record.sourceTable.storeCurrentData);
|
|
925
|
+
|
|
926
|
+
if (anyTableStoresCurrentData && !this.options.skip_existing_rows) {
|
|
927
|
+
// We skip this step if no tables store current_data, since the sizes will
|
|
897
928
|
// always be small in that case.
|
|
898
929
|
|
|
899
930
|
// With skipExistingRows, we don't load the full documents into memory,
|
|
@@ -901,15 +932,19 @@ export class PostgresBucketBatch
|
|
|
901
932
|
|
|
902
933
|
// Find sizes of current_data documents, to assist in intelligent batching without
|
|
903
934
|
// exceeding memory limits.
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
935
|
+
// Within this branch the batch stores current_data, so the per-table flag is the
|
|
936
|
+
// effective value - only look up sizes for tables that actually store current_data.
|
|
937
|
+
const sizeLookups = batch.batch
|
|
938
|
+
.filter((r) => r.record.sourceTable.storeCurrentData)
|
|
939
|
+
.map((r) => {
|
|
940
|
+
return {
|
|
941
|
+
source_table: postgresTableId(r.record.sourceTable.id),
|
|
942
|
+
/**
|
|
943
|
+
* Encode to hex in order to pass a jsonb
|
|
944
|
+
*/
|
|
945
|
+
source_key: storage.serializeReplicaId(r.beforeId).toString('hex')
|
|
946
|
+
};
|
|
947
|
+
});
|
|
913
948
|
|
|
914
949
|
sizes = new Map<string, number>();
|
|
915
950
|
|
|
@@ -1006,15 +1041,14 @@ export class PostgresBucketBatch
|
|
|
1006
1041
|
}
|
|
1007
1042
|
}
|
|
1008
1043
|
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
}
|
|
1044
|
+
// The error itself is cleared by the caller, outside the replication transaction - see flushInner.
|
|
1045
|
+
const clearedError = didFlush && !this.clearedError;
|
|
1012
1046
|
|
|
1013
1047
|
// Don't return empty batches
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1048
|
+
return {
|
|
1049
|
+
resumeBatch: resumeBatch?.batch.length ? resumeBatch : null,
|
|
1050
|
+
clearedError
|
|
1051
|
+
};
|
|
1018
1052
|
}
|
|
1019
1053
|
|
|
1020
1054
|
protected async saveOperation(
|
|
@@ -1028,6 +1062,9 @@ export class PostgresBucketBatch
|
|
|
1028
1062
|
const afterId = operation.afterId;
|
|
1029
1063
|
let after = record.after;
|
|
1030
1064
|
const sourceTable = record.sourceTable;
|
|
1065
|
+
// Effective per-record flag: store current_data only if both the batch (source-level)
|
|
1066
|
+
// and the table (e.g. non-FULL replica identity) require it.
|
|
1067
|
+
const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
|
|
1031
1068
|
|
|
1032
1069
|
let existingBuckets: CurrentBucket[] = [];
|
|
1033
1070
|
let newBuckets: CurrentBucket[] = [];
|
|
@@ -1056,7 +1093,7 @@ export class PostgresBucketBatch
|
|
|
1056
1093
|
existingLookups = [];
|
|
1057
1094
|
// Log to help with debugging if there was a consistency issue
|
|
1058
1095
|
|
|
1059
|
-
if (
|
|
1096
|
+
if (storeCurrentData) {
|
|
1060
1097
|
if (this.markRecordUnavailable != null) {
|
|
1061
1098
|
// This will trigger a "resnapshot" of the record.
|
|
1062
1099
|
// This is not relevant if storeCurrentData is false, since we'll get the full row
|
|
@@ -1072,7 +1109,7 @@ export class PostgresBucketBatch
|
|
|
1072
1109
|
} else {
|
|
1073
1110
|
existingBuckets = result.buckets;
|
|
1074
1111
|
existingLookups = result.lookups;
|
|
1075
|
-
if (
|
|
1112
|
+
if (storeCurrentData) {
|
|
1076
1113
|
const data = storage.deserializeBson(result.data) as sync_rules.SqliteRow;
|
|
1077
1114
|
after = storage.mergeToast(after!, data);
|
|
1078
1115
|
}
|
|
@@ -1083,7 +1120,9 @@ export class PostgresBucketBatch
|
|
|
1083
1120
|
// Not an error if we re-apply a transaction
|
|
1084
1121
|
existingBuckets = [];
|
|
1085
1122
|
existingLookups = [];
|
|
1086
|
-
// Log to help with debugging if there was a consistency issue
|
|
1123
|
+
// Log to help with debugging if there was a consistency issue.
|
|
1124
|
+
// Gate on the batch-level flag: FULL tables (per-record flag false) still get a
|
|
1125
|
+
// current_data entry, so a missing record on DELETE is meaningful for them too.
|
|
1087
1126
|
if (this.options.store_current_data && this.markRecordUnavailable == null) {
|
|
1088
1127
|
this.logger.warn(
|
|
1089
1128
|
`Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
|
|
@@ -1096,7 +1135,7 @@ export class PostgresBucketBatch
|
|
|
1096
1135
|
}
|
|
1097
1136
|
|
|
1098
1137
|
let afterData: Buffer<ArrayBuffer> | undefined;
|
|
1099
|
-
if (afterId != null && !
|
|
1138
|
+
if (afterId != null && !storeCurrentData) {
|
|
1100
1139
|
afterData = storage.serializeBson({});
|
|
1101
1140
|
} else if (afterId != null) {
|
|
1102
1141
|
try {
|
|
@@ -1118,7 +1157,7 @@ export class PostgresBucketBatch
|
|
|
1118
1157
|
{
|
|
1119
1158
|
level: errors.ErrorSeverity.WARNING,
|
|
1120
1159
|
metadata: {
|
|
1121
|
-
replication_slot: this.options.
|
|
1160
|
+
replication_slot: this.options.replicationStreamName,
|
|
1122
1161
|
table: record.sourceTable.qualifiedName
|
|
1123
1162
|
}
|
|
1124
1163
|
}
|
|
@@ -1159,7 +1198,7 @@ export class PostgresBucketBatch
|
|
|
1159
1198
|
// However, it will be valid by the end of the transaction.
|
|
1160
1199
|
//
|
|
1161
1200
|
// In this case, we don't save the op, but we do save the current data.
|
|
1162
|
-
if (afterId && after && utils.isCompleteRow(
|
|
1201
|
+
if (afterId && after && utils.isCompleteRow(storeCurrentData, after)) {
|
|
1163
1202
|
// Insert or update
|
|
1164
1203
|
if (sourceTable.syncData) {
|
|
1165
1204
|
const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({
|
|
@@ -1174,7 +1213,7 @@ export class PostgresBucketBatch
|
|
|
1174
1213
|
{
|
|
1175
1214
|
level: errors.ErrorSeverity.WARNING,
|
|
1176
1215
|
metadata: {
|
|
1177
|
-
replication_slot: this.options.
|
|
1216
|
+
replication_slot: this.options.replicationStreamName,
|
|
1178
1217
|
table: record.sourceTable.qualifiedName
|
|
1179
1218
|
}
|
|
1180
1219
|
}
|
|
@@ -1215,7 +1254,7 @@ export class PostgresBucketBatch
|
|
|
1215
1254
|
{
|
|
1216
1255
|
level: errors.ErrorSeverity.WARNING,
|
|
1217
1256
|
metadata: {
|
|
1218
|
-
replication_slot: this.options.
|
|
1257
|
+
replication_slot: this.options.replicationStreamName,
|
|
1219
1258
|
table: record.sourceTable.qualifiedName
|
|
1220
1259
|
}
|
|
1221
1260
|
}
|
|
@@ -1272,14 +1311,13 @@ export class PostgresBucketBatch
|
|
|
1272
1311
|
*
|
|
1273
1312
|
* Called on new commits.
|
|
1274
1313
|
*/
|
|
1275
|
-
private async autoActivate(lsn: string): Promise<void> {
|
|
1314
|
+
private async autoActivate(lsn: string, checkpoint: models.ActiveCheckpointDecoded): Promise<void> {
|
|
1276
1315
|
if (!this.needsActivation) {
|
|
1277
1316
|
// Already activated
|
|
1278
1317
|
return;
|
|
1279
1318
|
}
|
|
1280
1319
|
|
|
1281
|
-
|
|
1282
|
-
await this.db.transaction(async (db) => {
|
|
1320
|
+
const activationResult = await this.db.transaction(async (db) => {
|
|
1283
1321
|
const syncRulesRow = await db.sql`
|
|
1284
1322
|
SELECT
|
|
1285
1323
|
state,
|
|
@@ -1313,13 +1351,18 @@ export class PostgresBucketBatch
|
|
|
1313
1351
|
)
|
|
1314
1352
|
AND id != ${{ type: 'int4', value: this.group_id }}
|
|
1315
1353
|
`.execute();
|
|
1316
|
-
|
|
1317
|
-
|
|
1354
|
+
await notifySyncRulesUpdate(db, checkpoint);
|
|
1355
|
+
return 'activated';
|
|
1318
1356
|
} else if (syncRulesRow?.state != storage.SyncRuleState.PROCESSING) {
|
|
1319
|
-
|
|
1357
|
+
return 'not-processing';
|
|
1320
1358
|
}
|
|
1359
|
+
return 'pending';
|
|
1321
1360
|
});
|
|
1322
|
-
|
|
1361
|
+
|
|
1362
|
+
if (activationResult != 'pending') {
|
|
1363
|
+
this.needsActivation = false;
|
|
1364
|
+
}
|
|
1365
|
+
if (activationResult == 'activated') {
|
|
1323
1366
|
this.logger.info(`Activated new replication stream at ${lsn}`);
|
|
1324
1367
|
}
|
|
1325
1368
|
}
|
|
@@ -1374,11 +1417,6 @@ export class PostgresBucketBatch
|
|
|
1374
1417
|
protected async clearError(
|
|
1375
1418
|
db: lib_postgres.AbstractPostgresConnection | lib_postgres.DatabaseClient = this.db
|
|
1376
1419
|
): Promise<void> {
|
|
1377
|
-
// No need to clear an error more than once per batch, since an error would always result in restarting the batch.
|
|
1378
|
-
if (this.clearedError) {
|
|
1379
|
-
return;
|
|
1380
|
-
}
|
|
1381
|
-
|
|
1382
1420
|
await db.sql`
|
|
1383
1421
|
UPDATE sync_rules
|
|
1384
1422
|
SET
|
|
@@ -1386,7 +1424,6 @@ export class PostgresBucketBatch
|
|
|
1386
1424
|
WHERE
|
|
1387
1425
|
id = ${{ type: 'int4', value: this.group_id }}
|
|
1388
1426
|
`.execute();
|
|
1389
|
-
this.clearedError = true;
|
|
1390
1427
|
}
|
|
1391
1428
|
|
|
1392
1429
|
private async getLastOpIdSequence(db: lib_postgres.AbstractPostgresConnection) {
|
|
@@ -1411,12 +1448,17 @@ export class PostgresBucketBatch
|
|
|
1411
1448
|
* Uses Postgres' NOTIFY functionality to update different processes when the
|
|
1412
1449
|
* active checkpoint has been updated.
|
|
1413
1450
|
*/
|
|
1414
|
-
export const notifySyncRulesUpdate = async (
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1451
|
+
export const notifySyncRulesUpdate = async (
|
|
1452
|
+
db: lib_postgres.AbstractPostgresConnection,
|
|
1453
|
+
update: models.ActiveCheckpointDecoded
|
|
1454
|
+
) => {
|
|
1455
|
+
const payload = models.ActiveCheckpointNotification.encode({ active_checkpoint: update });
|
|
1456
|
+
|
|
1457
|
+
await db.sql`
|
|
1458
|
+
SELECT
|
|
1459
|
+
pg_notify (
|
|
1460
|
+
${{ type: 'varchar', value: NOTIFICATION_CHANNEL }},
|
|
1461
|
+
${{ type: 'varchar', value: payload }}
|
|
1462
|
+
)
|
|
1463
|
+
`.execute();
|
|
1422
1464
|
};
|