@powersync/service-module-postgres-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/CHANGELOG.md +101 -6
  2. package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.d.ts +3 -0
  3. package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js +42 -0
  4. package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js.map +1 -0
  5. package/dist/{@types/storage → storage}/PostgresBucketStorageFactory.d.ts +13 -12
  6. package/dist/storage/PostgresBucketStorageFactory.js +59 -55
  7. package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
  8. package/dist/{@types/storage → storage}/PostgresCompactor.d.ts +3 -1
  9. package/dist/storage/PostgresCompactor.js +21 -1
  10. package/dist/storage/PostgresCompactor.js.map +1 -1
  11. package/dist/storage/PostgresReportStorage.js.map +1 -1
  12. package/dist/storage/PostgresStorageProvider.js +2 -1
  13. package/dist/storage/PostgresStorageProvider.js.map +1 -1
  14. package/dist/{@types/storage → storage}/PostgresSyncRulesStorage.d.ts +36 -11
  15. package/dist/storage/PostgresSyncRulesStorage.js +250 -177
  16. package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
  17. package/dist/{@types/storage → storage}/batch/PostgresBucketBatch.d.ts +8 -24
  18. package/dist/storage/batch/PostgresBucketBatch.js +229 -194
  19. package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
  20. package/dist/{@types/storage → storage}/checkpoints/PostgresWriteCheckpointAPI.d.ts +1 -1
  21. package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js +161 -24
  22. package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map +1 -1
  23. package/dist/{@types/storage → storage}/current-data-store.d.ts +8 -2
  24. package/dist/storage/current-data-store.js +66 -11
  25. package/dist/storage/current-data-store.js.map +1 -1
  26. package/dist/{@types/storage → storage}/storage-index.d.ts +1 -0
  27. package/dist/storage/storage-index.js +1 -0
  28. package/dist/storage/storage-index.js.map +1 -1
  29. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.d.ts +17 -0
  30. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js +90 -0
  31. package/dist/storage/sync-rules/PostgresPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/{@types/types → types}/models/WriteCheckpoint.d.ts +2 -0
  33. package/dist/types/models/WriteCheckpoint.js +5 -2
  34. package/dist/types/models/WriteCheckpoint.js.map +1 -1
  35. package/dist/utils/checkpoints.d.ts +9 -0
  36. package/dist/utils/checkpoints.js +26 -0
  37. package/dist/utils/checkpoints.js.map +1 -0
  38. package/dist/utils/test-utils.js +2 -2
  39. package/dist/utils/test-utils.js.map +1 -1
  40. package/package.json +10 -22
  41. package/src/migrations/scripts/1782950400000-checkpoint-requested-at.ts +51 -0
  42. package/src/storage/PostgresBucketStorageFactory.ts +67 -58
  43. package/src/storage/PostgresCompactor.ts +25 -1
  44. package/src/storage/PostgresReportStorage.ts +1 -2
  45. package/src/storage/PostgresStorageProvider.ts +2 -1
  46. package/src/storage/PostgresSyncRulesStorage.ts +275 -191
  47. package/src/storage/batch/PostgresBucketBatch.ts +256 -214
  48. package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +174 -24
  49. package/src/storage/current-data-store.ts +66 -11
  50. package/src/storage/storage-index.ts +1 -0
  51. package/src/storage/sync-rules/PostgresPersistedSyncConfigContent.ts +98 -0
  52. package/src/types/models/WriteCheckpoint.ts +5 -2
  53. package/src/utils/checkpoints.ts +31 -0
  54. package/src/utils/test-utils.ts +2 -2
  55. package/test/src/__snapshots__/storage_sync.test.ts.snap +0 -582
  56. package/test/src/checkpoint_notifications.test.ts +522 -0
  57. package/test/src/migrations.test.ts +1 -1
  58. package/test/src/storage.test.ts +226 -10
  59. package/test/src/storage_compacting.test.ts +15 -7
  60. package/test/src/storage_sync.test.ts +11 -8
  61. package/test/tsconfig.json +1 -3
  62. package/tsconfig.json +1 -4
  63. package/tsconfig.tsbuildinfo +1 -0
  64. package/dist/.tsbuildinfo +0 -1
  65. package/dist/@types/storage/sync-rules/PostgresPersistedSyncRulesContent.d.ts +0 -9
  66. package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js +0 -49
  67. package/dist/storage/sync-rules/PostgresPersistedSyncRulesContent.js.map +0 -1
  68. package/src/storage/sync-rules/PostgresPersistedSyncRulesContent.ts +0 -54
  69. /package/dist/{@types/index.d.ts → index.d.ts} +0 -0
  70. /package/dist/{@types/migrations → migrations}/PostgresMigrationAgent.d.ts +0 -0
  71. /package/dist/{@types/migrations → migrations}/PostgresMigrationStore.d.ts +0 -0
  72. /package/dist/{@types/migrations → migrations}/migration-utils.d.ts +0 -0
  73. /package/dist/{@types/migrations → migrations}/scripts/1684951997326-init.d.ts +0 -0
  74. /package/dist/{@types/migrations → migrations}/scripts/1749024804042-snapshot-progress.d.ts +0 -0
  75. /package/dist/{@types/migrations → migrations}/scripts/1756282360128-connection-reporting.d.ts +0 -0
  76. /package/dist/{@types/migrations → migrations}/scripts/1771232439485-storage-version.d.ts +0 -0
  77. /package/dist/{@types/migrations → migrations}/scripts/1771424826685-current-data-pending-deletes.d.ts +0 -0
  78. /package/dist/{@types/migrations → migrations}/scripts/1771491856000-sync-plan.d.ts +0 -0
  79. /package/dist/{@types/module → module}/PostgresStorageModule.d.ts +0 -0
  80. /package/dist/{@types/storage → storage}/PostgresReportStorage.d.ts +0 -0
  81. /package/dist/{@types/storage → storage}/PostgresStorageProvider.d.ts +0 -0
  82. /package/dist/{@types/storage → storage}/batch/OperationBatch.d.ts +0 -0
  83. /package/dist/{@types/storage → storage}/batch/PostgresPersistedBatch.d.ts +0 -0
  84. /package/dist/{@types/storage → storage}/table-id.d.ts +0 -0
  85. /package/dist/{@types/types → types}/codecs.d.ts +0 -0
  86. /package/dist/{@types/types → types}/models/ActiveCheckpoint.d.ts +0 -0
  87. /package/dist/{@types/types → types}/models/ActiveCheckpointNotification.d.ts +0 -0
  88. /package/dist/{@types/types → types}/models/BucketData.d.ts +0 -0
  89. /package/dist/{@types/types → types}/models/BucketParameters.d.ts +0 -0
  90. /package/dist/{@types/types → types}/models/CurrentData.d.ts +0 -0
  91. /package/dist/{@types/types → types}/models/Instance.d.ts +0 -0
  92. /package/dist/{@types/types → types}/models/Migration.d.ts +0 -0
  93. /package/dist/{@types/types → types}/models/SdkReporting.d.ts +0 -0
  94. /package/dist/{@types/types → types}/models/SourceTable.d.ts +0 -0
  95. /package/dist/{@types/types → types}/models/SyncRules.d.ts +0 -0
  96. /package/dist/{@types/types → types}/models/json.d.ts +0 -0
  97. /package/dist/{@types/types → types}/models/models-index.d.ts +0 -0
  98. /package/dist/{@types/types → types}/types.d.ts +0 -0
  99. /package/dist/{@types/utils → utils}/application-name.d.ts +0 -0
  100. /package/dist/{@types/utils → utils}/bson.d.ts +0 -0
  101. /package/dist/{@types/utils → utils}/bucket-data.d.ts +0 -0
  102. /package/dist/{@types/utils → utils}/db.d.ts +0 -0
  103. /package/dist/{@types/utils → utils}/test-utils.d.ts +0 -0
  104. /package/dist/{@types/utils → utils}/ts-codec.d.ts +0 -0
  105. /package/dist/{@types/utils → utils}/utils-index.d.ts +0 -0
@@ -38,8 +38,8 @@ export interface PostgresBucketBatchOptions {
38
38
  logger: Logger;
39
39
  db: lib_postgres.DatabaseClient;
40
40
  sync_rules: sync_rules.HydratedSyncConfig;
41
- group_id: number;
42
- slot_name: string;
41
+ replicationStreamId: number;
42
+ replicationStreamName: string;
43
43
  last_checkpoint_lsn: string | null;
44
44
  store_current_data: boolean;
45
45
  keep_alive_op?: InternalOpId | null;
@@ -60,7 +60,6 @@ export interface PostgresBucketBatchOptions {
60
60
  * via the Postgres NOTIFY protocol.
61
61
  */
62
62
  const StatefulCheckpoint = models.ActiveCheckpoint.and(t.object({ state: t.Enum(storage.SyncRuleState) }));
63
- type StatefulCheckpointDecoded = t.Decoded<typeof StatefulCheckpoint>;
64
63
 
65
64
  const CheckpointWithStatus = StatefulCheckpoint.and(
66
65
  t.object({
@@ -112,7 +111,7 @@ export class PostgresBucketBatch
112
111
  super();
113
112
  this.logger = options.logger;
114
113
  this.db = options.db;
115
- this.group_id = options.group_id;
114
+ this.group_id = options.replicationStreamId;
116
115
  this.last_checkpoint_lsn = options.last_checkpoint_lsn;
117
116
  this.resumeFromLsn = options.resumeFromLsn;
118
117
  this.skipExistingRows = options.skip_existing_rows;
@@ -129,10 +128,6 @@ export class PostgresBucketBatch
129
128
  }
130
129
  }
131
130
 
132
- get lastCheckpointLsn() {
133
- return this.last_checkpoint_lsn;
134
- }
135
-
136
131
  async [Symbol.asyncDispose]() {
137
132
  if (this.batch != null || this.write_checkpoint_batch.length > 0) {
138
133
  // We don't error here, since:
@@ -148,15 +143,16 @@ export class PostgresBucketBatch
148
143
  }
149
144
 
150
145
  async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
151
- const syncRules = options.syncRules ?? this.sync_rules;
146
+ const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
152
147
  const { connection_id, source } = options;
153
- const { schema, name: table, objectId, replicaIdColumns, connectionTag } = source;
148
+ const { schema, name: table, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
154
149
 
155
150
  const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
156
151
  name: column.name,
157
152
  type: column.type,
158
153
  type_oid: typeof column.typeId !== 'undefined' ? Number(column.typeId) : column.typeId
159
154
  }));
155
+
160
156
  return this.db.transaction(async (db) => {
161
157
  let sourceTableRow: SourceTableDecoded | null;
162
158
  if (objectId != null) {
@@ -240,6 +236,7 @@ export class PostgresBucketBatch
240
236
  sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
241
237
  sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
242
238
  sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
239
+ sourceTable.storeCurrentData = sendsCompleteRows !== true;
243
240
 
244
241
  let truncatedTables: SourceTableDecoded[] = [];
245
242
  if (objectId != null) {
@@ -359,19 +356,23 @@ export class PostgresBucketBatch
359
356
  async save(record: storage.SaveOptions): Promise<storage.FlushedResult | null> {
360
357
  // TODO maybe share with abstract class
361
358
  const { after, before, sourceTable, tag } = record;
362
- for (const event of this.getTableEvents(sourceTable)) {
363
- this.iterateListeners((cb) =>
364
- cb.replicationEvent?.({
365
- batch: this,
366
- table: sourceTable,
367
- data: {
368
- op: tag,
369
- after: after && utils.isCompleteRow(this.options.store_current_data, after) ? after : undefined,
370
- before: before && utils.isCompleteRow(this.options.store_current_data, before) ? before : undefined
371
- },
372
- event
373
- })
374
- );
359
+ const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
360
+ // Only the table designated as event carrier by resolveTables may fire events.
361
+ if (sourceTable.syncEvent) {
362
+ for (const event of this.getTableEvents(sourceTable)) {
363
+ this.iterateListeners((cb) =>
364
+ cb.replicationEvent?.({
365
+ batch: this,
366
+ table: sourceTable,
367
+ data: {
368
+ op: tag,
369
+ after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined,
370
+ before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined
371
+ },
372
+ event
373
+ })
374
+ );
375
+ }
375
376
  }
376
377
  /**
377
378
  * Return if the table is just an event table
@@ -420,7 +421,9 @@ export class PostgresBucketBatch
420
421
  let processedCount = 0;
421
422
  while (lastBatchCount == BATCH_LIMIT) {
422
423
  lastBatchCount = 0;
424
+ let clearedError = false;
423
425
  await this.withReplicationTransaction(async (db) => {
426
+ clearedError = false;
424
427
  const persistedBatch = new PostgresPersistedBatch({
425
428
  group_id: this.group_id,
426
429
  storageConfig: this.storageConfig,
@@ -460,10 +463,14 @@ export class PostgresBucketBatch
460
463
  }
461
464
  }
462
465
  const { flushedAny } = await persistedBatch.flush(db);
463
- if (flushedAny) {
464
- await this.clearError(db);
465
- }
466
+ clearedError = flushedAny && !this.clearedError;
466
467
  });
468
+ if (clearedError) {
469
+ // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
470
+ // Cleared outside the replication transaction - see flushInner.
471
+ await this.clearError();
472
+ this.clearedError = true;
473
+ }
467
474
  }
468
475
  if (processedCount == 0) {
469
476
  // The op sequence should not have progressed
@@ -516,13 +523,25 @@ export class PostgresBucketBatch
516
523
  await this.hooks?.beforeBatchFlush?.(this);
517
524
 
518
525
  let resumeBatch: OperationBatch | null = null;
526
+ let clearedError = false;
519
527
 
520
528
  const lastOp = await this.withReplicationTransaction(async (db) => {
521
- resumeBatch = await this.replicateBatch(db, batch);
529
+ clearedError = false;
530
+ const result = await this.replicateBatch(db, batch);
531
+ resumeBatch = result.resumeBatch;
532
+ clearedError ||= result.clearedError;
522
533
 
523
534
  return this.getLastOpIdSequence(db);
524
535
  });
525
536
 
537
+ if (clearedError) {
538
+ // Clear the error outside the replication transaction (plain autocommit update,
539
+ // like the keepalive), to avoid serialization conflicts on the sync_rules row
540
+ // when multiple writers flush concurrently.
541
+ await this.clearError();
542
+ this.clearedError = true;
543
+ }
544
+
526
545
  // null if done, set if we need another flush
527
546
  this.batch = resumeBatch;
528
547
 
@@ -545,136 +564,144 @@ export class PostgresBucketBatch
545
564
 
546
565
  const persisted_op = this.persisted_op ?? null;
547
566
 
548
- const result = await this.db.sql`
549
- WITH
550
- selected AS (
551
- SELECT
552
- id,
553
- state,
554
- last_checkpoint,
555
- last_checkpoint_lsn,
556
- snapshot_done,
557
- no_checkpoint_before,
558
- keepalive_op,
559
- (
560
- snapshot_done = TRUE
561
- AND (
562
- last_checkpoint_lsn IS NULL
563
- OR last_checkpoint_lsn <= ${{ type: 'varchar', value: lsn }}
564
- )
567
+ // Transaction failures restart replication from the last durable checkpoint.
568
+ const result = await this.db.transaction(async (db) => {
569
+ const checkpoint = await db.sql`
570
+ WITH
571
+ selected AS (
572
+ SELECT
573
+ id,
574
+ state,
575
+ last_checkpoint,
576
+ last_checkpoint_lsn,
577
+ snapshot_done,
578
+ no_checkpoint_before,
579
+ keepalive_op,
580
+ (
581
+ snapshot_done = TRUE
582
+ AND (
583
+ last_checkpoint_lsn IS NULL
584
+ OR last_checkpoint_lsn <= ${{ type: 'varchar', value: lsn }}
585
+ )
586
+ AND (
587
+ no_checkpoint_before IS NULL
588
+ OR no_checkpoint_before <= ${{ type: 'varchar', value: lsn }}
589
+ )
590
+ ) AS can_checkpoint
591
+ FROM
592
+ sync_rules
593
+ WHERE
594
+ id = ${{ type: 'int4', value: this.group_id }}
595
+ FOR UPDATE
596
+ ),
597
+ computed AS (
598
+ SELECT
599
+ selected.*,
600
+ CASE
601
+ WHEN selected.can_checkpoint THEN GREATEST(
602
+ selected.last_checkpoint,
603
+ ${{ type: 'int8', value: persisted_op }},
604
+ selected.keepalive_op,
605
+ 0
606
+ )
607
+ ELSE selected.last_checkpoint
608
+ END AS new_last_checkpoint,
609
+ CASE
610
+ WHEN selected.can_checkpoint THEN NULL
611
+ ELSE GREATEST(
612
+ selected.keepalive_op,
613
+ ${{ type: 'int8', value: persisted_op }},
614
+ 0
615
+ )
616
+ END AS new_keepalive_op
617
+ FROM
618
+ selected
619
+ ),
620
+ updated AS (
621
+ UPDATE sync_rules AS sr
622
+ SET
623
+ last_checkpoint_lsn = CASE
624
+ WHEN computed.can_checkpoint THEN ${{ type: 'varchar', value: lsn }}
625
+ ELSE sr.last_checkpoint_lsn
626
+ END,
627
+ last_checkpoint_ts = CASE
628
+ WHEN computed.can_checkpoint THEN ${{ type: 1184, value: now }}
629
+ ELSE sr.last_checkpoint_ts
630
+ END,
631
+ last_keepalive_ts = ${{ type: 1184, value: now }},
632
+ last_fatal_error = CASE
633
+ WHEN computed.can_checkpoint THEN NULL
634
+ ELSE sr.last_fatal_error
635
+ END,
636
+ keepalive_op = computed.new_keepalive_op,
637
+ last_checkpoint = computed.new_last_checkpoint,
638
+ snapshot_lsn = CASE
639
+ WHEN computed.can_checkpoint THEN NULL
640
+ ELSE sr.snapshot_lsn
641
+ END
642
+ FROM
643
+ computed
644
+ WHERE
645
+ sr.id = computed.id
565
646
  AND (
566
- no_checkpoint_before IS NULL
567
- OR no_checkpoint_before <= ${{ type: 'varchar', value: lsn }}
568
- )
569
- ) AS can_checkpoint
570
- FROM
571
- sync_rules
572
- WHERE
573
- id = ${{ type: 'int4', value: this.group_id }}
574
- FOR UPDATE
575
- ),
576
- computed AS (
577
- SELECT
578
- selected.*,
579
- CASE
580
- WHEN selected.can_checkpoint THEN GREATEST(
581
- selected.last_checkpoint,
582
- ${{ type: 'int8', value: persisted_op }},
583
- selected.keepalive_op,
584
- 0
585
- )
586
- ELSE selected.last_checkpoint
587
- END AS new_last_checkpoint,
588
- CASE
589
- WHEN selected.can_checkpoint THEN NULL
590
- ELSE GREATEST(
591
- selected.keepalive_op,
592
- ${{ type: 'int8', value: persisted_op }},
593
- 0
647
+ sr.keepalive_op IS DISTINCT FROM computed.new_keepalive_op
648
+ OR sr.last_checkpoint IS DISTINCT FROM computed.new_last_checkpoint
649
+ OR ${{ type: 'bool', value: createEmptyCheckpoints }}
594
650
  )
595
- END AS new_keepalive_op
596
- FROM
597
- selected
598
- ),
599
- updated AS (
600
- UPDATE sync_rules AS sr
601
- SET
602
- last_checkpoint_lsn = CASE
603
- WHEN computed.can_checkpoint THEN ${{ type: 'varchar', value: lsn }}
604
- ELSE sr.last_checkpoint_lsn
605
- END,
606
- last_checkpoint_ts = CASE
607
- WHEN computed.can_checkpoint THEN ${{ type: 1184, value: now }}
608
- ELSE sr.last_checkpoint_ts
609
- END,
610
- last_keepalive_ts = ${{ type: 1184, value: now }},
611
- last_fatal_error = CASE
612
- WHEN computed.can_checkpoint THEN NULL
613
- ELSE sr.last_fatal_error
614
- END,
615
- keepalive_op = computed.new_keepalive_op,
616
- last_checkpoint = computed.new_last_checkpoint,
617
- snapshot_lsn = CASE
618
- WHEN computed.can_checkpoint THEN NULL
619
- ELSE sr.snapshot_lsn
620
- END
621
- FROM
622
- computed
623
- WHERE
624
- sr.id = computed.id
625
- AND (
626
- sr.keepalive_op IS DISTINCT FROM computed.new_keepalive_op
627
- OR sr.last_checkpoint IS DISTINCT FROM computed.new_last_checkpoint
628
- OR ${{ type: 'bool', value: createEmptyCheckpoints }}
629
- )
630
- RETURNING
631
- sr.id,
632
- sr.state,
633
- sr.last_checkpoint,
634
- sr.last_checkpoint_lsn,
635
- sr.snapshot_done,
636
- sr.no_checkpoint_before,
637
- computed.can_checkpoint,
638
- computed.keepalive_op,
639
- computed.new_last_checkpoint
640
- )
641
- SELECT
642
- id,
643
- state,
644
- last_checkpoint,
645
- last_checkpoint_lsn,
646
- snapshot_done,
647
- no_checkpoint_before,
648
- can_checkpoint,
649
- keepalive_op,
650
- new_last_checkpoint,
651
- TRUE AS created_checkpoint
652
- FROM
653
- updated
654
- UNION ALL
655
- SELECT
656
- id,
657
- state,
658
- new_last_checkpoint AS last_checkpoint,
659
- last_checkpoint_lsn,
660
- snapshot_done,
661
- no_checkpoint_before,
662
- can_checkpoint,
663
- keepalive_op,
664
- new_last_checkpoint,
665
- FALSE AS created_checkpoint
666
- FROM
667
- computed
668
- WHERE
669
- NOT EXISTS (
670
- SELECT
671
- 1
672
- FROM
673
- updated
674
- )
675
- `
676
- .decoded(CheckpointWithStatus)
677
- .first();
651
+ RETURNING
652
+ sr.id,
653
+ sr.state,
654
+ sr.last_checkpoint,
655
+ sr.last_checkpoint_lsn,
656
+ sr.snapshot_done,
657
+ sr.no_checkpoint_before,
658
+ computed.can_checkpoint,
659
+ computed.keepalive_op,
660
+ computed.new_last_checkpoint
661
+ )
662
+ SELECT
663
+ id,
664
+ state,
665
+ last_checkpoint,
666
+ last_checkpoint_lsn,
667
+ snapshot_done,
668
+ no_checkpoint_before,
669
+ can_checkpoint,
670
+ keepalive_op,
671
+ new_last_checkpoint,
672
+ TRUE AS created_checkpoint
673
+ FROM
674
+ updated
675
+ UNION ALL
676
+ SELECT
677
+ id,
678
+ state,
679
+ new_last_checkpoint AS last_checkpoint,
680
+ last_checkpoint_lsn,
681
+ snapshot_done,
682
+ no_checkpoint_before,
683
+ can_checkpoint,
684
+ keepalive_op,
685
+ new_last_checkpoint,
686
+ FALSE AS created_checkpoint
687
+ FROM
688
+ computed
689
+ WHERE
690
+ NOT EXISTS (
691
+ SELECT
692
+ 1
693
+ FROM
694
+ updated
695
+ )
696
+ `
697
+ .decoded(CheckpointWithStatus)
698
+ .first();
699
+
700
+ if (checkpoint?.can_checkpoint && checkpoint.state == storage.SyncRuleState.ACTIVE) {
701
+ await notifySyncRulesUpdate(db, checkpoint);
702
+ }
703
+ return checkpoint;
704
+ });
678
705
 
679
706
  if (result == null) {
680
707
  throw new ReplicationAssertionError('Failed to update sync_rules during checkpoint');
@@ -706,10 +733,8 @@ export class PostgresBucketBatch
706
733
  });
707
734
  }
708
735
  }
709
- await this.autoActivate(lsn);
710
- await notifySyncRulesUpdate(this.db, {
736
+ await this.autoActivate(lsn, {
711
737
  id: result.id,
712
- state: result.state,
713
738
  last_checkpoint: result.last_checkpoint,
714
739
  last_checkpoint_lsn: result.last_checkpoint_lsn
715
740
  });
@@ -758,23 +783,22 @@ export class PostgresBucketBatch
758
783
 
759
784
  async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void> {
760
785
  await this.db.transaction(async (db) => {
761
- const snapshotRequiredCount = await db.sql`
786
+ const blockingTables = await db.sql`
762
787
  SELECT
763
- COUNT(*) AS count
788
+ schema_name,
789
+ table_name
764
790
  FROM
765
791
  source_tables
766
792
  WHERE
767
793
  group_id = ${{ type: 'int4', value: this.group_id }}
768
794
  AND snapshot_done = FALSE
769
795
  `
770
- .decoded(t.object({ count: bigint }))
771
- .first();
772
- if ((snapshotRequiredCount?.count ?? 0n) > 0n) {
796
+ .decoded(t.object({ schema_name: t.string, table_name: t.string }))
797
+ .rows();
798
+ if (blockingTables.length > 0) {
773
799
  if (options?.throwOnConflict ?? true) {
774
800
  throw new ReplicationAssertionError(
775
- `Cannot mark snapshot done while ${snapshotRequiredCount?.count} source table${
776
- snapshotRequiredCount?.count == 1n ? '' : 's'
777
- } still require snapshotting`
801
+ `Cannot mark snapshot done while source tables still require snapshotting. Tables: ${blockingTables.map((t) => `${t.schema_name}.${t.table_name}`).join(', ')}`
778
802
  );
779
803
  } else {
780
804
  return;
@@ -890,10 +914,17 @@ export class PostgresBucketBatch
890
914
  });
891
915
  }
892
916
 
893
- protected async replicateBatch(db: lib_postgres.WrappedConnection, batch: OperationBatch) {
917
+ protected async replicateBatch(
918
+ db: lib_postgres.WrappedConnection,
919
+ batch: OperationBatch
920
+ ): Promise<{ resumeBatch: OperationBatch | null; clearedError: boolean }> {
894
921
  let sizes: Map<string, number> | undefined = undefined;
895
- if (this.options.store_current_data && !this.options.skip_existing_rows) {
896
- // We skip this step if we don't store current_data, since the sizes will
922
+ // Check if any table in this batch needs to store current_data
923
+ const anyTableStoresCurrentData =
924
+ this.options.store_current_data && batch.batch.some((r) => r.record.sourceTable.storeCurrentData);
925
+
926
+ if (anyTableStoresCurrentData && !this.options.skip_existing_rows) {
927
+ // We skip this step if no tables store current_data, since the sizes will
897
928
  // always be small in that case.
898
929
 
899
930
  // With skipExistingRows, we don't load the full documents into memory,
@@ -901,15 +932,19 @@ export class PostgresBucketBatch
901
932
 
902
933
  // Find sizes of current_data documents, to assist in intelligent batching without
903
934
  // exceeding memory limits.
904
- const sizeLookups = batch.batch.map((r) => {
905
- return {
906
- source_table: postgresTableId(r.record.sourceTable.id),
907
- /**
908
- * Encode to hex in order to pass a jsonb
909
- */
910
- source_key: storage.serializeReplicaId(r.beforeId).toString('hex')
911
- };
912
- });
935
+ // Within this branch the batch stores current_data, so the per-table flag is the
936
+ // effective value - only look up sizes for tables that actually store current_data.
937
+ const sizeLookups = batch.batch
938
+ .filter((r) => r.record.sourceTable.storeCurrentData)
939
+ .map((r) => {
940
+ return {
941
+ source_table: postgresTableId(r.record.sourceTable.id),
942
+ /**
943
+ * Encode to hex in order to pass a jsonb
944
+ */
945
+ source_key: storage.serializeReplicaId(r.beforeId).toString('hex')
946
+ };
947
+ });
913
948
 
914
949
  sizes = new Map<string, number>();
915
950
 
@@ -1006,15 +1041,14 @@ export class PostgresBucketBatch
1006
1041
  }
1007
1042
  }
1008
1043
 
1009
- if (didFlush) {
1010
- await this.clearError(db);
1011
- }
1044
+ // The error itself is cleared by the caller, outside the replication transaction - see flushInner.
1045
+ const clearedError = didFlush && !this.clearedError;
1012
1046
 
1013
1047
  // Don't return empty batches
1014
- if (resumeBatch?.batch.length) {
1015
- return resumeBatch;
1016
- }
1017
- return null;
1048
+ return {
1049
+ resumeBatch: resumeBatch?.batch.length ? resumeBatch : null,
1050
+ clearedError
1051
+ };
1018
1052
  }
1019
1053
 
1020
1054
  protected async saveOperation(
@@ -1028,6 +1062,9 @@ export class PostgresBucketBatch
1028
1062
  const afterId = operation.afterId;
1029
1063
  let after = record.after;
1030
1064
  const sourceTable = record.sourceTable;
1065
+ // Effective per-record flag: store current_data only if both the batch (source-level)
1066
+ // and the table (e.g. non-FULL replica identity) require it.
1067
+ const storeCurrentData = this.options.store_current_data && sourceTable.storeCurrentData;
1031
1068
 
1032
1069
  let existingBuckets: CurrentBucket[] = [];
1033
1070
  let newBuckets: CurrentBucket[] = [];
@@ -1056,7 +1093,7 @@ export class PostgresBucketBatch
1056
1093
  existingLookups = [];
1057
1094
  // Log to help with debugging if there was a consistency issue
1058
1095
 
1059
- if (this.options.store_current_data) {
1096
+ if (storeCurrentData) {
1060
1097
  if (this.markRecordUnavailable != null) {
1061
1098
  // This will trigger a "resnapshot" of the record.
1062
1099
  // This is not relevant if storeCurrentData is false, since we'll get the full row
@@ -1072,7 +1109,7 @@ export class PostgresBucketBatch
1072
1109
  } else {
1073
1110
  existingBuckets = result.buckets;
1074
1111
  existingLookups = result.lookups;
1075
- if (this.options.store_current_data) {
1112
+ if (storeCurrentData) {
1076
1113
  const data = storage.deserializeBson(result.data) as sync_rules.SqliteRow;
1077
1114
  after = storage.mergeToast(after!, data);
1078
1115
  }
@@ -1083,7 +1120,9 @@ export class PostgresBucketBatch
1083
1120
  // Not an error if we re-apply a transaction
1084
1121
  existingBuckets = [];
1085
1122
  existingLookups = [];
1086
- // Log to help with debugging if there was a consistency issue
1123
+ // Log to help with debugging if there was a consistency issue.
1124
+ // Gate on the batch-level flag: FULL tables (per-record flag false) still get a
1125
+ // current_data entry, so a missing record on DELETE is meaningful for them too.
1087
1126
  if (this.options.store_current_data && this.markRecordUnavailable == null) {
1088
1127
  this.logger.warn(
1089
1128
  `Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
@@ -1096,7 +1135,7 @@ export class PostgresBucketBatch
1096
1135
  }
1097
1136
 
1098
1137
  let afterData: Buffer<ArrayBuffer> | undefined;
1099
- if (afterId != null && !this.options.store_current_data) {
1138
+ if (afterId != null && !storeCurrentData) {
1100
1139
  afterData = storage.serializeBson({});
1101
1140
  } else if (afterId != null) {
1102
1141
  try {
@@ -1118,7 +1157,7 @@ export class PostgresBucketBatch
1118
1157
  {
1119
1158
  level: errors.ErrorSeverity.WARNING,
1120
1159
  metadata: {
1121
- replication_slot: this.options.slot_name,
1160
+ replication_slot: this.options.replicationStreamName,
1122
1161
  table: record.sourceTable.qualifiedName
1123
1162
  }
1124
1163
  }
@@ -1159,7 +1198,7 @@ export class PostgresBucketBatch
1159
1198
  // However, it will be valid by the end of the transaction.
1160
1199
  //
1161
1200
  // In this case, we don't save the op, but we do save the current data.
1162
- if (afterId && after && utils.isCompleteRow(this.options.store_current_data, after)) {
1201
+ if (afterId && after && utils.isCompleteRow(storeCurrentData, after)) {
1163
1202
  // Insert or update
1164
1203
  if (sourceTable.syncData) {
1165
1204
  const { results: evaluated, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({
@@ -1174,7 +1213,7 @@ export class PostgresBucketBatch
1174
1213
  {
1175
1214
  level: errors.ErrorSeverity.WARNING,
1176
1215
  metadata: {
1177
- replication_slot: this.options.slot_name,
1216
+ replication_slot: this.options.replicationStreamName,
1178
1217
  table: record.sourceTable.qualifiedName
1179
1218
  }
1180
1219
  }
@@ -1215,7 +1254,7 @@ export class PostgresBucketBatch
1215
1254
  {
1216
1255
  level: errors.ErrorSeverity.WARNING,
1217
1256
  metadata: {
1218
- replication_slot: this.options.slot_name,
1257
+ replication_slot: this.options.replicationStreamName,
1219
1258
  table: record.sourceTable.qualifiedName
1220
1259
  }
1221
1260
  }
@@ -1272,14 +1311,13 @@ export class PostgresBucketBatch
1272
1311
  *
1273
1312
  * Called on new commits.
1274
1313
  */
1275
- private async autoActivate(lsn: string): Promise<void> {
1314
+ private async autoActivate(lsn: string, checkpoint: models.ActiveCheckpointDecoded): Promise<void> {
1276
1315
  if (!this.needsActivation) {
1277
1316
  // Already activated
1278
1317
  return;
1279
1318
  }
1280
1319
 
1281
- let didActivate = false;
1282
- await this.db.transaction(async (db) => {
1320
+ const activationResult = await this.db.transaction(async (db) => {
1283
1321
  const syncRulesRow = await db.sql`
1284
1322
  SELECT
1285
1323
  state,
@@ -1313,13 +1351,18 @@ export class PostgresBucketBatch
1313
1351
  )
1314
1352
  AND id != ${{ type: 'int4', value: this.group_id }}
1315
1353
  `.execute();
1316
- didActivate = true;
1317
- this.needsActivation = false;
1354
+ await notifySyncRulesUpdate(db, checkpoint);
1355
+ return 'activated';
1318
1356
  } else if (syncRulesRow?.state != storage.SyncRuleState.PROCESSING) {
1319
- this.needsActivation = false;
1357
+ return 'not-processing';
1320
1358
  }
1359
+ return 'pending';
1321
1360
  });
1322
- if (didActivate) {
1361
+
1362
+ if (activationResult != 'pending') {
1363
+ this.needsActivation = false;
1364
+ }
1365
+ if (activationResult == 'activated') {
1323
1366
  this.logger.info(`Activated new replication stream at ${lsn}`);
1324
1367
  }
1325
1368
  }
@@ -1374,11 +1417,6 @@ export class PostgresBucketBatch
1374
1417
  protected async clearError(
1375
1418
  db: lib_postgres.AbstractPostgresConnection | lib_postgres.DatabaseClient = this.db
1376
1419
  ): Promise<void> {
1377
- // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
1378
- if (this.clearedError) {
1379
- return;
1380
- }
1381
-
1382
1420
  await db.sql`
1383
1421
  UPDATE sync_rules
1384
1422
  SET
@@ -1386,7 +1424,6 @@ export class PostgresBucketBatch
1386
1424
  WHERE
1387
1425
  id = ${{ type: 'int4', value: this.group_id }}
1388
1426
  `.execute();
1389
- this.clearedError = true;
1390
1427
  }
1391
1428
 
1392
1429
  private async getLastOpIdSequence(db: lib_postgres.AbstractPostgresConnection) {
@@ -1411,12 +1448,17 @@ export class PostgresBucketBatch
1411
1448
  * Uses Postgres' NOTIFY functionality to update different processes when the
1412
1449
  * active checkpoint has been updated.
1413
1450
  */
1414
- export const notifySyncRulesUpdate = async (db: lib_postgres.DatabaseClient, update: StatefulCheckpointDecoded) => {
1415
- if (update.state != storage.SyncRuleState.ACTIVE) {
1416
- return;
1417
- }
1418
-
1419
- await db.query({
1420
- statement: `NOTIFY ${NOTIFICATION_CHANNEL}, '${models.ActiveCheckpointNotification.encode({ active_checkpoint: update })}'`
1421
- });
1451
+ export const notifySyncRulesUpdate = async (
1452
+ db: lib_postgres.AbstractPostgresConnection,
1453
+ update: models.ActiveCheckpointDecoded
1454
+ ) => {
1455
+ const payload = models.ActiveCheckpointNotification.encode({ active_checkpoint: update });
1456
+
1457
+ await db.sql`
1458
+ SELECT
1459
+ pg_notify (
1460
+ ${{ type: 'varchar', value: NOTIFICATION_CHANNEL }},
1461
+ ${{ type: 'varchar', value: payload }}
1462
+ )
1463
+ `.execute();
1422
1464
  };