@powersync/service-module-postgres-storage 0.16.3 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CHANGELOG.md +60 -0
  2. package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.d.ts +3 -0
  3. package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js +42 -0
  4. package/dist/migrations/scripts/1782950400000-checkpoint-requested-at.js.map +1 -0
  5. package/dist/migrations/scripts/1784900000000-source-metadata.d.ts +3 -0
  6. package/dist/migrations/scripts/1784900000000-source-metadata.js +22 -0
  7. package/dist/migrations/scripts/1784900000000-source-metadata.js.map +1 -0
  8. package/dist/storage/PostgresBucketStorageFactory.js +0 -2
  9. package/dist/storage/PostgresBucketStorageFactory.js.map +1 -1
  10. package/dist/storage/PostgresCompactor.d.ts +2 -0
  11. package/dist/storage/PostgresCompactor.js +20 -0
  12. package/dist/storage/PostgresCompactor.js.map +1 -1
  13. package/dist/storage/PostgresSyncRulesStorage.d.ts +16 -1
  14. package/dist/storage/PostgresSyncRulesStorage.js +200 -131
  15. package/dist/storage/PostgresSyncRulesStorage.js.map +1 -1
  16. package/dist/storage/batch/PostgresBucketBatch.d.ts +2 -20
  17. package/dist/storage/batch/PostgresBucketBatch.js +217 -240
  18. package/dist/storage/batch/PostgresBucketBatch.js.map +1 -1
  19. package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.d.ts +1 -1
  20. package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js +156 -36
  21. package/dist/storage/checkpoints/PostgresWriteCheckpointAPI.js.map +1 -1
  22. package/dist/storage/current-data-store.d.ts +8 -2
  23. package/dist/storage/current-data-store.js +66 -11
  24. package/dist/storage/current-data-store.js.map +1 -1
  25. package/dist/types/models/SourceTable.d.ts +7 -2
  26. package/dist/types/models/SourceTable.js +6 -2
  27. package/dist/types/models/SourceTable.js.map +1 -1
  28. package/dist/types/models/WriteCheckpoint.d.ts +2 -0
  29. package/dist/types/models/WriteCheckpoint.js +5 -2
  30. package/dist/types/models/WriteCheckpoint.js.map +1 -1
  31. package/dist/utils/checkpoints.d.ts +9 -0
  32. package/dist/utils/checkpoints.js +26 -0
  33. package/dist/utils/checkpoints.js.map +1 -0
  34. package/package.json +8 -8
  35. package/src/migrations/scripts/1782950400000-checkpoint-requested-at.ts +51 -0
  36. package/src/migrations/scripts/1784900000000-source-metadata.ts +31 -0
  37. package/src/storage/PostgresBucketStorageFactory.ts +0 -3
  38. package/src/storage/PostgresCompactor.ts +24 -0
  39. package/src/storage/PostgresSyncRulesStorage.ts +219 -137
  40. package/src/storage/batch/PostgresBucketBatch.ts +227 -246
  41. package/src/storage/checkpoints/PostgresWriteCheckpointAPI.ts +165 -37
  42. package/src/storage/current-data-store.ts +66 -11
  43. package/src/types/models/SourceTable.ts +7 -2
  44. package/src/types/models/WriteCheckpoint.ts +5 -2
  45. package/src/utils/checkpoints.ts +31 -0
  46. package/test/src/__snapshots__/storage_sync.test.ts.snap +0 -582
  47. package/test/src/checkpoint_notifications.test.ts +522 -0
  48. package/test/src/storage.test.ts +214 -5
  49. package/test/src/storage_compacting.test.ts +3 -3
  50. package/test/src/storage_sync.test.ts +48 -4
  51. package/test/tsconfig.json +1 -1
  52. package/tsconfig.tsbuildinfo +1 -1
package/test/src/__snapshots__/storage_sync.test.ts.snap CHANGED
@@ -589,200 +589,6 @@ exports[`sync - postgres > storage v1 > sync global data 1`] = `
589
589
  ]
590
590
  `;
591
591
 
592
- exports[`sync - postgres > storage v1 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = `
593
- [
594
- {
595
- "checkpoint": {
596
- "buckets": [
597
- {
598
- "bucket": "b0a[]",
599
- "checksum": -659831575,
600
- "count": 2000,
601
- "priority": 2,
602
- "subscriptions": [
603
- {
604
- "default": 0,
605
- },
606
- ],
607
- },
608
- {
609
- "bucket": "b0b[]",
610
- "checksum": -659831575,
611
- "count": 2000,
612
- "priority": 2,
613
- "subscriptions": [
614
- {
615
- "default": 1,
616
- },
617
- ],
618
- },
619
- {
620
- "bucket": "b1[]",
621
- "checksum": -1096116670,
622
- "count": 1,
623
- "priority": 1,
624
- "subscriptions": [
625
- {
626
- "default": 2,
627
- },
628
- ],
629
- },
630
- ],
631
- "last_op_id": "4001",
632
- "streams": [
633
- {
634
- "errors": [],
635
- "is_default": true,
636
- "name": "b0a",
637
- },
638
- {
639
- "errors": [],
640
- "is_default": true,
641
- "name": "b0b",
642
- },
643
- {
644
- "errors": [],
645
- "is_default": true,
646
- "name": "b1",
647
- },
648
- ],
649
- "write_checkpoint": undefined,
650
- },
651
- },
652
- {
653
- "data": {
654
- "after": "0",
655
- "bucket": "b1[]",
656
- "data": undefined,
657
- "has_more": false,
658
- "next_after": "1",
659
- },
660
- },
661
- {
662
- "partial_checkpoint_complete": {
663
- "last_op_id": "4001",
664
- "priority": 1,
665
- },
666
- },
667
- {
668
- "data": {
669
- "after": "0",
670
- "bucket": "b0a[]",
671
- "data": undefined,
672
- "has_more": true,
673
- "next_after": "2000",
674
- },
675
- },
676
- {
677
- "data": {
678
- "after": "2000",
679
- "bucket": "b0a[]",
680
- "data": undefined,
681
- "has_more": true,
682
- "next_after": "4000",
683
- },
684
- },
685
- {
686
- "checkpoint_diff": {
687
- "last_op_id": "4004",
688
- "removed_buckets": [],
689
- "updated_buckets": [
690
- {
691
- "bucket": "b0a[]",
692
- "checksum": 883076828,
693
- "count": 2001,
694
- "priority": 2,
695
- "subscriptions": [
696
- {
697
- "default": 0,
698
- },
699
- ],
700
- },
701
- {
702
- "bucket": "b0b[]",
703
- "checksum": 883076828,
704
- "count": 2001,
705
- "priority": 2,
706
- "subscriptions": [
707
- {
708
- "default": 1,
709
- },
710
- ],
711
- },
712
- {
713
- "bucket": "b1[]",
714
- "checksum": 1841937527,
715
- "count": 2,
716
- "priority": 1,
717
- "subscriptions": [
718
- {
719
- "default": 2,
720
- },
721
- ],
722
- },
723
- ],
724
- "write_checkpoint": undefined,
725
- },
726
- },
727
- {
728
- "data": {
729
- "after": "1",
730
- "bucket": "b1[]",
731
- "data": undefined,
732
- "has_more": false,
733
- "next_after": "4002",
734
- },
735
- },
736
- {
737
- "partial_checkpoint_complete": {
738
- "last_op_id": "4004",
739
- "priority": 1,
740
- },
741
- },
742
- {
743
- "data": {
744
- "after": "4000",
745
- "bucket": "b0a[]",
746
- "data": undefined,
747
- "has_more": false,
748
- "next_after": "4003",
749
- },
750
- },
751
- {
752
- "data": {
753
- "after": "0",
754
- "bucket": "b0b[]",
755
- "data": undefined,
756
- "has_more": true,
757
- "next_after": "1999",
758
- },
759
- },
760
- {
761
- "data": {
762
- "after": "1999",
763
- "bucket": "b0b[]",
764
- "data": undefined,
765
- "has_more": true,
766
- "next_after": "3999",
767
- },
768
- },
769
- {
770
- "data": {
771
- "after": "3999",
772
- "bucket": "b0b[]",
773
- "data": undefined,
774
- "has_more": false,
775
- "next_after": "4004",
776
- },
777
- },
778
- {
779
- "checkpoint_complete": {
780
- "last_op_id": "4004",
781
- },
782
- },
783
- ]
784
- `;
785
-
786
592
  exports[`sync - postgres > storage v1 > sync legacy non-raw data 1`] = `
787
593
  [
788
594
  {
@@ -1778,200 +1584,6 @@ exports[`sync - postgres > storage v2 > sync global data 1`] = `
1778
1584
  ]
1779
1585
  `;
1780
1586
 
1781
- exports[`sync - postgres > storage v2 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = `
1782
- [
1783
- {
1784
- "checkpoint": {
1785
- "buckets": [
1786
- {
1787
- "bucket": "1#b0a[]",
1788
- "checksum": -659831575,
1789
- "count": 2000,
1790
- "priority": 2,
1791
- "subscriptions": [
1792
- {
1793
- "default": 0,
1794
- },
1795
- ],
1796
- },
1797
- {
1798
- "bucket": "1#b0b[]",
1799
- "checksum": -659831575,
1800
- "count": 2000,
1801
- "priority": 2,
1802
- "subscriptions": [
1803
- {
1804
- "default": 1,
1805
- },
1806
- ],
1807
- },
1808
- {
1809
- "bucket": "1#b1[]",
1810
- "checksum": -1096116670,
1811
- "count": 1,
1812
- "priority": 1,
1813
- "subscriptions": [
1814
- {
1815
- "default": 2,
1816
- },
1817
- ],
1818
- },
1819
- ],
1820
- "last_op_id": "4001",
1821
- "streams": [
1822
- {
1823
- "errors": [],
1824
- "is_default": true,
1825
- "name": "b0a",
1826
- },
1827
- {
1828
- "errors": [],
1829
- "is_default": true,
1830
- "name": "b0b",
1831
- },
1832
- {
1833
- "errors": [],
1834
- "is_default": true,
1835
- "name": "b1",
1836
- },
1837
- ],
1838
- "write_checkpoint": undefined,
1839
- },
1840
- },
1841
- {
1842
- "data": {
1843
- "after": "0",
1844
- "bucket": "1#b1[]",
1845
- "data": undefined,
1846
- "has_more": false,
1847
- "next_after": "1",
1848
- },
1849
- },
1850
- {
1851
- "partial_checkpoint_complete": {
1852
- "last_op_id": "4001",
1853
- "priority": 1,
1854
- },
1855
- },
1856
- {
1857
- "data": {
1858
- "after": "0",
1859
- "bucket": "1#b0a[]",
1860
- "data": undefined,
1861
- "has_more": true,
1862
- "next_after": "2000",
1863
- },
1864
- },
1865
- {
1866
- "data": {
1867
- "after": "2000",
1868
- "bucket": "1#b0a[]",
1869
- "data": undefined,
1870
- "has_more": true,
1871
- "next_after": "4000",
1872
- },
1873
- },
1874
- {
1875
- "checkpoint_diff": {
1876
- "last_op_id": "4004",
1877
- "removed_buckets": [],
1878
- "updated_buckets": [
1879
- {
1880
- "bucket": "1#b0a[]",
1881
- "checksum": 883076828,
1882
- "count": 2001,
1883
- "priority": 2,
1884
- "subscriptions": [
1885
- {
1886
- "default": 0,
1887
- },
1888
- ],
1889
- },
1890
- {
1891
- "bucket": "1#b0b[]",
1892
- "checksum": 883076828,
1893
- "count": 2001,
1894
- "priority": 2,
1895
- "subscriptions": [
1896
- {
1897
- "default": 1,
1898
- },
1899
- ],
1900
- },
1901
- {
1902
- "bucket": "1#b1[]",
1903
- "checksum": 1841937527,
1904
- "count": 2,
1905
- "priority": 1,
1906
- "subscriptions": [
1907
- {
1908
- "default": 2,
1909
- },
1910
- ],
1911
- },
1912
- ],
1913
- "write_checkpoint": undefined,
1914
- },
1915
- },
1916
- {
1917
- "data": {
1918
- "after": "1",
1919
- "bucket": "1#b1[]",
1920
- "data": undefined,
1921
- "has_more": false,
1922
- "next_after": "4002",
1923
- },
1924
- },
1925
- {
1926
- "partial_checkpoint_complete": {
1927
- "last_op_id": "4004",
1928
- "priority": 1,
1929
- },
1930
- },
1931
- {
1932
- "data": {
1933
- "after": "4000",
1934
- "bucket": "1#b0a[]",
1935
- "data": undefined,
1936
- "has_more": false,
1937
- "next_after": "4003",
1938
- },
1939
- },
1940
- {
1941
- "data": {
1942
- "after": "0",
1943
- "bucket": "1#b0b[]",
1944
- "data": undefined,
1945
- "has_more": true,
1946
- "next_after": "1999",
1947
- },
1948
- },
1949
- {
1950
- "data": {
1951
- "after": "1999",
1952
- "bucket": "1#b0b[]",
1953
- "data": undefined,
1954
- "has_more": true,
1955
- "next_after": "3999",
1956
- },
1957
- },
1958
- {
1959
- "data": {
1960
- "after": "3999",
1961
- "bucket": "1#b0b[]",
1962
- "data": undefined,
1963
- "has_more": false,
1964
- "next_after": "4004",
1965
- },
1966
- },
1967
- {
1968
- "checkpoint_complete": {
1969
- "last_op_id": "4004",
1970
- },
1971
- },
1972
- ]
1973
- `;
1974
-
1975
1587
  exports[`sync - postgres > storage v2 > sync legacy non-raw data 1`] = `
1976
1588
  [
1977
1589
  {
@@ -2967,200 +2579,6 @@ exports[`sync - postgres > storage v3 > sync global data 1`] = `
2967
2579
  ]
2968
2580
  `;
2969
2581
 
2970
- exports[`sync - postgres > storage v3 > sync interrupts low-priority buckets on new checkpoints (2) 1`] = `
2971
- [
2972
- {
2973
- "checkpoint": {
2974
- "buckets": [
2975
- {
2976
- "bucket": "1#b0a[]",
2977
- "checksum": -659831575,
2978
- "count": 2000,
2979
- "priority": 2,
2980
- "subscriptions": [
2981
- {
2982
- "default": 0,
2983
- },
2984
- ],
2985
- },
2986
- {
2987
- "bucket": "1#b0b[]",
2988
- "checksum": -659831575,
2989
- "count": 2000,
2990
- "priority": 2,
2991
- "subscriptions": [
2992
- {
2993
- "default": 1,
2994
- },
2995
- ],
2996
- },
2997
- {
2998
- "bucket": "1#b1[]",
2999
- "checksum": -1096116670,
3000
- "count": 1,
3001
- "priority": 1,
3002
- "subscriptions": [
3003
- {
3004
- "default": 2,
3005
- },
3006
- ],
3007
- },
3008
- ],
3009
- "last_op_id": "4001",
3010
- "streams": [
3011
- {
3012
- "errors": [],
3013
- "is_default": true,
3014
- "name": "b0a",
3015
- },
3016
- {
3017
- "errors": [],
3018
- "is_default": true,
3019
- "name": "b0b",
3020
- },
3021
- {
3022
- "errors": [],
3023
- "is_default": true,
3024
- "name": "b1",
3025
- },
3026
- ],
3027
- "write_checkpoint": undefined,
3028
- },
3029
- },
3030
- {
3031
- "data": {
3032
- "after": "0",
3033
- "bucket": "1#b1[]",
3034
- "data": undefined,
3035
- "has_more": false,
3036
- "next_after": "1",
3037
- },
3038
- },
3039
- {
3040
- "partial_checkpoint_complete": {
3041
- "last_op_id": "4001",
3042
- "priority": 1,
3043
- },
3044
- },
3045
- {
3046
- "data": {
3047
- "after": "0",
3048
- "bucket": "1#b0a[]",
3049
- "data": undefined,
3050
- "has_more": true,
3051
- "next_after": "2000",
3052
- },
3053
- },
3054
- {
3055
- "data": {
3056
- "after": "2000",
3057
- "bucket": "1#b0a[]",
3058
- "data": undefined,
3059
- "has_more": true,
3060
- "next_after": "4000",
3061
- },
3062
- },
3063
- {
3064
- "checkpoint_diff": {
3065
- "last_op_id": "4004",
3066
- "removed_buckets": [],
3067
- "updated_buckets": [
3068
- {
3069
- "bucket": "1#b0a[]",
3070
- "checksum": 883076828,
3071
- "count": 2001,
3072
- "priority": 2,
3073
- "subscriptions": [
3074
- {
3075
- "default": 0,
3076
- },
3077
- ],
3078
- },
3079
- {
3080
- "bucket": "1#b0b[]",
3081
- "checksum": 883076828,
3082
- "count": 2001,
3083
- "priority": 2,
3084
- "subscriptions": [
3085
- {
3086
- "default": 1,
3087
- },
3088
- ],
3089
- },
3090
- {
3091
- "bucket": "1#b1[]",
3092
- "checksum": 1841937527,
3093
- "count": 2,
3094
- "priority": 1,
3095
- "subscriptions": [
3096
- {
3097
- "default": 2,
3098
- },
3099
- ],
3100
- },
3101
- ],
3102
- "write_checkpoint": undefined,
3103
- },
3104
- },
3105
- {
3106
- "data": {
3107
- "after": "1",
3108
- "bucket": "1#b1[]",
3109
- "data": undefined,
3110
- "has_more": false,
3111
- "next_after": "4002",
3112
- },
3113
- },
3114
- {
3115
- "partial_checkpoint_complete": {
3116
- "last_op_id": "4004",
3117
- "priority": 1,
3118
- },
3119
- },
3120
- {
3121
- "data": {
3122
- "after": "4000",
3123
- "bucket": "1#b0a[]",
3124
- "data": undefined,
3125
- "has_more": false,
3126
- "next_after": "4003",
3127
- },
3128
- },
3129
- {
3130
- "data": {
3131
- "after": "0",
3132
- "bucket": "1#b0b[]",
3133
- "data": undefined,
3134
- "has_more": true,
3135
- "next_after": "1999",
3136
- },
3137
- },
3138
- {
3139
- "data": {
3140
- "after": "1999",
3141
- "bucket": "1#b0b[]",
3142
- "data": undefined,
3143
- "has_more": true,
3144
- "next_after": "3999",
3145
- },
3146
- },
3147
- {
3148
- "data": {
3149
- "after": "3999",
3150
- "bucket": "1#b0b[]",
3151
- "data": undefined,
3152
- "has_more": false,
3153
- "next_after": "4004",
3154
- },
3155
- },
3156
- {
3157
- "checkpoint_complete": {
3158
- "last_op_id": "4004",
3159
- },
3160
- },
3161
- ]
3162
- `;
3163
-
3164
2582
  exports[`sync - postgres > storage v3 > sync legacy non-raw data 1`] = `
3165
2583
  [
3166
2584
  {