@powersync/service-module-mongodb-storage 0.16.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/CHANGELOG.md +53 -0
  2. package/dist/storage/MongoBucketStorage.d.ts +35 -15
  3. package/dist/storage/MongoBucketStorage.js +309 -86
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/CheckpointState.d.ts +20 -0
  6. package/dist/storage/implementation/CheckpointState.js +31 -0
  7. package/dist/storage/implementation/CheckpointState.js.map +1 -0
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +69 -39
  9. package/dist/storage/implementation/MongoBucketBatch.js +104 -313
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +11 -9
  12. package/dist/storage/implementation/MongoChecksums.js +3 -111
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +3 -1
  15. package/dist/storage/implementation/MongoCompactor.js +18 -9
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +6 -5
  21. package/dist/storage/implementation/MongoParameterCompactor.js +24 -1
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +121 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +5 -2
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +50 -21
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +62 -199
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +5 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +9 -5
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +14 -0
  41. package/dist/storage/implementation/SyncRuleStateUpdate.js +36 -0
  42. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -0
  43. package/dist/storage/implementation/common/BucketDataDoc.d.ts +1 -1
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +6 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  47. package/dist/storage/implementation/common/PersistedBatch.d.ts +2 -3
  48. package/dist/storage/implementation/common/PersistedBatch.js +5 -0
  49. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  50. package/dist/storage/implementation/common/SourceRecordStore.d.ts +1 -2
  51. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +1 -1
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  53. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  54. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  55. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  56. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  57. package/dist/storage/implementation/db.d.ts +10 -2
  58. package/dist/storage/implementation/db.js +9 -14
  59. package/dist/storage/implementation/db.js.map +1 -1
  60. package/dist/storage/implementation/models.d.ts +25 -50
  61. package/dist/storage/implementation/models.js.map +1 -1
  62. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +23 -1
  63. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +403 -2
  64. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  65. package/dist/storage/implementation/v1/MongoChecksumsV1.js +114 -3
  66. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +1 -1
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +22 -15
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +100 -27
  70. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  71. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +1 -2
  72. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +21 -1
  77. package/dist/storage/implementation/v1/models.js.map +1 -1
  78. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +21 -0
  79. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +459 -3
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  81. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +27 -7
  82. package/dist/storage/implementation/v3/MongoChecksumsV3.js +202 -18
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  84. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +20 -3
  85. package/dist/storage/implementation/v3/MongoCompactorV3.js +511 -22
  86. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  87. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +62 -0
  88. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +350 -0
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  90. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +50 -16
  91. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +292 -75
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  93. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +2 -3
  94. package/dist/storage/implementation/v3/PersistedBatchV3.js +43 -20
  95. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  96. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +52 -16
  97. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +16 -14
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +4 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +58 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +28 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +111 -10
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  114. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  115. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  116. package/dist/storage/storage-index.d.ts +4 -4
  117. package/dist/storage/storage-index.js +4 -4
  118. package/dist/storage/storage-index.js.map +1 -1
  119. package/dist/types/types.d.ts +2 -2
  120. package/dist/utils/test-utils.d.ts +1 -2
  121. package/dist/utils/test-utils.js +5 -1
  122. package/dist/utils/test-utils.js.map +1 -1
  123. package/dist/utils/util.d.ts +1 -1
  124. package/dist/utils/util.js +3 -2
  125. package/dist/utils/util.js.map +1 -1
  126. package/package.json +9 -19
  127. package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +1 -1
  128. package/src/storage/MongoBucketStorage.ts +464 -95
  129. package/src/storage/implementation/CheckpointState.ts +59 -0
  130. package/src/storage/implementation/MongoBucketBatch.ts +159 -405
  131. package/src/storage/implementation/MongoChecksums.ts +7 -138
  132. package/src/storage/implementation/MongoCompactor.ts +20 -10
  133. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  134. package/src/storage/implementation/MongoParameterCompactor.ts +24 -5
  135. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  136. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  137. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +152 -0
  138. package/src/storage/implementation/MongoStorageProvider.ts +5 -2
  139. package/src/storage/implementation/MongoSyncBucketStorage.ts +89 -250
  140. package/src/storage/implementation/MongoSyncRulesLock.ts +14 -5
  141. package/src/storage/implementation/SyncRuleStateUpdate.ts +38 -0
  142. package/src/storage/implementation/common/BucketDataDoc.ts +1 -1
  143. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +7 -0
  144. package/src/storage/implementation/common/PersistedBatch.ts +8 -4
  145. package/src/storage/implementation/common/SourceRecordStore.ts +1 -2
  146. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  147. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  148. package/src/storage/implementation/db.ts +16 -21
  149. package/src/storage/implementation/models.ts +28 -61
  150. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +503 -5
  151. package/src/storage/implementation/v1/MongoChecksumsV1.ts +137 -4
  152. package/src/storage/implementation/v1/MongoCompactorV1.ts +1 -1
  153. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +150 -49
  154. package/src/storage/implementation/v1/PersistedBatchV1.ts +1 -2
  155. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  156. package/src/storage/implementation/v1/models.ts +25 -0
  157. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +611 -4
  158. package/src/storage/implementation/v3/MongoChecksumsV3.ts +262 -26
  159. package/src/storage/implementation/v3/MongoCompactorV3.ts +647 -30
  160. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +480 -0
  161. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +397 -111
  162. package/src/storage/implementation/v3/PersistedBatchV3.ts +63 -38
  163. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +59 -19
  164. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  165. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +48 -38
  166. package/src/storage/implementation/v3/bucket-format.ts +70 -0
  167. package/src/storage/implementation/v3/chunking.ts +38 -0
  168. package/src/storage/implementation/v3/models.ts +150 -44
  169. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  170. package/src/storage/storage-index.ts +12 -4
  171. package/src/utils/test-utils.ts +6 -3
  172. package/src/utils/util.ts +3 -2
  173. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  174. package/test/src/__snapshots__/storage_sync.test.ts.snap +48 -48
  175. package/test/src/cleanup-stopped-sync-configs.test.ts +648 -0
  176. package/test/src/storage.test.ts +11 -11
  177. package/test/src/storage_compacting.test.ts +1956 -6
  178. package/test/src/storage_sync.test.ts +1429 -42
  179. package/test/src/storeCurrentData.test.ts +221 -0
  180. package/test/src/util.ts +2 -1
  181. package/tsconfig.json +1 -4
  182. package/tsconfig.tsbuildinfo +1 -1
  183. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -17
  184. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  185. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  186. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  187. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -64
  188. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  189. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -15
  190. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -42
  191. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  192. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  193. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  194. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  195. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  196. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  197. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  198. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -5
  199. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  200. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  201. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -72
  202. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -76
  203. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -56
  204. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  205. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  206. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -12
@@ -0,0 +1,59 @@
1
+ export interface CheckpointStateInput {
2
+ lsn: string;
3
+ snapshotDone: boolean;
4
+ lastCheckpointLsn: string | null;
5
+ noCheckpointBefore: string | null;
6
+ keepaliveOp: bigint | null;
7
+ lastCheckpoint: bigint | null;
8
+ persistedOp: bigint | null;
9
+ createEmptyCheckpoints: boolean;
10
+ }
11
+
12
+ export interface CheckpointStateResult {
13
+ canCheckpoint: boolean;
14
+ checkpointBlocked: boolean;
15
+ checkpointCreated: boolean;
16
+ notEmpty: boolean;
17
+ newKeepaliveOp: bigint | null;
18
+ newLastCheckpoint: bigint | null;
19
+ }
20
+
21
+ function maxOpId(...values: (bigint | null | undefined)[]): bigint {
22
+ let max = 0n;
23
+ for (const value of values) {
24
+ if (value != null && value > max) {
25
+ max = value;
26
+ }
27
+ }
28
+ return max;
29
+ }
30
+
31
+ export function canCheckpointState(
32
+ lsn: string,
33
+ state: Pick<CheckpointStateInput, 'snapshotDone' | 'lastCheckpointLsn' | 'noCheckpointBefore'>
34
+ ): boolean {
35
+ return (
36
+ state.snapshotDone === true &&
37
+ (state.lastCheckpointLsn == null || state.lastCheckpointLsn <= lsn) &&
38
+ (state.noCheckpointBefore == null || state.noCheckpointBefore <= lsn)
39
+ );
40
+ }
41
+
42
+ export function calculateCheckpointState(input: CheckpointStateInput): CheckpointStateResult {
43
+ const canCheckpoint = canCheckpointState(input.lsn, input);
44
+ const newKeepaliveOp = canCheckpoint ? null : maxOpId(input.keepaliveOp, input.persistedOp);
45
+ const newLastCheckpoint = canCheckpoint
46
+ ? maxOpId(input.lastCheckpoint, input.persistedOp, input.keepaliveOp)
47
+ : input.lastCheckpoint;
48
+ const notEmpty =
49
+ input.createEmptyCheckpoints || input.keepaliveOp !== newKeepaliveOp || input.lastCheckpoint !== newLastCheckpoint;
50
+
51
+ return {
52
+ canCheckpoint,
53
+ checkpointBlocked: !canCheckpoint,
54
+ checkpointCreated: canCheckpoint && notEmpty,
55
+ notEmpty,
56
+ newKeepaliveOp,
57
+ newLastCheckpoint
58
+ };
59
+ }