@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (275) hide show
  1. package/CHANGELOG.md +177 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +71 -18
  6. package/dist/storage/MongoBucketStorage.js +516 -202
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
  9. package/dist/storage/implementation/MongoBucketBatch.js +140 -66
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
  15. package/dist/storage/implementation/MongoCompactor.js +87 -481
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
  21. package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +41 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  44. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  45. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
  47. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  48. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  49. package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
  50. package/dist/storage/implementation/common/PersistedBatch.js +63 -10
  51. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  53. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  54. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  55. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  56. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  57. package/dist/storage/implementation/db.d.ts +4 -1
  58. package/dist/storage/implementation/db.js +31 -16
  59. package/dist/storage/implementation/db.js.map +1 -1
  60. package/dist/storage/implementation/models.d.ts +55 -13
  61. package/dist/storage/implementation/models.js.map +1 -1
  62. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
  63. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
  64. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  65. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  66. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  67. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  68. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
  69. package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
  70. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  71. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  72. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  73. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  74. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
  75. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
  76. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  77. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  78. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  79. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  80. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  81. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  82. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  83. package/dist/storage/implementation/v1/models.d.ts +10 -1
  84. package/dist/storage/implementation/v1/models.js +1 -0
  85. package/dist/storage/implementation/v1/models.js.map +1 -1
  86. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  87. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  88. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  89. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  90. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  91. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
  93. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
  94. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  96. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  97. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
  99. package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
  100. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
  102. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
  103. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
  105. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
  106. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  107. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
  108. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
  109. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  110. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  111. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  112. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  113. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
  114. package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
  115. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  116. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  117. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  118. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  119. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
  120. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
  121. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  122. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  123. package/dist/storage/implementation/v3/bucket-format.js +79 -0
  124. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  125. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  126. package/dist/storage/implementation/v3/chunking.js +30 -0
  127. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  128. package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
  129. package/dist/storage/implementation/v3/compact-utils.js +216 -0
  130. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  131. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  132. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  133. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  134. package/dist/storage/implementation/v3/models.d.ts +187 -23
  135. package/dist/storage/implementation/v3/models.js +11 -28
  136. package/dist/storage/implementation/v3/models.js.map +1 -1
  137. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
  138. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
  139. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  140. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
  141. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  142. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  143. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
  144. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
  145. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  146. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
  147. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
  148. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  149. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
  150. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
  151. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  152. package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
  153. package/dist/storage/implementation/v3/source-table-utils.js +206 -0
  154. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  155. package/dist/storage/storage-index.d.ts +5 -4
  156. package/dist/storage/storage-index.js +5 -4
  157. package/dist/storage/storage-index.js.map +1 -1
  158. package/dist/types/types.d.ts +71 -3
  159. package/dist/types/types.js +58 -2
  160. package/dist/types/types.js.map +1 -1
  161. package/dist/utils/test-utils.d.ts +1 -2
  162. package/dist/utils/test-utils.js +7 -1
  163. package/dist/utils/test-utils.js.map +1 -1
  164. package/dist/utils/util.d.ts +9 -2
  165. package/dist/utils/util.js +131 -5
  166. package/dist/utils/util.js.map +1 -1
  167. package/package.json +13 -19
  168. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  169. package/src/storage/MongoBucketStorage.ts +728 -242
  170. package/src/storage/implementation/MongoBucketBatch.ts +196 -91
  171. package/src/storage/implementation/MongoChecksums.ts +93 -153
  172. package/src/storage/implementation/MongoCompactor.ts +106 -631
  173. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  174. package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
  175. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
  176. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  177. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
  178. package/src/storage/implementation/MongoStorageProvider.ts +48 -4
  179. package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
  180. package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
  181. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
  182. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  183. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
  184. package/src/storage/implementation/common/PersistedBatch.ts +83 -15
  185. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  186. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  187. package/src/storage/implementation/db.ts +40 -22
  188. package/src/storage/implementation/models.ts +58 -14
  189. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
  190. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  191. package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
  192. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  193. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
  194. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  195. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  196. package/src/storage/implementation/v1/models.ts +12 -1
  197. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  198. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  199. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
  200. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  201. package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
  202. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
  203. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
  204. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
  205. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  206. package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
  207. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  208. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
  209. package/src/storage/implementation/v3/bucket-format.ts +99 -0
  210. package/src/storage/implementation/v3/chunking.ts +40 -0
  211. package/src/storage/implementation/v3/compact-utils.ts +320 -0
  212. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  213. package/src/storage/implementation/v3/models.ts +228 -58
  214. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
  215. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
  216. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
  217. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
  218. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
  219. package/src/storage/implementation/v3/source-table-utils.ts +398 -0
  220. package/src/storage/storage-index.ts +14 -4
  221. package/src/types/types.ts +76 -2
  222. package/src/utils/test-utils.ts +8 -3
  223. package/src/utils/util.ts +188 -7
  224. package/test/src/__snapshots__/storage.test.ts.snap +4 -54
  225. package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
  226. package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
  227. package/test/src/compact-utils.test.ts +226 -0
  228. package/test/src/helpers/MemoryObjectStorage.ts +61 -0
  229. package/test/src/helpers/s3TestFactory.ts +69 -0
  230. package/test/src/object_storage_lifecycle.test.ts +59 -0
  231. package/test/src/object_storage_usage.test.ts +255 -0
  232. package/test/src/parameter_compacting_v1.test.ts +120 -0
  233. package/test/src/parameter_compacting_v3.test.ts +192 -0
  234. package/test/src/parameter_compaction_fence.test.ts +276 -0
  235. package/test/src/setup.ts +6 -1
  236. package/test/src/storage.test.ts +478 -13
  237. package/test/src/storage_compacting.test.ts +2516 -107
  238. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  239. package/test/src/storage_s3_checksums.test.ts +190 -0
  240. package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
  241. package/test/src/storage_s3_reading.test.ts +768 -0
  242. package/test/src/storage_s3_writing.test.ts +136 -0
  243. package/test/src/storage_sync.test.ts +1507 -76
  244. package/test/src/storeCurrentData.test.ts +227 -0
  245. package/test/src/util.ts +3 -2
  246. package/tsconfig.json +1 -4
  247. package/tsconfig.tsbuildinfo +1 -1
  248. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  249. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  250. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  251. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  252. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  253. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  254. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  255. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  256. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  257. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  258. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  259. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  260. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  261. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  262. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  263. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  264. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  265. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  266. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  267. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  268. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  269. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  270. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  271. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  272. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  273. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  274. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  275. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -1,58 +1,50 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
2
  import { InternalOpId } from '@powersync/service-core';
3
3
  import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
4
- import {
5
- BucketDataDocumentGeneric,
6
- BucketDataDocumentGenericId,
7
- SingleBucketStore
8
- } from '../common/SingleBucketStore.js';
9
4
  import { BucketDataProperties } from '../models.js';
10
5
  import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
11
6
  import { BucketDataDocumentV1, BucketDataKeyV1, serializeBucketDataV1 } from './models.js';
12
7
 
13
- export class SingleBucketStoreV1 implements SingleBucketStore {
14
- public readonly collection: mongo.Collection<BucketDataDocumentGeneric>;
8
+ export class SingleBucketStoreV1 {
9
+ public readonly collection: mongo.Collection<BucketDataDocumentV1>;
15
10
 
16
11
  constructor(
17
- private db: VersionedPowerSyncMongoV1,
12
+ db: VersionedPowerSyncMongoV1,
18
13
  public readonly key: BucketKey
19
14
  ) {
20
- this.collection = db.bucketDataV1 as unknown as mongo.Collection<BucketDataDocumentGeneric>;
15
+ this.collection = db.bucketDataV1;
21
16
  }
22
17
 
23
- docId(o: InternalOpId): BucketDataDocumentGenericId {
24
- // `satisfies BucketDataKeyV1` checks that we use the correct type for V1 storage
25
- // `as anyt` is to allow casting to the interface virtual type
18
+ docId(o: InternalOpId): BucketDataKeyV1 {
26
19
  return {
27
20
  g: this.key.replicationStreamId,
28
21
  b: this.key.bucket,
29
22
  o
30
- } satisfies BucketDataKeyV1 as any;
23
+ };
31
24
  }
32
25
 
33
- get minId(): BucketDataDocumentGenericId {
26
+ get minId(): BucketDataKeyV1 {
34
27
  return {
35
28
  g: this.key.replicationStreamId,
36
29
  b: this.key.bucket,
37
- o: new mongo.MinKey()
38
- } as any;
30
+ o: new mongo.MinKey() as any
31
+ };
39
32
  }
40
33
 
41
- get maxId(): BucketDataDocumentGenericId {
34
+ get maxId(): BucketDataKeyV1 {
42
35
  return {
43
36
  g: this.key.replicationStreamId,
44
37
  b: this.key.bucket,
45
- o: new mongo.MaxKey()
46
- } as any;
38
+ o: new mongo.MaxKey() as any
39
+ };
47
40
  }
48
41
 
49
- toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentGeneric {
50
- return serializeBucketDataV1({ bucketKey: this.key, ...source }) as unknown as BucketDataDocumentGeneric;
42
+ toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentV1 {
43
+ return serializeBucketDataV1({ bucketKey: this.key, ...source });
51
44
  }
52
45
 
53
- fromPersistedDocument(doc: BucketDataDocumentGeneric): BucketDataDoc {
54
- const document = doc as unknown as BucketDataDocumentV1;
55
- const { _id, ...rest } = document;
46
+ fromPersistedDocument(doc: BucketDataDocumentV1): BucketDataDoc {
47
+ const { _id, ...rest } = doc;
56
48
  return {
57
49
  bucketKey: this.key,
58
50
  o: _id.o,
@@ -61,10 +53,9 @@ export class SingleBucketStoreV1 implements SingleBucketStore {
61
53
  }
62
54
 
63
55
  fromPartialPersistedDocument<T extends keyof BucketDataProperties>(
64
- doc: Pick<BucketDataDocumentGeneric, '_id' | T>
56
+ doc: Pick<BucketDataDocumentV1, '_id' | T>
65
57
  ): Pick<BucketDataDoc, 'bucketKey' | 'o' | T> {
66
- const document = doc as Pick<BucketDataDocumentV1, '_id' | T>;
67
- const { _id, ...rest } = document;
58
+ const { _id, ...rest } = doc;
68
59
  return {
69
60
  bucketKey: this.key,
70
61
  o: _id.o,
@@ -1,7 +1,13 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
2
  import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
3
3
  import { CommonSourceTableDocument } from '../models.js';
4
- import { BucketDataDocumentV1, BucketParameterDocument, BucketStateDocumentV1, CurrentDataDocument } from './models.js';
4
+ import {
5
+ BucketDataDocumentV1,
6
+ BucketParameterDocument,
7
+ BucketStateDocumentV1,
8
+ CurrentDataDocument,
9
+ SourceTableDocumentV1
10
+ } from './models.js';
5
11
 
6
12
  export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
7
13
  get sourceRecordsV1(): mongo.Collection<CurrentDataDocument> {
@@ -16,6 +22,10 @@ export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
16
22
  return this.upstream.source_tables as any as mongo.Collection<CommonSourceTableDocument>;
17
23
  }
18
24
 
25
+ sourceTablesV1(_replicationStreamId: number): mongo.Collection<SourceTableDocumentV1> {
26
+ return this.upstream.source_tables as any as mongo.Collection<SourceTableDocumentV1>;
27
+ }
28
+
19
29
  async initializeStreamStorage(_replicationStreamId: number): Promise<void> {}
20
30
 
21
31
  get bucketDataV1(): mongo.Collection<BucketDataDocumentV1> {
@@ -48,6 +48,7 @@ export function serializeBucketDataV1(document: BucketDataDoc): BucketDataDocume
48
48
  op: document.op,
49
49
  source_table: document.source_table,
50
50
  source_key: document.source_key,
51
+ subkey: document.subkey,
51
52
  table: document.table,
52
53
  row_id: document.row_id,
53
54
  checksum: document.checksum,
@@ -78,7 +79,7 @@ export interface SourceTableDocumentV1 extends SourceTableDocument {
78
79
  group_id: number;
79
80
  }
80
81
 
81
- export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckpointFields<string | null> {
82
+ export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckpointFields {
82
83
  content: string;
83
84
  serialized_plan?: SerializedSyncPlan | null;
84
85
 
@@ -88,6 +89,16 @@ export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckp
88
89
  * Can only be false if state == PROCESSING.
89
90
  */
90
91
  snapshot_done: boolean;
92
+
93
+ /**
94
+ * Goes together with no_checkpoint_before.
95
+ *
96
+ * If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
97
+ * then the checkpoint must be equal to this keepalive_op.
98
+ *
99
+ * string for legacy reasons - it's always a number.
100
+ */
101
+ keepalive_op: string | null;
91
102
  }
92
103
 
93
104
  export interface BucketStateDocumentV1 extends BucketStateDocumentBase {
@@ -0,0 +1,32 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+ import { InternalOpId } from '@powersync/service-core';
3
+ import { BucketKey } from '../common/BucketDataDoc.js';
4
+ import { BucketDataKey } from '../models.js';
5
+ import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
6
+ import { BucketDataDocumentV3 } from './models.js';
7
+
8
+ export class BucketDataContextV3 {
9
+ public readonly collection: mongo.Collection<BucketDataDocumentV3>;
10
+
11
+ constructor(
12
+ db: VersionedPowerSyncMongoV3,
13
+ public readonly key: BucketKey
14
+ ) {
15
+ this.collection = db.bucketData(key.replicationStreamId, key.definitionId);
16
+ }
17
+
18
+ docId(o: InternalOpId): BucketDataKey {
19
+ return {
20
+ b: this.key.bucket,
21
+ o
22
+ };
23
+ }
24
+
25
+ get minId(): BucketDataKey {
26
+ // MongoDB's MinKey sentinel does not match the bigint type for _id.o.
27
+ return {
28
+ b: this.key.bucket,
29
+ o: new mongo.MinKey()
30
+ } as unknown as BucketDataKey;
31
+ }
32
+ }
@@ -0,0 +1,152 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+ import { InternalOpId } from '@powersync/service-core';
3
+ import { CompactionLeaseLostError } from '../MongoCompactor.js';
4
+ import { BucketStateDocumentV3 } from './models.js';
5
+
6
+ const LEASE_RENEW_INTERVAL_MS = 60 * 1000;
7
+
8
+ /**
9
+ * Filters for buckets that have no lease or an expired lease.
10
+ */
11
+ export const AVAILABLE_LEASE_EXPR = {
12
+ // $$NOW is evaluated by MongoDB, avoiding lease expiry races due
13
+ // to clocks on separate compact workers.
14
+ $expr: {
15
+ $or: [{ $eq: [{ $type: '$compact_lease' }, 'missing'] }, { $lte: ['$compact_lease.expires_at', '$$NOW'] }]
16
+ }
17
+ };
18
+
19
+ /**
20
+ * Owns one V3 bucket-compaction lease, including its server-time renewal and
21
+ * the owner-fenced operations which release it.
22
+ *
23
+ * This is intended as a way to reduce redundant work if multiple jobs attempt to
24
+ * compact the same bucket concurrently, but it's not an absolute safety guarantee.
25
+ * The individual operations must still be designed to be safe with concurrent compacting.
26
+ */
27
+ export class CompactionLease implements AsyncDisposable {
28
+ readonly startedAt: Date;
29
+ readonly lastOp: InternalOpId;
30
+
31
+ private timer: NodeJS.Timeout | undefined;
32
+ private renewalInFlight = false;
33
+ private renewalError: unknown;
34
+ private finalizing = false;
35
+ private finished = false;
36
+
37
+ private constructor(
38
+ private readonly collection: mongo.Collection<BucketStateDocumentV3>,
39
+ readonly state: BucketStateDocumentV3,
40
+ readonly id: mongo.ObjectId,
41
+ private readonly durationMs: number
42
+ ) {
43
+ this.startedAt = new Date(state.compact_lease!.expires_at.getTime() - durationMs);
44
+ this.lastOp = state.last_op;
45
+ }
46
+
47
+ static async claim(
48
+ collection: mongo.Collection<BucketStateDocumentV3>,
49
+ filter: mongo.Filter<BucketStateDocumentV3>,
50
+ sort: mongo.Sort | undefined,
51
+ durationMs: number
52
+ ): Promise<CompactionLease | null> {
53
+ const id = new mongo.ObjectId();
54
+ const state = await collection.findOneAndUpdate(
55
+ {
56
+ $and: [filter, AVAILABLE_LEASE_EXPR]
57
+ },
58
+ [
59
+ {
60
+ $set: {
61
+ compact_lease: {
62
+ id,
63
+ expires_at: { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: durationMs } }
64
+ }
65
+ }
66
+ }
67
+ ],
68
+ { sort, returnDocument: 'after' }
69
+ );
70
+ return state == null ? null : new CompactionLease(collection, state, id, durationMs);
71
+ }
72
+
73
+ startRenewal() {
74
+ const interval = Math.max(1, Math.min(LEASE_RENEW_INTERVAL_MS, Math.floor(this.durationMs / 2)));
75
+ this.timer = setInterval(() => {
76
+ if (this.renewalInFlight) {
77
+ return;
78
+ }
79
+ this.renewalInFlight = true;
80
+ void this.renew()
81
+ .catch((error) => {
82
+ this.renewalError = error;
83
+ })
84
+ .finally(() => {
85
+ this.renewalInFlight = false;
86
+ });
87
+ }, interval);
88
+ this.timer.unref();
89
+ }
90
+
91
+ async throwIfLost() {
92
+ if (this.renewalError != null) {
93
+ throw this.renewalError;
94
+ }
95
+ }
96
+
97
+ async reschedule(nextCompactCheck: mongo.Document) {
98
+ await this.finish([{ $set: { next_compact_check: nextCompactCheck } }, { $unset: 'compact_lease' }]);
99
+ }
100
+
101
+ async finalize(update: mongo.Document) {
102
+ await this.finish([{ $set: update }, { $unset: 'compact_lease' }]);
103
+ }
104
+
105
+ async [Symbol.asyncDispose]() {
106
+ if (this.timer != null) {
107
+ clearInterval(this.timer);
108
+ this.timer = undefined;
109
+ }
110
+ if (this.finished) {
111
+ return;
112
+ }
113
+ this.finalizing = true;
114
+ await this.collection.updateOne(this.filter, { $unset: { compact_lease: '' } });
115
+ this.finished = true;
116
+ }
117
+
118
+ private get filter(): mongo.Filter<BucketStateDocumentV3> {
119
+ return { _id: this.state._id, 'compact_lease.id': this.id };
120
+ }
121
+
122
+ private async renew() {
123
+ const result = await this.collection.updateOne(this.filter, [
124
+ {
125
+ $set: {
126
+ 'compact_lease.expires_at': {
127
+ $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.durationMs }
128
+ }
129
+ }
130
+ }
131
+ ]);
132
+ // A finalization in progress owns the authoritative fenced result.
133
+ if (result.matchedCount != 1 && !this.finalizing) {
134
+ throw new CompactionLeaseLostError(`Lost compaction lease for bucket ${this.state._id.b}`);
135
+ }
136
+ // A successful renewal confirms that a transient failure has passed. A
137
+ // lease-loss error remains sticky: it means another worker may own it.
138
+ if (!(this.renewalError instanceof CompactionLeaseLostError)) {
139
+ this.renewalError = undefined;
140
+ }
141
+ }
142
+
143
+ private async finish(update: mongo.Document[]) {
144
+ await this.throwIfLost();
145
+ this.finalizing = true;
146
+ const result = await this.collection.updateOne(this.filter, update);
147
+ if (result.matchedCount != 1) {
148
+ throw new CompactionLeaseLostError(`Lost compaction lease for bucket ${this.state._id.b}`);
149
+ }
150
+ this.finished = true;
151
+ }
152
+ }