@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/CHANGELOG.md +101 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +41 -16
  6. package/dist/storage/MongoBucketStorage.js +226 -71
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
  9. package/dist/storage/implementation/MongoBucketBatch.js +110 -62
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
  15. package/dist/storage/implementation/MongoCompactor.js +80 -263
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
  21. package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +39 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  46. package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
  47. package/dist/storage/implementation/common/PersistedBatch.js +12 -0
  48. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  49. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  51. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  52. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  53. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  54. package/dist/storage/implementation/db.js +17 -15
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +20 -13
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  62. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  63. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
  65. package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
  66. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  71. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  72. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +10 -1
  77. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  78. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  79. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  81. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
  82. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  84. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  85. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  86. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
  87. package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
  88. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
  90. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
  91. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
  93. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
  94. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
  96. package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +77 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +30 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +101 -21
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
  115. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  126. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  128. package/dist/storage/storage-index.d.ts +4 -4
  129. package/dist/storage/storage-index.js +4 -4
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +49 -3
  132. package/dist/types/types.js +48 -1
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +1 -2
  135. package/dist/utils/test-utils.js +7 -1
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.d.ts +9 -2
  138. package/dist/utils/util.js +128 -4
  139. package/dist/utils/util.js.map +1 -1
  140. package/package.json +12 -19
  141. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  142. package/src/storage/MongoBucketStorage.ts +347 -88
  143. package/src/storage/implementation/MongoBucketBatch.ts +147 -82
  144. package/src/storage/implementation/MongoChecksums.ts +93 -153
  145. package/src/storage/implementation/MongoCompactor.ts +102 -316
  146. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  147. package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
  148. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  149. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  150. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  151. package/src/storage/implementation/MongoStorageProvider.ts +46 -4
  152. package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
  153. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  154. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
  156. package/src/storage/implementation/common/PersistedBatch.ts +19 -3
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  158. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  159. package/src/storage/implementation/db.ts +19 -20
  160. package/src/storage/implementation/models.ts +20 -14
  161. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
  162. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  163. package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
  164. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
  165. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  166. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  167. package/src/storage/implementation/v1/models.ts +11 -1
  168. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  169. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
  170. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  171. package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
  172. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
  173. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
  174. package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
  175. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  176. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
  177. package/src/storage/implementation/v3/bucket-format.ts +97 -0
  178. package/src/storage/implementation/v3/chunking.ts +40 -0
  179. package/src/storage/implementation/v3/models.ts +132 -55
  180. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
  181. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
  184. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  185. package/src/storage/storage-index.ts +12 -4
  186. package/src/types/types.ts +58 -1
  187. package/src/utils/test-utils.ts +8 -3
  188. package/src/utils/util.ts +185 -6
  189. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
  191. package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
  192. package/test/src/helpers/MemoryObjectStorage.ts +53 -0
  193. package/test/src/helpers/s3TestFactory.ts +69 -0
  194. package/test/src/object_storage_lifecycle.test.ts +59 -0
  195. package/test/src/setup.ts +6 -1
  196. package/test/src/storage.test.ts +227 -13
  197. package/test/src/storage_compacting.test.ts +1951 -12
  198. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  199. package/test/src/storage_s3_checksums.test.ts +191 -0
  200. package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
  201. package/test/src/storage_s3_reading.test.ts +526 -0
  202. package/test/src/storage_s3_writing.test.ts +135 -0
  203. package/test/src/storage_sync.test.ts +1180 -73
  204. package/test/src/storeCurrentData.test.ts +227 -0
  205. package/test/src/util.ts +2 -1
  206. package/tsconfig.json +1 -4
  207. package/tsconfig.tsbuildinfo +1 -1
  208. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  209. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  210. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  211. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  212. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  213. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  214. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  215. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  216. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  217. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  218. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  219. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  220. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  221. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  222. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  223. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  224. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  225. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  226. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  227. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  228. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  229. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  230. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  231. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  232. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  233. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  234. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  235. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  236. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  237. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  238. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  239. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -1 +1 @@
1
- {"version":3,"file":"VersionedPowerSyncMongoV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2BAA2B,EAAE,MAAM,0CAA0C,CAAC;AAWvF,MAAM,OAAO,yBAA0B,SAAQ,2BAA2B;IACxE,IAAI,qBAAqB;QACvB,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAuB,aAAa,CAAC,CAAC;IACjE,CAAC;IAED,eAAe,CAAC,mBAA2B,EAAE,aAA6B;QACxE,MAAM,cAAc,GAAG,IAAI,CAAC,2BAA2B,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;QAC5F,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAwB,cAAc,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,6BAA6B,CAAC,mBAA2B;QAC7D,OAAO,IAAI,CAAC,uBAAuB,CAAwB,kBAAkB,mBAAmB,GAAG,CAAC,CAAC;IACvG,CAAC;IAED,KAAK,CAAC,iCAAiC,CAAC,mBAA2B,EAAE,aAA6B;QAChG,MAAM,IAAI,CAAC,eAAe,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC,WAAW,CACxE;YACE,cAAc,EAAE,CAAC;SAClB,EACD;YACE,uBAAuB,EAAE,EAAE,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;YAC9D,IAAI,EAAE,gBAAgB;SACvB,CACF,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,mBAA2B;QAC5C,OAAO,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAgD,CAAC;IACjG,CAAC;IAED,aAAa,CAAC,mBAA2B;QACvC,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,gBAAgB,mBAAmB,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,gBAAgB,CACd,mBAA2B,EAC3B,OAAyB;QAEzB,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,mBAAmB,IAAI,OAAO,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,cAAc,CAAC,mBAA2B;QACxC,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAwB,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC,CAAC;IACxG,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,mBAA2B;QACvD,MAAM,YAAY,GAAG,IAAI,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;QAC9D,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,mBAAmB,CAAC,CAAC;QAC5D,MAAM,YAAY,CAAC,WAAW,CAC5B;YACE,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,CAAC;SACf,EACD;YACE,IAAI,EAAE,eAAe;SACtB,CACF,CAAC;QACF,MAAM,YAAY,CAAC,WAAW,CAC5B;YACE,qBAAqB,EAAE,CAAC;SACzB,EACD;YACE,uBAAuB,EAAE,EAAE,qBAAqB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;YACrE,IAAI,EAAE,uBAAuB;SAC9B,CACF,CAAC;QACF,MAAM,WAAW,CAAC,WAAW,CAC3B;YACE,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,CACzC,CAAC;QACF,MAAM,WAAW,CAAC,WAAW,CAC3B;YACE,8BAA8B,EAAE,CAAC,CAAC;SACnC,EACD,EAAE,IAAI,EAAE,aAAa,EAAE,CACxB,CAAC;IACJ,CAAC;IAED,YAAY,CAAC,mBAA2B,EAAE,YAAgC;QACxE,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAuB,eAAe,mBAAmB,IAAI,YAAY,EAAE,CAAC,CAAC;IACxG,CAAC;IAED,2BAA2B,CAAC,mBAA2B;QACrD,OAAO,IAAI,CAAC,QAAQ,CAAC,2BAA2B,CAAC,mBAAmB,CAAC,CAAC;IACxE,CAAC;IAED,KAAK,CAAC,+BAA+B,CACnC,mBAA2B;QAE3B,MAAM,MAAM,GAAG,mBAAmB,mBAAmB,GAAG,CAAC;QACzD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAEpH,OAAO,WAAW;aACf,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACpB,UAAU,EAAE,IAAI,CAAC,EAAE,CAAC,UAAU,CAA4B,UAAU,CAAC,IAAI,CAAC;YAC1E,OAAO,EAAE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;SAC9C,CAAC,CAAC,CAAC;IACR,CAAC;CACF"}
1
+ {"version":3,"file":"VersionedPowerSyncMongoV3.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,2BAA2B,EAAE,MAAM,0CAA0C,CAAC;AAYvF,MAAM,OAAO,yBAA0B,SAAQ,2BAA2B;IACxE,YACE,QAAsE,EACtE,aAA2E;QAE3E,KAAK,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,qBAAqB;QACvB,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAuB,aAAa,CAAC,CAAC;IACjE,CAAC;IAED,aAAa,CAAC,mBAA2B,EAAE,aAA6B;QACtE,MAAM,cAAc,GAAG,IAAI,CAAC,2BAA2B,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;QAC5F,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAwB,cAAc,CAAC,CAAC;IACnE,CAAC;IAED,KAAK,CAAC,2BAA2B,CAAC,mBAA2B;QAC3D,OAAO,IAAI,CAAC,uBAAuB,CAAwB,kBAAkB,mBAAmB,GAAG,CAAC,CAAC;IACvG,CAAC;IAED,KAAK,CAAC,iCAAiC,CACrC,mBAA2B,EAC3B,aAA6B,EAC7B,OAA6B;QAE7B,MAAM,IAAI,CAAC,aAAa,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC,WAAW,CACtE;YACE,cAAc,EAAE,CAAC;SAClB,EACD;YACE,uBAAuB,EAAE,EAAE,cAAc,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;YAC9D,IAAI,EAAE,gBAAgB;YACtB,OAAO;SACR,CACF,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,mBAA2B;QAC5C,OAAO,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAuD,CAAC;IACtG,CAAC;IAED,WAAW,CAAC,mBAA2B;QACrC,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,gBAAgB,mBAAmB,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,cAAc,CAAC,mBAA2B,EAAE,OAAyB;QACnE,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,mBAAmB,IAAI,OAAO,EAAE,CAAC,CAAC;IACjF,CAAC;IAED,YAAY,CAAC,mBAA2B;QACtC,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAwB,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC,CAAC;IACxG,CAAC;IAED,UAAU,CAAC,mBAA2B,EAAE,YAAgC;QACtE,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAAuB,eAAe,mBAAmB,IAAI,YAAY,EAAE,CAAC,CAAC;IACxG,CAAC;IAED,2BAA2B,CAAC,mBAA2B;QACrD,OAAO,IAAI,CAAC,EAAE,CAAC,UAAU,CAA8B,kCAAkC,mBAAmB,EAAE,CAAC,CAAC;IAClH,CAAC;IAED,yBAAyB,CAAC,mBAA2B;QACnD,OAAO,IAAI,CAAC,uBAAuB,CAAC,eAAe,mBAAmB,GAAG,CAAC,CAAC;IAC7E,CAAC;IAED,KAAK,CAAC,6BAA6B,CAAC,mBAA2B;QAC7D,MAAM,MAAM,GAAG,mBAAmB,mBAAmB,GAAG,CAAC;QACzD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,uBAAuB,CAAC,MAAM,CAAC,CAAC;QAE/D,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACtC,UAAU;YACV,OAAO,EAAE,UAAU,CAAC,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;SACxD,CAAC,CAAC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,mBAA2B;QACvD,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,mBAAmB,CAAC,CAAC;QAC5D,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,mBAAmB,CAAC,CAAC;QAC1D,MAAM,YAAY,CAAC,WAAW,CAC5B;YACE,aAAa,EAAE,CAAC;YAChB,WAAW,EAAE,CAAC;YACd,UAAU,EAAE,CAAC;YACb,WAAW,EAAE,CAAC;SACf,EACD;YACE,IAAI,EAAE,eAAe;SACtB,CACF,CAAC;QACF,MAAM,YAAY,CAAC,WAAW,CAC5B;YACE,qBAAqB,EAAE,CAAC;SACzB,EACD;YACE,uBAAuB,EAAE,EAAE,qBAAqB,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;YACrE,IAAI,EAAE,uBAAuB;SAC9B,CACF,CAAC;QACF,MAAM,WAAW,CAAC,WAAW,CAC3B;YACE,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,CACzC,CAAC;QACF,MAAM,WAAW,CAAC,WAAW,CAC3B;YACE,8BAA8B,EAAE,CAAC,CAAC;SACnC,EACD,EAAE,IAAI,EAAE,aAAa,EAAE,CACxB,CAAC;IACJ,CAAC;CACF"}
@@ -0,0 +1,13 @@
1
+ import { InternalOpId } from '@powersync/service-core';
2
+ import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
3
+ import { BucketDataDocumentV3 } from './models.js';
4
+ /**
5
+ * Serialize a non-empty list of operations in strictly ascending `o` order.
6
+ * Preserving that order establishes the {@link BucketDataDocumentV3} range
7
+ * invariants used by metadata-only queries.
8
+ */
9
+ export declare function serializeBucketData(bucket: string, operations: BucketDataDoc[], options?: {
10
+ targetOp?: InternalOpId | null;
11
+ }): BucketDataDocumentV3;
12
+ export declare function loadBucketDataDocument(context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>, doc: BucketDataDocumentV3): Generator<BucketDataDoc>;
13
+ export declare function maxOpId(a: InternalOpId | null | undefined, b: InternalOpId | null | undefined): InternalOpId | null;
@@ -0,0 +1,77 @@
1
+ import { ServiceAssertionError } from '@powersync/lib-services-framework';
2
+ import { bson } from '@powersync/service-core';
3
+ /**
4
+ * Serialize a non-empty list of operations in strictly ascending `o` order.
5
+ * Preserving that order establishes the {@link BucketDataDocumentV3} range
6
+ * invariants used by metadata-only queries.
7
+ */
8
+ export function serializeBucketData(bucket, operations, options) {
9
+ const minOp = operations[0].o;
10
+ const maxOp = operations[operations.length - 1].o;
11
+ let totalChecksum = 0n;
12
+ let maxTargetOp = options?.targetOp ?? null;
13
+ let hasClearOp = false;
14
+ const ops = operations.map((op) => {
15
+ totalChecksum += op.checksum;
16
+ if (op.op == 'CLEAR') {
17
+ hasClearOp = true;
18
+ }
19
+ return {
20
+ o: op.o,
21
+ op: op.op,
22
+ source_table: op.source_table,
23
+ source_key: op.source_key,
24
+ table: op.table,
25
+ row_id: op.row_id,
26
+ checksum: op.checksum,
27
+ data: op.data
28
+ };
29
+ });
30
+ const size = bson.calculateObjectSize(ops);
31
+ return {
32
+ _id: {
33
+ b: bucket,
34
+ o: maxOp
35
+ },
36
+ min_op: minOp,
37
+ checksum: totalChecksum,
38
+ count: operations.length,
39
+ size,
40
+ target_op: maxTargetOp,
41
+ ...(hasClearOp ? { has_clear_op: true } : {}),
42
+ ops
43
+ };
44
+ }
45
+ export function* loadBucketDataDocument(context, doc) {
46
+ const { _id, ops } = doc;
47
+ if (!ops) {
48
+ throw new ServiceAssertionError(`Missing ops array on BucketDataDocumentV3 at _id.o=${_id.o}. Callers must patch doc.ops from S3 before calling this function.`);
49
+ }
50
+ const bucketKey = {
51
+ ...context,
52
+ bucket: _id.b
53
+ };
54
+ for (const op of ops) {
55
+ yield {
56
+ bucketKey,
57
+ o: op.o,
58
+ op: op.op,
59
+ source_table: op.source_table,
60
+ source_key: op.source_key,
61
+ table: op.table,
62
+ row_id: op.row_id,
63
+ checksum: op.checksum,
64
+ data: op.data
65
+ };
66
+ }
67
+ }
68
+ export function maxOpId(a, b) {
69
+ if (a == null) {
70
+ return b ?? null;
71
+ }
72
+ if (b == null) {
73
+ return a ?? null;
74
+ }
75
+ return a > b ? a : b;
76
+ }
77
+ //# sourceMappingURL=bucket-format.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"bucket-format.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/bucket-format.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAC1E,OAAO,EAAE,IAAI,EAAgB,MAAM,yBAAyB,CAAC;AAI7D;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CACjC,MAAc,EACd,UAA2B,EAC3B,OAA4C;IAE5C,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAElD,IAAI,aAAa,GAAG,EAAE,CAAC;IACvB,IAAI,WAAW,GAAwB,OAAO,EAAE,QAAQ,IAAI,IAAI,CAAC;IACjE,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,MAAM,GAAG,GAAsB,UAAU,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QACnD,aAAa,IAAI,EAAE,CAAC,QAAQ,CAAC;QAE7B,IAAI,EAAE,CAAC,EAAE,IAAI,OAAO,EAAE,CAAC;YACrB,UAAU,GAAG,IAAI,CAAC;QACpB,CAAC;QAED,OAAO;YACL,CAAC,EAAE,EAAE,CAAC,CAAC;YACP,EAAE,EAAE,EAAE,CAAC,EAAE;YACT,YAAY,EAAE,EAAE,CAAC,YAAY;YAC7B,UAAU,EAAE,EAAE,CAAC,UAAU;YACzB,KAAK,EAAE,EAAE,CAAC,KAAK;YACf,MAAM,EAAE,EAAE,CAAC,MAAM;YACjB,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,IAAI,EAAE,EAAE,CAAC,IAAI;SACd,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,MAAM,IAAI,GAAG,IAAI,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAC;IAE3C,OAAO;QACL,GAAG,EAAE;YACH,CAAC,EAAE,MAAM;YACT,CAAC,EAAE,KAAK;SACT;QACD,MAAM,EAAE,KAAK;QACb,QAAQ,EAAE,aAAa;QACvB,KAAK,EAAE,UAAU,CAAC,MAAM;QACxB,IAAI;QACJ,SAAS,EAAE,WAAW;QACtB,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,IAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,GAAG;KACJ,CAAC;AACJ,CAAC;AAED,MAAM,SAAS,CAAC,CAAC,sBAAsB,CACrC,OAAgE,EAChE,GAAyB;IAEzB,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC;IACzB,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,MAAM,IAAI,qBAAqB,CAC7B,sDAAsD,GAAG,CAAC,CAAC,oEAAoE,CAChI,CAAC;IACJ,CAAC;IACD,MAAM,SAAS,GAAG;QAChB,GAAG,OAAO;QACV,MAAM,EAAE,GAAG,CAAC,CAAC;KACd,CAAC;IAEF,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM;YACJ,SAAS;YACT,CAAC,EAAE,EAAE,CAAC,CAAC;YACP,EAAE,EAAE,EAAE,CAAC,EAAE;YACT,YAAY,EAAE,EAAE,CAAC,YAAY;YAC7B,UAAU,EAAE,EAAE,CAAC,UAAU;YACzB,KAAK,EAAE,EAAE,CAAC,KAAK;YACf,MAAM,EAAE,EAAE,CAAC,MAAM;YACjB,QAAQ,EAAE,EAAE,CAAC,QAAQ;YACrB,IAAI,EAAE,EAAE,CAAC,IAAI;SACd,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,CAAkC,EAAE,CAAkC;IAC5F,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,IAAI,IAAI,CAAC;IACnB,CAAC;IACD,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QACd,OAAO,CAAC,IAAI,IAAI,CAAC;IACnB,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACvB,CAAC"}
@@ -0,0 +1,10 @@
1
+ import { BucketDataDoc } from '../common/BucketDataDoc.js';
2
+ export declare const DEFAULT_MAX_DOC_SIZE_BYTES: number;
3
+ /**
4
+ * Split an array of bucket data operations into chunks where each chunk's
5
+ * total data size stays close to `maxDocSizeBytes`.
6
+ *
7
+ * The threshold is a target, not a hard ceiling — a single operation whose
8
+ * `data` field exceeds the limit is placed in its own chunk and is NOT split.
9
+ */
10
+ export declare function chunkBucketData(operations: BucketDataDoc[], maxDocSizeBytes?: number): BucketDataDoc[][];
@@ -0,0 +1,30 @@
1
+ import { bson } from '@powersync/service-core';
2
+ export const DEFAULT_MAX_DOC_SIZE_BYTES = 1024 * 1024; // 1MB
3
+ /**
4
+ * Split an array of bucket data operations into chunks where each chunk's
5
+ * total data size stays close to `maxDocSizeBytes`.
6
+ *
7
+ * The threshold is a target, not a hard ceiling — a single operation whose
8
+ * `data` field exceeds the limit is placed in its own chunk and is NOT split.
9
+ */
10
+ export function chunkBucketData(operations, maxDocSizeBytes = DEFAULT_MAX_DOC_SIZE_BYTES) {
11
+ const chunks = [];
12
+ let currentChunk = [];
13
+ let currentSize = 0;
14
+ for (const op of operations) {
15
+ // Add 5 to account for array index up to 4 digits (9999) plus null byte.
16
+ const opSize = bson.calculateObjectSize(op) + 5;
17
+ if (currentSize + opSize > maxDocSizeBytes && currentChunk.length > 0) {
18
+ chunks.push(currentChunk);
19
+ currentChunk = [];
20
+ currentSize = 0;
21
+ }
22
+ currentChunk.push(op);
23
+ currentSize += opSize;
24
+ }
25
+ if (currentChunk.length > 0) {
26
+ chunks.push(currentChunk);
27
+ }
28
+ return chunks;
29
+ }
30
+ //# sourceMappingURL=chunking.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"chunking.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/chunking.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,yBAAyB,CAAC;AAG/C,MAAM,CAAC,MAAM,0BAA0B,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,MAAM;AAE7D;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAC7B,UAA2B,EAC3B,eAAe,GAAG,0BAA0B;IAE5C,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,IAAI,YAAY,GAAoB,EAAE,CAAC;IACvC,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;QAC5B,yEAAyE;QACzE,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;QAEhD,IAAI,WAAW,GAAG,MAAM,GAAG,eAAe,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YAC1B,YAAY,GAAG,EAAE,CAAC;YAClB,WAAW,GAAG,CAAC,CAAC;QAClB,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtB,WAAW,IAAI,MAAM,CAAC;IACxB,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC5B,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
@@ -1,12 +1,11 @@
1
- import { InternalOpId, SerializedSyncPlan, SyncRuleState } from '@powersync/service-core';
2
- import { BucketDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
1
+ import { InternalOpId, PersistedDefinitionMapping, SerializedSyncPlan, SyncRuleState } from '@powersync/service-core';
2
+ import { BucketDefinitionId, ParameterIndexId, ScopedParameterLookup, SqliteJsonValue } from '@powersync/service-sync-rules';
3
3
  import * as bson from 'bson';
4
- import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
5
- import { BucketDataDocumentBase, BucketDataKey, BucketParameterDocumentBase, BucketStateDocumentBase, CurrentBucket, ReplicaId, SourceTableDocument, SourceTableKey, SyncRuleCheckpointFields, SyncRuleDocumentBase, TaggedBucketParameterDocument } from '../models.js';
4
+ import { BucketDataKey, BucketParameterDocumentBase, BucketStateDocumentBase, CurrentBucket, OpType, ReplicaId, SourceTableDocumentSnapshotStatus, SourceTableKey, SyncRuleCheckpointFields, SyncRuleDocumentBase, TaggedBucketParameterDocument } from '../models.js';
6
5
  /**
7
6
  * Embedded in sync_rules.sync_configs.
8
7
  */
9
- export interface SyncRuleConfigStateV3 extends SyncRuleCheckpointFields<bigint | null> {
8
+ export interface SyncRuleConfigStateV3 extends SyncRuleCheckpointFields {
10
9
  _id: bson.ObjectId;
11
10
  /**
12
11
  * If false, we cannot create any checkpoints.
@@ -30,6 +29,29 @@ export interface ReplicationStreamDocumentV3 extends SyncRuleDocumentBase {
30
29
  * but the model allows multiple configs in any state.
31
30
  */
32
31
  sync_configs: SyncRuleConfigStateV3[];
32
+ /**
33
+ * The monotonic head of the stream's op sequence: the highest op id persisted to bucket data,
34
+ * whether or not yet covered by a checkpoint.
35
+ *
36
+ * This is shared across all sync configs of the stream (they share the global op sequence).
37
+ * It is never cleared, only `$max`-advanced. A newly-appended config that replicates nothing
38
+ * adopts this value as its checkpoint rather than starting at 0.
39
+ *
40
+ * Stored as a mongo Long, nullable.
41
+ */
42
+ last_persisted_op?: bigint | null;
43
+ /**
44
+ * The stream's replication position: all source changes up to this LSN have been processed,
45
+ * and the resulting ops persisted. Replication resumes from here.
46
+ *
47
+ * Like {@link last_persisted_op}, this is shared across all sync configs of the stream -
48
+ * per-config last_checkpoint_lsn values are consistency markers, not replication positions.
49
+ *
50
+ * Set via setResumeLsn() (snapshot start, and per-batch progress during streaming), and
51
+ * advanced on every commit/keepalive - including checkpoint-blocked ones, since commit
52
+ * flushes first and blocking only delays consistency markers, not data persistence.
53
+ */
54
+ resume_lsn?: string | null;
33
55
  }
34
56
  /**
35
57
  * Static sync config definition.
@@ -50,16 +72,7 @@ export interface SyncConfigDefinition {
50
72
  replication_stream_id: number;
51
73
  content: string;
52
74
  serialized_plan?: SerializedSyncPlan | null;
53
- rule_mapping: {
54
- /**
55
- * Map of uniqueName -> id, unique per replication stream.
56
- */
57
- definitions: Record<string, string>;
58
- /**
59
- * Map of (lookupName, queryId) -> id, unique per replication stream.
60
- */
61
- parameter_indexes: Record<string, string>;
62
- };
75
+ rule_mapping: PersistedDefinitionMapping;
63
76
  }
64
77
  export interface CurrentBucketV3 extends CurrentBucket {
65
78
  def: BucketDefinitionId;
@@ -83,13 +96,21 @@ export interface CurrentDataDocumentV3 {
83
96
  export interface BucketParameterDocumentV3 extends BucketParameterDocumentBase<SourceTableKey> {
84
97
  }
85
98
  export type BucketDataKeyV3 = BucketDataKey;
86
- export interface BucketDataDocumentV3 extends BucketDataDocumentBase {
87
- _id: BucketDataKeyV3;
88
- }
89
- export declare function serializeBucketDataV3(document: BucketDataDoc): BucketDataDocumentV3;
90
- export declare function loadBucketDataDocumentV3(context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>, doc: BucketDataDocumentV3): BucketDataDoc;
91
99
  export declare function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3;
92
- export interface SourceTableDocumentV3 extends SourceTableDocument {
100
+ export interface ReplicaIdColumn {
101
+ name: string;
102
+ type_oid?: number;
103
+ type?: string;
104
+ }
105
+ export interface SourceTableDocumentV3 {
106
+ _id: bson.ObjectId;
107
+ connection_id: number;
108
+ relation_id: number | string | undefined;
109
+ schema_name: string;
110
+ table_name: string;
111
+ replica_id_columns: ReplicaIdColumn[];
112
+ snapshot_done: boolean;
113
+ snapshot_status: SourceTableDocumentSnapshotStatus | undefined;
93
114
  bucket_data_source_ids: BucketDefinitionId[];
94
115
  parameter_lookup_source_ids: ParameterIndexId[];
95
116
  latest_pending_delete?: InternalOpId | undefined;
@@ -99,3 +120,62 @@ export interface BucketStateDocumentV3 extends BucketStateDocumentBase {
99
120
  d: BucketDefinitionId;
100
121
  };
101
122
  }
123
+ export interface BucketOperation {
124
+ o: bigint;
125
+ op: OpType;
126
+ source_table?: bson.ObjectId;
127
+ source_key?: ReplicaId;
128
+ table?: string;
129
+ row_id?: string;
130
+ checksum: bigint;
131
+ data: string | null;
132
+ }
133
+ export interface StorageRef {
134
+ path: string;
135
+ file_size: number;
136
+ }
137
+ /** An S3 object that may be deleted once its grace period has elapsed. */
138
+ export interface ObjectStorageDeletionMarker {
139
+ _id: bson.ObjectId;
140
+ path: string;
141
+ delete_after: Date;
142
+ }
143
+ /**
144
+ * A non-empty chunk of a bucket's ordered operation stream.
145
+ *
146
+ * All writers maintain these invariants:
147
+ *
148
+ * - `ops` is non-empty and strictly ordered by `o`.
149
+ * - `min_op` is `ops[0].o` and `_id.o` is `ops[ops.length - 1].o`.
150
+ * - Documents for the same bucket have ordered, disjoint operation ranges,
151
+ * although gaps between ranges are allowed.
152
+ * - `checksum` is the sum of every operation checksum and `count` is
153
+ * `ops.length`.
154
+ *
155
+ * Metadata-only checksum and data queries rely on these invariants to identify
156
+ * boundary-straddling documents without reading `ops`.
157
+ */
158
+ export interface BucketDataDocumentV3 {
159
+ _id: BucketDataKey;
160
+ min_op: bigint;
161
+ checksum: bigint;
162
+ count: number;
163
+ size: number;
164
+ /**
165
+ * The greatest operation boundary that influenced this document's contents.
166
+ * Bucket-data reads propagate it so serving an earlier checkpoint can be
167
+ * invalidated when compaction changed the operations being returned.
168
+ */
169
+ target_op?: bigint | null;
170
+ ops?: BucketOperation[];
171
+ storage_ref?: StorageRef;
172
+ /**
173
+ * Present (and always true) when this document contains a CLEAR operation.
174
+ * In that case, this is the first document in the bucket: all preceding
175
+ * documents have been removed.
176
+ */
177
+ has_clear_op?: true;
178
+ }
179
+ export declare function serializeParameterLookup(lookup: ScopedParameterLookup): bson.Binary;
180
+ export declare function deserializeParameterLookup(lookup: bson.Binary, indexId: ParameterIndexId): SqliteJsonValue[];
181
+ export declare function taggedBucketParameterDocumentToTagged(document: TaggedBucketParameterDocument): BucketParameterDocumentV3;
@@ -1,33 +1,16 @@
1
- export function serializeBucketDataV3(document) {
2
- const { bucketKey, o } = document;
3
- return {
4
- _id: {
5
- b: bucketKey.bucket,
6
- o: o
7
- },
8
- // List fields directly, so that we don't accidentally persist any unknown fields
9
- op: document.op,
10
- source_table: document.source_table,
11
- source_key: document.source_key,
12
- table: document.table,
13
- row_id: document.row_id,
14
- checksum: document.checksum,
15
- data: document.data,
16
- target_op: document.target_op
17
- };
1
+ import { deserializeParameterLookup as deserializeParameterLookupCore } from '@powersync/service-core';
2
+ import * as bson from 'bson';
3
+ export function taggedBucketParameterDocumentToV3(document) {
4
+ const { index: _index, ...rest } = document;
5
+ return rest;
18
6
  }
19
- export function loadBucketDataDocumentV3(context, doc) {
20
- const { _id, ...rest } = doc;
21
- return {
22
- bucketKey: {
23
- ...context,
24
- bucket: _id.b
25
- },
26
- o: _id.o,
27
- ...rest
28
- };
7
+ export function serializeParameterLookup(lookup) {
8
+ return new bson.Binary(bson.serialize({ l: lookup.values.slice(2) }));
29
9
  }
30
- export function taggedBucketParameterDocumentToV3(document) {
10
+ export function deserializeParameterLookup(lookup, indexId) {
11
+ return [indexId, '', ...deserializeParameterLookupCore(lookup)];
12
+ }
13
+ export function taggedBucketParameterDocumentToTagged(document) {
31
14
  const { index: _index, ...rest } = document;
32
15
  return rest;
33
16
  }
@@ -1 +1 @@
1
- {"version":3,"file":"models.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/models.ts"],"names":[],"mappings":"AAkHA,MAAM,UAAU,qBAAqB,CAAC,QAAuB;IAC3D,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,QAAQ,CAAC;IAClC,OAAO;QACL,GAAG,EAAE;YACH,CAAC,EAAE,SAAS,CAAC,MAAM;YACnB,CAAC,EAAE,CAAC;SACL;QACD,iFAAiF;QACjF,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,OAAgE,EAChE,GAAyB;IAEzB,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC;IAC7B,OAAO;QACL,SAAS,EAAE;YACT,GAAG,OAAO;YACV,MAAM,EAAE,GAAG,CAAC,CAAC;SACd;QACD,CAAC,EAAE,GAAG,CAAC,CAAC;QACR,GAAG,IAAI;KACR,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAAC,QAAuC;IACvF,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;IAC5C,OAAO,IAAiC,CAAC;AAC3C,CAAC"}
1
+ {"version":3,"file":"models.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/models.ts"],"names":[],"mappings":"AAAA,OAAO,EAKL,0BAA0B,IAAI,8BAA8B,EAC7D,MAAM,yBAAyB,CAAC;AAOjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AA2H7B,MAAM,UAAU,iCAAiC,CAAC,QAAuC;IACvF,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;IAC5C,OAAO,IAAiC,CAAC;AAC3C,CAAC;AAwFD,MAAM,UAAU,wBAAwB,CAAC,MAA6B;IACpE,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,MAAmB,EAAE,OAAyB;IACvF,OAAO,CAAC,OAAO,EAAE,EAAE,EAAE,GAAG,8BAA8B,CAAC,MAAM,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,MAAM,UAAU,qCAAqC,CACnD,QAAuC;IAEvC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;IAC5C,OAAO,IAAiC,CAAC;AAC3C,CAAC"}
@@ -0,0 +1,21 @@
1
+ import { BucketDataDocumentV3, BucketOperation } from '../models.js';
2
+ import { ObjectStorage } from './ObjectStorage.js';
3
+ export declare class BucketDataObjectStorage {
4
+ private readonly storage;
5
+ constructor(storage: ObjectStorage);
6
+ store(path: string, ops: BucketOperation[]): Promise<{
7
+ fileSize: number;
8
+ }>;
9
+ retrieve(path: string, options: {
10
+ signal?: AbortSignal;
11
+ }): Promise<BucketOperation[]>;
12
+ delete(paths: string[]): Promise<void>;
13
+ }
14
+ /**
15
+ * Load offloaded operations and patch them onto their MongoDB metadata documents.
16
+ * The object-storage implementation is responsible for limiting request
17
+ * concurrency across all callers.
18
+ */
19
+ export declare function hydrateBucketDataDocuments(documents: BucketDataDocumentV3[], objectStorage: ObjectStorage | undefined, options: {
20
+ signal?: AbortSignal;
21
+ }): Promise<void>;
@@ -0,0 +1,44 @@
1
+ import { storage } from '@powersync/service-core';
2
+ import * as bson from 'bson';
3
+ export class BucketDataObjectStorage {
4
+ storage;
5
+ constructor(storage) {
6
+ this.storage = storage;
7
+ }
8
+ async store(path, ops) {
9
+ const bsonBuffer = bson.serialize({ ops });
10
+ await this.storage.put(path, bsonBuffer, { contentType: 'application/bson', contentEncoding: null });
11
+ return { fileSize: bsonBuffer.byteLength };
12
+ }
13
+ async retrieve(path, options) {
14
+ const { data, metadata } = await this.storage.get(path, { signal: options.signal });
15
+ if (metadata.contentEncoding != null) {
16
+ throw new Error(`Unexpected content encoding: ${metadata.contentEncoding}`);
17
+ }
18
+ if (metadata.contentType !== 'application/bson') {
19
+ throw new Error(`Unexpected content type: ${metadata.contentType}`);
20
+ }
21
+ const wrapper = bson.deserialize(data, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS);
22
+ return wrapper.ops;
23
+ }
24
+ async delete(paths) {
25
+ return this.storage.delete(paths);
26
+ }
27
+ }
28
+ /**
29
+ * Load offloaded operations and patch them onto their MongoDB metadata documents.
30
+ * The object-storage implementation is responsible for limiting request
31
+ * concurrency across all callers.
32
+ */
33
+ export async function hydrateBucketDataDocuments(documents, objectStorage, options) {
34
+ if (!objectStorage) {
35
+ return;
36
+ }
37
+ options.signal?.throwIfAborted();
38
+ const store = new BucketDataObjectStorage(objectStorage);
39
+ const storedDocuments = documents.filter((document) => document.storage_ref);
40
+ await Promise.all(storedDocuments.map(async (document) => {
41
+ document.ops = await store.retrieve(document.storage_ref.path, { signal: options.signal });
42
+ }));
43
+ }
44
+ //# sourceMappingURL=BucketDataObjectStorage.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"BucketDataObjectStorage.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAClD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAI7B,MAAM,OAAO,uBAAuB;IACL;IAA7B,YAA6B,OAAsB;QAAtB,YAAO,GAAP,OAAO,CAAe;IAAG,CAAC;IAEvD,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,GAAsB;QAC9C,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3C,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,EAAE,WAAW,EAAE,kBAAkB,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QACrG,OAAO,EAAE,QAAQ,EAAE,UAAU,CAAC,UAAU,EAAE,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,IAAY,EAAE,OAAiC;QAC5D,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACpF,IAAI,QAAQ,CAAC,eAAe,IAAI,IAAI,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,QAAQ,CAAC,WAAW,KAAK,kBAAkB,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;QACtE,CAAC;QACD,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,iCAAiC,CAAC,CAAC;QAClF,OAAO,OAAO,CAAC,GAAG,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAe;QAC1B,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpC,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,SAAiC,EACjC,aAAwC,EACxC,OAAiC;IAEjC,IAAI,CAAC,aAAa,EAAE,CAAC;QACnB,OAAO;IACT,CAAC;IAED,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;IACjC,MAAM,KAAK,GAAG,IAAI,uBAAuB,CAAC,aAAa,CAAC,CAAC;IACzD,MAAM,eAAe,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC7E,MAAM,OAAO,CAAC,GAAG,CACf,eAAe,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QACrC,QAAQ,CAAC,GAAG,GAAG,MAAM,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,WAAY,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9F,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}
@@ -0,0 +1,30 @@
1
+ export interface ObjectStoragePutMetadata {
2
+ contentType: string;
3
+ contentEncoding: string | null;
4
+ }
5
+ export declare class ObjectStorageError extends Error {
6
+ readonly retryable: boolean;
7
+ constructor(message: string, options: {
8
+ cause: unknown;
9
+ retryable: boolean;
10
+ });
11
+ }
12
+ export declare function isRetryableObjectStorageError(error: unknown): error is ObjectStorageError;
13
+ export interface ObjectStorage {
14
+ put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata): Promise<void>;
15
+ get(path: string, options?: {
16
+ signal?: AbortSignal;
17
+ }): Promise<{
18
+ data: Uint8Array;
19
+ metadata: ObjectStoragePutMetadata;
20
+ }>;
21
+ list(prefix: string, options?: {
22
+ signal?: AbortSignal;
23
+ }): AsyncIterable<string>;
24
+ delete(paths: string[]): Promise<void>;
25
+ deletePrefix(prefix: string, options?: {
26
+ signal?: AbortSignal;
27
+ }): Promise<{
28
+ objectCount: number;
29
+ }>;
30
+ }
@@ -0,0 +1,12 @@
1
+ export class ObjectStorageError extends Error {
2
+ retryable;
3
+ constructor(message, options) {
4
+ super(message, { cause: options.cause });
5
+ this.name = 'ObjectStorageError';
6
+ this.retryable = options.retryable;
7
+ }
8
+ }
9
+ export function isRetryableObjectStorageError(error) {
10
+ return error instanceof ObjectStorageError && error.retryable;
11
+ }
12
+ //# sourceMappingURL=ObjectStorage.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"ObjectStorage.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/ObjectStorage.ts"],"names":[],"mappings":"AAKA,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IAClC,SAAS,CAAU;IAE5B,YAAY,OAAe,EAAE,OAA+C;QAC1E,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACrC,CAAC;CACF;AAED,MAAM,UAAU,6BAA6B,CAAC,KAAc;IAC1D,OAAO,KAAK,YAAY,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC;AAChE,CAAC"}
@@ -0,0 +1,70 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+ import { Logger } from '@powersync/lib-services-framework';
3
+ import * as bson from 'bson';
4
+ import { VersionedPowerSyncMongoV3 } from '../VersionedPowerSyncMongoV3.js';
5
+ import { BucketDataObjectStorage } from './BucketDataObjectStorage.js';
6
+ import { ObjectStorage } from './ObjectStorage.js';
7
+ export declare const OBJECT_STORAGE_UPLOAD_LEASE_MS: number;
8
+ export declare const OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS: number;
9
+ export declare const OBJECT_STORAGE_REFERENCE_GRACE_MS: number;
10
+ export interface PreparedObjectStorageUpload {
11
+ markerId: bson.ObjectId;
12
+ path: string;
13
+ deleteAfter: Date;
14
+ }
15
+ /**
16
+ * Coordinates the MongoDB outbox used to make object-storage changes recoverable.
17
+ *
18
+ * A MongoDB collection is used to track files that will be uploaded, or should be deleted.
19
+ * By using a MongoDB collection, we can use the same transaction that creates or removes
20
+ * references to these files, making the operations atomic.
21
+ *
22
+ * These markers indicate which files should be deleted - either by being orphaned during create,
23
+ * or after references to them are removed.
24
+ */
25
+ export declare class ObjectStorageLifecycle {
26
+ private readonly db;
27
+ private readonly replicationStreamId;
28
+ private readonly objectStorage;
29
+ readonly bucketData: BucketDataObjectStorage;
30
+ constructor(db: VersionedPowerSyncMongoV3, replicationStreamId: number, objectStorage: ObjectStorage);
31
+ streamPrefix(): string;
32
+ definitionPrefix(definitionId: string): string;
33
+ allocatePath(definitionId: string, bucket: string, minOp: bigint, maxOp: bigint): string;
34
+ /**
35
+ * Indicate that files will be uploaded to S3.
36
+ *
37
+ * These markers are expected to be removed using publishUploads(). If that does not happen within
38
+ * the grace period, the files will be deleted.
39
+ */
40
+ prepareUploads(paths: string[], now?: Date): Promise<PreparedObjectStorageUpload[]>;
41
+ canPublish(upload: PreparedObjectStorageUpload, now?: Date): boolean;
42
+ /**
43
+ * Remove markers for files that have finished uploading, and are now referenced from other documents.
44
+ *
45
+ * Call this in the same transaction as the one creating the references.
46
+ */
47
+ publishUploads(uploads: PreparedObjectStorageUpload[], session: mongo.ClientSession): Promise<void>;
48
+ /**
49
+ * Mark old paths for deletion.
50
+ *
51
+ * By marking for deletion instead of immediately deleting:
52
+ * 1. This avoids inconsistencies/orphaned files between MongoDB and S3, by allowing the markers to
53
+ * be persisted transactionally, while the deletes happen asynchronously and can be retried.
54
+ * 2. The deletes are delayed, giving clients a grace period to continue using the files for
55
+ * in-progress requests.
56
+ *
57
+ * Call this in the same transaction as the one removing the references to these files.
58
+ */
59
+ retire(paths: Iterable<string>, session: mongo.ClientSession, now?: Date): Promise<void>;
60
+ /**
61
+ * Discover and delete objects by their storage prefix
62
+ * without touching the database.
63
+ */
64
+ deletePrefix(prefix: string, options?: {
65
+ signal?: AbortSignal;
66
+ }): Promise<{
67
+ objectCount: number;
68
+ }>;
69
+ cleanup(logger: Logger): Promise<void>;
70
+ }