@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/CHANGELOG.md +101 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +41 -16
  6. package/dist/storage/MongoBucketStorage.js +226 -71
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
  9. package/dist/storage/implementation/MongoBucketBatch.js +110 -62
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
  15. package/dist/storage/implementation/MongoCompactor.js +80 -263
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
  21. package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +39 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  46. package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
  47. package/dist/storage/implementation/common/PersistedBatch.js +12 -0
  48. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  49. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  51. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  52. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  53. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  54. package/dist/storage/implementation/db.js +17 -15
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +20 -13
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  62. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  63. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
  65. package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
  66. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  71. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  72. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +10 -1
  77. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  78. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  79. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  81. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
  82. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  84. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  85. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  86. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
  87. package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
  88. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
  90. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
  91. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
  93. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
  94. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
  96. package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +77 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +30 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +101 -21
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
  115. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  126. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  128. package/dist/storage/storage-index.d.ts +4 -4
  129. package/dist/storage/storage-index.js +4 -4
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +49 -3
  132. package/dist/types/types.js +48 -1
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +1 -2
  135. package/dist/utils/test-utils.js +7 -1
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.d.ts +9 -2
  138. package/dist/utils/util.js +128 -4
  139. package/dist/utils/util.js.map +1 -1
  140. package/package.json +12 -19
  141. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  142. package/src/storage/MongoBucketStorage.ts +347 -88
  143. package/src/storage/implementation/MongoBucketBatch.ts +147 -82
  144. package/src/storage/implementation/MongoChecksums.ts +93 -153
  145. package/src/storage/implementation/MongoCompactor.ts +102 -316
  146. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  147. package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
  148. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  149. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  150. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  151. package/src/storage/implementation/MongoStorageProvider.ts +46 -4
  152. package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
  153. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  154. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
  156. package/src/storage/implementation/common/PersistedBatch.ts +19 -3
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  158. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  159. package/src/storage/implementation/db.ts +19 -20
  160. package/src/storage/implementation/models.ts +20 -14
  161. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
  162. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  163. package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
  164. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
  165. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  166. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  167. package/src/storage/implementation/v1/models.ts +11 -1
  168. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  169. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
  170. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  171. package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
  172. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
  173. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
  174. package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
  175. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  176. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
  177. package/src/storage/implementation/v3/bucket-format.ts +97 -0
  178. package/src/storage/implementation/v3/chunking.ts +40 -0
  179. package/src/storage/implementation/v3/models.ts +132 -55
  180. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
  181. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
  184. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  185. package/src/storage/storage-index.ts +12 -4
  186. package/src/types/types.ts +58 -1
  187. package/src/utils/test-utils.ts +8 -3
  188. package/src/utils/util.ts +185 -6
  189. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
  191. package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
  192. package/test/src/helpers/MemoryObjectStorage.ts +53 -0
  193. package/test/src/helpers/s3TestFactory.ts +69 -0
  194. package/test/src/object_storage_lifecycle.test.ts +59 -0
  195. package/test/src/setup.ts +6 -1
  196. package/test/src/storage.test.ts +227 -13
  197. package/test/src/storage_compacting.test.ts +1951 -12
  198. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  199. package/test/src/storage_s3_checksums.test.ts +191 -0
  200. package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
  201. package/test/src/storage_s3_reading.test.ts +526 -0
  202. package/test/src/storage_s3_writing.test.ts +135 -0
  203. package/test/src/storage_sync.test.ts +1180 -73
  204. package/test/src/storeCurrentData.test.ts +227 -0
  205. package/test/src/util.ts +2 -1
  206. package/tsconfig.json +1 -4
  207. package/tsconfig.tsbuildinfo +1 -1
  208. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  209. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  210. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  211. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  212. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  213. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  214. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  215. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  216. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  217. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  218. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  219. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  220. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  221. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  222. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  223. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  224. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  225. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  226. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  227. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  228. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  229. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  230. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  231. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  232. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  233. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  234. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  235. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  236. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  237. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  238. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  239. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -6,9 +6,9 @@ import {
6
6
  GetCheckpointChangesOptions,
7
7
  InternalOpId,
8
8
  internalToExternalOpId,
9
- maxLsn,
10
9
  ParameterSetLimitExceededError,
11
10
  ProtocolOpId,
11
+ SingleSyncConfigBucketDefinitionMapping,
12
12
  storage,
13
13
  utils
14
14
  } from '@powersync/service-core';
@@ -17,57 +17,114 @@ import { ParameterLookupRows, ScopedParameterLookup, SqliteJsonRow } from '@powe
17
17
  import * as bson from 'bson';
18
18
  import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
19
19
  import { MongoBucketStorage } from '../../MongoBucketStorage.js';
20
- import {
21
- MongoSyncBucketStorageCheckpoint,
22
- MongoSyncBucketStorageContext
23
- } from '../common/MongoSyncBucketStorageContext.js';
24
- import { MongoBucketBatchOptions } from '../MongoBucketBatch.js';
20
+ import { BucketDataDoc } from '../common/BucketDataDoc.js';
21
+ import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js';
25
22
  import { MongoChecksums } from '../MongoChecksums.js';
26
23
  import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
27
24
  import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
28
- import {
29
- MongoPersistedSyncRulesContentV1,
30
- MongoPersistedSyncRulesContentV3
31
- } from '../MongoPersistedSyncRulesContent.js';
25
+ import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js';
32
26
  import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
27
+ import { loadBucketDataDocument, maxOpId } from './bucket-format.js';
33
28
  import {
34
29
  BucketDataDocumentV3,
35
30
  BucketParameterDocumentV3,
36
- loadBucketDataDocumentV3,
31
+ deserializeParameterLookup,
37
32
  ReplicationStreamDocumentV3,
33
+ serializeParameterLookup,
38
34
  SyncRuleConfigStateV3
39
35
  } from './models.js';
40
36
  import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js';
41
37
  import { MongoChecksumsV3 } from './MongoChecksumsV3.js';
42
38
  import { MongoCompactorV3 } from './MongoCompactorV3.js';
43
- import { MongoParameterCompactorV3 } from './MongoParameterCompactorV3.js';
44
- import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './MongoParameterLookupV3.js';
39
+ import { MongoStoppedSyncConfigCleanup } from './MongoStoppedSyncConfigCleanup.js';
40
+ import { hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js';
41
+ import { ObjectStorage } from './object-storage/ObjectStorage.js';
42
+ import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
45
43
  import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
46
44
 
45
+ export interface MongoSyncBucketStorageContextV3 {
46
+ db: VersionedPowerSyncMongoV3;
47
+ replicationStreamId: number;
48
+ readPreference?: mongo.ReadPreference;
49
+ objectStorage: ObjectStorage | undefined;
50
+ /**
51
+ * Persisted mapping of the single sync config that read operations are served from.
52
+ *
53
+ * Implemented as a lazy getter: accessing it on a storage instance with multiple sync
54
+ * configs throws, but operations that don't use it remain unaffected.
55
+ */
56
+ readonly mapping: SingleSyncConfigBucketDefinitionMapping;
57
+ }
58
+
59
+ const BUCKET_DATA_FETCH_BATCH_LIMIT_BYTES = 16 * 1024 * 1024;
60
+
61
+ /**
62
+ * Keep the documents hydrated for one sync response within a bounded payload or slightly higher.
63
+ *
64
+ * This deserializes on-demand, so no deserialization is performed for discarded data.
65
+ */
66
+ function cutBucketDataBatch(rawDocuments: Buffer[]): {
67
+ documents: BucketDataDocumentV3[];
68
+ wasCut: boolean;
69
+ } {
70
+ let cumulativeBytes = 0;
71
+ let documents: BucketDataDocumentV3[] = [];
72
+ for (const raw of rawDocuments) {
73
+ const doc = bson.deserialize(raw, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV3;
74
+ documents.push(doc);
75
+ cumulativeBytes += doc.size;
76
+ if (cumulativeBytes > BUCKET_DATA_FETCH_BATCH_LIMIT_BYTES) {
77
+ return {
78
+ documents,
79
+ wasCut: documents.length < rawDocuments.length
80
+ };
81
+ }
82
+ }
83
+ return { documents, wasCut: false };
84
+ }
85
+
86
+ function extractRowsFromDocument(
87
+ doc: BucketDataDocumentV3,
88
+ context: { replicationStreamId: number; definitionId: string },
89
+ bucketStart: InternalOpId,
90
+ endOpId: InternalOpId
91
+ ): BucketDataDoc[] {
92
+ const rows: BucketDataDoc[] = [];
93
+ for (const row of loadBucketDataDocument(context, doc)) {
94
+ // In theory a binary search could be faster than a linear scan to find the start.
95
+ // In practice, most cases should not filter out anything here.
96
+ if (row.o <= bucketStart) {
97
+ continue;
98
+ }
99
+ if (row.o > endOpId) {
100
+ break;
101
+ }
102
+
103
+ rows.push(row);
104
+ }
105
+ return rows;
106
+ }
107
+
47
108
  export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
48
- // Declare types to be more specific
49
109
  declare readonly db: VersionedPowerSyncMongoV3;
50
110
  declare readonly checksums: MongoChecksumsV3;
51
111
 
52
- private readonly syncRulesV3: MongoPersistedSyncRulesContentV3;
53
-
54
112
  constructor(
55
113
  factory: MongoBucketStorage,
56
- group_id: number,
57
- sync_rules: MongoPersistedSyncRulesContentV1,
58
- slot_name: string,
114
+ replicationStreamId: number,
115
+ replicationStream: MongoPersistedReplicationStream,
116
+ replicationStreamName: string,
59
117
  writeCheckpointMode: storage.WriteCheckpointMode | undefined,
60
118
  options: MongoSyncBucketStorageOptions
61
119
  ) {
62
- super(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
63
- if (!(sync_rules instanceof MongoPersistedSyncRulesContentV3)) {
120
+ super(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
121
+ if (replicationStream.syncConfigIds.length == 0) {
64
122
  throw new ServiceAssertionError('Missing sync config id for storage v3');
65
123
  }
66
- this.syncRulesV3 = sync_rules;
67
124
  }
68
125
 
69
- private get syncConfigId(): bson.ObjectId {
70
- return this.syncRulesV3.syncConfigId;
126
+ private get syncConfigIds(): bson.ObjectId[] {
127
+ return this.replicationStream.syncConfigIds;
71
128
  }
72
129
 
73
130
  private get syncRulesCollection(): mongo.Collection<ReplicationStreamDocumentV3> {
@@ -76,10 +133,10 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
76
133
 
77
134
  private syncConfigMatch(extra: mongo.Document = {}): mongo.Filter<ReplicationStreamDocumentV3> {
78
135
  return {
79
- _id: this.group_id,
136
+ _id: this.replicationStreamId,
80
137
  sync_configs: {
81
138
  $elemMatch: {
82
- _id: this.syncConfigId,
139
+ _id: { $in: this.syncConfigIds },
83
140
  ...extra
84
141
  }
85
142
  }
@@ -89,44 +146,27 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
89
146
  private syncConfigProjection(extra: mongo.Document = {}): mongo.Document {
90
147
  return {
91
148
  ...extra,
92
- sync_configs: {
93
- $elemMatch: {
94
- _id: this.syncConfigId
95
- }
96
- }
149
+ sync_configs: 1
97
150
  };
98
151
  }
99
152
 
100
- private syncConfigArrayFilters(): mongo.UpdateOptions['arrayFilters'] {
101
- return [{ 'config._id': this.syncConfigId }];
102
- }
103
-
104
- /**
105
- * For now, we only support a single sync config per replication stream.
106
- *
107
- * In the future we'll add support for multiple.
108
- */
109
- private selectedSyncConfig(
110
- doc: Pick<ReplicationStreamDocumentV3, 'sync_configs'> | null
111
- ): SyncRuleConfigStateV3 | null {
112
- return doc?.sync_configs?.[0] ?? null;
153
+ private selectedSyncConfigs(doc: Pick<ReplicationStreamDocumentV3, 'sync_configs'> | null): SyncRuleConfigStateV3[] {
154
+ return doc?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ?? [];
113
155
  }
114
156
 
115
157
  protected async initializeVersionStorage(): Promise<void> {
116
- const mapping = this.mapping;
117
- for (let source of mapping.allBucketDefinitionIds()) {
118
- const collection = this.db.bucketDataV3(this.group_id, source).collectionName;
119
- await this.db.db
120
- .createCollection(collection, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } })
121
- .catch((error) => {
122
- if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceExists') {
123
- return;
124
- }
125
- throw error;
126
- });
158
+ const storageIds = this.storageIds;
159
+ for (const source of storageIds.bucketDefinitionIds) {
160
+ const collection = this.db.bucketData(this.replicationStreamId, source).collectionName;
161
+ await this.db.db.createCollection(collection, {}).catch((error) => {
162
+ if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceExists') {
163
+ return;
164
+ }
165
+ throw error;
166
+ });
127
167
  }
128
- for (let indexId of mapping.allParameterIndexIds()) {
129
- await this.db.parameterIndexV3(this.group_id, indexId).createIndex(
168
+ for (const indexId of storageIds.parameterIndexIds) {
169
+ await this.db.parameterIndex(this.replicationStreamId, indexId).createIndex(
130
170
  {
131
171
  lookup: 1,
132
172
  key: 1,
@@ -140,10 +180,11 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
140
180
  }
141
181
 
142
182
  protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums {
143
- return new MongoChecksumsV3(this.db, this.group_id, {
183
+ return new MongoChecksumsV3(this.db, this.replicationStreamId, {
144
184
  ...options.checksumOptions,
185
+ checksumCacheTtlMs: options.checksumCacheTtlMs,
145
186
  storageConfig: options?.storageConfig,
146
- mapping: this.sync_rules.mapping
187
+ syncConfigMapping: () => this.singleSyncConfigMapping()
147
188
  });
148
189
  }
149
190
 
@@ -155,11 +196,36 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
155
196
  checkpoint: InternalOpId,
156
197
  options: storage.CompactOptions
157
198
  ): MongoParameterCompactor {
158
- return new MongoParameterCompactorV3(this.db, this.group_id, checkpoint, options);
199
+ return new MongoParameterCompactor(this.db, this.replicationStreamId, checkpoint, options, () =>
200
+ this.db
201
+ .listParameterIndexCollections(this.replicationStreamId)
202
+ .then((collections) =>
203
+ collections.map((c) => c.collection as unknown as lib_mongo.mongo.Collection<lib_mongo.mongo.Document>)
204
+ )
205
+ );
159
206
  }
160
207
 
161
- protected createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch {
162
- return new MongoBucketBatchV3(batchOptions);
208
+ protected async fetchPersistedOpHead(): Promise<InternalOpId | null> {
209
+ const doc = await this.syncRulesCollection.findOne(
210
+ { _id: this.replicationStreamId },
211
+ { projection: { last_persisted_op: 1 } }
212
+ );
213
+ return doc?.last_persisted_op == null ? null : BigInt(doc.last_persisted_op);
214
+ }
215
+
216
+ protected async createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
217
+ const doc = await this.syncRulesCollection.findOne(
218
+ { _id: this.replicationStreamId },
219
+ { projection: { resume_lsn: 1 } }
220
+ );
221
+
222
+ return new MongoBucketBatchV3({
223
+ ...this.writerBatchOptions(options),
224
+ // The stream-level replication position - per-config checkpoint LSNs are consistency
225
+ // markers and do not affect where replication resumes.
226
+ resumeFromLsn: doc?.resume_lsn ?? null,
227
+ syncConfigIds: this.syncConfigIds
228
+ });
163
229
  }
164
230
 
165
231
  protected async fetchCheckpointState(
@@ -174,8 +240,20 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
174
240
  projection: this.syncConfigProjection()
175
241
  }
176
242
  );
177
- const syncConfig = this.selectedSyncConfig(doc);
178
- if (!syncConfig?.snapshot_done) {
243
+ // Checkpoints are served from the single active config. A PROCESSING config in the same
244
+ // stream (incremental reprocessing) does not affect checkpoints until it is activated.
245
+ const syncConfigs = this.selectedSyncConfigs(doc).filter((config) =>
246
+ [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED].includes(config.state)
247
+ );
248
+ if (syncConfigs.length > 1) {
249
+ // Activation atomically replaces the previous active config, so this cannot happen unless
250
+ // the stored state is corrupt.
251
+ throw new ServiceAssertionError(
252
+ `Expected a single active sync config, got ${syncConfigs.map((config) => config._id.toHexString()).join(', ')}`
253
+ );
254
+ }
255
+ const syncConfig = syncConfigs[0];
256
+ if (syncConfig == null || !syncConfig.snapshot_done) {
179
257
  return null;
180
258
  }
181
259
  return {
@@ -184,24 +262,10 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
184
262
  };
185
263
  }
186
264
 
187
- protected async getWriterSyncState() {
188
- const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), {
189
- projection: this.syncConfigProjection({ snapshot_lsn: 1 })
190
- });
191
- const syncConfig = this.selectedSyncConfig(doc);
192
- const checkpointLsn = syncConfig?.last_checkpoint_lsn ?? null;
193
- return {
194
- lastCheckpointLsn: checkpointLsn,
195
- resumeFromLsn: maxLsn(checkpointLsn, doc?.snapshot_lsn),
196
- keepaliveOp: syncConfig?.keepalive_op ?? null,
197
- syncConfigId: this.syncConfigId
198
- };
199
- }
200
-
201
265
  protected async terminateSyncRuleState(): Promise<void> {
202
266
  await this.db.sync_rules.updateOne(
203
267
  {
204
- _id: this.group_id
268
+ _id: this.replicationStreamId
205
269
  },
206
270
  {
207
271
  $set: {
@@ -213,52 +277,82 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
213
277
  );
214
278
  }
215
279
 
216
- protected async getStatusImpl(): Promise<storage.SyncRuleStatus> {
280
+ protected async getStatusImpl(): Promise<storage.ReplicationStreamStatus> {
217
281
  const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), {
218
- projection: this.syncConfigProjection({ state: 1, snapshot_lsn: 1, keepalive_op: 1 })
282
+ projection: this.syncConfigProjection({ resume_lsn: 1 })
219
283
  });
220
- const syncConfig = this.selectedSyncConfig(doc);
221
- if (doc == null || syncConfig == null) {
284
+ const syncConfigs = this.selectedSyncConfigs(doc);
285
+ if (doc == null || syncConfigs.length == 0) {
222
286
  throw new ServiceAssertionError('Cannot find replication stream status');
223
287
  }
224
288
 
225
289
  return {
226
- snapshot_done: syncConfig.snapshot_done ?? false,
227
- snapshot_lsn: doc.snapshot_lsn ?? null,
228
- active: doc.state == storage.SyncRuleState.ACTIVE && syncConfig.state == storage.SyncRuleState.ACTIVE,
229
- checkpoint_lsn: syncConfig.last_checkpoint_lsn ?? null,
230
- keepalive_op: syncConfig.keepalive_op ?? null
290
+ snapshotDone:
291
+ syncConfigs.every((config) => config.snapshot_done ?? false) &&
292
+ syncConfigs.every((config) => config.last_checkpoint_lsn != null),
293
+ resumeLsn: doc.resume_lsn ?? null
231
294
  };
232
295
  }
233
296
 
234
297
  protected async clearSyncRuleState(): Promise<void> {
298
+ // Clearing resets the entire replication stream (bucket data and the op sequence), so reset
299
+ // the checkpoint state for _all_ embedded sync configs, not only the ones selected for this
300
+ // storage instance. This maintains the invariant that no config has a last_checkpoint past
301
+ // the stream-level last_persisted_op.
235
302
  await this.syncRulesCollection.updateOne(
236
- this.syncConfigMatch(),
303
+ { _id: this.replicationStreamId },
237
304
  {
238
305
  $set: {
239
306
  persisted_lsn: null,
240
- 'sync_configs.$[config].snapshot_done': false,
241
- 'sync_configs.$[config].last_checkpoint_lsn': null,
242
- 'sync_configs.$[config].last_checkpoint': null,
243
- 'sync_configs.$[config].no_checkpoint_before': null,
244
- 'sync_configs.$[config].keepalive_op': null
307
+ 'sync_configs.$[].snapshot_done': false,
308
+ 'sync_configs.$[].last_checkpoint_lsn': null,
309
+ 'sync_configs.$[].last_checkpoint': null,
310
+ 'sync_configs.$[].no_checkpoint_before': null
245
311
  },
246
312
  $unset: {
247
- snapshot_lsn: 1
313
+ resume_lsn: 1,
314
+ last_persisted_op: 1
248
315
  }
249
316
  },
250
317
  {
251
- maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
252
- arrayFilters: this.syncConfigArrayFilters()
318
+ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
253
319
  }
254
320
  );
255
321
  }
256
322
 
257
- protected override get versionContext(): MongoSyncBucketStorageContext<VersionedPowerSyncMongoV3> {
323
+ /**
324
+ * The persisted mapping of the single sync config that read operations are served from.
325
+ *
326
+ * Reads always operate on a single sync config: read-path storage instances are
327
+ * constructed for the active sync config only (see MongoBucketStorage.getActiveSyncConfig),
328
+ * and checkpoints are served from the single active config (see fetchCheckpointState).
329
+ * Within a single sync config, unique names are the persistence key of its rule mapping,
330
+ * so its name-keyed {@link SingleSyncConfigBucketDefinitionMapping} resolves sources from
331
+ * any parse of that config unambiguously - no parsed-set identity is required.
332
+ *
333
+ * Throws on storage instances with multiple sync configs (replication-side instances),
334
+ * which must not serve reads.
335
+ */
336
+ private singleSyncConfigMapping(): SingleSyncConfigBucketDefinitionMapping {
337
+ const content = this.replicationStream.syncConfigContent;
338
+ if (content.length != 1) {
339
+ throw new ServiceAssertionError(
340
+ `Read operations require a storage instance with a single sync config, got ${content.length}`
341
+ );
342
+ }
343
+ return content[0].mapping;
344
+ }
345
+
346
+ protected get versionContext(): MongoSyncBucketStorageContextV3 {
347
+ const self = this;
258
348
  return {
259
349
  db: this.db,
260
- group_id: this.group_id,
261
- mapping: this.mapping
350
+ objectStorage: this.objectStorage,
351
+ replicationStreamId: this.replicationStreamId,
352
+ readPreference: this.readPreference,
353
+ get mapping() {
354
+ return self.singleSyncConfigMapping();
355
+ }
262
356
  };
263
357
  }
264
358
 
@@ -271,34 +365,47 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
271
365
  }
272
366
 
273
367
  protected getBucketDataBatchImpl(
274
- checkpoint: utils.InternalOpId,
368
+ checkpoint: MongoSyncBucketStorageCheckpoint,
275
369
  dataBuckets: storage.BucketDataRequest[],
276
370
  options?: storage.BucketDataBatchOptions
277
- ): AsyncIterable<storage.SyncBucketDataChunk> {
371
+ ): AsyncIterable<storage.SyncBucketDataChunk | storage.SyncBucketDataBatchEnd> {
278
372
  return getBucketDataBatchV3(this.versionContext, checkpoint, dataBuckets, options);
279
373
  }
280
374
 
281
- protected async clearBucketData(_signal?: AbortSignal): Promise<void> {
282
- for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) {
375
+ protected async clearBucketData(signal?: AbortSignal): Promise<void> {
376
+ for (const collection of await this.db.listBucketDataCollections(this.replicationStreamId)) {
283
377
  await collection.drop();
284
378
  }
379
+ if (this.objectStorage) {
380
+ const lifecycle = new ObjectStorageLifecycle(this.db, this.replicationStreamId, this.objectStorage);
381
+ await lifecycle.deletePrefix(lifecycle.streamPrefix(), { signal });
382
+ }
383
+ await this.db
384
+ .pendingObjectStorageDeletes(this.replicationStreamId)
385
+ .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
386
+ .catch((error) => {
387
+ if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
388
+ return;
389
+ }
390
+ throw error;
391
+ });
285
392
  }
286
393
 
287
394
  protected async clearParameterIndexes(_signal?: AbortSignal): Promise<void> {
288
- for (const collection of await this.db.listParameterIndexCollectionsV3(this.group_id)) {
395
+ for (const collection of await this.db.listParameterIndexCollections(this.replicationStreamId)) {
289
396
  await collection.collection.drop();
290
397
  }
291
398
  }
292
399
 
293
400
  protected async clearSourceRecords(_signal?: AbortSignal): Promise<void> {
294
- for (const collection of await this.db.listSourceRecordCollectionsV3(this.group_id)) {
401
+ for (const collection of await this.db.listSourceRecordCollections(this.replicationStreamId)) {
295
402
  await collection.drop();
296
403
  }
297
404
  }
298
405
 
299
406
  protected async clearBucketState(_signal?: AbortSignal): Promise<void> {
300
407
  await this.db
301
- .bucketStateV3(this.group_id)
408
+ .bucketState(this.replicationStreamId)
302
409
  .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
303
410
  .catch((error) => {
304
411
  if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
@@ -310,7 +417,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
310
417
 
311
418
  protected async clearSourceTables(_signal?: AbortSignal): Promise<void> {
312
419
  await this.db
313
- .sourceTablesV3(this.group_id)
420
+ .sourceTables(this.replicationStreamId)
314
421
  .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
315
422
  .catch((error) => {
316
423
  if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
@@ -320,6 +427,21 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
320
427
  });
321
428
  }
322
429
 
430
+ async cleanupStoppedSyncConfigs(
431
+ options: storage.CleanupStoppedSyncConfigsOptions
432
+ ): Promise<storage.CleanupStoppedSyncConfigsResult> {
433
+ return new MongoStoppedSyncConfigCleanup({
434
+ db: this.db,
435
+ replicationStreamId: this.replicationStreamId,
436
+ signal: options.signal,
437
+ logger: options.logger ?? this.logger,
438
+ defaultSchema: options.defaultSchema,
439
+ sourceConnectionTag: options.sourceConnectionTag,
440
+ objectStorage: this.objectStorage,
441
+ clearBatchThrottleRate: this.clearBatchThrottleRate
442
+ }).run();
443
+ }
444
+
323
445
  protected getDataBucketChangesImpl(
324
446
  options: GetCheckpointChangesOptions
325
447
  ): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>> {
@@ -334,7 +456,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
334
456
  }
335
457
 
336
458
  export async function getParameterSetsV3(
337
- ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV3>,
459
+ ctx: MongoSyncBucketStorageContextV3,
338
460
  checkpoint: MongoSyncBucketStorageCheckpoint,
339
461
  lookups: ScopedParameterLookup[],
340
462
  limit: number
@@ -350,8 +472,8 @@ export async function getParameterSetsV3(
350
472
  pipeline: mongo.Document[];
351
473
  } => {
352
474
  const indexId = lookup.indexId;
353
- const collection = ctx.db.parameterIndexV3(ctx.group_id, indexId);
354
- const lookupFilter = serializeParameterLookupV3(lookup);
475
+ const collection = ctx.db.parameterIndex(ctx.replicationStreamId, indexId);
476
+ const lookupFilter = serializeParameterLookup(lookup);
355
477
 
356
478
  return {
357
479
  collection,
@@ -434,24 +556,38 @@ export async function getParameterSetsV3(
434
556
  }
435
557
 
436
558
  export async function* getBucketDataBatchV3(
437
- ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV3>,
438
- checkpoint: utils.InternalOpId,
559
+ ctx: MongoSyncBucketStorageContextV3,
560
+ checkpoint: MongoSyncBucketStorageCheckpoint,
439
561
  dataBuckets: storage.BucketDataRequest[],
440
562
  options?: storage.BucketDataBatchOptions
441
- ): AsyncIterable<storage.SyncBucketDataChunk> {
563
+ ): AsyncIterable<storage.SyncBucketDataChunk | storage.SyncBucketDataBatchEnd> {
442
564
  if (dataBuckets.length == 0) {
443
565
  return;
444
566
  }
445
567
 
446
- if (checkpoint == null) {
568
+ if (checkpoint.checkpoint == null) {
447
569
  throw new Error('checkpoint is null');
448
570
  }
449
571
 
572
+ const readPreference = options?.requestHint == 'bulk' ? ctx.readPreference : undefined;
573
+ const readConcern = ctx.readPreference == null ? undefined : 'majority';
574
+ const session =
575
+ readPreference == null || checkpoint.snapshotTime == null
576
+ ? undefined
577
+ : ctx.db.client.startSession({ causalConsistency: true });
578
+ await using _ = { [Symbol.asyncDispose]: async () => session?.endSession() };
579
+
580
+ if (session != null) {
581
+ session.advanceOperationTime(checkpoint.snapshotTime);
582
+ session.advanceClusterTime(checkpoint.clusterTime);
583
+ }
584
+
450
585
  const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
451
586
  const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
452
- const end = checkpoint;
453
- let remainingLimit = batchLimit;
587
+ const end = checkpoint.checkpoint;
454
588
 
589
+ // Group requests by definition, so that we can query each definition's bucket data in a single query.
590
+ // We only return the results of a single query per batch.
455
591
  const requestsByDefinition = new Map<string, storage.BucketDataRequest[]>();
456
592
  for (const request of dataBuckets) {
457
593
  const definitionId = ctx.mapping.bucketSourceId(request.source);
@@ -461,62 +597,80 @@ export async function* getBucketDataBatchV3(
461
597
  }
462
598
 
463
599
  const definitionGroups = Array.from(requestsByDefinition.entries());
464
- for (let groupIndex = 0; groupIndex < definitionGroups.length && remainingLimit > 0; groupIndex++) {
465
- const [definitionId, requests] = definitionGroups[groupIndex];
600
+ for (const [groupIndex, [definitionId, requests]] of definitionGroups.entries()) {
466
601
  const hasLaterDefinitionGroups = groupIndex < definitionGroups.length - 1;
467
602
  const bucketMap = new Map(requests.map((request) => [request.bucket, request.start]));
468
- const filters: mongo.Filter<BucketDataDocumentV3>[] = Array.from(bucketMap.entries()).map(([bucket, start]) => ({
603
+ const filters = Array.from(bucketMap.entries()).map(([bucket, start]) => ({
469
604
  _id: {
470
- $gt: {
471
- b: bucket,
472
- o: start
473
- },
474
- $lte: {
475
- b: bucket,
476
- o: end as any
477
- }
478
- }
479
- }));
480
-
481
- const cursor = ctx.db.bucketDataV3(ctx.group_id, definitionId).find(
482
- {
483
- $or: filters
605
+ $gt: { b: bucket, o: start },
606
+ $lte: { b: bucket, o: new bson.MaxKey() }
484
607
  },
485
- {
486
- session: undefined,
487
- sort: { _id: 1 },
488
- limit: remainingLimit,
489
- batchSize: remainingLimit + 1,
490
- raw: true,
491
- maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
492
- }
493
- ) as unknown as mongo.FindCursor<Buffer>;
494
-
495
- let { data, hasMore: batchHasMore } = await readSingleBatch(cursor).catch((e) => {
608
+ min_op: { $lte: end }
609
+ // MongoDB Filter<T> doesn't accept compound _id ranges or dotted field paths in its type.
610
+ })) as unknown as mongo.Filter<BucketDataDocumentV3>[];
611
+
612
+ const collection = ctx.db.bucketData(ctx.replicationStreamId, definitionId);
613
+ // MongoDB Filter<T> doesn't accept the $or operator in its type.
614
+ const filter = { $or: filters } as unknown as mongo.Filter<BucketDataDocumentV3>;
615
+ const context = { replicationStreamId: ctx.replicationStreamId, definitionId };
616
+
617
+ const cursorOptions = { limit: batchLimit, batchSize: batchLimit + 1 };
618
+
619
+ // raw: true returns Buffers, but the driver typing doesn't reflect that
620
+ // without an explicit cast to FindCursor<Buffer>.
621
+ const cursor = collection.find(filter, {
622
+ session,
623
+ readPreference,
624
+ readConcern,
625
+ sort: { _id: 1 },
626
+ raw: true,
627
+ maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS,
628
+ ...cursorOptions
629
+ }) as unknown as mongo.FindCursor<Buffer>;
630
+
631
+ let { data: rawData, hasMore } = await readSingleBatch(cursor).catch((e: unknown) => {
496
632
  throw lib_mongo.mapQueryError(e, 'while reading bucket data');
497
633
  });
498
- if (data.length == remainingLimit) {
499
- batchHasMore = true;
634
+
635
+ if (rawData.length >= cursorOptions.limit) {
636
+ hasMore = true;
500
637
  }
501
- if (data.length == 0) {
502
- continue;
638
+
639
+ // Deserialize the raw documents and cut the batch to a bounded size. Any data not
640
+ // making the cut will be read in the next round, using a fresh query.
641
+ const cutBatch = cutBucketDataBatch(rawData);
642
+ const docs = cutBatch.documents;
643
+ if (cutBatch.wasCut) {
644
+ hasMore = true;
503
645
  }
504
646
 
505
- remainingLimit -= data.length;
647
+ // Hydrate any operations from object storage.
648
+ // In the future we can do this in a more pipelined fashion, but for now we hydrate
649
+ // the entire batch at once.
650
+ await hydrateBucketDataDocuments(docs, ctx.objectStorage, { signal: options?.signal });
506
651
 
507
- let chunkSizeBytes = 0;
652
+ let currentChunkSizeBytes = 0;
508
653
  let currentChunk: utils.SyncBucketData | null = null;
509
654
  let targetOp: InternalOpId | null = null;
510
-
511
- for (let rawData of data) {
512
- const row = loadBucketDataDocumentV3(
513
- { replicationStreamId: ctx.group_id, definitionId },
514
- bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV3
515
- );
516
- const bucket = row.bucketKey.bucket;
517
-
518
- if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) {
519
- let start: ProtocolOpId | undefined = undefined;
655
+ let seenBuckets = new Set<string>();
656
+ const batchHasMore = hasMore;
657
+
658
+ for (const doc of docs) {
659
+ const bucket = doc._id.b;
660
+ seenBuckets.add(bucket);
661
+ const bucketStart = bucketMap.get(bucket);
662
+ if (bucketStart == null) {
663
+ throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`);
664
+ }
665
+ let start: ProtocolOpId | undefined = undefined;
666
+
667
+ // Reached a new bucket or size limit: yield the current chunk and start a new one.
668
+ if (
669
+ currentChunk == null ||
670
+ currentChunk.bucket != bucket ||
671
+ currentChunkSizeBytes >= chunkSizeLimitBytes ||
672
+ currentChunk.data.length >= batchLimit
673
+ ) {
520
674
  if (currentChunk != null) {
521
675
  if (currentChunk.bucket == bucket) {
522
676
  currentChunk.has_more = true;
@@ -525,18 +679,15 @@ export async function* getBucketDataBatchV3(
525
679
 
526
680
  const yieldChunk = currentChunk;
527
681
  currentChunk = null;
528
- chunkSizeBytes = 0;
529
- yield { chunkData: yieldChunk, targetOp: targetOp };
682
+ currentChunkSizeBytes = 0;
683
+ yield { chunkData: yieldChunk, targetOp };
530
684
  targetOp = null;
531
685
  }
532
686
 
533
687
  if (start == null) {
534
- const startOpId = bucketMap.get(bucket);
535
- if (startOpId == null) {
536
- throw new Error(`data for unexpected bucket: ${bucket}`);
537
- }
538
- start = internalToExternalOpId(startOpId);
688
+ start = internalToExternalOpId(bucketStart);
539
689
  }
690
+
540
691
  currentChunk = {
541
692
  bucket,
542
693
  after: start,
@@ -546,35 +697,58 @@ export async function* getBucketDataBatchV3(
546
697
  };
547
698
  }
548
699
 
549
- const entry = mapOpEntry(row);
550
- if (row.target_op != null && (targetOp == null || row.target_op > targetOp)) {
551
- targetOp = row.target_op;
552
- }
553
-
554
- currentChunk.data.push(entry);
555
- currentChunk.next_after = entry.op_id;
556
- chunkSizeBytes += rawData.byteLength;
700
+ const rows = extractRowsFromDocument(doc, context, bucketStart, end);
701
+ currentChunk.data.push(...rows.map(mapOpEntry));
702
+ currentChunk.next_after = currentChunk.data.at(-1)?.op_id ?? internalToExternalOpId(end);
703
+ targetOp = maxOpId(targetOp, doc.target_op);
704
+ currentChunkSizeBytes += doc.size;
557
705
  }
558
706
 
559
707
  if (currentChunk != null) {
560
708
  const yieldChunk = currentChunk;
561
- yieldChunk.has_more = batchHasMore || (remainingLimit <= 0 && hasLaterDefinitionGroups);
562
- yield { chunkData: yieldChunk, targetOp: targetOp };
709
+ // The last chunk may contain more data that was cut in this batch.
710
+ yieldChunk.has_more = batchHasMore;
711
+ yield { chunkData: yieldChunk, targetOp };
563
712
  }
564
713
 
565
- if (batchHasMore || remainingLimit <= 0) {
566
- return;
714
+ if (!batchHasMore) {
715
+ for (const bucket of bucketMap.keys()) {
716
+ if (!seenBuckets.has(bucket)) {
717
+ // We processed everything for this definition group, but this bucket had no data in the batch.
718
+ // Yield an empty chunk to indicate that it is complete.
719
+ // This prevents re-querying the same bucket in the next batch.
720
+ yield {
721
+ chunkData: {
722
+ bucket,
723
+ after: internalToExternalOpId(bucketMap.get(bucket)!),
724
+ has_more: false,
725
+ data: [],
726
+ next_after: internalToExternalOpId(end)
727
+ },
728
+ targetOp
729
+ };
730
+ }
731
+ }
732
+ }
733
+
734
+ if (currentChunk != null) {
735
+ // We yielded data in this group (aside from empty buckets).
736
+ // Return to the caller to allow them to process it before continuing to the next group.
737
+ yield { hasMore: batchHasMore || hasLaterDefinitionGroups };
738
+ break;
739
+ } else {
740
+ // No data in this definition group - continue in the next group.
567
741
  }
568
742
  }
569
743
  }
570
744
 
571
745
  export async function getDataBucketChangesV3(
572
- ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV3>,
746
+ ctx: MongoSyncBucketStorageContextV3,
573
747
  options: GetCheckpointChangesOptions
574
748
  ): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>> {
575
749
  const limit = 1000;
576
750
  const bucketStateUpdates = await ctx.db
577
- .bucketStateV3(ctx.group_id)
751
+ .bucketState(ctx.replicationStreamId)
578
752
  .aggregate<{ _id: string; last_op: bigint }>(
579
753
  [
580
754
  {
@@ -611,14 +785,14 @@ export async function getDataBucketChangesV3(
611
785
  }
612
786
 
613
787
  export async function getParameterBucketChangesV3(
614
- ctx: MongoSyncBucketStorageContext<VersionedPowerSyncMongoV3>,
788
+ ctx: MongoSyncBucketStorageContextV3,
615
789
  options: GetCheckpointChangesOptions
616
790
  ): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
617
791
  const limit = 1000;
618
792
  const indexIds = ctx.mapping.allParameterIndexIds();
619
793
  const collections = indexIds.map((indexId) => ({
620
794
  indexId,
621
- collection: ctx.db.parameterIndexV3(ctx.group_id, indexId)
795
+ collection: ctx.db.parameterIndex(ctx.replicationStreamId, indexId)
622
796
  }));
623
797
  if (collections.length == 0) {
624
798
  return {
@@ -671,8 +845,6 @@ export async function getParameterBucketChangesV3(
671
845
  invalidateParameterBuckets: invalidateParameterUpdates,
672
846
  updatedParameterLookups: invalidateParameterUpdates
673
847
  ? new Set<string>()
674
- : new Set<string>(
675
- parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookupV3(p.lookup, p.indexId)))
676
- )
848
+ : new Set<string>(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookup(p.lookup, p.indexId))))
677
849
  };
678
850
  }