@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/CHANGELOG.md +101 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +41 -16
  6. package/dist/storage/MongoBucketStorage.js +226 -71
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
  9. package/dist/storage/implementation/MongoBucketBatch.js +110 -62
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
  15. package/dist/storage/implementation/MongoCompactor.js +80 -263
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
  21. package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +39 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  46. package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
  47. package/dist/storage/implementation/common/PersistedBatch.js +12 -0
  48. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  49. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  51. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  52. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  53. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  54. package/dist/storage/implementation/db.js +17 -15
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +20 -13
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  62. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  63. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
  65. package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
  66. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  71. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  72. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +10 -1
  77. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  78. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  79. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  81. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
  82. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  84. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  85. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  86. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
  87. package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
  88. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
  90. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
  91. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
  93. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
  94. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
  96. package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +77 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +30 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +101 -21
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
  115. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  126. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  128. package/dist/storage/storage-index.d.ts +4 -4
  129. package/dist/storage/storage-index.js +4 -4
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +49 -3
  132. package/dist/types/types.js +48 -1
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +1 -2
  135. package/dist/utils/test-utils.js +7 -1
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.d.ts +9 -2
  138. package/dist/utils/util.js +128 -4
  139. package/dist/utils/util.js.map +1 -1
  140. package/package.json +12 -19
  141. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  142. package/src/storage/MongoBucketStorage.ts +347 -88
  143. package/src/storage/implementation/MongoBucketBatch.ts +147 -82
  144. package/src/storage/implementation/MongoChecksums.ts +93 -153
  145. package/src/storage/implementation/MongoCompactor.ts +102 -316
  146. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  147. package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
  148. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  149. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  150. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  151. package/src/storage/implementation/MongoStorageProvider.ts +46 -4
  152. package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
  153. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  154. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
  156. package/src/storage/implementation/common/PersistedBatch.ts +19 -3
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  158. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  159. package/src/storage/implementation/db.ts +19 -20
  160. package/src/storage/implementation/models.ts +20 -14
  161. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
  162. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  163. package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
  164. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
  165. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  166. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  167. package/src/storage/implementation/v1/models.ts +11 -1
  168. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  169. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
  170. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  171. package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
  172. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
  173. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
  174. package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
  175. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  176. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
  177. package/src/storage/implementation/v3/bucket-format.ts +97 -0
  178. package/src/storage/implementation/v3/chunking.ts +40 -0
  179. package/src/storage/implementation/v3/models.ts +132 -55
  180. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
  181. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
  184. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  185. package/src/storage/storage-index.ts +12 -4
  186. package/src/types/types.ts +58 -1
  187. package/src/utils/test-utils.ts +8 -3
  188. package/src/utils/util.ts +185 -6
  189. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
  191. package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
  192. package/test/src/helpers/MemoryObjectStorage.ts +53 -0
  193. package/test/src/helpers/s3TestFactory.ts +69 -0
  194. package/test/src/object_storage_lifecycle.test.ts +59 -0
  195. package/test/src/setup.ts +6 -1
  196. package/test/src/storage.test.ts +227 -13
  197. package/test/src/storage_compacting.test.ts +1951 -12
  198. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  199. package/test/src/storage_s3_checksums.test.ts +191 -0
  200. package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
  201. package/test/src/storage_s3_reading.test.ts +526 -0
  202. package/test/src/storage_s3_writing.test.ts +135 -0
  203. package/test/src/storage_sync.test.ts +1180 -73
  204. package/test/src/storeCurrentData.test.ts +227 -0
  205. package/test/src/util.ts +2 -1
  206. package/tsconfig.json +1 -4
  207. package/tsconfig.tsbuildinfo +1 -1
  208. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  209. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  210. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  211. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  212. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  213. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  214. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  215. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  216. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  217. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  218. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  219. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  220. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  221. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  222. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  223. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  224. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  225. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  226. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  227. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  228. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  229. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  230. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  231. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  232. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  233. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  234. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  235. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  236. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  237. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  238. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  239. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -1,38 +1,75 @@
1
1
  import * as lib_mongo from '@powersync/lib-service-mongodb';
2
2
  import { ServiceAssertionError } from '@powersync/lib-services-framework';
3
- import { internalToExternalOpId, maxLsn, ParameterSetLimitExceededError, storage } from '@powersync/service-core';
3
+ import { internalToExternalOpId, ParameterSetLimitExceededError, storage } from '@powersync/service-core';
4
4
  import { JSONBig } from '@powersync/service-jsonbig';
5
5
  import * as bson from 'bson';
6
6
  import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
7
- import { MongoPersistedSyncRulesContentV3 } from '../MongoPersistedSyncRulesContent.js';
7
+ import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
8
8
  import { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js';
9
- import { loadBucketDataDocumentV3 } from './models.js';
9
+ import { loadBucketDataDocument, maxOpId } from './bucket-format.js';
10
+ import { deserializeParameterLookup, serializeParameterLookup } from './models.js';
10
11
  import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js';
11
12
  import { MongoChecksumsV3 } from './MongoChecksumsV3.js';
12
13
  import { MongoCompactorV3 } from './MongoCompactorV3.js';
13
- import { MongoParameterCompactorV3 } from './MongoParameterCompactorV3.js';
14
- import { deserializeParameterLookupV3, serializeParameterLookupV3 } from './MongoParameterLookupV3.js';
14
+ import { MongoStoppedSyncConfigCleanup } from './MongoStoppedSyncConfigCleanup.js';
15
+ import { hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js';
16
+ import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
17
+ const BUCKET_DATA_FETCH_BATCH_LIMIT_BYTES = 16 * 1024 * 1024;
18
+ /**
19
+ * Keep the documents hydrated for one sync response within a bounded payload or slightly higher.
20
+ *
21
+ * This deserializes on-demand, so no deserialization is performed for discarded data.
22
+ */
23
+ function cutBucketDataBatch(rawDocuments) {
24
+ let cumulativeBytes = 0;
25
+ let documents = [];
26
+ for (const raw of rawDocuments) {
27
+ const doc = bson.deserialize(raw, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS);
28
+ documents.push(doc);
29
+ cumulativeBytes += doc.size;
30
+ if (cumulativeBytes > BUCKET_DATA_FETCH_BATCH_LIMIT_BYTES) {
31
+ return {
32
+ documents,
33
+ wasCut: documents.length < rawDocuments.length
34
+ };
35
+ }
36
+ }
37
+ return { documents, wasCut: false };
38
+ }
39
+ function extractRowsFromDocument(doc, context, bucketStart, endOpId) {
40
+ const rows = [];
41
+ for (const row of loadBucketDataDocument(context, doc)) {
42
+ // In theory a binary search could be faster than a linear scan to find the start.
43
+ // In practice, most cases should not filter out anything here.
44
+ if (row.o <= bucketStart) {
45
+ continue;
46
+ }
47
+ if (row.o > endOpId) {
48
+ break;
49
+ }
50
+ rows.push(row);
51
+ }
52
+ return rows;
53
+ }
15
54
  export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
16
- syncRulesV3;
17
- constructor(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options) {
18
- super(factory, group_id, sync_rules, slot_name, writeCheckpointMode, options);
19
- if (!(sync_rules instanceof MongoPersistedSyncRulesContentV3)) {
55
+ constructor(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options) {
56
+ super(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
57
+ if (replicationStream.syncConfigIds.length == 0) {
20
58
  throw new ServiceAssertionError('Missing sync config id for storage v3');
21
59
  }
22
- this.syncRulesV3 = sync_rules;
23
60
  }
24
- get syncConfigId() {
25
- return this.syncRulesV3.syncConfigId;
61
+ get syncConfigIds() {
62
+ return this.replicationStream.syncConfigIds;
26
63
  }
27
64
  get syncRulesCollection() {
28
65
  return this.db.sync_rules;
29
66
  }
30
67
  syncConfigMatch(extra = {}) {
31
68
  return {
32
- _id: this.group_id,
69
+ _id: this.replicationStreamId,
33
70
  sync_configs: {
34
71
  $elemMatch: {
35
- _id: this.syncConfigId,
72
+ _id: { $in: this.syncConfigIds },
36
73
  ...extra
37
74
  }
38
75
  }
@@ -41,39 +78,25 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
41
78
  syncConfigProjection(extra = {}) {
42
79
  return {
43
80
  ...extra,
44
- sync_configs: {
45
- $elemMatch: {
46
- _id: this.syncConfigId
47
- }
48
- }
81
+ sync_configs: 1
49
82
  };
50
83
  }
51
- syncConfigArrayFilters() {
52
- return [{ 'config._id': this.syncConfigId }];
53
- }
54
- /**
55
- * For now, we only support a single sync config per replication stream.
56
- *
57
- * In the future we'll add support for multiple.
58
- */
59
- selectedSyncConfig(doc) {
60
- return doc?.sync_configs?.[0] ?? null;
84
+ selectedSyncConfigs(doc) {
85
+ return doc?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ?? [];
61
86
  }
62
87
  async initializeVersionStorage() {
63
- const mapping = this.mapping;
64
- for (let source of mapping.allBucketDefinitionIds()) {
65
- const collection = this.db.bucketDataV3(this.group_id, source).collectionName;
66
- await this.db.db
67
- .createCollection(collection, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } })
68
- .catch((error) => {
88
+ const storageIds = this.storageIds;
89
+ for (const source of storageIds.bucketDefinitionIds) {
90
+ const collection = this.db.bucketData(this.replicationStreamId, source).collectionName;
91
+ await this.db.db.createCollection(collection, {}).catch((error) => {
69
92
  if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceExists') {
70
93
  return;
71
94
  }
72
95
  throw error;
73
96
  });
74
97
  }
75
- for (let indexId of mapping.allParameterIndexIds()) {
76
- await this.db.parameterIndexV3(this.group_id, indexId).createIndex({
98
+ for (const indexId of storageIds.parameterIndexIds) {
99
+ await this.db.parameterIndex(this.replicationStreamId, indexId).createIndex({
77
100
  lookup: 1,
78
101
  key: 1,
79
102
  _id: -1
@@ -83,20 +106,34 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
83
106
  }
84
107
  }
85
108
  createMongoChecksums(options) {
86
- return new MongoChecksumsV3(this.db, this.group_id, {
109
+ return new MongoChecksumsV3(this.db, this.replicationStreamId, {
87
110
  ...options.checksumOptions,
111
+ checksumCacheTtlMs: options.checksumCacheTtlMs,
88
112
  storageConfig: options?.storageConfig,
89
- mapping: this.sync_rules.mapping
113
+ syncConfigMapping: () => this.singleSyncConfigMapping()
90
114
  });
91
115
  }
92
116
  createMongoCompactor(options) {
93
117
  return new MongoCompactorV3(this, this.db, options);
94
118
  }
95
119
  createMongoParameterCompactor(checkpoint, options) {
96
- return new MongoParameterCompactorV3(this.db, this.group_id, checkpoint, options);
97
- }
98
- createWriterImpl(batchOptions) {
99
- return new MongoBucketBatchV3(batchOptions);
120
+ return new MongoParameterCompactor(this.db, this.replicationStreamId, checkpoint, options, () => this.db
121
+ .listParameterIndexCollections(this.replicationStreamId)
122
+ .then((collections) => collections.map((c) => c.collection)));
123
+ }
124
+ async fetchPersistedOpHead() {
125
+ const doc = await this.syncRulesCollection.findOne({ _id: this.replicationStreamId }, { projection: { last_persisted_op: 1 } });
126
+ return doc?.last_persisted_op == null ? null : BigInt(doc.last_persisted_op);
127
+ }
128
+ async createWriterImpl(options) {
129
+ const doc = await this.syncRulesCollection.findOne({ _id: this.replicationStreamId }, { projection: { resume_lsn: 1 } });
130
+ return new MongoBucketBatchV3({
131
+ ...this.writerBatchOptions(options),
132
+ // The stream-level replication position - per-config checkpoint LSNs are consistency
133
+ // markers and do not affect where replication resumes.
134
+ resumeFromLsn: doc?.resume_lsn ?? null,
135
+ syncConfigIds: this.syncConfigIds
136
+ });
100
137
  }
101
138
  async fetchCheckpointState(session) {
102
139
  const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch({
@@ -105,8 +142,16 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
105
142
  session,
106
143
  projection: this.syncConfigProjection()
107
144
  });
108
- const syncConfig = this.selectedSyncConfig(doc);
109
- if (!syncConfig?.snapshot_done) {
145
+ // Checkpoints are served from the single active config. A PROCESSING config in the same
146
+ // stream (incremental reprocessing) does not affect checkpoints until it is activated.
147
+ const syncConfigs = this.selectedSyncConfigs(doc).filter((config) => [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED].includes(config.state));
148
+ if (syncConfigs.length > 1) {
149
+ // Activation atomically replaces the previous active config, so this cannot happen unless
150
+ // the stored state is corrupt.
151
+ throw new ServiceAssertionError(`Expected a single active sync config, got ${syncConfigs.map((config) => config._id.toHexString()).join(', ')}`);
152
+ }
153
+ const syncConfig = syncConfigs[0];
154
+ if (syncConfig == null || !syncConfig.snapshot_done) {
110
155
  return null;
111
156
  }
112
157
  return {
@@ -114,22 +159,9 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
114
159
  lsn: syncConfig.last_checkpoint_lsn ?? null
115
160
  };
116
161
  }
117
- async getWriterSyncState() {
118
- const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), {
119
- projection: this.syncConfigProjection({ snapshot_lsn: 1 })
120
- });
121
- const syncConfig = this.selectedSyncConfig(doc);
122
- const checkpointLsn = syncConfig?.last_checkpoint_lsn ?? null;
123
- return {
124
- lastCheckpointLsn: checkpointLsn,
125
- resumeFromLsn: maxLsn(checkpointLsn, doc?.snapshot_lsn),
126
- keepaliveOp: syncConfig?.keepalive_op ?? null,
127
- syncConfigId: this.syncConfigId
128
- };
129
- }
130
162
  async terminateSyncRuleState() {
131
163
  await this.db.sync_rules.updateOne({
132
- _id: this.group_id
164
+ _id: this.replicationStreamId
133
165
  }, {
134
166
  $set: {
135
167
  state: storage.SyncRuleState.TERMINATED,
@@ -140,43 +172,69 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
140
172
  }
141
173
  async getStatusImpl() {
142
174
  const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), {
143
- projection: this.syncConfigProjection({ state: 1, snapshot_lsn: 1, keepalive_op: 1 })
175
+ projection: this.syncConfigProjection({ resume_lsn: 1 })
144
176
  });
145
- const syncConfig = this.selectedSyncConfig(doc);
146
- if (doc == null || syncConfig == null) {
177
+ const syncConfigs = this.selectedSyncConfigs(doc);
178
+ if (doc == null || syncConfigs.length == 0) {
147
179
  throw new ServiceAssertionError('Cannot find replication stream status');
148
180
  }
149
181
  return {
150
- snapshot_done: syncConfig.snapshot_done ?? false,
151
- snapshot_lsn: doc.snapshot_lsn ?? null,
152
- active: doc.state == storage.SyncRuleState.ACTIVE && syncConfig.state == storage.SyncRuleState.ACTIVE,
153
- checkpoint_lsn: syncConfig.last_checkpoint_lsn ?? null,
154
- keepalive_op: syncConfig.keepalive_op ?? null
182
+ snapshotDone: syncConfigs.every((config) => config.snapshot_done ?? false) &&
183
+ syncConfigs.every((config) => config.last_checkpoint_lsn != null),
184
+ resumeLsn: doc.resume_lsn ?? null
155
185
  };
156
186
  }
157
187
  async clearSyncRuleState() {
158
- await this.syncRulesCollection.updateOne(this.syncConfigMatch(), {
188
+ // Clearing resets the entire replication stream (bucket data and the op sequence), so reset
189
+ // the checkpoint state for _all_ embedded sync configs, not only the ones selected for this
190
+ // storage instance. This maintains the invariant that no config has a last_checkpoint past
191
+ // the stream-level last_persisted_op.
192
+ await this.syncRulesCollection.updateOne({ _id: this.replicationStreamId }, {
159
193
  $set: {
160
194
  persisted_lsn: null,
161
- 'sync_configs.$[config].snapshot_done': false,
162
- 'sync_configs.$[config].last_checkpoint_lsn': null,
163
- 'sync_configs.$[config].last_checkpoint': null,
164
- 'sync_configs.$[config].no_checkpoint_before': null,
165
- 'sync_configs.$[config].keepalive_op': null
195
+ 'sync_configs.$[].snapshot_done': false,
196
+ 'sync_configs.$[].last_checkpoint_lsn': null,
197
+ 'sync_configs.$[].last_checkpoint': null,
198
+ 'sync_configs.$[].no_checkpoint_before': null
166
199
  },
167
200
  $unset: {
168
- snapshot_lsn: 1
201
+ resume_lsn: 1,
202
+ last_persisted_op: 1
169
203
  }
170
204
  }, {
171
- maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
172
- arrayFilters: this.syncConfigArrayFilters()
205
+ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
173
206
  });
174
207
  }
208
+ /**
209
+ * The persisted mapping of the single sync config that read operations are served from.
210
+ *
211
+ * Reads always operate on a single sync config: read-path storage instances are
212
+ * constructed for the active sync config only (see MongoBucketStorage.getActiveSyncConfig),
213
+ * and checkpoints are served from the single active config (see fetchCheckpointState).
214
+ * Within a single sync config, unique names are the persistence key of its rule mapping,
215
+ * so its name-keyed {@link SingleSyncConfigBucketDefinitionMapping} resolves sources from
216
+ * any parse of that config unambiguously - no parsed-set identity is required.
217
+ *
218
+ * Throws on storage instances with multiple sync configs (replication-side instances),
219
+ * which must not serve reads.
220
+ */
221
+ singleSyncConfigMapping() {
222
+ const content = this.replicationStream.syncConfigContent;
223
+ if (content.length != 1) {
224
+ throw new ServiceAssertionError(`Read operations require a storage instance with a single sync config, got ${content.length}`);
225
+ }
226
+ return content[0].mapping;
227
+ }
175
228
  get versionContext() {
229
+ const self = this;
176
230
  return {
177
231
  db: this.db,
178
- group_id: this.group_id,
179
- mapping: this.mapping
232
+ objectStorage: this.objectStorage,
233
+ replicationStreamId: this.replicationStreamId,
234
+ readPreference: this.readPreference,
235
+ get mapping() {
236
+ return self.singleSyncConfigMapping();
237
+ }
180
238
  };
181
239
  }
182
240
  getParameterSetsImpl(checkpoint, lookups, limit) {
@@ -185,24 +243,37 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
185
243
  getBucketDataBatchImpl(checkpoint, dataBuckets, options) {
186
244
  return getBucketDataBatchV3(this.versionContext, checkpoint, dataBuckets, options);
187
245
  }
188
- async clearBucketData(_signal) {
189
- for (const collection of await this.db.listBucketDataCollectionsV3(this.group_id)) {
246
+ async clearBucketData(signal) {
247
+ for (const collection of await this.db.listBucketDataCollections(this.replicationStreamId)) {
190
248
  await collection.drop();
191
249
  }
250
+ if (this.objectStorage) {
251
+ const lifecycle = new ObjectStorageLifecycle(this.db, this.replicationStreamId, this.objectStorage);
252
+ await lifecycle.deletePrefix(lifecycle.streamPrefix(), { signal });
253
+ }
254
+ await this.db
255
+ .pendingObjectStorageDeletes(this.replicationStreamId)
256
+ .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
257
+ .catch((error) => {
258
+ if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
259
+ return;
260
+ }
261
+ throw error;
262
+ });
192
263
  }
193
264
  async clearParameterIndexes(_signal) {
194
- for (const collection of await this.db.listParameterIndexCollectionsV3(this.group_id)) {
265
+ for (const collection of await this.db.listParameterIndexCollections(this.replicationStreamId)) {
195
266
  await collection.collection.drop();
196
267
  }
197
268
  }
198
269
  async clearSourceRecords(_signal) {
199
- for (const collection of await this.db.listSourceRecordCollectionsV3(this.group_id)) {
270
+ for (const collection of await this.db.listSourceRecordCollections(this.replicationStreamId)) {
200
271
  await collection.drop();
201
272
  }
202
273
  }
203
274
  async clearBucketState(_signal) {
204
275
  await this.db
205
- .bucketStateV3(this.group_id)
276
+ .bucketState(this.replicationStreamId)
206
277
  .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
207
278
  .catch((error) => {
208
279
  if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
@@ -213,7 +284,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
213
284
  }
214
285
  async clearSourceTables(_signal) {
215
286
  await this.db
216
- .sourceTablesV3(this.group_id)
287
+ .sourceTables(this.replicationStreamId)
217
288
  .drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
218
289
  .catch((error) => {
219
290
  if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
@@ -222,6 +293,18 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
222
293
  throw error;
223
294
  });
224
295
  }
296
+ async cleanupStoppedSyncConfigs(options) {
297
+ return new MongoStoppedSyncConfigCleanup({
298
+ db: this.db,
299
+ replicationStreamId: this.replicationStreamId,
300
+ signal: options.signal,
301
+ logger: options.logger ?? this.logger,
302
+ defaultSchema: options.defaultSchema,
303
+ sourceConnectionTag: options.sourceConnectionTag,
304
+ objectStorage: this.objectStorage,
305
+ clearBatchThrottleRate: this.clearBatchThrottleRate
306
+ }).run();
307
+ }
225
308
  getDataBucketChangesImpl(options) {
226
309
  return getDataBucketChangesV3(this.versionContext, options);
227
310
  }
@@ -234,8 +317,8 @@ export async function getParameterSetsV3(ctx, checkpoint, lookups, limit) {
234
317
  setSessionSnapshotTime(session, checkpoint.snapshotTime);
235
318
  const buildLookupPipeline = (lookup, index) => {
236
319
  const indexId = lookup.indexId;
237
- const collection = ctx.db.parameterIndexV3(ctx.group_id, indexId);
238
- const lookupFilter = serializeParameterLookupV3(lookup);
320
+ const collection = ctx.db.parameterIndex(ctx.replicationStreamId, indexId);
321
+ const lookupFilter = serializeParameterLookup(lookup);
239
322
  return {
240
323
  collection,
241
324
  pipeline: [
@@ -313,13 +396,24 @@ export async function* getBucketDataBatchV3(ctx, checkpoint, dataBuckets, option
313
396
  if (dataBuckets.length == 0) {
314
397
  return;
315
398
  }
316
- if (checkpoint == null) {
399
+ if (checkpoint.checkpoint == null) {
317
400
  throw new Error('checkpoint is null');
318
401
  }
402
+ const readPreference = options?.requestHint == 'bulk' ? ctx.readPreference : undefined;
403
+ const readConcern = ctx.readPreference == null ? undefined : 'majority';
404
+ const session = readPreference == null || checkpoint.snapshotTime == null
405
+ ? undefined
406
+ : ctx.db.client.startSession({ causalConsistency: true });
407
+ await using _ = { [Symbol.asyncDispose]: async () => session?.endSession() };
408
+ if (session != null) {
409
+ session.advanceOperationTime(checkpoint.snapshotTime);
410
+ session.advanceClusterTime(checkpoint.clusterTime);
411
+ }
319
412
  const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
320
413
  const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
321
- const end = checkpoint;
322
- let remainingLimit = batchLimit;
414
+ const end = checkpoint.checkpoint;
415
+ // Group requests by definition, so that we can query each definition's bucket data in a single query.
416
+ // We only return the results of a single query per batch.
323
417
  const requestsByDefinition = new Map();
324
418
  for (const request of dataBuckets) {
325
419
  const definitionId = ctx.mapping.bucketSourceId(request.source);
@@ -328,50 +422,68 @@ export async function* getBucketDataBatchV3(ctx, checkpoint, dataBuckets, option
328
422
  requestsByDefinition.set(definitionId, requests);
329
423
  }
330
424
  const definitionGroups = Array.from(requestsByDefinition.entries());
331
- for (let groupIndex = 0; groupIndex < definitionGroups.length && remainingLimit > 0; groupIndex++) {
332
- const [definitionId, requests] = definitionGroups[groupIndex];
425
+ for (const [groupIndex, [definitionId, requests]] of definitionGroups.entries()) {
333
426
  const hasLaterDefinitionGroups = groupIndex < definitionGroups.length - 1;
334
427
  const bucketMap = new Map(requests.map((request) => [request.bucket, request.start]));
335
428
  const filters = Array.from(bucketMap.entries()).map(([bucket, start]) => ({
336
429
  _id: {
337
- $gt: {
338
- b: bucket,
339
- o: start
340
- },
341
- $lte: {
342
- b: bucket,
343
- o: end
344
- }
345
- }
430
+ $gt: { b: bucket, o: start },
431
+ $lte: { b: bucket, o: new bson.MaxKey() }
432
+ },
433
+ min_op: { $lte: end }
434
+ // MongoDB Filter<T> doesn't accept compound _id ranges or dotted field paths in its type.
346
435
  }));
347
- const cursor = ctx.db.bucketDataV3(ctx.group_id, definitionId).find({
348
- $or: filters
349
- }, {
350
- session: undefined,
436
+ const collection = ctx.db.bucketData(ctx.replicationStreamId, definitionId);
437
+ // MongoDB Filter<T> doesn't accept the $or operator in its type.
438
+ const filter = { $or: filters };
439
+ const context = { replicationStreamId: ctx.replicationStreamId, definitionId };
440
+ const cursorOptions = { limit: batchLimit, batchSize: batchLimit + 1 };
441
+ // raw: true returns Buffers, but the driver typing doesn't reflect that
442
+ // without an explicit cast to FindCursor<Buffer>.
443
+ const cursor = collection.find(filter, {
444
+ session,
445
+ readPreference,
446
+ readConcern,
351
447
  sort: { _id: 1 },
352
- limit: remainingLimit,
353
- batchSize: remainingLimit + 1,
354
448
  raw: true,
355
- maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
449
+ maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS,
450
+ ...cursorOptions
356
451
  });
357
- let { data, hasMore: batchHasMore } = await readSingleBatch(cursor).catch((e) => {
452
+ let { data: rawData, hasMore } = await readSingleBatch(cursor).catch((e) => {
358
453
  throw lib_mongo.mapQueryError(e, 'while reading bucket data');
359
454
  });
360
- if (data.length == remainingLimit) {
361
- batchHasMore = true;
455
+ if (rawData.length >= cursorOptions.limit) {
456
+ hasMore = true;
362
457
  }
363
- if (data.length == 0) {
364
- continue;
458
+ // Deserialize the raw documents and cut the batch to a bounded size. Any data not
459
+ // making the cut will be read in the next round, using a fresh query.
460
+ const cutBatch = cutBucketDataBatch(rawData);
461
+ const docs = cutBatch.documents;
462
+ if (cutBatch.wasCut) {
463
+ hasMore = true;
365
464
  }
366
- remainingLimit -= data.length;
367
- let chunkSizeBytes = 0;
465
+ // Hydrate any operations from object storage.
466
+ // In the future we can do this in a more pipelined fashion, but for now we hydrate
467
+ // the entire batch at once.
468
+ await hydrateBucketDataDocuments(docs, ctx.objectStorage, { signal: options?.signal });
469
+ let currentChunkSizeBytes = 0;
368
470
  let currentChunk = null;
369
471
  let targetOp = null;
370
- for (let rawData of data) {
371
- const row = loadBucketDataDocumentV3({ replicationStreamId: ctx.group_id, definitionId }, bson.deserialize(rawData, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS));
372
- const bucket = row.bucketKey.bucket;
373
- if (currentChunk == null || currentChunk.bucket != bucket || chunkSizeBytes >= chunkSizeLimitBytes) {
374
- let start = undefined;
472
+ let seenBuckets = new Set();
473
+ const batchHasMore = hasMore;
474
+ for (const doc of docs) {
475
+ const bucket = doc._id.b;
476
+ seenBuckets.add(bucket);
477
+ const bucketStart = bucketMap.get(bucket);
478
+ if (bucketStart == null) {
479
+ throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`);
480
+ }
481
+ let start = undefined;
482
+ // Reached a new bucket or size limit: yield the current chunk and start a new one.
483
+ if (currentChunk == null ||
484
+ currentChunk.bucket != bucket ||
485
+ currentChunkSizeBytes >= chunkSizeLimitBytes ||
486
+ currentChunk.data.length >= batchLimit) {
375
487
  if (currentChunk != null) {
376
488
  if (currentChunk.bucket == bucket) {
377
489
  currentChunk.has_more = true;
@@ -379,16 +491,12 @@ export async function* getBucketDataBatchV3(ctx, checkpoint, dataBuckets, option
379
491
  }
380
492
  const yieldChunk = currentChunk;
381
493
  currentChunk = null;
382
- chunkSizeBytes = 0;
383
- yield { chunkData: yieldChunk, targetOp: targetOp };
494
+ currentChunkSizeBytes = 0;
495
+ yield { chunkData: yieldChunk, targetOp };
384
496
  targetOp = null;
385
497
  }
386
498
  if (start == null) {
387
- const startOpId = bucketMap.get(bucket);
388
- if (startOpId == null) {
389
- throw new Error(`data for unexpected bucket: ${bucket}`);
390
- }
391
- start = internalToExternalOpId(startOpId);
499
+ start = internalToExternalOpId(bucketStart);
392
500
  }
393
501
  currentChunk = {
394
502
  bucket,
@@ -398,28 +506,52 @@ export async function* getBucketDataBatchV3(ctx, checkpoint, dataBuckets, option
398
506
  next_after: start
399
507
  };
400
508
  }
401
- const entry = mapOpEntry(row);
402
- if (row.target_op != null && (targetOp == null || row.target_op > targetOp)) {
403
- targetOp = row.target_op;
404
- }
405
- currentChunk.data.push(entry);
406
- currentChunk.next_after = entry.op_id;
407
- chunkSizeBytes += rawData.byteLength;
509
+ const rows = extractRowsFromDocument(doc, context, bucketStart, end);
510
+ currentChunk.data.push(...rows.map(mapOpEntry));
511
+ currentChunk.next_after = currentChunk.data.at(-1)?.op_id ?? internalToExternalOpId(end);
512
+ targetOp = maxOpId(targetOp, doc.target_op);
513
+ currentChunkSizeBytes += doc.size;
408
514
  }
409
515
  if (currentChunk != null) {
410
516
  const yieldChunk = currentChunk;
411
- yieldChunk.has_more = batchHasMore || (remainingLimit <= 0 && hasLaterDefinitionGroups);
412
- yield { chunkData: yieldChunk, targetOp: targetOp };
517
+ // The last chunk may contain more data that was cut in this batch.
518
+ yieldChunk.has_more = batchHasMore;
519
+ yield { chunkData: yieldChunk, targetOp };
520
+ }
521
+ if (!batchHasMore) {
522
+ for (const bucket of bucketMap.keys()) {
523
+ if (!seenBuckets.has(bucket)) {
524
+ // We processed everything for this definition group, but this bucket had no data in the batch.
525
+ // Yield an empty chunk to indicate that it is complete.
526
+ // This prevents re-querying the same bucket in the next batch.
527
+ yield {
528
+ chunkData: {
529
+ bucket,
530
+ after: internalToExternalOpId(bucketMap.get(bucket)),
531
+ has_more: false,
532
+ data: [],
533
+ next_after: internalToExternalOpId(end)
534
+ },
535
+ targetOp
536
+ };
537
+ }
538
+ }
539
+ }
540
+ if (currentChunk != null) {
541
+ // We yielded data in this group (aside from empty buckets).
542
+ // Return to the caller to allow them to process it before continuing to the next group.
543
+ yield { hasMore: batchHasMore || hasLaterDefinitionGroups };
544
+ break;
413
545
  }
414
- if (batchHasMore || remainingLimit <= 0) {
415
- return;
546
+ else {
547
+ // No data in this definition group - continue in the next group.
416
548
  }
417
549
  }
418
550
  }
419
551
  export async function getDataBucketChangesV3(ctx, options) {
420
552
  const limit = 1000;
421
553
  const bucketStateUpdates = await ctx.db
422
- .bucketStateV3(ctx.group_id)
554
+ .bucketState(ctx.replicationStreamId)
423
555
  .aggregate([
424
556
  {
425
557
  $match: {
@@ -454,7 +586,7 @@ export async function getParameterBucketChangesV3(ctx, options) {
454
586
  const indexIds = ctx.mapping.allParameterIndexIds();
455
587
  const collections = indexIds.map((indexId) => ({
456
588
  indexId,
457
- collection: ctx.db.parameterIndexV3(ctx.group_id, indexId)
589
+ collection: ctx.db.parameterIndex(ctx.replicationStreamId, indexId)
458
590
  }));
459
591
  if (collections.length == 0) {
460
592
  return {
@@ -502,7 +634,7 @@ export async function getParameterBucketChangesV3(ctx, options) {
502
634
  invalidateParameterBuckets: invalidateParameterUpdates,
503
635
  updatedParameterLookups: invalidateParameterUpdates
504
636
  ? new Set()
505
- : new Set(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookupV3(p.lookup, p.indexId))))
637
+ : new Set(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookup(p.lookup, p.indexId))))
506
638
  };
507
639
  }
508
640
  //# sourceMappingURL=MongoSyncBucketStorageV3.js.map