@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/CHANGELOG.md +101 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +41 -16
  6. package/dist/storage/MongoBucketStorage.js +226 -71
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
  9. package/dist/storage/implementation/MongoBucketBatch.js +110 -62
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
  15. package/dist/storage/implementation/MongoCompactor.js +80 -263
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
  21. package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +39 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  46. package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
  47. package/dist/storage/implementation/common/PersistedBatch.js +12 -0
  48. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  49. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  51. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  52. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  53. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  54. package/dist/storage/implementation/db.js +17 -15
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +20 -13
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  62. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  63. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
  65. package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
  66. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  71. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  72. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +10 -1
  77. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  78. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  79. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  81. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
  82. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  84. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  85. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  86. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
  87. package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
  88. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
  90. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
  91. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
  93. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
  94. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
  96. package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +77 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +30 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +101 -21
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
  115. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  126. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  128. package/dist/storage/storage-index.d.ts +4 -4
  129. package/dist/storage/storage-index.js +4 -4
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +49 -3
  132. package/dist/types/types.js +48 -1
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +1 -2
  135. package/dist/utils/test-utils.js +7 -1
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.d.ts +9 -2
  138. package/dist/utils/util.js +128 -4
  139. package/dist/utils/util.js.map +1 -1
  140. package/package.json +12 -19
  141. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  142. package/src/storage/MongoBucketStorage.ts +347 -88
  143. package/src/storage/implementation/MongoBucketBatch.ts +147 -82
  144. package/src/storage/implementation/MongoChecksums.ts +93 -153
  145. package/src/storage/implementation/MongoCompactor.ts +102 -316
  146. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  147. package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
  148. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  149. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  150. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  151. package/src/storage/implementation/MongoStorageProvider.ts +46 -4
  152. package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
  153. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  154. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
  156. package/src/storage/implementation/common/PersistedBatch.ts +19 -3
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  158. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  159. package/src/storage/implementation/db.ts +19 -20
  160. package/src/storage/implementation/models.ts +20 -14
  161. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
  162. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  163. package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
  164. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
  165. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  166. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  167. package/src/storage/implementation/v1/models.ts +11 -1
  168. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  169. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
  170. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  171. package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
  172. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
  173. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
  174. package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
  175. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  176. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
  177. package/src/storage/implementation/v3/bucket-format.ts +97 -0
  178. package/src/storage/implementation/v3/chunking.ts +40 -0
  179. package/src/storage/implementation/v3/models.ts +132 -55
  180. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
  181. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
  184. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  185. package/src/storage/storage-index.ts +12 -4
  186. package/src/types/types.ts +58 -1
  187. package/src/utils/test-utils.ts +8 -3
  188. package/src/utils/util.ts +185 -6
  189. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
  191. package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
  192. package/test/src/helpers/MemoryObjectStorage.ts +53 -0
  193. package/test/src/helpers/s3TestFactory.ts +69 -0
  194. package/test/src/object_storage_lifecycle.test.ts +59 -0
  195. package/test/src/setup.ts +6 -1
  196. package/test/src/storage.test.ts +227 -13
  197. package/test/src/storage_compacting.test.ts +1951 -12
  198. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  199. package/test/src/storage_s3_checksums.test.ts +191 -0
  200. package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
  201. package/test/src/storage_s3_reading.test.ts +526 -0
  202. package/test/src/storage_s3_writing.test.ts +135 -0
  203. package/test/src/storage_sync.test.ts +1180 -73
  204. package/test/src/storeCurrentData.test.ts +227 -0
  205. package/test/src/util.ts +2 -1
  206. package/tsconfig.json +1 -4
  207. package/tsconfig.tsbuildinfo +1 -1
  208. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  209. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  210. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  211. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  212. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  213. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  214. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  215. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  216. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  217. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  218. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  219. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  220. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  221. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  222. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  223. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  224. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  225. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  226. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  227. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  228. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  229. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  230. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  231. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  232. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  233. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  234. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  235. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  236. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  237. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  238. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  239. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -1,13 +1,24 @@
1
- import { deserializeParameterLookup, JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
1
+ import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/MongoSyncBucketStorageV3.js';
2
+ import {
3
+ deserializeParameterLookup,
4
+ JwtPayload,
5
+ SingleSyncConfigBucketDefinitionMapping,
6
+ storage,
7
+ updateSyncRulesFromYaml
8
+ } from '@powersync/service-core';
2
9
  import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
3
- import { DEFAULT_HYDRATION_STATE, nodeSqlite, RequestParameters, SqlSyncRules } from '@powersync/service-sync-rules';
10
+ import { RequestParameters, ScopedParameterLookup, SqlSyncRules } from '@powersync/service-sync-rules';
4
11
  import * as bson from 'bson';
5
- import * as sqlite from 'node:sqlite';
6
12
  import { describe, expect, test } from 'vitest';
7
13
  import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
14
+ import { MongoParsedSyncConfigSet } from '../../src/storage/implementation/MongoParsedSyncConfigSet.js';
15
+ import { MongoPersistedSyncConfigContentV3 } from '../../src/storage/implementation/MongoPersistedSyncConfigContent.js';
16
+ import { BucketDataDoc, BucketKey } from '../../src/storage/implementation/common/BucketDataDoc.js';
8
17
  import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
18
+ import { getMongoStorageConfig } from '../../src/storage/implementation/models.js';
9
19
  import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js';
10
20
  import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
21
+ import { serializeBucketData } from '../../src/storage/implementation/v3/bucket-format.js';
11
22
  import {
12
23
  CurrentBucketV3,
13
24
  ReplicationStreamDocumentV3,
@@ -53,10 +64,54 @@ function objectIdGenerator(id: string) {
53
64
  };
54
65
  }
55
66
 
56
- function hydratedRulesFor(yaml: string) {
67
+ /**
68
+ * Build a standalone parsed sync config set for the given yaml, with its own mapping.
69
+ *
70
+ * This keeps the sync rules and the bucket definition mapping paired, as required by
71
+ * `ResolveTablesOptions.parsedSyncConfig`.
72
+ */
73
+ function parsedSyncConfigSetFor(yaml: string, storageVersion: number) {
57
74
  const parsed = SqlSyncRules.fromYaml(yaml, test_utils.PARSE_OPTIONS);
58
75
  expect(parsed.errors).toEqual([]);
59
- return parsed.config.hydrate({ hydrationState: DEFAULT_HYDRATION_STATE, sqlite: nodeSqlite(sqlite) });
76
+ return new MongoParsedSyncConfigSet(1, getMongoStorageConfig(storageVersion), 'test_slot', [
77
+ {
78
+ syncConfigId: new bson.ObjectId().toHexString(),
79
+ syncConfig: parsed,
80
+ mapping: SingleSyncConfigBucketDefinitionMapping.fromParsedSyncConfig(parsed)
81
+ }
82
+ ]);
83
+ }
84
+
85
+ async function getMongoSyncConfigContents(factory: storage.BucketStorageFactory, replicationStreamId: number) {
86
+ const mongoFactory = factory as MongoBucketStorage;
87
+ const doc = (await mongoFactory.db.sync_rules.findOne({
88
+ _id: replicationStreamId
89
+ })) as ReplicationStreamDocumentV3 | null;
90
+ if (doc == null) {
91
+ return [];
92
+ }
93
+
94
+ const db = mongoFactory.db.versioned(getMongoStorageConfig(doc.storage_version)) as VersionedPowerSyncMongoV3;
95
+ const syncConfigDocs = await db.syncConfigDefinitions
96
+ .find({
97
+ _id: { $in: doc.sync_configs.map((config) => config._id) }
98
+ })
99
+ .toArray();
100
+
101
+ return syncConfigDocs.map((config) => new MongoPersistedSyncConfigContentV3(mongoFactory.db, doc, config));
102
+ }
103
+
104
+ /**
105
+ * Get a MongoDB V3 storage bucket definition id, from a bucket name.
106
+ *
107
+ * This relies on internals of bucket naming, but helps for simplifying tests.
108
+ */
109
+ function bucketDefinitionId(bucket: string) {
110
+ const match = bucket.match(/^\w+\|\w+\.\w+\.(\w+)\[/);
111
+ if (match == null) {
112
+ throw new Error(`Expected versioned bucket name, got ${bucket}`);
113
+ }
114
+ return parseInt(match[1], 16);
60
115
  }
61
116
 
62
117
  function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) {
@@ -82,6 +137,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
82
137
  )
83
138
  );
84
139
  const bucketStorage = factory.getInstance(syncRules);
140
+ const syncRulesContent = syncRules.syncConfigContent[0];
85
141
 
86
142
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
87
143
 
@@ -135,8 +191,12 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
135
191
  const checkpoint = flushResult!.flushed_op;
136
192
 
137
193
  const options: storage.BucketDataBatchOptions = {};
138
- const batch1 = await test_utils.fromAsync(
139
- bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)], options)
194
+ const batch1 = await test_utils.getBatchArray(
195
+ bucketStorage.getBucketDataBatch(
196
+ test_utils.testCheckpoint(checkpoint),
197
+ [bucketRequest(syncRulesContent, 'global[]', 0n)],
198
+ options
199
+ )
140
200
  );
141
201
  expect(test_utils.getBatchData(batch1)).toEqual([
142
202
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
@@ -148,10 +208,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
148
208
  next_after: '2'
149
209
  });
150
210
 
151
- const batch2 = await test_utils.fromAsync(
211
+ const batch2 = await test_utils.getBatchArray(
152
212
  bucketStorage.getBucketDataBatch(
153
- checkpoint,
154
- [bucketRequest(syncRules, 'global[]', batch1[0].chunkData.next_after)],
213
+ test_utils.testCheckpoint(checkpoint),
214
+ [bucketRequest(syncRulesContent, 'global[]', batch1[0].chunkData.next_after)],
155
215
  options
156
216
  )
157
217
  );
@@ -164,10 +224,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
164
224
  next_after: '3'
165
225
  });
166
226
 
167
- const batch3 = await test_utils.fromAsync(
227
+ const batch3 = await test_utils.getBatchArray(
168
228
  bucketStorage.getBucketDataBatch(
169
- checkpoint,
170
- [bucketRequest(syncRules, 'global[]', batch2[0].chunkData.next_after)],
229
+ test_utils.testCheckpoint(checkpoint),
230
+ [bucketRequest(syncRulesContent, 'global[]', batch2[0].chunkData.next_after)],
171
231
  options
172
232
  )
173
233
  );
@@ -189,7 +249,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
189
249
  await Promise.all(
190
250
  (
191
251
  await mongoFactory.db.db
192
- .listCollections({ name: new RegExp(`^bucket_data_${syncRules.id}_`) }, { nameOnly: true })
252
+ .listCollections(
253
+ { name: new RegExp(`^bucket_data_${syncRules.replicationStreamId}_`) },
254
+ { nameOnly: true }
255
+ )
193
256
  .toArray()
194
257
  ).map((collection: { name: string }) =>
195
258
  mongoFactory.db.db
@@ -388,8 +451,8 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
388
451
  }
389
452
  );
390
453
 
391
- test.runIf(storageVersion >= 3)('resolveTables handles v3 source membership additions and removals', async () => {
392
- // Tests the behavior of resolveTables when bucket data sources and parameter index creators are added or removed.
454
+ test.runIf(storageVersion >= 3)('resolveTables handles v3 source membership additions', async () => {
455
+ // Tests the behavior of resolveTables when bucket data sources and parameter index creators are added.
393
456
  // These are not end-to-end tests yet, since we don't have a full incremental reprocessing implementation.
394
457
  // This just tests the specific resolveTables behavior.
395
458
 
@@ -434,20 +497,19 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
434
497
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(fullRulesYaml, { storageVersion }));
435
498
  const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
436
499
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
437
- const fullRules = hydratedRulesFor(fullRulesYaml);
438
- const dataOnlyRules = hydratedRulesFor(dataOnlyRulesYaml);
439
- const parameterOnlyRules = hydratedRulesFor(parameterOnlyRulesYaml);
440
- const eventOnlyRules = hydratedRulesFor(eventOnlyRulesYaml);
500
+ const fullRules = parsedSyncConfigSetFor(fullRulesYaml, storageVersion);
501
+ const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyRulesYaml, storageVersion);
502
+ const parameterOnlyRules = parsedSyncConfigSetFor(parameterOnlyRulesYaml, storageVersion);
503
+ const eventOnlyRules = parsedSyncConfigSetFor(eventOnlyRulesYaml, storageVersion);
441
504
  const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
442
505
  const dataOnlyTableId = new bson.ObjectId('6544e3899293153fa7b38348');
443
506
  const addedParameterTableId = new bson.ObjectId('6544e3899293153fa7b38349');
444
- const removedDataTableId = new bson.ObjectId('6544e3899293153fa7b3834a');
445
507
 
446
508
  const dataOnly = await writer.resolveTables({
447
509
  connection_id: 1,
448
510
  source,
449
511
  idGenerator: () => dataOnlyTableId,
450
- syncRules: dataOnlyRules
512
+ parsedSyncConfig: dataOnlyRules
451
513
  });
452
514
  expect(dataOnly.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
453
515
  expect(dataOnly.dropTables.map((table) => table.id)).toEqual([]);
@@ -458,7 +520,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
458
520
  connection_id: 1,
459
521
  source,
460
522
  idGenerator: () => addedParameterTableId,
461
- syncRules: fullRules
523
+ parsedSyncConfig: fullRules
462
524
  });
463
525
  // Adding a definition always creates a new SourceTable
464
526
  expect(addedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId, addedParameterTableId]);
@@ -472,29 +534,29 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
472
534
  idGenerator: () => {
473
535
  throw new Error('data-only resolve should reuse existing v3 source table');
474
536
  },
475
- syncRules: dataOnlyRules
537
+ parsedSyncConfig: dataOnlyRules
476
538
  });
477
539
  expect(removedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
478
- // Now this sourceTable is unused & dropped
479
- expect(removedParameter.dropTables.map((table) => table.id)).toEqual([addedParameterTableId]);
540
+ // Same-identity tables that no longer cover desired memberships are left in place.
541
+ expect(removedParameter.dropTables.map((table) => table.id)).toEqual([]);
480
542
  expect(removedParameter.tables[0].bucketDataSources).toHaveLength(1);
481
543
  expect(removedParameter.tables[0].parameterLookupSources).toHaveLength(0);
482
- await writer.drop(removedParameter.dropTables);
483
544
 
484
545
  const removedData = await writer.resolveTables({
485
546
  connection_id: 1,
486
547
  source,
487
- idGenerator: () => removedDataTableId,
488
- syncRules: parameterOnlyRules
548
+ idGenerator: () => {
549
+ throw new Error('parameter-only resolve should reuse existing v3 source table');
550
+ },
551
+ parsedSyncConfig: parameterOnlyRules
489
552
  });
490
553
 
491
- // This goes from dataOnlyRules -> parameterOnlyRules, which adds one definition and removes another.
492
- // This generates a new SourceTable again, and removes all others.
493
- expect(removedData.tables.map((table) => table.id)).toEqual([removedDataTableId]);
494
- expect(removedData.dropTables.map((table) => table.id)).toEqual([dataOnlyTableId]);
554
+ // This goes from dataOnlyRules -> parameterOnlyRules, which removes one definition but
555
+ // reuses the existing parameter-only source table.
556
+ expect(removedData.tables.map((table) => table.id)).toEqual([addedParameterTableId]);
557
+ expect(removedData.dropTables.map((table) => table.id)).toEqual([]);
495
558
  expect(removedData.tables[0].bucketDataSources).toHaveLength(0);
496
559
  expect(removedData.tables[0].parameterLookupSources).toHaveLength(1);
497
- await writer.drop(removedData.dropTables);
498
560
 
499
561
  const eventOnly = await writer.resolveTables({
500
562
  connection_id: 1,
@@ -502,11 +564,12 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
502
564
  idGenerator: () => {
503
565
  throw new Error('resolve should reuse existing v3 source table');
504
566
  },
505
- syncRules: eventOnlyRules
567
+ parsedSyncConfig: eventOnlyRules
506
568
  });
507
569
 
508
570
  // Event-only table can re-use any existing table.
509
- expect(eventOnly.tables.map((table) => table.id)).toEqual([removedDataTableId]);
571
+ expect(eventOnly.tables).toHaveLength(1);
572
+ expect([dataOnlyTableId.toString(), addedParameterTableId.toString()]).toContain(eventOnly.tables[0].id.toString());
510
573
  expect(eventOnly.dropTables.map((table) => table.id)).toEqual([]);
511
574
  expect(eventOnly.tables[0].bucketDataSources).toHaveLength(0);
512
575
  expect(eventOnly.tables[0].parameterLookupSources).toHaveLength(0);
@@ -515,6 +578,76 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
515
578
  expect(eventOnly.tables[0].syncEvent).toBe(true);
516
579
  });
517
580
 
581
+ test.runIf(storageVersion >= 3)(
582
+ 'resolveTables designates a single event carrier across split source tables',
583
+ async () => {
584
+ // When memberships are split over multiple SourceTables for the same ref, a row change
585
+ // is saved once per table. Only one table may fire events, otherwise the same event
586
+ // would fire once per table for every row change.
587
+ const dataOnlyEventYaml = `
588
+ bucket_definitions:
589
+ by_owner:
590
+ parameters:
591
+ - SELECT token_parameters.owner_id as owner_id
592
+ data:
593
+ - SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
594
+
595
+ event_definitions:
596
+ write_checkpoints:
597
+ payloads:
598
+ - SELECT id, owner_id FROM memberships
599
+ `;
600
+ const fullEventYaml = `
601
+ bucket_definitions:
602
+ by_owner:
603
+ parameters:
604
+ - SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
605
+ data:
606
+ - SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
607
+
608
+ event_definitions:
609
+ write_checkpoints:
610
+ payloads:
611
+ - SELECT id, owner_id FROM memberships
612
+ `;
613
+
614
+ await using factory = await storageConfig.factory();
615
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(dataOnlyEventYaml, { storageVersion }));
616
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
617
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
618
+ const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyEventYaml, storageVersion);
619
+ const fullRules = parsedSyncConfigSetFor(fullEventYaml, storageVersion);
620
+ const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
621
+
622
+ const initial = await writer.resolveTables({
623
+ connection_id: 1,
624
+ source,
625
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834b'),
626
+ parsedSyncConfig: dataOnlyRules
627
+ });
628
+ expect(initial.tables).toHaveLength(1);
629
+ expect(initial.tables[0].syncEvent).toBe(true);
630
+
631
+ // Adding the table-based parameter lookup creates a second SourceTable for the same ref.
632
+ const split = await writer.resolveTables({
633
+ connection_id: 1,
634
+ source,
635
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834c'),
636
+ parsedSyncConfig: fullRules
637
+ });
638
+ expect(split.tables).toHaveLength(2);
639
+ // Both tables match the event by ref, but only one may carry it.
640
+ const carriers = split.tables.filter((table) => table.syncEvent);
641
+ expect(carriers).toHaveLength(1);
642
+
643
+ // getSourceTableStatus preserves the carrier designation rather than recomputing it
644
+ // from the ref, so refreshing a non-carrier table does not make it fire events.
645
+ const nonCarrier = split.tables.find((table) => !table.syncEvent)!;
646
+ const refreshed = await writer.getSourceTableStatus(nonCarrier);
647
+ expect(refreshed!.syncEvent).toBe(false);
648
+ }
649
+ );
650
+
518
651
  test.runIf(storageVersion >= 3)('uses v3 mongodb model shapes', async () => {
519
652
  await using factory = await storageConfig.factory();
520
653
  const syncRules = await factory.updateSyncRules(
@@ -531,7 +664,8 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
531
664
  )
532
665
  );
533
666
  const bucketStorage = factory.getInstance(syncRules);
534
- const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig();
667
+ const syncRulesContent = syncRules.syncConfigContent[0];
668
+ const sync_rules = syncRulesContent.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig;
535
669
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
536
670
  const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
537
671
 
@@ -552,7 +686,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
552
686
  const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { test: 'shape-check' } }), {});
553
687
  const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
554
688
  const buckets = await querier.queryDynamicBucketDescriptions({
555
- async getParameterSets(lookups) {
689
+ async getParameterSets(lookups: ScopedParameterLookup[]) {
556
690
  expect(lookups.map((l) => l.indexKey)).toEqual([['shape-check']]);
557
691
  expect(lookups[0].indexId).toEqual('1');
558
692
 
@@ -561,33 +695,635 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
561
695
  return parameter_sets;
562
696
  }
563
697
  });
564
- expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'global["user-1"]').bucket]);
698
+ expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRulesContent, 'global["user-1"]').bucket]);
565
699
 
566
700
  const mongoFactory = factory as MongoBucketStorage;
567
701
  const db = (bucketStorage as MongoSyncBucketStorage).db as VersionedPowerSyncMongoV3;
568
- const currentDataCollections = await db.listSourceRecordCollectionsV3(syncRules.id);
702
+ const currentDataCollections = await db.listSourceRecordCollections(syncRules.replicationStreamId);
569
703
  const currentData = await currentDataCollections[0]?.findOne({});
570
704
  const firstBucket: CurrentBucketV3 | undefined = currentData?.buckets[0] as CurrentBucketV3 | undefined;
571
705
  expect(firstBucket?.def).toMatch(/^[0-9a-f]+$/);
572
706
 
573
707
  const bucketCollections = await mongoFactory.db.db
574
- .listCollections({ name: new RegExp(`^bucket_data_${syncRules.id}_`) }, { nameOnly: true })
708
+ .listCollections({ name: new RegExp(`^bucket_data_${syncRules.replicationStreamId}_`) }, { nameOnly: true })
575
709
  .toArray();
576
710
  expect(
577
- bucketCollections.some((collection) => collection.name === `bucket_data_${syncRules.id}_${firstBucket?.def}`)
711
+ bucketCollections.some(
712
+ (collection) => collection.name === `bucket_data_${syncRules.replicationStreamId}_${firstBucket?.def}`
713
+ )
578
714
  ).toBe(true);
579
715
 
580
- const syncRule = (await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id })) as ReplicationStreamDocumentV3;
716
+ const syncRule = (await mongoFactory.db.sync_rules.findOne({
717
+ _id: syncRules.replicationStreamId
718
+ })) as ReplicationStreamDocumentV3;
581
719
  const syncConfig = await db.syncConfigDefinitions.findOne({ _id: syncRule.sync_configs[0]._id });
582
720
  const ruleMapping: SyncConfigDefinition['rule_mapping'] | undefined = syncConfig?.rule_mapping;
583
721
  expect(Object.keys(ruleMapping?.definitions ?? {})).not.toHaveLength(0);
584
722
 
585
723
  const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0] as string | undefined;
586
724
  expect(parameterIndexId).toBeDefined();
587
- const parameterEntry = await db.parameterIndexV3(syncRules.id, parameterIndexId!).findOne({});
725
+ const parameterEntry = await db.parameterIndex(syncRules.replicationStreamId, parameterIndexId!).findOne({});
588
726
  expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']);
589
727
  });
590
728
 
729
+ test.runIf(storageVersion >= 3)('replaces an existing deploying sync config', async () => {
730
+ await using factory = await storageConfig.factory();
731
+
732
+ const first = await factory.updateSyncRules(
733
+ updateSyncRulesFromYaml(
734
+ `
735
+ config:
736
+ edition: 2
737
+
738
+ streams:
739
+ by_owner:
740
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
741
+ `,
742
+ { storageVersion }
743
+ )
744
+ );
745
+ const second = await factory.updateSyncRules(
746
+ updateSyncRulesFromYaml(
747
+ `
748
+ config:
749
+ edition: 2
750
+
751
+ streams:
752
+ by_project:
753
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
754
+ `,
755
+ { storageVersion }
756
+ )
757
+ );
758
+
759
+ expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
760
+ expect(
761
+ (await factory.getStoppedReplicationStreams()).find(
762
+ (stream) => stream.replicationStreamId == first.replicationStreamId
763
+ )?.state
764
+ ).toBe(storage.SyncRuleState.STOP);
765
+
766
+ const replicatingStreams = await factory.getReplicatingReplicationStreams();
767
+ expect(replicatingStreams).toHaveLength(1);
768
+ expect(replicatingStreams[0].replicationStreamId).toEqual(second.replicationStreamId);
769
+
770
+ const configs = second.syncConfigContent;
771
+ expect(configs).toHaveLength(1);
772
+ const statuses = await Promise.all(configs.map((config) => config.getSyncConfigStatus()));
773
+ expect(statuses.map((status) => status?.state)).toEqual([storage.SyncRuleState.PROCESSING]);
774
+ expect(statuses.map((status) => status?.id)).toEqual(configs.map((config) => config.syncConfigId));
775
+ const parsed = replicatingStreams[0].parsed(test_utils.PARSE_OPTIONS);
776
+ expect(parsed.syncConfigs).toHaveLength(1);
777
+ expect(parsed.hydratedSyncConfig.bucketDataSources).toHaveLength(1);
778
+
779
+ const bucketStorage = factory.getInstance(replicatingStreams[0]);
780
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
781
+ const resolved = await writer.resolveTables({
782
+ connection_id: 1,
783
+ source: sourceDescriptor('todos', { objectId: 'todos-relation' }),
784
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834b')
785
+ });
786
+ expect(resolved.tables).toHaveLength(1);
787
+ expect(resolved.tables[0].bucketDataSources).toHaveLength(1);
788
+ });
789
+
790
+ test.runIf(storageVersion >= 3)('removing one config keeps shared source-table membership', async () => {
791
+ await using factory = await storageConfig.factory();
792
+
793
+ const first = await factory.updateSyncRules(
794
+ updateSyncRulesFromYaml(
795
+ `
796
+ config:
797
+ edition: 3
798
+
799
+ streams:
800
+ by_owner:
801
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
802
+ `,
803
+ { storageVersion }
804
+ )
805
+ );
806
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
807
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
808
+ await firstWriter.markAllSnapshotDone('1/1');
809
+ await firstWriter.commit('1/1');
810
+
811
+ const second = await factory.updateSyncRules(
812
+ updateSyncRulesFromYaml(
813
+ `
814
+ config:
815
+ edition: 3
816
+
817
+ streams:
818
+ by_project:
819
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
820
+ `,
821
+ { storageVersion }
822
+ )
823
+ );
824
+ expect(second.replicationStreamId).toEqual(first.replicationStreamId);
825
+
826
+ const bucketStorage = factory.getInstance(second);
827
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
828
+ const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
829
+
830
+ const resolved = await writer.resolveTables({
831
+ connection_id: 1,
832
+ source,
833
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834c')
834
+ });
835
+ expect(resolved.tables).toHaveLength(1);
836
+ expect(resolved.tables[0].bucketDataSources).toHaveLength(2);
837
+
838
+ await writer.markAllSnapshotDone('2/1');
839
+ await writer.commit('2/1');
840
+
841
+ const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
842
+ await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
843
+ const activeStatus = await activeWriter.getSourceTableStatus(resolved.tables[0]);
844
+ expect(activeStatus?.bucketDataSources).toHaveLength(1);
845
+ });
846
+
847
+ test.runIf(storageVersion >= 3)(
848
+ 'editing a stream query on the same table does not drop the active source table',
849
+ async () => {
850
+ await using factory = await storageConfig.factory();
851
+
852
+ const first = await factory.updateSyncRules(
853
+ updateSyncRulesFromYaml(
854
+ `
855
+ config:
856
+ edition: 3
857
+
858
+ streams:
859
+ todos:
860
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
861
+ `,
862
+ { storageVersion }
863
+ )
864
+ );
865
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
866
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
867
+ const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
868
+ const firstResolved = await firstWriter.resolveTables({
869
+ connection_id: 1,
870
+ source,
871
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834d')
872
+ });
873
+ expect(firstResolved.dropTables).toHaveLength(0);
874
+ await firstWriter.markAllSnapshotDone('1/1');
875
+ await firstWriter.commit('1/1');
876
+
877
+ const second = await factory.updateSyncRules(
878
+ updateSyncRulesFromYaml(
879
+ `
880
+ config:
881
+ edition: 3
882
+
883
+ streams:
884
+ todos:
885
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
886
+ `,
887
+ { storageVersion }
888
+ )
889
+ );
890
+ expect(second.replicationStreamId).toBe(first.replicationStreamId);
891
+
892
+ const deploying = await factory.getDeployingSyncConfig();
893
+ expect(deploying?.replicationStream.replicationStreamId).toBe(first.replicationStreamId);
894
+ const bucketStorage = deploying!.storage;
895
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
896
+ const resolved = await writer.resolveTables({
897
+ connection_id: 1,
898
+ source,
899
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834e')
900
+ });
901
+
902
+ expect(resolved.dropTables).toHaveLength(0);
903
+ expect(resolved.tables.map((table) => table.id.toString())).toEqual(['6544e3899293153fa7b3834e']);
904
+ expect(resolved.tables[0].bucketDataSources).toHaveLength(1);
905
+
906
+ const activeStatus = await firstWriter.getSourceTableStatus(firstResolved.tables[0]);
907
+ expect(activeStatus?.bucketDataSources).toHaveLength(1);
908
+ }
909
+ );
910
+
911
+ test.runIf(storageVersion >= 3)('reserves historical mapping ids without reusing stopped configs', async () => {
912
+ await using factory = await storageConfig.factory();
913
+
914
+ const ownerRules = `
915
+ config:
916
+ edition: 3
917
+
918
+ streams:
919
+ by_owner:
920
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
921
+ `;
922
+ const projectRules = `
923
+ config:
924
+ edition: 3
925
+
926
+ streams:
927
+ by_project:
928
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
929
+ `;
930
+ const statusRules = `
931
+ config:
932
+ edition: 3
933
+
934
+ streams:
935
+ by_status:
936
+ query: SELECT * FROM todos WHERE status = subscription.parameter('status')
937
+ `;
938
+
939
+ const first = await factory.updateSyncRules(updateSyncRulesFromYaml(ownerRules, { storageVersion }));
940
+ const firstStorage = factory.getInstance(first);
941
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
942
+ await firstWriter.markAllSnapshotDone('1/1');
943
+ await firstWriter.commit('1/1');
944
+
945
+ const second = await factory.updateSyncRules(updateSyncRulesFromYaml(projectRules, { storageVersion }));
946
+ expect(second.replicationStreamId).toBe(first.replicationStreamId);
947
+
948
+ const third = await factory.updateSyncRules(updateSyncRulesFromYaml(statusRules, { storageVersion }));
949
+ expect(third.replicationStreamId).toBe(first.replicationStreamId);
950
+
951
+ const configs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
952
+ const statuses = await Promise.all(
953
+ configs.map(async (config) => [config, await config.getSyncConfigStatus()] as const)
954
+ );
955
+ // ownerRules
956
+ const activeConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.ACTIVE)?.[0];
957
+ // projectRules
958
+ const stoppedConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.STOP)?.[0];
959
+ // statusRules
960
+ const processingConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.PROCESSING)?.[0];
961
+ expect(stoppedConfig).toBeDefined();
962
+ expect(activeConfig).toBeDefined();
963
+ expect(processingConfig).toBeDefined();
964
+
965
+ const activePrefix = bucketRequest(activeConfig!, 'by_owner|0["owner"]').bucket;
966
+ const stoppedPrefix = bucketRequest(stoppedConfig!, 'by_project|0["project"]').bucket;
967
+ const processingPrefix = bucketRequest(processingConfig!, 'by_status|0["status"]').bucket;
968
+
969
+ expect(processingPrefix).not.toBe(stoppedPrefix);
970
+ expect(processingPrefix).not.toBe(activePrefix);
971
+ expect(bucketDefinitionId(processingPrefix)).toBeGreaterThan(
972
+ Math.max(bucketDefinitionId(stoppedPrefix), bucketDefinitionId(activePrefix))
973
+ );
974
+ });
975
+
976
+ test.runIf(storageVersion >= 3)('table snapshot status only affects sync configs using that table', async () => {
977
+ await using factory = await storageConfig.factory();
978
+
979
+ const first = await factory.updateSyncRules(
980
+ updateSyncRulesFromYaml(
981
+ `
982
+ config:
983
+ edition: 3
984
+
985
+ streams:
986
+ by_owner:
987
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
988
+ `,
989
+ { storageVersion }
990
+ )
991
+ );
992
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
993
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
994
+ await firstWriter.markAllSnapshotDone('1/1');
995
+ await firstWriter.commit('1/1');
996
+
997
+ const second = await factory.updateSyncRules(
998
+ updateSyncRulesFromYaml(
999
+ `
1000
+ config:
1001
+ edition: 3
1002
+
1003
+ streams:
1004
+ by_owner:
1005
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1006
+ by_project:
1007
+ query: SELECT * FROM projects WHERE id = subscription.parameter('project_id')
1008
+ `,
1009
+ { storageVersion }
1010
+ )
1011
+ );
1012
+ expect(second.replicationStreamId).toBe(first.replicationStreamId);
1013
+
1014
+ const bucketStorage = factory.getInstance(second);
1015
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1016
+ const resolved = await writer.resolveTables({
1017
+ connection_id: 1,
1018
+ source: sourceDescriptor('projects', { objectId: 'projects-relation' }),
1019
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834d')
1020
+ });
1021
+ expect(resolved.tables).toHaveLength(1);
1022
+
1023
+ await writer.markAllSnapshotDone('2/1');
1024
+ await writer.markTableSnapshotRequired(resolved.tables[0]);
1025
+
1026
+ const configsAfterRequired = await getMongoSyncConfigContents(factory, first.replicationStreamId);
1027
+ const statusesAfterRequired = await Promise.all(configsAfterRequired.map((config) => config.getSyncConfigStatus()));
1028
+ const ownerConfigAfterRequired = statusesAfterRequired.find(
1029
+ (status) => status?.state == storage.SyncRuleState.ACTIVE
1030
+ );
1031
+ const projectConfigAfterRequired = statusesAfterRequired.find(
1032
+ (status) => status?.state == storage.SyncRuleState.PROCESSING
1033
+ );
1034
+ expect(ownerConfigAfterRequired?.snapshot_done).toBe(true);
1035
+ expect(projectConfigAfterRequired?.snapshot_done).toBe(false);
1036
+
1037
+ await writer.markTableSnapshotDone(resolved.tables, '3/1');
1038
+
1039
+ const configsAfterDone = await getMongoSyncConfigContents(factory, first.replicationStreamId);
1040
+ const statusesAfterDone = await Promise.all(configsAfterDone.map((config) => config.getSyncConfigStatus()));
1041
+ const ownerConfigAfterDone = statusesAfterDone.find((status) => status?.id == ownerConfigAfterRequired!.id);
1042
+ const projectConfigAfterDone = statusesAfterDone.find((status) => status?.id == projectConfigAfterRequired!.id);
1043
+ expect(ownerConfigAfterDone?.snapshot_done).toBe(true);
1044
+ expect(projectConfigAfterDone?.snapshot_done).toBe(false);
1045
+ });
1046
+
1047
+ test.runIf(storageVersion >= 3)(
1048
+ 'keeps compatible active and deploying sync configs in one replication stream',
1049
+ async () => {
1050
+ await using factory = await storageConfig.factory();
1051
+
1052
+ const first = await factory.updateSyncRules(
1053
+ updateSyncRulesFromYaml(
1054
+ `
1055
+ config:
1056
+ edition: 3
1057
+
1058
+ streams:
1059
+ by_owner:
1060
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1061
+ `,
1062
+ { storageVersion }
1063
+ )
1064
+ );
1065
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1066
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1067
+ await firstWriter.markAllSnapshotDone('1/1');
1068
+ await firstWriter.commit('1/1');
1069
+ const initialActiveStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
1070
+
1071
+ let configs = first.syncConfigContent;
1072
+ const firstConfigId = configs[0].syncConfigId;
1073
+ expect((await factory.getActiveSyncConfig())?.content.syncConfigId).toBe(firstConfigId);
1074
+
1075
+ const second = await factory.updateSyncRules(
1076
+ updateSyncRulesFromYaml(
1077
+ `
1078
+ config:
1079
+ edition: 3
1080
+
1081
+ streams:
1082
+ by_project:
1083
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
1084
+ `,
1085
+ { storageVersion }
1086
+ )
1087
+ );
1088
+ expect(second.replicationStreamId).toEqual(first.replicationStreamId);
1089
+
1090
+ configs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
1091
+ const deploying = await factory.getDeployingSyncConfig();
1092
+ expect(configs).toHaveLength(2);
1093
+ expect(deploying).not.toBeNull();
1094
+ expect((await factory.getActiveSyncConfig())?.content.syncConfigId).toBe(firstConfigId);
1095
+
1096
+ const stream = (await factory.getActiveSyncConfig())?.replicationStream;
1097
+ expect(stream?.state).toBe(storage.SyncRuleState.ACTIVE);
1098
+ expect(
1099
+ (await Promise.all(configs.map((config) => config.getSyncConfigStatus()))).map((status) => status?.state).sort()
1100
+ ).toEqual([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.PROCESSING]);
1101
+
1102
+ const replicatingStreams = await factory.getReplicatingReplicationStreams();
1103
+ expect(replicatingStreams).toHaveLength(1);
1104
+ for (const config of configs) {
1105
+ expect(replicatingStreams[0].replicationJobId).toContain(config.syncConfigId);
1106
+ }
1107
+
1108
+ const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage;
1109
+ await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
1110
+ await secondWriter.markAllSnapshotDone('2/1');
1111
+ await secondWriter.commit('2/1');
1112
+
1113
+ const updatedStream = (await factory.getActiveSyncConfig())?.replicationStream;
1114
+ expect(updatedStream?.state).toBe(storage.SyncRuleState.ACTIVE);
1115
+ const updatedConfigs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
1116
+ expect(
1117
+ (await Promise.all(updatedConfigs.map((config) => config.getSyncConfigStatus())))
1118
+ .map((status) => status?.state)
1119
+ .sort()
1120
+ ).toEqual([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.STOP]);
1121
+ expect(await factory.getDeployingSyncConfig()).toBeNull();
1122
+ expect((await factory.getActiveSyncConfig())?.content.syncConfigId).not.toBe(firstConfigId);
1123
+ const updatedActiveStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
1124
+ expect(updatedActiveStorage.replicationStream.replicationJobId).not.toBe(
1125
+ initialActiveStorage.replicationStream.replicationJobId
1126
+ );
1127
+ expect(updatedActiveStorage.replicationStream.replicationJobId).toContain(
1128
+ (await factory.getActiveSyncConfig())!.content.syncConfigId
1129
+ );
1130
+ }
1131
+ );
1132
+
1133
+ test.runIf(storageVersion >= 3)(
1134
+ 'appended compatible config adopts the stream checkpoint instead of regressing to 0',
1135
+ async () => {
1136
+ await using factory = await storageConfig.factory();
1137
+
1138
+ // First config replicates some data, advancing the stream-level op head and its checkpoint.
1139
+ const first = await factory.updateSyncRules(
1140
+ updateSyncRulesFromYaml(
1141
+ `
1142
+ config:
1143
+ edition: 3
1144
+
1145
+ streams:
1146
+ by_owner:
1147
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1148
+ `,
1149
+ { storageVersion }
1150
+ )
1151
+ );
1152
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1153
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1154
+ const sourceTable = await test_utils.resolveTestTable(
1155
+ firstWriter,
1156
+ 'todos',
1157
+ ['id'],
1158
+ INITIALIZED_MONGO_STORAGE_FACTORY
1159
+ );
1160
+ await firstWriter.save({
1161
+ sourceTable,
1162
+ tag: storage.SaveOperationTag.INSERT,
1163
+ after: {
1164
+ id: 'todo-1',
1165
+ owner_id: 'user-1',
1166
+ project_id: 'project-1'
1167
+ },
1168
+ afterReplicaId: test_utils.rid('todo-1')
1169
+ });
1170
+ await firstWriter.markAllSnapshotDone('1/1');
1171
+ await firstWriter.commit('1/1');
1172
+
1173
+ const firstCheckpoint = (await firstStorage.getCheckpoint()).checkpoint;
1174
+ expect(firstCheckpoint).toBeGreaterThan(0n);
1175
+
1176
+ const mongoFactory = factory as MongoBucketStorage;
1177
+ const streamDocBefore = (await mongoFactory.db.sync_rules.findOne({
1178
+ _id: first.replicationStreamId
1179
+ })) as ReplicationStreamDocumentV3;
1180
+ // The stream-level head was advanced durably to (at least) the first config's checkpoint.
1181
+ expect(streamDocBefore.last_persisted_op).not.toBeNull();
1182
+ expect(BigInt(streamDocBefore.last_persisted_op!)).toBeGreaterThanOrEqual(firstCheckpoint);
1183
+
1184
+ // Append a compatible second config that replicates nothing new.
1185
+ const second = await factory.updateSyncRules(
1186
+ updateSyncRulesFromYaml(
1187
+ `
1188
+ config:
1189
+ edition: 3
1190
+
1191
+ streams:
1192
+ by_project:
1193
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
1194
+ `,
1195
+ { storageVersion }
1196
+ )
1197
+ );
1198
+ expect(second.replicationStreamId).toEqual(first.replicationStreamId);
1199
+
1200
+ const replicatingStreams = await factory.getReplicatingReplicationStreams();
1201
+ expect(replicatingStreams).toHaveLength(1);
1202
+ const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage;
1203
+ await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
1204
+ // No new data replicated - just complete the snapshot and commit.
1205
+ await secondWriter.markAllSnapshotDone('2/1');
1206
+ await secondWriter.commit('2/1');
1207
+
1208
+ // The appended config's checkpoint must adopt the stream head, not regress to 0.
1209
+ const streamDocAfter = (await mongoFactory.db.sync_rules.findOne({
1210
+ _id: first.replicationStreamId
1211
+ })) as ReplicationStreamDocumentV3;
1212
+ const head = BigInt(streamDocAfter.last_persisted_op!);
1213
+ for (const config of streamDocAfter.sync_configs) {
1214
+ expect(config.last_checkpoint).not.toBeNull();
1215
+ expect(BigInt(config.last_checkpoint!)).toEqual(head);
1216
+ }
1217
+ // All configs share the same checkpoint, equal to the stream head.
1218
+ expect(head).toBeGreaterThanOrEqual(firstCheckpoint);
1219
+
1220
+ // After activation, the active config's checkpoint does not regress.
1221
+ const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
1222
+ const activeCheckpoint = (await activeStorage.getCheckpoint()).checkpoint;
1223
+ expect(activeCheckpoint).toBeGreaterThanOrEqual(firstCheckpoint);
1224
+ }
1225
+ );
1226
+
1227
+ test.runIf(storageVersion >= 3)('creates a new replication stream when compatibility options differ', async () => {
1228
+ await using factory = await storageConfig.factory();
1229
+
1230
+ const first = await factory.updateSyncRules(
1231
+ updateSyncRulesFromYaml(
1232
+ `
1233
+ config:
1234
+ edition: 3
1235
+
1236
+ streams:
1237
+ by_owner:
1238
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1239
+ `,
1240
+ { storageVersion }
1241
+ )
1242
+ );
1243
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1244
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1245
+ await firstWriter.markAllSnapshotDone('1/1');
1246
+ await firstWriter.commit('1/1');
1247
+
1248
+ // Same streams, but a different compatibility edition - must not append to the active stream.
1249
+ const second = await factory.updateSyncRules(
1250
+ updateSyncRulesFromYaml(
1251
+ `
1252
+ config:
1253
+ edition: 2
1254
+
1255
+ streams:
1256
+ by_owner:
1257
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1258
+ `,
1259
+ { storageVersion }
1260
+ )
1261
+ );
1262
+ expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
1263
+
1264
+ // The first stream stays active until the replacement has replicated.
1265
+ const active = await factory.getActiveSyncConfig();
1266
+ expect(active?.replicationStream.replicationStreamId).toEqual(first.replicationStreamId);
1267
+ expect((await factory.getDeployingSyncConfig())?.replicationStream.replicationStreamId).toEqual(
1268
+ second.replicationStreamId
1269
+ );
1270
+ });
1271
+
1272
+ test.runIf(storageVersion >= 3)(
1273
+ 'creates a new replication stream for legacy sync rules without a serialized plan',
1274
+ async () => {
1275
+ await using factory = await storageConfig.factory();
1276
+
1277
+ const first = await factory.updateSyncRules(
1278
+ updateSyncRulesFromYaml(
1279
+ `
1280
+ config:
1281
+ edition: 3
1282
+
1283
+ streams:
1284
+ by_owner:
1285
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1286
+ `,
1287
+ { storageVersion }
1288
+ )
1289
+ );
1290
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1291
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1292
+ await firstWriter.markAllSnapshotDone('1/1');
1293
+ await firstWriter.commit('1/1');
1294
+
1295
+ // Legacy sync rules have no serialized plan - must not append to the active stream.
1296
+ const second = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
1297
+ expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
1298
+ }
1299
+ );
1300
+
1301
+ test.runIf(storageVersion >= 3)('does not append to an active legacy sync config', async () => {
1302
+ await using factory = await storageConfig.factory();
1303
+
1304
+ const first = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
1305
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1306
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1307
+ await firstWriter.markAllSnapshotDone('1/1');
1308
+ await firstWriter.commit('1/1');
1309
+
1310
+ // The active config has no serialized plan, so a new streams config must not be appended to it.
1311
+ const second = await factory.updateSyncRules(
1312
+ updateSyncRulesFromYaml(
1313
+ `
1314
+ config:
1315
+ edition: 3
1316
+
1317
+ streams:
1318
+ by_owner:
1319
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1320
+ `,
1321
+ { storageVersion }
1322
+ )
1323
+ );
1324
+ expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
1325
+ });
1326
+
591
1327
  test.runIf(storageVersion < 3)('can replace processing legacy sync rules', async () => {
592
1328
  await using factory = await storageConfig.factory();
593
1329
 
@@ -600,7 +1336,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
600
1336
  ).resolves.toBeDefined();
601
1337
 
602
1338
  const mongoFactory = factory as MongoBucketStorage;
603
- expect((await mongoFactory.db.sync_rules.findOne({ _id: firstSyncRules.id }))?.state).toBe(
1339
+ expect((await mongoFactory.db.sync_rules.findOne({ _id: firstSyncRules.replicationStreamId }))?.state).toBe(
604
1340
  storage.SyncRuleState.STOP
605
1341
  );
606
1342
  });
@@ -612,10 +1348,48 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
612
1348
  updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion, lock: true })
613
1349
  );
614
1350
 
615
- expect(syncRules.current_lock?.sync_rules_id).toBe(syncRules.id);
1351
+ expect(syncRules.current_lock?.sync_rules_id).toBe(syncRules.replicationStreamId);
616
1352
  await syncRules.current_lock?.release();
617
1353
  });
618
1354
 
1355
+ test.runIf(storageVersion >= 3)('does not lock when appending a sync config to an existing stream', async () => {
1356
+ await using factory = await storageConfig.factory();
1357
+
1358
+ const firstRules = `
1359
+ config:
1360
+ edition: 3
1361
+
1362
+ streams:
1363
+ by_owner:
1364
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1365
+ `;
1366
+ const secondRules = `
1367
+ config:
1368
+ edition: 3
1369
+
1370
+ streams:
1371
+ by_project:
1372
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
1373
+ `;
1374
+
1375
+ const first = await factory.updateSyncRules(updateSyncRulesFromYaml(firstRules, { storageVersion, lock: true }));
1376
+ expect(first.current_lock?.sync_rules_id).toBe(first.replicationStreamId);
1377
+ try {
1378
+ const firstStorage = factory.getInstance(first);
1379
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1380
+ await firstWriter.markAllSnapshotDone('1/1');
1381
+ await firstWriter.commit('1/1');
1382
+
1383
+ const second = await factory.updateSyncRules(
1384
+ updateSyncRulesFromYaml(secondRules, { storageVersion, lock: true })
1385
+ );
1386
+ expect(second.replicationStreamId).toBe(first.replicationStreamId);
1387
+ expect(second.current_lock).toBeNull();
1388
+ } finally {
1389
+ await first.current_lock?.release();
1390
+ }
1391
+ });
1392
+
619
1393
  test.runIf(storageVersion < 3)('uses a single current_data collection for v1 source records', async () => {
620
1394
  await using factory = await storageConfig.factory();
621
1395
  const syncRules = await factory.updateSyncRules(
@@ -647,10 +1421,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
647
1421
  await writer.commit('1/1');
648
1422
 
649
1423
  const mongoFactory = factory as MongoBucketStorage;
650
- expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1);
1424
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
651
1425
 
652
1426
  const sourceRecordCollections = await mongoFactory.db.db
653
- .listCollections({ name: new RegExp(`^source_records_${syncRules.id}_`) }, { nameOnly: true })
1427
+ .listCollections({ name: new RegExp(`^source_records_${syncRules.replicationStreamId}_`) }, { nameOnly: true })
654
1428
  .toArray();
655
1429
  expect(sourceRecordCollections).toEqual([]);
656
1430
  });
@@ -686,13 +1460,103 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
686
1460
  await writer.commit('1/1');
687
1461
 
688
1462
  const mongoFactory = factory as MongoBucketStorage;
689
- expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1);
1463
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
690
1464
 
691
1465
  await bucketStorage.clear();
692
1466
 
693
- expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(0);
1467
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(0);
694
1468
  });
695
1469
 
1470
+ test.runIf(storageVersion < 3)(
1471
+ 'clear removes large v1 current_data sets without affecting other streams',
1472
+ async () => {
1473
+ await using factory = await storageConfig.factory();
1474
+ const syncRules = await factory.updateSyncRules(
1475
+ updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, {
1476
+ storageVersion
1477
+ })
1478
+ );
1479
+ const bucketStorage = factory.getInstance(syncRules);
1480
+ const mongoFactory = factory as MongoBucketStorage;
1481
+ const sourceTableId = new bson.ObjectId();
1482
+ const documents = Array.from({ length: 10_002 }, (_, index) => ({
1483
+ _id: {
1484
+ g: syncRules.replicationStreamId,
1485
+ t: sourceTableId,
1486
+ k: index
1487
+ },
1488
+ data: new bson.Binary(),
1489
+ buckets: [],
1490
+ lookups: []
1491
+ }));
1492
+ await mongoFactory.db.current_data.insertMany([
1493
+ ...documents,
1494
+ {
1495
+ _id: {
1496
+ g: syncRules.replicationStreamId + 1,
1497
+ t: sourceTableId,
1498
+ k: 0
1499
+ },
1500
+ data: new bson.Binary(),
1501
+ buckets: [],
1502
+ lookups: []
1503
+ }
1504
+ ]);
1505
+
1506
+ await bucketStorage.clear();
1507
+
1508
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(0);
1509
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId + 1 })).toBe(1);
1510
+ }
1511
+ );
1512
+
1513
+ test.runIf(storageVersion < 3)(
1514
+ 'clear removes large v1 parameter index sets without affecting other streams',
1515
+ async () => {
1516
+ await using factory = await storageConfig.factory();
1517
+ const syncRules = await factory.updateSyncRules(
1518
+ updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, {
1519
+ storageVersion
1520
+ })
1521
+ );
1522
+ const bucketStorage = factory.getInstance(syncRules);
1523
+ const mongoFactory = factory as MongoBucketStorage;
1524
+ const sourceTableId = new bson.ObjectId();
1525
+ const documents = Array.from({ length: 10_002 }, (_, index) => ({
1526
+ _id: BigInt(index * 2),
1527
+ key: {
1528
+ g: syncRules.replicationStreamId,
1529
+ t: sourceTableId,
1530
+ k: index
1531
+ },
1532
+ lookup: new bson.Binary(),
1533
+ bucket_parameters: []
1534
+ }));
1535
+ await mongoFactory.db.bucket_parameters.insertMany([
1536
+ ...documents,
1537
+ {
1538
+ _id: 1n,
1539
+ key: {
1540
+ g: syncRules.replicationStreamId + 1,
1541
+ t: sourceTableId,
1542
+ k: 0
1543
+ },
1544
+ lookup: new bson.Binary(),
1545
+ bucket_parameters: []
1546
+ }
1547
+ ]);
1548
+
1549
+ await bucketStorage.clear();
1550
+
1551
+ expect(await mongoFactory.db.bucket_parameters.countDocuments({ 'key.g': syncRules.replicationStreamId })).toBe(
1552
+ 0
1553
+ );
1554
+ expect(
1555
+ await mongoFactory.db.bucket_parameters.countDocuments({ 'key.g': syncRules.replicationStreamId + 1 })
1556
+ ).toBe(1);
1557
+ }
1558
+ );
1559
+
696
1560
  test.runIf(storageVersion < 3)('storage metrics include v1 current_data', async () => {
697
1561
  await using factory = await storageConfig.factory();
698
1562
  const syncRules = await factory.updateSyncRules(
@@ -725,7 +1589,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
725
1589
  await writer.commit('1/1');
726
1590
 
727
1591
  const mongoFactory = factory as MongoBucketStorage;
728
- expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1);
1592
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
729
1593
 
730
1594
  const metricsAfter = await factory.getStorageMetrics();
731
1595
  expect(metricsAfter.replication_size_bytes).toBeGreaterThan(metricsBefore.replication_size_bytes);
@@ -783,7 +1647,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
783
1647
  }
784
1648
  );
785
1649
 
786
- test.runIf(storageVersion >= 3)('cleans pending deletes only for tracked v3 source tables', async () => {
1650
+ test.runIf(storageVersion == 3)('cleans pending deletes only for tracked v3 source tables', async () => {
787
1651
  await using factory = await storageConfig.factory();
788
1652
  const syncRules = await factory.updateSyncRules(
789
1653
  updateSyncRulesFromYaml(
@@ -798,21 +1662,20 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
798
1662
  );
799
1663
 
800
1664
  const mongoFactory = factory as MongoBucketStorage;
801
- const bucketStorage = mongoFactory.getInstance(syncRules) as any;
802
- const db = bucketStorage.db;
803
- await db.initializeStreamStorage(syncRules.id);
1665
+ const bucketStorage = mongoFactory.getInstance(syncRules);
1666
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
1667
+ await db.initializeStreamStorage(syncRules.replicationStreamId);
804
1668
 
805
1669
  const sourceTableA = new bson.ObjectId();
806
1670
  const sourceTableB = new bson.ObjectId();
807
- await db.sourceTablesV3(syncRules.id).insertMany([
1671
+ await db.sourceTables(syncRules.replicationStreamId).insertMany([
808
1672
  {
809
1673
  _id: sourceTableA,
810
1674
  connection_id: 1,
811
1675
  relation_id: 'a',
812
1676
  schema_name: 'public',
813
1677
  table_name: 'table_a',
814
- replica_id_columns: null,
815
- replica_id_columns2: [],
1678
+ replica_id_columns: [],
816
1679
  snapshot_done: true,
817
1680
  snapshot_status: undefined,
818
1681
  bucket_data_source_ids: [],
@@ -825,8 +1688,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
825
1688
  relation_id: 'b',
826
1689
  schema_name: 'public',
827
1690
  table_name: 'table_b',
828
- replica_id_columns: null,
829
- replica_id_columns2: [],
1691
+ replica_id_columns: [],
830
1692
  snapshot_done: true,
831
1693
  snapshot_status: undefined,
832
1694
  bucket_data_source_ids: [],
@@ -835,35 +1697,53 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
835
1697
  }
836
1698
  ]);
837
1699
 
838
- await db.sourceRecordsV3(syncRules.id, sourceTableA).insertMany([
1700
+ await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).insertMany([
839
1701
  { _id: 'deleted-1', data: null, buckets: [], lookups: [], pending_delete: 5n },
840
1702
  { _id: 'deleted-2', data: null, buckets: [], lookups: [], pending_delete: 9n },
841
1703
  { _id: 'active', data: null, buckets: [], lookups: [] }
842
1704
  ]);
843
1705
  await db
844
- .sourceRecordsV3(syncRules.id, sourceTableB)
1706
+ .sourceRecords(syncRules.replicationStreamId, sourceTableB)
845
1707
  .insertMany([{ _id: 'later-delete', data: null, buckets: [], lookups: [], pending_delete: 12n }]);
846
1708
 
847
- const store = new SourceRecordStoreV3(db, syncRules.id, bucketStorage.sync_rules.mapping);
1709
+ const store = new SourceRecordStoreV3(
1710
+ db,
1711
+ syncRules.replicationStreamId,
1712
+ bucketStorage.replicationStream.syncConfigContent[0].mapping
1713
+ );
848
1714
  const logger = { info() {} } as any;
849
1715
 
850
1716
  await store.postCommitCleanup(6n, logger);
851
1717
 
852
- expect(await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: 5n })).toBe(0);
853
- expect(await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: 9n })).toBe(1);
854
- expect(await db.sourceRecordsV3(syncRules.id, sourceTableB).countDocuments({ pending_delete: 12n })).toBe(1);
855
- expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableA }))?.latest_pending_delete).toBe(9n);
856
- expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableB }))?.latest_pending_delete).toBe(12n);
1718
+ expect(
1719
+ await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).countDocuments({ pending_delete: 5n })
1720
+ ).toBe(0);
1721
+ expect(
1722
+ await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).countDocuments({ pending_delete: 9n })
1723
+ ).toBe(1);
1724
+ expect(
1725
+ await db.sourceRecords(syncRules.replicationStreamId, sourceTableB).countDocuments({ pending_delete: 12n })
1726
+ ).toBe(1);
1727
+ expect(
1728
+ (await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableA }))?.latest_pending_delete
1729
+ ).toBe(9n);
1730
+ expect(
1731
+ (await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableB }))?.latest_pending_delete
1732
+ ).toBe(12n);
857
1733
 
858
1734
  await store.postCommitCleanup(10n, logger);
859
1735
 
860
1736
  expect(
861
- await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: { $exists: true } })
1737
+ await db
1738
+ .sourceRecords(syncRules.replicationStreamId, sourceTableA)
1739
+ .countDocuments({ pending_delete: { $exists: true } })
862
1740
  ).toBe(0);
863
1741
  expect(
864
- (await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableA }))?.latest_pending_delete
1742
+ (await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableA }))?.latest_pending_delete
865
1743
  ).toBeUndefined();
866
- expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableB }))?.latest_pending_delete).toBe(12n);
1744
+ expect(
1745
+ (await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableB }))?.latest_pending_delete
1746
+ ).toBe(12n);
867
1747
  });
868
1748
  }
869
1749
 
@@ -880,7 +1760,7 @@ describe('sync - mongodb', () => {
880
1760
  await legacyWriter.markAllSnapshotDone('1/1');
881
1761
  await legacyWriter.commit('1/1');
882
1762
 
883
- expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.id }))?.state).toBe(
1763
+ expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.replicationStreamId }))?.state).toBe(
884
1764
  storage.SyncRuleState.ACTIVE
885
1765
  );
886
1766
 
@@ -892,7 +1772,7 @@ describe('sync - mongodb', () => {
892
1772
  await v3Writer.markAllSnapshotDone('2/1');
893
1773
  await v3Writer.commit('2/1');
894
1774
 
895
- expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.id }))?.state).toBe(
1775
+ expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.replicationStreamId }))?.state).toBe(
896
1776
  storage.SyncRuleState.STOP
897
1777
  );
898
1778
  });
@@ -900,6 +1780,233 @@ describe('sync - mongodb', () => {
900
1780
  for (const storageVersion of TEST_STORAGE_VERSIONS) {
901
1781
  describe(`storage v${storageVersion}`, () => {
902
1782
  registerSyncStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion);
1783
+
1784
+ describe.runIf(storageVersion == 3)('V3 read filtering boundaries', () => {
1785
+ async function setupFilteringTest() {
1786
+ await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
1787
+ const syncRules = await factory.updateSyncRules(
1788
+ updateSyncRulesFromYaml(
1789
+ `
1790
+ bucket_definitions:
1791
+ global:
1792
+ data:
1793
+ - SELECT id, description FROM test
1794
+ `,
1795
+ { storageVersion }
1796
+ )
1797
+ );
1798
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
1799
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
1800
+ const mapping = syncRules.syncConfigContent[0].mapping;
1801
+
1802
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
1803
+ const definitionId = mapping.bucketSourceId(request.source);
1804
+ const collection = db.bucketData(syncRules.replicationStreamId, definitionId);
1805
+
1806
+ const bucketName = request.bucket;
1807
+ const sourceTable = new bson.ObjectId();
1808
+ const bucketKey: BucketKey = {
1809
+ replicationStreamId: syncRules.replicationStreamId,
1810
+ definitionId,
1811
+ bucket: bucketName
1812
+ };
1813
+
1814
+ function makeOps(opIds: bigint[]): BucketDataDoc[] {
1815
+ return opIds.map((opId) => ({
1816
+ bucketKey,
1817
+ o: opId,
1818
+ op: 'PUT' as const,
1819
+ source_table: sourceTable,
1820
+ source_key: test_utils.rid(`row-${opId}`),
1821
+ table: 'items',
1822
+ row_id: `row-${opId}`,
1823
+ checksum: BigInt(opId) * 10n,
1824
+ data: `{"id":"row-${opId}"}`
1825
+ }));
1826
+ }
1827
+
1828
+ const docA = serializeBucketData(bucketName, makeOps([10n, 20n, 30n]));
1829
+ const docB = serializeBucketData(bucketName, makeOps([40n, 50n, 60n]));
1830
+ const docC = serializeBucketData(bucketName, makeOps([70n, 80n, 90n]));
1831
+
1832
+ await collection.insertMany([docA, docB, docC]);
1833
+
1834
+ return { factory, syncRules, bucketStorage, bucketName };
1835
+ }
1836
+
1837
+ async function getFilteredOps(start: number, checkpoint: number): Promise<bigint[]> {
1838
+ const { syncRules, bucketStorage } = await setupFilteringTest();
1839
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', BigInt(start));
1840
+ const batch = await test_utils.getBatchArray(
1841
+ bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(BigInt(checkpoint)), [request])
1842
+ );
1843
+ const ops = batch.flatMap((b) => b.chunkData.data.map((d) => BigInt(d.op_id)));
1844
+ return ops;
1845
+ }
1846
+
1847
+ test('case 1: start=5, checkpoint=95 → all ops', async () => {
1848
+ const ops = await getFilteredOps(5, 95);
1849
+ expect(ops).toEqual([10n, 20n, 30n, 40n, 50n, 60n, 70n, 80n, 90n]);
1850
+ });
1851
+
1852
+ test('case 2: start=10, checkpoint=90 → ops in (10,90]', async () => {
1853
+ const ops = await getFilteredOps(10, 90);
1854
+ expect(ops).toEqual([20n, 30n, 40n, 50n, 60n, 70n, 80n, 90n]);
1855
+ });
1856
+
1857
+ test('case 3: start=15, checkpoint=85 → partial doc boundaries', async () => {
1858
+ const ops = await getFilteredOps(15, 85);
1859
+ expect(ops).toEqual([20n, 30n, 40n, 50n, 60n, 70n, 80n]);
1860
+ });
1861
+
1862
+ test('case 4: start=25, checkpoint=55 → spans two docs', async () => {
1863
+ const ops = await getFilteredOps(25, 55);
1864
+ expect(ops).toEqual([30n, 40n, 50n]);
1865
+ });
1866
+
1867
+ test('case 5: start=35, checkpoint=45 → single op within doc', async () => {
1868
+ const ops = await getFilteredOps(35, 45);
1869
+ expect(ops).toEqual([40n]);
1870
+ });
1871
+
1872
+ test('case 6: start=35, checkpoint=65 → full doc B', async () => {
1873
+ const ops = await getFilteredOps(35, 65);
1874
+ expect(ops).toEqual([40n, 50n, 60n]);
1875
+ });
1876
+
1877
+ test('case 7: start=25, checkpoint=35 → single op from doc A', async () => {
1878
+ const ops = await getFilteredOps(25, 35);
1879
+ expect(ops).toEqual([30n]);
1880
+ });
1881
+
1882
+ test('case 8: start=30, checkpoint=40 → op at checkpoint from next doc', async () => {
1883
+ const ops = await getFilteredOps(30, 40);
1884
+ expect(ops).toEqual([40n]);
1885
+ });
1886
+
1887
+ test('case 9: start=100, checkpoint=200 → beyond all docs', async () => {
1888
+ const ops = await getFilteredOps(100, 200);
1889
+ expect(ops).toEqual([]);
1890
+ });
1891
+
1892
+ test('case 10: start=0, checkpoint=5 → before all docs', async () => {
1893
+ const ops = await getFilteredOps(0, 5);
1894
+ expect(ops).toEqual([]);
1895
+ });
1896
+
1897
+ test('case 11: start=50, checkpoint=50 → zero-width range', async () => {
1898
+ const ops = await getFilteredOps(50, 50);
1899
+ expect(ops).toEqual([]);
1900
+ });
1901
+
1902
+ test('case 12: start=45, checkpoint=50 → op at checkpoint boundary', async () => {
1903
+ const ops = await getFilteredOps(45, 50);
1904
+ expect(ops).toEqual([50n]);
1905
+ });
1906
+
1907
+ test('case 13: start=50, checkpoint=55 → no ops strictly after start', async () => {
1908
+ const ops = await getFilteredOps(50, 55);
1909
+ expect(ops).toEqual([]);
1910
+ });
1911
+
1912
+ test('all-filtered first batch still returns data behind the batch boundary', async () => {
1913
+ // Documents straddling the requested (start, end] window are matched by the
1914
+ // query, but contribute no rows after filtering. If an entire server batch
1915
+ // (~101 documents) consists of such straddlers, the remaining documents in
1916
+ // the cursor must still be reachable. Storage reports the straddler buckets
1917
+ // as complete via empty chunks, and the caller re-requests the rest.
1918
+ await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
1919
+ const syncRules = await factory.updateSyncRules(
1920
+ updateSyncRulesFromYaml(
1921
+ `
1922
+ bucket_definitions:
1923
+ by_user:
1924
+ parameters: select request.user_id() as user_id
1925
+ data: [select * from test where owner_id = bucket.user_id]
1926
+ `,
1927
+ { storageVersion }
1928
+ )
1929
+ );
1930
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorageV3;
1931
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
1932
+
1933
+ const start = 5n;
1934
+ const end = 50n;
1935
+
1936
+ // 150 buckets sorted before the data bucket, each with a single document
1937
+ // containing ops at 1 and 100: matched (_id.o=100 > start, min_op=1 <= end),
1938
+ // but no op in (5, 50].
1939
+ const straddlerNames = Array.from({ length: 150 }, (_, i) => `b${`${i}`.padStart(3, '0')}`);
1940
+ const requests = [...straddlerNames, 'zzz'].map((id) =>
1941
+ bucketRequest(syncRules.syncConfigContent[0], `by_user["${id}"]`, start)
1942
+ );
1943
+ const mapping = syncRules.syncConfigContent[0].mapping;
1944
+ const definitionId = mapping.bucketSourceId(requests[0].source);
1945
+ const collection = db.bucketData(syncRules.replicationStreamId, definitionId);
1946
+ const sourceTable = new bson.ObjectId();
1947
+
1948
+ function makeOps(bucket: string, opIds: bigint[]): BucketDataDoc[] {
1949
+ const bucketKey: BucketKey = {
1950
+ replicationStreamId: syncRules.replicationStreamId,
1951
+ definitionId,
1952
+ bucket
1953
+ };
1954
+ return opIds.map((opId) => ({
1955
+ bucketKey,
1956
+ o: opId,
1957
+ op: 'PUT' as const,
1958
+ source_table: sourceTable,
1959
+ source_key: test_utils.rid(`row-${opId}`),
1960
+ table: 'test',
1961
+ row_id: `row-${opId}`,
1962
+ checksum: BigInt(opId) * 10n,
1963
+ data: `{"id":"row-${opId}"}`
1964
+ }));
1965
+ }
1966
+
1967
+ const straddlerDocs = requests
1968
+ .slice(0, -1)
1969
+ .map((request) => serializeBucketData(request.bucket, makeOps(request.bucket, [1n, 100n])));
1970
+ const dataBucket = requests[requests.length - 1].bucket;
1971
+ const dataDoc = serializeBucketData(dataBucket, makeOps(dataBucket, [10n]));
1972
+ await collection.insertMany([...straddlerDocs, dataDoc]);
1973
+
1974
+ // Emulate the caller loop in sync.ts / BucketChecksumState: advance bucket
1975
+ // positions from each chunk, drop completed buckets, and re-request while
1976
+ // any chunk reported has_more.
1977
+ const positions = new Map(requests.map((request) => [request.bucket, request.start]));
1978
+ const pending = new Set(positions.keys());
1979
+ const receivedOps: bigint[] = [];
1980
+ let rounds = 0;
1981
+
1982
+ while (rounds < 10) {
1983
+ rounds++;
1984
+ const roundRequests = requests
1985
+ .filter((request) => pending.has(request.bucket))
1986
+ .map((request) => ({ ...request, start: positions.get(request.bucket)! }));
1987
+ const batch = await test_utils.getBatchArray(
1988
+ bucketStorage.getBucketDataBatch(test_utils.testCheckpoint(end), roundRequests)
1989
+ );
1990
+ let anyHasMore = false;
1991
+ for (const { chunkData } of batch) {
1992
+ positions.set(chunkData.bucket, BigInt(chunkData.next_after));
1993
+ if (chunkData.has_more) {
1994
+ anyHasMore = true;
1995
+ } else {
1996
+ pending.delete(chunkData.bucket);
1997
+ }
1998
+ receivedOps.push(...chunkData.data.map((entry) => BigInt(entry.op_id)));
1999
+ }
2000
+ if (!anyHasMore) {
2001
+ break;
2002
+ }
2003
+ }
2004
+
2005
+ // The op behind the all-straddler first batch must be returned.
2006
+ expect(receivedOps).toEqual([10n]);
2007
+ expect(rounds).toBeLessThan(10);
2008
+ });
2009
+ });
903
2010
  });
904
2011
  }
905
2012
  });