@powersync/service-module-mongodb-storage 0.16.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (206) hide show
  1. package/CHANGELOG.md +53 -0
  2. package/dist/storage/MongoBucketStorage.d.ts +35 -15
  3. package/dist/storage/MongoBucketStorage.js +309 -86
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/CheckpointState.d.ts +20 -0
  6. package/dist/storage/implementation/CheckpointState.js +31 -0
  7. package/dist/storage/implementation/CheckpointState.js.map +1 -0
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +69 -39
  9. package/dist/storage/implementation/MongoBucketBatch.js +104 -313
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +11 -9
  12. package/dist/storage/implementation/MongoChecksums.js +3 -111
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +3 -1
  15. package/dist/storage/implementation/MongoCompactor.js +18 -9
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +6 -5
  21. package/dist/storage/implementation/MongoParameterCompactor.js +24 -1
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +121 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +5 -2
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +50 -21
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +62 -199
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +5 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +9 -5
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +14 -0
  41. package/dist/storage/implementation/SyncRuleStateUpdate.js +36 -0
  42. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -0
  43. package/dist/storage/implementation/common/BucketDataDoc.d.ts +1 -1
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +6 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  47. package/dist/storage/implementation/common/PersistedBatch.d.ts +2 -3
  48. package/dist/storage/implementation/common/PersistedBatch.js +5 -0
  49. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  50. package/dist/storage/implementation/common/SourceRecordStore.d.ts +1 -2
  51. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +1 -1
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  53. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  54. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  55. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  56. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  57. package/dist/storage/implementation/db.d.ts +10 -2
  58. package/dist/storage/implementation/db.js +9 -14
  59. package/dist/storage/implementation/db.js.map +1 -1
  60. package/dist/storage/implementation/models.d.ts +25 -50
  61. package/dist/storage/implementation/models.js.map +1 -1
  62. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +23 -1
  63. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +403 -2
  64. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  65. package/dist/storage/implementation/v1/MongoChecksumsV1.js +114 -3
  66. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +1 -1
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +22 -15
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +100 -27
  70. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  71. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +1 -2
  72. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +21 -1
  77. package/dist/storage/implementation/v1/models.js.map +1 -1
  78. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +21 -0
  79. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +459 -3
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  81. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +27 -7
  82. package/dist/storage/implementation/v3/MongoChecksumsV3.js +202 -18
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  84. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +20 -3
  85. package/dist/storage/implementation/v3/MongoCompactorV3.js +511 -22
  86. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  87. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +62 -0
  88. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +350 -0
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  90. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +50 -16
  91. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +292 -75
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  93. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +2 -3
  94. package/dist/storage/implementation/v3/PersistedBatchV3.js +43 -20
  95. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  96. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +52 -16
  97. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +16 -14
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +4 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +58 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +28 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +111 -10
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  114. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  115. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  116. package/dist/storage/storage-index.d.ts +4 -4
  117. package/dist/storage/storage-index.js +4 -4
  118. package/dist/storage/storage-index.js.map +1 -1
  119. package/dist/types/types.d.ts +2 -2
  120. package/dist/utils/test-utils.d.ts +1 -2
  121. package/dist/utils/test-utils.js +5 -1
  122. package/dist/utils/test-utils.js.map +1 -1
  123. package/dist/utils/util.d.ts +1 -1
  124. package/dist/utils/util.js +3 -2
  125. package/dist/utils/util.js.map +1 -1
  126. package/package.json +9 -19
  127. package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +1 -1
  128. package/src/storage/MongoBucketStorage.ts +464 -95
  129. package/src/storage/implementation/CheckpointState.ts +59 -0
  130. package/src/storage/implementation/MongoBucketBatch.ts +159 -405
  131. package/src/storage/implementation/MongoChecksums.ts +7 -138
  132. package/src/storage/implementation/MongoCompactor.ts +20 -10
  133. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  134. package/src/storage/implementation/MongoParameterCompactor.ts +24 -5
  135. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  136. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  137. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +152 -0
  138. package/src/storage/implementation/MongoStorageProvider.ts +5 -2
  139. package/src/storage/implementation/MongoSyncBucketStorage.ts +89 -250
  140. package/src/storage/implementation/MongoSyncRulesLock.ts +14 -5
  141. package/src/storage/implementation/SyncRuleStateUpdate.ts +38 -0
  142. package/src/storage/implementation/common/BucketDataDoc.ts +1 -1
  143. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +7 -0
  144. package/src/storage/implementation/common/PersistedBatch.ts +8 -4
  145. package/src/storage/implementation/common/SourceRecordStore.ts +1 -2
  146. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  147. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  148. package/src/storage/implementation/db.ts +16 -21
  149. package/src/storage/implementation/models.ts +28 -61
  150. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +503 -5
  151. package/src/storage/implementation/v1/MongoChecksumsV1.ts +137 -4
  152. package/src/storage/implementation/v1/MongoCompactorV1.ts +1 -1
  153. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +150 -49
  154. package/src/storage/implementation/v1/PersistedBatchV1.ts +1 -2
  155. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  156. package/src/storage/implementation/v1/models.ts +25 -0
  157. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +611 -4
  158. package/src/storage/implementation/v3/MongoChecksumsV3.ts +262 -26
  159. package/src/storage/implementation/v3/MongoCompactorV3.ts +647 -30
  160. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +480 -0
  161. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +397 -111
  162. package/src/storage/implementation/v3/PersistedBatchV3.ts +63 -38
  163. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +59 -19
  164. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  165. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +48 -38
  166. package/src/storage/implementation/v3/bucket-format.ts +70 -0
  167. package/src/storage/implementation/v3/chunking.ts +38 -0
  168. package/src/storage/implementation/v3/models.ts +150 -44
  169. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  170. package/src/storage/storage-index.ts +12 -4
  171. package/src/utils/test-utils.ts +6 -3
  172. package/src/utils/util.ts +3 -2
  173. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  174. package/test/src/__snapshots__/storage_sync.test.ts.snap +48 -48
  175. package/test/src/cleanup-stopped-sync-configs.test.ts +648 -0
  176. package/test/src/storage.test.ts +11 -11
  177. package/test/src/storage_compacting.test.ts +1956 -6
  178. package/test/src/storage_sync.test.ts +1429 -42
  179. package/test/src/storeCurrentData.test.ts +221 -0
  180. package/test/src/util.ts +2 -1
  181. package/tsconfig.json +1 -4
  182. package/tsconfig.tsbuildinfo +1 -1
  183. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -17
  184. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  185. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  186. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  187. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -64
  188. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  189. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -15
  190. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -42
  191. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  192. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  193. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  194. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  195. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  196. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  197. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  198. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -5
  199. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  200. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  201. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -72
  202. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -76
  203. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -56
  204. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  205. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  206. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -12
@@ -1,16 +1,119 @@
1
- import { deserializeParameterLookup, JwtPayload, storage, updateSyncRulesFromYaml } from '@powersync/service-core';
1
+ import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/MongoSyncBucketStorageV3.js';
2
+ import {
3
+ deserializeParameterLookup,
4
+ JwtPayload,
5
+ SingleSyncConfigBucketDefinitionMapping,
6
+ storage,
7
+ updateSyncRulesFromYaml
8
+ } from '@powersync/service-core';
2
9
  import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
3
- import { RequestParameters } from '@powersync/service-sync-rules';
10
+ import { RequestParameters, ScopedParameterLookup, SqlSyncRules } from '@powersync/service-sync-rules';
4
11
  import * as bson from 'bson';
5
12
  import { describe, expect, test } from 'vitest';
6
13
  import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js';
14
+ import { MongoParsedSyncConfigSet } from '../../src/storage/implementation/MongoParsedSyncConfigSet.js';
15
+ import { MongoPersistedSyncConfigContentV3 } from '../../src/storage/implementation/MongoPersistedSyncConfigContent.js';
16
+ import { BucketDataDoc, BucketKey } from '../../src/storage/implementation/common/BucketDataDoc.js';
7
17
  import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
8
- import { SyncRuleDocument } from '../../src/storage/implementation/models.js';
18
+ import { getMongoStorageConfig } from '../../src/storage/implementation/models.js';
9
19
  import { SourceRecordStoreV3 } from '../../src/storage/implementation/v3/SourceRecordStoreV3.js';
10
20
  import type { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
11
- import { CurrentBucketV3 } from '../../src/storage/implementation/v3/models.js';
21
+ import { serializeBucketData } from '../../src/storage/implementation/v3/bucket-format.js';
22
+ import {
23
+ CurrentBucketV3,
24
+ ReplicationStreamDocumentV3,
25
+ SyncConfigDefinition
26
+ } from '../../src/storage/implementation/v3/models.js';
12
27
  import { INITIALIZED_MONGO_STORAGE_FACTORY, TEST_STORAGE_VERSIONS } from './util.js';
13
28
 
29
+ const MINIMAL_SYNC_RULES = `
30
+ bucket_definitions:
31
+ global:
32
+ data:
33
+ - SELECT id FROM test
34
+ `;
35
+
36
+ function sourceDescriptor(
37
+ name: string,
38
+ options: {
39
+ objectId?: string;
40
+ replicaIdColumns?: string[];
41
+ } = {}
42
+ ): storage.SourceEntityDescriptor {
43
+ return {
44
+ connectionTag: storage.SourceTable.DEFAULT_TAG,
45
+ objectId: options.objectId ?? name,
46
+ schema: 'public',
47
+ name,
48
+ replicaIdColumns: (options.replicaIdColumns ?? ['id']).map((column) => ({
49
+ name: column,
50
+ type: 'VARCHAR',
51
+ typeId: 25
52
+ }))
53
+ };
54
+ }
55
+
56
+ function objectIdGenerator(id: string) {
57
+ let used = false;
58
+ return () => {
59
+ if (used) {
60
+ throw new Error(`Can only generate a single id using ${id}`);
61
+ }
62
+ used = true;
63
+ return new bson.ObjectId(id);
64
+ };
65
+ }
66
+
67
+ /**
68
+ * Build a standalone parsed sync config set for the given yaml, with its own mapping.
69
+ *
70
+ * This keeps the sync rules and the bucket definition mapping paired, as required by
71
+ * `ResolveTablesOptions.parsedSyncConfig`.
72
+ */
73
+ function parsedSyncConfigSetFor(yaml: string, storageVersion: number) {
74
+ const parsed = SqlSyncRules.fromYaml(yaml, test_utils.PARSE_OPTIONS);
75
+ expect(parsed.errors).toEqual([]);
76
+ return new MongoParsedSyncConfigSet(1, getMongoStorageConfig(storageVersion), 'test_slot', [
77
+ {
78
+ syncConfigId: new bson.ObjectId().toHexString(),
79
+ syncConfig: parsed,
80
+ mapping: SingleSyncConfigBucketDefinitionMapping.fromParsedSyncConfig(parsed)
81
+ }
82
+ ]);
83
+ }
84
+
85
+ async function getMongoSyncConfigContents(factory: storage.BucketStorageFactory, replicationStreamId: number) {
86
+ const mongoFactory = factory as MongoBucketStorage;
87
+ const doc = (await mongoFactory.db.sync_rules.findOne({
88
+ _id: replicationStreamId
89
+ })) as ReplicationStreamDocumentV3 | null;
90
+ if (doc == null) {
91
+ return [];
92
+ }
93
+
94
+ const db = mongoFactory.db.versioned(getMongoStorageConfig(doc.storage_version)) as VersionedPowerSyncMongoV3;
95
+ const syncConfigDocs = await db.syncConfigDefinitions
96
+ .find({
97
+ _id: { $in: doc.sync_configs.map((config) => config._id) }
98
+ })
99
+ .toArray();
100
+
101
+ return syncConfigDocs.map((config) => new MongoPersistedSyncConfigContentV3(mongoFactory.db, doc, config));
102
+ }
103
+
104
+ /**
105
+ * Get a MongoDB V3 storage bucket definition id, from a bucket name.
106
+ *
107
+ * This relies on internals of bucket naming, but helps for simplifying tests.
108
+ */
109
+ function bucketDefinitionId(bucket: string) {
110
+ const match = bucket.match(/^\w+\|\w+\.\w+\.(\w+)\[/);
111
+ if (match == null) {
112
+ throw new Error(`Expected versioned bucket name, got ${bucket}`);
113
+ }
114
+ return parseInt(match[1], 16);
115
+ }
116
+
14
117
  function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, storageVersion: number) {
15
118
  register.registerSyncTests(storageConfig.factory, {
16
119
  storageVersion,
@@ -34,6 +137,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
34
137
  )
35
138
  );
36
139
  const bucketStorage = factory.getInstance(syncRules);
140
+ const syncRulesContent = syncRules.syncConfigContent[0];
37
141
 
38
142
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
39
143
 
@@ -88,7 +192,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
88
192
 
89
193
  const options: storage.BucketDataBatchOptions = {};
90
194
  const batch1 = await test_utils.fromAsync(
91
- bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)], options)
195
+ bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)], options)
92
196
  );
93
197
  expect(test_utils.getBatchData(batch1)).toEqual([
94
198
  { op_id: '1', op: 'PUT', object_id: 'test1', checksum: 2871785649 },
@@ -103,7 +207,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
103
207
  const batch2 = await test_utils.fromAsync(
104
208
  bucketStorage.getBucketDataBatch(
105
209
  checkpoint,
106
- [bucketRequest(syncRules, 'global[]', batch1[0].chunkData.next_after)],
210
+ [bucketRequest(syncRulesContent, 'global[]', batch1[0].chunkData.next_after)],
107
211
  options
108
212
  )
109
213
  );
@@ -119,7 +223,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
119
223
  const batch3 = await test_utils.fromAsync(
120
224
  bucketStorage.getBucketDataBatch(
121
225
  checkpoint,
122
- [bucketRequest(syncRules, 'global[]', batch2[0].chunkData.next_after)],
226
+ [bucketRequest(syncRulesContent, 'global[]', batch2[0].chunkData.next_after)],
123
227
  options
124
228
  )
125
229
  );
@@ -141,7 +245,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
141
245
  await Promise.all(
142
246
  (
143
247
  await mongoFactory.db.db
144
- .listCollections({ name: new RegExp(`^bucket_data_${syncRules.id}_`) }, { nameOnly: true })
248
+ .listCollections(
249
+ { name: new RegExp(`^bucket_data_${syncRules.replicationStreamId}_`) },
250
+ { nameOnly: true }
251
+ )
145
252
  .toArray()
146
253
  ).map((collection: { name: string }) =>
147
254
  mongoFactory.db.db
@@ -157,6 +264,386 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
157
264
  expect(checksumTypes).toEqual([{ _id: 'long', count: 4 }]);
158
265
  });
159
266
 
267
+ test('resolveTables populates matching data and parameter sources', async () => {
268
+ await using factory = await storageConfig.factory();
269
+ const syncRules = await factory.updateSyncRules(
270
+ updateSyncRulesFromYaml(
271
+ `
272
+ bucket_definitions:
273
+ by_owner:
274
+ parameters:
275
+ - SELECT owner_id FROM test WHERE id = token_parameters.test_id
276
+ data:
277
+ - SELECT id, owner_id FROM test WHERE owner_id = bucket.owner_id
278
+ `,
279
+ { storageVersion }
280
+ )
281
+ );
282
+ const bucketStorage = factory.getInstance(syncRules);
283
+
284
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
285
+ const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
286
+
287
+ expect(sourceTable.bucketDataSources).toHaveLength(1);
288
+ expect(sourceTable.parameterLookupSources).toHaveLength(1);
289
+ expect(sourceTable.syncData).toBe(true);
290
+ expect(sourceTable.syncParameters).toBe(true);
291
+ expect(sourceTable.syncEvent).toBe(false);
292
+ });
293
+
294
+ test('resolveTables drops old table when table name changes for the same objectId', async () => {
295
+ await using factory = await storageConfig.factory();
296
+ const syncRules = await factory.updateSyncRules(
297
+ updateSyncRulesFromYaml(
298
+ `
299
+ bucket_definitions:
300
+ global:
301
+ data:
302
+ - SELECT id FROM "%"
303
+ `,
304
+ { storageVersion }
305
+ )
306
+ );
307
+ const bucketStorage = factory.getInstance(syncRules);
308
+
309
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
310
+ const before = await writer.resolveTables({
311
+ connection_id: 1,
312
+ source: sourceDescriptor('orders', { objectId: 'orders-relation' }),
313
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38342')
314
+ });
315
+ const after = await writer.resolveTables({
316
+ connection_id: 1,
317
+ source: sourceDescriptor('renamed_orders', { objectId: 'orders-relation' }),
318
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38343')
319
+ });
320
+
321
+ expect(after.tables).toHaveLength(1);
322
+ expect(after.tables[0].id).not.toEqual(before.tables[0].id);
323
+ expect(after.tables[0].bucketDataSources).toHaveLength(1);
324
+ expect(after.tables[0].parameterLookupSources).toHaveLength(0);
325
+ expect(after.dropTables.map((table) => ({ id: table.id, name: table.name }))).toEqual([
326
+ { id: before.tables[0].id, name: 'orders' }
327
+ ]);
328
+ expect(after.dropTables[0].bucketDataSources).toHaveLength(1);
329
+ expect(after.dropTables[0].parameterLookupSources).toHaveLength(0);
330
+ });
331
+
332
+ test('resolveTables drops old table when objectId changes for the same table name', async () => {
333
+ await using factory = await storageConfig.factory();
334
+ const syncRules = await factory.updateSyncRules(
335
+ updateSyncRulesFromYaml(
336
+ `
337
+ bucket_definitions:
338
+ global:
339
+ data:
340
+ - SELECT id FROM "%"
341
+ `,
342
+ { storageVersion }
343
+ )
344
+ );
345
+ const bucketStorage = factory.getInstance(syncRules);
346
+
347
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
348
+ const before = await writer.resolveTables({
349
+ connection_id: 1,
350
+ source: sourceDescriptor('accounts', { objectId: 'accounts-relation-old' }),
351
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38344')
352
+ });
353
+ const after = await writer.resolveTables({
354
+ connection_id: 1,
355
+ source: sourceDescriptor('accounts', { objectId: 'accounts-relation-new' }),
356
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38345')
357
+ });
358
+
359
+ expect(after.tables).toHaveLength(1);
360
+ expect(after.tables[0].id).not.toEqual(before.tables[0].id);
361
+ expect(after.tables[0].bucketDataSources).toHaveLength(1);
362
+ expect(after.dropTables.map((table) => ({ id: table.id, objectId: table.objectId }))).toEqual([
363
+ { id: before.tables[0].id, objectId: 'accounts-relation-old' }
364
+ ]);
365
+ expect(after.dropTables[0].bucketDataSources).toHaveLength(1);
366
+ });
367
+
368
+ test('resolveTables drops old table when replica id columns change', async () => {
369
+ await using factory = await storageConfig.factory();
370
+ const syncRules = await factory.updateSyncRules(
371
+ updateSyncRulesFromYaml(
372
+ `
373
+ bucket_definitions:
374
+ global:
375
+ data:
376
+ - SELECT id FROM "%"
377
+ `,
378
+ { storageVersion }
379
+ )
380
+ );
381
+ const bucketStorage = factory.getInstance(syncRules);
382
+
383
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
384
+ const before = await writer.resolveTables({
385
+ connection_id: 1,
386
+ source: sourceDescriptor('items', { objectId: 'items-relation', replicaIdColumns: ['id'] }),
387
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38346')
388
+ });
389
+ const after = await writer.resolveTables({
390
+ connection_id: 1,
391
+ source: sourceDescriptor('items', { objectId: 'items-relation', replicaIdColumns: ['tenant_id', 'id'] }),
392
+ idGenerator: objectIdGenerator('6544e3899293153fa7b38347')
393
+ });
394
+
395
+ expect(after.tables).toHaveLength(1);
396
+ expect(after.tables[0].id).not.toEqual(before.tables[0].id);
397
+ expect(after.tables[0].replicaIdColumns.map((column) => column.name)).toEqual(['tenant_id', 'id']);
398
+ expect(after.tables[0].bucketDataSources).toHaveLength(1);
399
+ expect(
400
+ after.dropTables.map((table) => ({ id: table.id, columns: table.replicaIdColumns.map((c) => c.name) }))
401
+ ).toEqual([{ id: before.tables[0].id, columns: ['id'] }]);
402
+ expect(after.dropTables[0].bucketDataSources).toHaveLength(1);
403
+ });
404
+
405
+ test.runIf(storageVersion >= 3)(
406
+ 'resolveTables resolves v3 event-only tables without source memberships',
407
+ async () => {
408
+ await using factory = await storageConfig.factory();
409
+ const syncRules = await factory.updateSyncRules(
410
+ updateSyncRulesFromYaml(
411
+ `
412
+ bucket_definitions:
413
+ by_owner:
414
+ data:
415
+ - SELECT id FROM users
416
+
417
+ event_definitions:
418
+ write_checkpoints:
419
+ payloads:
420
+ - SELECT user_id, checkpoint FROM checkpoints
421
+ `,
422
+ { storageVersion }
423
+ )
424
+ );
425
+ const bucketStorage = factory.getInstance(syncRules);
426
+
427
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
428
+ const resolved = await writer.resolveTables({
429
+ connection_id: 1,
430
+ source: {
431
+ connectionTag: storage.SourceTable.DEFAULT_TAG,
432
+ objectId: 'checkpoints',
433
+ schema: 'public',
434
+ name: 'checkpoints',
435
+ replicaIdColumns: [{ name: 'id', type: 'VARCHAR', typeId: 25 }]
436
+ },
437
+ idGenerator: () => new bson.ObjectId('6544e3899293153fa7b38341')
438
+ });
439
+
440
+ expect(resolved.tables).toHaveLength(1);
441
+ expect(resolved.dropTables).toHaveLength(0);
442
+ expect(resolved.tables[0].bucketDataSources).toEqual([]);
443
+ expect(resolved.tables[0].parameterLookupSources).toEqual([]);
444
+ expect(resolved.tables[0].syncData).toBe(false);
445
+ expect(resolved.tables[0].syncParameters).toBe(false);
446
+ expect(resolved.tables[0].syncEvent).toBe(true);
447
+ }
448
+ );
449
+
450
+ test.runIf(storageVersion >= 3)('resolveTables handles v3 source membership additions', async () => {
451
+ // Tests the behavior of resolveTables when bucket data sources and parameter index creators are added.
452
+ // These are not end-to-end tests yet, since we don't have a full incremental reprocessing implementation.
453
+ // This just tests the specific resolveTables behavior.
454
+
455
+ // The same tests should work with sync streams, but legacy bucket_definitions make it easy
456
+ // to see the distinction between the parameter index queries and the data sources.
457
+ const fullRulesYaml = `
458
+ bucket_definitions:
459
+ by_owner:
460
+ parameters:
461
+ - SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
462
+ data:
463
+ - SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
464
+ `;
465
+ const dataOnlyRulesYaml = `
466
+ bucket_definitions:
467
+ by_owner:
468
+ parameters:
469
+ - SELECT token_parameters.owner_id as owner_id
470
+ data:
471
+ - SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
472
+ `;
473
+ const parameterOnlyRulesYaml = `
474
+ bucket_definitions:
475
+ by_owner:
476
+ parameters:
477
+ - SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
478
+ data: []
479
+ `;
480
+ const eventOnlyRulesYaml = `
481
+ bucket_definitions: {}
482
+
483
+ event_definitions:
484
+ write_checkpoints:
485
+ payloads:
486
+ - SELECT id, owner_id FROM memberships
487
+ `;
488
+
489
+ await using factory = await storageConfig.factory();
490
+ // This does not quite match what actual API usage would look like.
491
+ // Here we're persisting one sync config, then resolving tables with others.
492
+ // We're also using the default hydration state for them all.
493
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(fullRulesYaml, { storageVersion }));
494
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
495
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
496
+ const fullRules = parsedSyncConfigSetFor(fullRulesYaml, storageVersion);
497
+ const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyRulesYaml, storageVersion);
498
+ const parameterOnlyRules = parsedSyncConfigSetFor(parameterOnlyRulesYaml, storageVersion);
499
+ const eventOnlyRules = parsedSyncConfigSetFor(eventOnlyRulesYaml, storageVersion);
500
+ const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
501
+ const dataOnlyTableId = new bson.ObjectId('6544e3899293153fa7b38348');
502
+ const addedParameterTableId = new bson.ObjectId('6544e3899293153fa7b38349');
503
+
504
+ const dataOnly = await writer.resolveTables({
505
+ connection_id: 1,
506
+ source,
507
+ idGenerator: () => dataOnlyTableId,
508
+ parsedSyncConfig: dataOnlyRules
509
+ });
510
+ expect(dataOnly.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
511
+ expect(dataOnly.dropTables.map((table) => table.id)).toEqual([]);
512
+ expect(dataOnly.tables[0].bucketDataSources).toHaveLength(1);
513
+ expect(dataOnly.tables[0].parameterLookupSources).toHaveLength(0);
514
+
515
+ const addedParameter = await writer.resolveTables({
516
+ connection_id: 1,
517
+ source,
518
+ idGenerator: () => addedParameterTableId,
519
+ parsedSyncConfig: fullRules
520
+ });
521
+ // Adding a definition always creates a new SourceTable
522
+ expect(addedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId, addedParameterTableId]);
523
+ expect(addedParameter.tables.map((table) => table.bucketDataSources.length).sort()).toEqual([0, 1]);
524
+ expect(addedParameter.tables.map((table) => table.parameterLookupSources.length).sort()).toEqual([0, 1]);
525
+ expect(addedParameter.dropTables.map((table) => table.id)).toEqual([]);
526
+
527
+ const removedParameter = await writer.resolveTables({
528
+ connection_id: 1,
529
+ source,
530
+ idGenerator: () => {
531
+ throw new Error('data-only resolve should reuse existing v3 source table');
532
+ },
533
+ parsedSyncConfig: dataOnlyRules
534
+ });
535
+ expect(removedParameter.tables.map((table) => table.id)).toEqual([dataOnlyTableId]);
536
+ // Same-identity tables that no longer cover desired memberships are left in place.
537
+ expect(removedParameter.dropTables.map((table) => table.id)).toEqual([]);
538
+ expect(removedParameter.tables[0].bucketDataSources).toHaveLength(1);
539
+ expect(removedParameter.tables[0].parameterLookupSources).toHaveLength(0);
540
+
541
+ const removedData = await writer.resolveTables({
542
+ connection_id: 1,
543
+ source,
544
+ idGenerator: () => {
545
+ throw new Error('parameter-only resolve should reuse existing v3 source table');
546
+ },
547
+ parsedSyncConfig: parameterOnlyRules
548
+ });
549
+
550
+ // This goes from dataOnlyRules -> parameterOnlyRules, which removes one definition but
551
+ // reuses the existing parameter-only source table.
552
+ expect(removedData.tables.map((table) => table.id)).toEqual([addedParameterTableId]);
553
+ expect(removedData.dropTables.map((table) => table.id)).toEqual([]);
554
+ expect(removedData.tables[0].bucketDataSources).toHaveLength(0);
555
+ expect(removedData.tables[0].parameterLookupSources).toHaveLength(1);
556
+
557
+ const eventOnly = await writer.resolveTables({
558
+ connection_id: 1,
559
+ source,
560
+ idGenerator: () => {
561
+ throw new Error('resolve should reuse existing v3 source table');
562
+ },
563
+ parsedSyncConfig: eventOnlyRules
564
+ });
565
+
566
+ // Event-only table can re-use any existing table.
567
+ expect(eventOnly.tables).toHaveLength(1);
568
+ expect([dataOnlyTableId.toString(), addedParameterTableId.toString()]).toContain(eventOnly.tables[0].id.toString());
569
+ expect(eventOnly.dropTables.map((table) => table.id)).toEqual([]);
570
+ expect(eventOnly.tables[0].bucketDataSources).toHaveLength(0);
571
+ expect(eventOnly.tables[0].parameterLookupSources).toHaveLength(0);
572
+ expect(eventOnly.tables[0].syncData).toBe(false);
573
+ expect(eventOnly.tables[0].syncParameters).toBe(false);
574
+ expect(eventOnly.tables[0].syncEvent).toBe(true);
575
+ });
576
+
577
+ test.runIf(storageVersion >= 3)(
578
+ 'resolveTables designates a single event carrier across split source tables',
579
+ async () => {
580
+ // When memberships are split over multiple SourceTables for the same ref, a row change
581
+ // is saved once per table. Only one table may fire events, otherwise the same event
582
+ // would fire once per table for every row change.
583
+ const dataOnlyEventYaml = `
584
+ bucket_definitions:
585
+ by_owner:
586
+ parameters:
587
+ - SELECT token_parameters.owner_id as owner_id
588
+ data:
589
+ - SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
590
+
591
+ event_definitions:
592
+ write_checkpoints:
593
+ payloads:
594
+ - SELECT id, owner_id FROM memberships
595
+ `;
596
+ const fullEventYaml = `
597
+ bucket_definitions:
598
+ by_owner:
599
+ parameters:
600
+ - SELECT owner_id FROM memberships WHERE id = token_parameters.test_id
601
+ data:
602
+ - SELECT id, owner_id FROM memberships WHERE owner_id = bucket.owner_id
603
+
604
+ event_definitions:
605
+ write_checkpoints:
606
+ payloads:
607
+ - SELECT id, owner_id FROM memberships
608
+ `;
609
+
610
+ await using factory = await storageConfig.factory();
611
+ const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(dataOnlyEventYaml, { storageVersion }));
612
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
613
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
614
+ const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyEventYaml, storageVersion);
615
+ const fullRules = parsedSyncConfigSetFor(fullEventYaml, storageVersion);
616
+ const source = sourceDescriptor('memberships', { objectId: 'memberships-relation' });
617
+
618
+ const initial = await writer.resolveTables({
619
+ connection_id: 1,
620
+ source,
621
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834b'),
622
+ parsedSyncConfig: dataOnlyRules
623
+ });
624
+ expect(initial.tables).toHaveLength(1);
625
+ expect(initial.tables[0].syncEvent).toBe(true);
626
+
627
+ // Adding the table-based parameter lookup creates a second SourceTable for the same ref.
628
+ const split = await writer.resolveTables({
629
+ connection_id: 1,
630
+ source,
631
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834c'),
632
+ parsedSyncConfig: fullRules
633
+ });
634
+ expect(split.tables).toHaveLength(2);
635
+ // Both tables match the event by ref, but only one may carry it.
636
+ const carriers = split.tables.filter((table) => table.syncEvent);
637
+ expect(carriers).toHaveLength(1);
638
+
639
+ // getSourceTableStatus preserves the carrier designation rather than recomputing it
640
+ // from the ref, so refreshing a non-carrier table does not make it fire events.
641
+ const nonCarrier = split.tables.find((table) => !table.syncEvent)!;
642
+ const refreshed = await writer.getSourceTableStatus(nonCarrier);
643
+ expect(refreshed!.syncEvent).toBe(false);
644
+ }
645
+ );
646
+
160
647
  test.runIf(storageVersion >= 3)('uses v3 mongodb model shapes', async () => {
161
648
  await using factory = await storageConfig.factory();
162
649
  const syncRules = await factory.updateSyncRules(
@@ -173,7 +660,8 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
173
660
  )
174
661
  );
175
662
  const bucketStorage = factory.getInstance(syncRules);
176
- const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncRules();
663
+ const syncRulesContent = syncRules.syncConfigContent[0];
664
+ const sync_rules = syncRulesContent.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig;
177
665
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
178
666
  const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
179
667
 
@@ -194,7 +682,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
194
682
  const parameters = new RequestParameters(new JwtPayload({ sub: 'u1', parameters: { test: 'shape-check' } }), {});
195
683
  const querier = sync_rules.getBucketParameterQuerier(test_utils.querierOptions(parameters)).querier;
196
684
  const buckets = await querier.queryDynamicBucketDescriptions({
197
- async getParameterSets(lookups) {
685
+ async getParameterSets(lookups: ScopedParameterLookup[]) {
198
686
  expect(lookups.map((l) => l.indexKey)).toEqual([['shape-check']]);
199
687
  expect(lookups[0].indexId).toEqual('1');
200
688
 
@@ -203,32 +691,663 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
203
691
  return parameter_sets;
204
692
  }
205
693
  });
206
- expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRules, 'global["user-1"]').bucket]);
694
+ expect(buckets.map((b) => b.bucket)).toEqual([bucketRequest(syncRulesContent, 'global["user-1"]').bucket]);
207
695
 
208
696
  const mongoFactory = factory as MongoBucketStorage;
209
697
  const db = (bucketStorage as MongoSyncBucketStorage).db as VersionedPowerSyncMongoV3;
210
- const currentDataCollections = await db.listSourceRecordCollectionsV3(syncRules.id);
698
+ const currentDataCollections = await db.listSourceRecordCollections(syncRules.replicationStreamId);
211
699
  const currentData = await currentDataCollections[0]?.findOne({});
212
700
  const firstBucket: CurrentBucketV3 | undefined = currentData?.buckets[0] as CurrentBucketV3 | undefined;
213
701
  expect(firstBucket?.def).toMatch(/^[0-9a-f]+$/);
214
702
 
215
703
  const bucketCollections = await mongoFactory.db.db
216
- .listCollections({ name: new RegExp(`^bucket_data_${syncRules.id}_`) }, { nameOnly: true })
704
+ .listCollections({ name: new RegExp(`^bucket_data_${syncRules.replicationStreamId}_`) }, { nameOnly: true })
217
705
  .toArray();
218
706
  expect(
219
- bucketCollections.some((collection) => collection.name === `bucket_data_${syncRules.id}_${firstBucket?.def}`)
707
+ bucketCollections.some(
708
+ (collection) => collection.name === `bucket_data_${syncRules.replicationStreamId}_${firstBucket?.def}`
709
+ )
220
710
  ).toBe(true);
221
711
 
222
- const syncRule = await mongoFactory.db.sync_rules.findOne({ _id: syncRules.id });
223
- const ruleMapping: SyncRuleDocument['rule_mapping'] | undefined = syncRule?.rule_mapping;
712
+ const syncRule = (await mongoFactory.db.sync_rules.findOne({
713
+ _id: syncRules.replicationStreamId
714
+ })) as ReplicationStreamDocumentV3;
715
+ const syncConfig = await db.syncConfigDefinitions.findOne({ _id: syncRule.sync_configs[0]._id });
716
+ const ruleMapping: SyncConfigDefinition['rule_mapping'] | undefined = syncConfig?.rule_mapping;
224
717
  expect(Object.keys(ruleMapping?.definitions ?? {})).not.toHaveLength(0);
225
718
 
226
719
  const parameterIndexId = Object.values(ruleMapping?.parameter_indexes ?? {})[0] as string | undefined;
227
720
  expect(parameterIndexId).toBeDefined();
228
- const parameterEntry = await db.parameterIndexV3(syncRules.id, parameterIndexId!).findOne({});
721
+ const parameterEntry = await db.parameterIndex(syncRules.replicationStreamId, parameterIndexId!).findOne({});
229
722
  expect(deserializeParameterLookup(parameterEntry!.lookup)).toEqual(['shape-check']);
230
723
  });
231
724
 
725
+ test.runIf(storageVersion >= 3)('replaces an existing deploying sync config', async () => {
726
+ await using factory = await storageConfig.factory();
727
+
728
+ const first = await factory.updateSyncRules(
729
+ updateSyncRulesFromYaml(
730
+ `
731
+ config:
732
+ edition: 2
733
+
734
+ streams:
735
+ by_owner:
736
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
737
+ `,
738
+ { storageVersion }
739
+ )
740
+ );
741
+ const second = await factory.updateSyncRules(
742
+ updateSyncRulesFromYaml(
743
+ `
744
+ config:
745
+ edition: 2
746
+
747
+ streams:
748
+ by_project:
749
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
750
+ `,
751
+ { storageVersion }
752
+ )
753
+ );
754
+
755
+ expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
756
+ expect(
757
+ (await factory.getStoppedReplicationStreams()).find(
758
+ (stream) => stream.replicationStreamId == first.replicationStreamId
759
+ )?.state
760
+ ).toBe(storage.SyncRuleState.STOP);
761
+
762
+ const replicatingStreams = await factory.getReplicatingReplicationStreams();
763
+ expect(replicatingStreams).toHaveLength(1);
764
+ expect(replicatingStreams[0].replicationStreamId).toEqual(second.replicationStreamId);
765
+
766
+ const configs = second.syncConfigContent;
767
+ expect(configs).toHaveLength(1);
768
+ const statuses = await Promise.all(configs.map((config) => config.getSyncConfigStatus()));
769
+ expect(statuses.map((status) => status?.state)).toEqual([storage.SyncRuleState.PROCESSING]);
770
+ expect(statuses.map((status) => status?.id)).toEqual(configs.map((config) => config.syncConfigId));
771
+ const parsed = replicatingStreams[0].parsed(test_utils.PARSE_OPTIONS);
772
+ expect(parsed.syncConfigs).toHaveLength(1);
773
+ expect(parsed.hydratedSyncConfig.bucketDataSources).toHaveLength(1);
774
+
775
+ const bucketStorage = factory.getInstance(replicatingStreams[0]);
776
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
777
+ const resolved = await writer.resolveTables({
778
+ connection_id: 1,
779
+ source: sourceDescriptor('todos', { objectId: 'todos-relation' }),
780
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834b')
781
+ });
782
+ expect(resolved.tables).toHaveLength(1);
783
+ expect(resolved.tables[0].bucketDataSources).toHaveLength(1);
784
+ });
785
+
786
+ test.runIf(storageVersion >= 3)('removing one config keeps shared source-table membership', async () => {
787
+ await using factory = await storageConfig.factory();
788
+
789
+ const first = await factory.updateSyncRules(
790
+ updateSyncRulesFromYaml(
791
+ `
792
+ config:
793
+ edition: 3
794
+
795
+ streams:
796
+ by_owner:
797
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
798
+ `,
799
+ { storageVersion }
800
+ )
801
+ );
802
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
803
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
804
+ await firstWriter.markAllSnapshotDone('1/1');
805
+ await firstWriter.commit('1/1');
806
+
807
+ const second = await factory.updateSyncRules(
808
+ updateSyncRulesFromYaml(
809
+ `
810
+ config:
811
+ edition: 3
812
+
813
+ streams:
814
+ by_project:
815
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
816
+ `,
817
+ { storageVersion }
818
+ )
819
+ );
820
+ expect(second.replicationStreamId).toEqual(first.replicationStreamId);
821
+
822
+ const bucketStorage = factory.getInstance(second);
823
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
824
+ const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
825
+
826
+ const resolved = await writer.resolveTables({
827
+ connection_id: 1,
828
+ source,
829
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834c')
830
+ });
831
+ expect(resolved.tables).toHaveLength(1);
832
+ expect(resolved.tables[0].bucketDataSources).toHaveLength(2);
833
+
834
+ await writer.markAllSnapshotDone('2/1');
835
+ await writer.commit('2/1');
836
+
837
+ const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
838
+ await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
839
+ const activeStatus = await activeWriter.getSourceTableStatus(resolved.tables[0]);
840
+ expect(activeStatus?.bucketDataSources).toHaveLength(1);
841
+ });
842
+
843
+ test.runIf(storageVersion >= 3)(
844
+ 'editing a stream query on the same table does not drop the active source table',
845
+ async () => {
846
+ await using factory = await storageConfig.factory();
847
+
848
+ const first = await factory.updateSyncRules(
849
+ updateSyncRulesFromYaml(
850
+ `
851
+ config:
852
+ edition: 3
853
+
854
+ streams:
855
+ todos:
856
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
857
+ `,
858
+ { storageVersion }
859
+ )
860
+ );
861
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
862
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
863
+ const source = sourceDescriptor('todos', { objectId: 'todos-relation' });
864
+ const firstResolved = await firstWriter.resolveTables({
865
+ connection_id: 1,
866
+ source,
867
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834d')
868
+ });
869
+ expect(firstResolved.dropTables).toHaveLength(0);
870
+ await firstWriter.markAllSnapshotDone('1/1');
871
+ await firstWriter.commit('1/1');
872
+
873
+ const second = await factory.updateSyncRules(
874
+ updateSyncRulesFromYaml(
875
+ `
876
+ config:
877
+ edition: 3
878
+
879
+ streams:
880
+ todos:
881
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
882
+ `,
883
+ { storageVersion }
884
+ )
885
+ );
886
+ expect(second.replicationStreamId).toBe(first.replicationStreamId);
887
+
888
+ const deploying = await factory.getDeployingSyncConfig();
889
+ expect(deploying?.replicationStream.replicationStreamId).toBe(first.replicationStreamId);
890
+ const bucketStorage = deploying!.storage;
891
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
892
+ const resolved = await writer.resolveTables({
893
+ connection_id: 1,
894
+ source,
895
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834e')
896
+ });
897
+
898
+ expect(resolved.dropTables).toHaveLength(0);
899
+ expect(resolved.tables.map((table) => table.id.toString())).toEqual(['6544e3899293153fa7b3834e']);
900
+ expect(resolved.tables[0].bucketDataSources).toHaveLength(1);
901
+
902
+ const activeStatus = await firstWriter.getSourceTableStatus(firstResolved.tables[0]);
903
+ expect(activeStatus?.bucketDataSources).toHaveLength(1);
904
+ }
905
+ );
906
+
907
+ test.runIf(storageVersion >= 3)('reserves historical mapping ids without reusing stopped configs', async () => {
908
+ await using factory = await storageConfig.factory();
909
+
910
+ const ownerRules = `
911
+ config:
912
+ edition: 3
913
+
914
+ streams:
915
+ by_owner:
916
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
917
+ `;
918
+ const projectRules = `
919
+ config:
920
+ edition: 3
921
+
922
+ streams:
923
+ by_project:
924
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
925
+ `;
926
+ const statusRules = `
927
+ config:
928
+ edition: 3
929
+
930
+ streams:
931
+ by_status:
932
+ query: SELECT * FROM todos WHERE status = subscription.parameter('status')
933
+ `;
934
+
935
+ const first = await factory.updateSyncRules(updateSyncRulesFromYaml(ownerRules, { storageVersion }));
936
+ const firstStorage = factory.getInstance(first);
937
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
938
+ await firstWriter.markAllSnapshotDone('1/1');
939
+ await firstWriter.commit('1/1');
940
+
941
+ const second = await factory.updateSyncRules(updateSyncRulesFromYaml(projectRules, { storageVersion }));
942
+ expect(second.replicationStreamId).toBe(first.replicationStreamId);
943
+
944
+ const third = await factory.updateSyncRules(updateSyncRulesFromYaml(statusRules, { storageVersion }));
945
+ expect(third.replicationStreamId).toBe(first.replicationStreamId);
946
+
947
+ const configs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
948
+ const statuses = await Promise.all(
949
+ configs.map(async (config) => [config, await config.getSyncConfigStatus()] as const)
950
+ );
951
+ // ownerRules
952
+ const activeConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.ACTIVE)?.[0];
953
+ // projectRules
954
+ const stoppedConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.STOP)?.[0];
955
+ // statusRules
956
+ const processingConfig = statuses.find(([, status]) => status?.state == storage.SyncRuleState.PROCESSING)?.[0];
957
+ expect(stoppedConfig).toBeDefined();
958
+ expect(activeConfig).toBeDefined();
959
+ expect(processingConfig).toBeDefined();
960
+
961
+ const activePrefix = bucketRequest(activeConfig!, 'by_owner|0["owner"]').bucket;
962
+ const stoppedPrefix = bucketRequest(stoppedConfig!, 'by_project|0["project"]').bucket;
963
+ const processingPrefix = bucketRequest(processingConfig!, 'by_status|0["status"]').bucket;
964
+
965
+ expect(processingPrefix).not.toBe(stoppedPrefix);
966
+ expect(processingPrefix).not.toBe(activePrefix);
967
+ expect(bucketDefinitionId(processingPrefix)).toBeGreaterThan(
968
+ Math.max(bucketDefinitionId(stoppedPrefix), bucketDefinitionId(activePrefix))
969
+ );
970
+ });
971
+
972
+ test.runIf(storageVersion >= 3)('table snapshot status only affects sync configs using that table', async () => {
973
+ await using factory = await storageConfig.factory();
974
+
975
+ const first = await factory.updateSyncRules(
976
+ updateSyncRulesFromYaml(
977
+ `
978
+ config:
979
+ edition: 3
980
+
981
+ streams:
982
+ by_owner:
983
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
984
+ `,
985
+ { storageVersion }
986
+ )
987
+ );
988
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
989
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
990
+ await firstWriter.markAllSnapshotDone('1/1');
991
+ await firstWriter.commit('1/1');
992
+
993
+ const second = await factory.updateSyncRules(
994
+ updateSyncRulesFromYaml(
995
+ `
996
+ config:
997
+ edition: 3
998
+
999
+ streams:
1000
+ by_owner:
1001
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1002
+ by_project:
1003
+ query: SELECT * FROM projects WHERE id = subscription.parameter('project_id')
1004
+ `,
1005
+ { storageVersion }
1006
+ )
1007
+ );
1008
+ expect(second.replicationStreamId).toBe(first.replicationStreamId);
1009
+
1010
+ const bucketStorage = factory.getInstance(second);
1011
+ await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
1012
+ const resolved = await writer.resolveTables({
1013
+ connection_id: 1,
1014
+ source: sourceDescriptor('projects', { objectId: 'projects-relation' }),
1015
+ idGenerator: objectIdGenerator('6544e3899293153fa7b3834d')
1016
+ });
1017
+ expect(resolved.tables).toHaveLength(1);
1018
+
1019
+ await writer.markAllSnapshotDone('2/1');
1020
+ await writer.markTableSnapshotRequired(resolved.tables[0]);
1021
+
1022
+ const configsAfterRequired = await getMongoSyncConfigContents(factory, first.replicationStreamId);
1023
+ const statusesAfterRequired = await Promise.all(configsAfterRequired.map((config) => config.getSyncConfigStatus()));
1024
+ const ownerConfigAfterRequired = statusesAfterRequired.find(
1025
+ (status) => status?.state == storage.SyncRuleState.ACTIVE
1026
+ );
1027
+ const projectConfigAfterRequired = statusesAfterRequired.find(
1028
+ (status) => status?.state == storage.SyncRuleState.PROCESSING
1029
+ );
1030
+ expect(ownerConfigAfterRequired?.snapshot_done).toBe(true);
1031
+ expect(projectConfigAfterRequired?.snapshot_done).toBe(false);
1032
+
1033
+ await writer.markTableSnapshotDone(resolved.tables, '3/1');
1034
+
1035
+ const configsAfterDone = await getMongoSyncConfigContents(factory, first.replicationStreamId);
1036
+ const statusesAfterDone = await Promise.all(configsAfterDone.map((config) => config.getSyncConfigStatus()));
1037
+ const ownerConfigAfterDone = statusesAfterDone.find((status) => status?.id == ownerConfigAfterRequired!.id);
1038
+ const projectConfigAfterDone = statusesAfterDone.find((status) => status?.id == projectConfigAfterRequired!.id);
1039
+ expect(ownerConfigAfterDone?.snapshot_done).toBe(true);
1040
+ expect(projectConfigAfterDone?.snapshot_done).toBe(false);
1041
+ });
1042
+
1043
+ test.runIf(storageVersion >= 3)(
1044
+ 'keeps compatible active and deploying sync configs in one replication stream',
1045
+ async () => {
1046
+ await using factory = await storageConfig.factory();
1047
+
1048
+ const first = await factory.updateSyncRules(
1049
+ updateSyncRulesFromYaml(
1050
+ `
1051
+ config:
1052
+ edition: 3
1053
+
1054
+ streams:
1055
+ by_owner:
1056
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1057
+ `,
1058
+ { storageVersion }
1059
+ )
1060
+ );
1061
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1062
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1063
+ await firstWriter.markAllSnapshotDone('1/1');
1064
+ await firstWriter.commit('1/1');
1065
+ const initialActiveStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
1066
+
1067
+ let configs = first.syncConfigContent;
1068
+ const firstConfigId = configs[0].syncConfigId;
1069
+ expect((await factory.getActiveSyncConfig())?.content.syncConfigId).toBe(firstConfigId);
1070
+
1071
+ const second = await factory.updateSyncRules(
1072
+ updateSyncRulesFromYaml(
1073
+ `
1074
+ config:
1075
+ edition: 3
1076
+
1077
+ streams:
1078
+ by_project:
1079
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
1080
+ `,
1081
+ { storageVersion }
1082
+ )
1083
+ );
1084
+ expect(second.replicationStreamId).toEqual(first.replicationStreamId);
1085
+
1086
+ configs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
1087
+ const deploying = await factory.getDeployingSyncConfig();
1088
+ expect(configs).toHaveLength(2);
1089
+ expect(deploying).not.toBeNull();
1090
+ expect((await factory.getActiveSyncConfig())?.content.syncConfigId).toBe(firstConfigId);
1091
+
1092
+ const stream = (await factory.getActiveSyncConfig())?.replicationStream;
1093
+ expect(stream?.state).toBe(storage.SyncRuleState.ACTIVE);
1094
+ expect(
1095
+ (await Promise.all(configs.map((config) => config.getSyncConfigStatus()))).map((status) => status?.state).sort()
1096
+ ).toEqual([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.PROCESSING]);
1097
+
1098
+ const replicatingStreams = await factory.getReplicatingReplicationStreams();
1099
+ expect(replicatingStreams).toHaveLength(1);
1100
+ for (const config of configs) {
1101
+ expect(replicatingStreams[0].replicationJobId).toContain(config.syncConfigId);
1102
+ }
1103
+
1104
+ const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage;
1105
+ await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
1106
+ await secondWriter.markAllSnapshotDone('2/1');
1107
+ await secondWriter.commit('2/1');
1108
+
1109
+ const updatedStream = (await factory.getActiveSyncConfig())?.replicationStream;
1110
+ expect(updatedStream?.state).toBe(storage.SyncRuleState.ACTIVE);
1111
+ const updatedConfigs = await getMongoSyncConfigContents(factory, first.replicationStreamId);
1112
+ expect(
1113
+ (await Promise.all(updatedConfigs.map((config) => config.getSyncConfigStatus())))
1114
+ .map((status) => status?.state)
1115
+ .sort()
1116
+ ).toEqual([storage.SyncRuleState.ACTIVE, storage.SyncRuleState.STOP]);
1117
+ expect(await factory.getDeployingSyncConfig()).toBeNull();
1118
+ expect((await factory.getActiveSyncConfig())?.content.syncConfigId).not.toBe(firstConfigId);
1119
+ const updatedActiveStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
1120
+ expect(updatedActiveStorage.replicationStream.replicationJobId).not.toBe(
1121
+ initialActiveStorage.replicationStream.replicationJobId
1122
+ );
1123
+ expect(updatedActiveStorage.replicationStream.replicationJobId).toContain(
1124
+ (await factory.getActiveSyncConfig())!.content.syncConfigId
1125
+ );
1126
+ }
1127
+ );
1128
+
1129
+ test.runIf(storageVersion >= 3)(
1130
+ 'appended compatible config adopts the stream checkpoint instead of regressing to 0',
1131
+ async () => {
1132
+ await using factory = await storageConfig.factory();
1133
+
1134
+ // First config replicates some data, advancing the stream-level op head and its checkpoint.
1135
+ const first = await factory.updateSyncRules(
1136
+ updateSyncRulesFromYaml(
1137
+ `
1138
+ config:
1139
+ edition: 3
1140
+
1141
+ streams:
1142
+ by_owner:
1143
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1144
+ `,
1145
+ { storageVersion }
1146
+ )
1147
+ );
1148
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1149
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1150
+ const sourceTable = await test_utils.resolveTestTable(
1151
+ firstWriter,
1152
+ 'todos',
1153
+ ['id'],
1154
+ INITIALIZED_MONGO_STORAGE_FACTORY
1155
+ );
1156
+ await firstWriter.save({
1157
+ sourceTable,
1158
+ tag: storage.SaveOperationTag.INSERT,
1159
+ after: {
1160
+ id: 'todo-1',
1161
+ owner_id: 'user-1',
1162
+ project_id: 'project-1'
1163
+ },
1164
+ afterReplicaId: test_utils.rid('todo-1')
1165
+ });
1166
+ await firstWriter.markAllSnapshotDone('1/1');
1167
+ await firstWriter.commit('1/1');
1168
+
1169
+ const firstCheckpoint = (await firstStorage.getCheckpoint()).checkpoint;
1170
+ expect(firstCheckpoint).toBeGreaterThan(0n);
1171
+
1172
+ const mongoFactory = factory as MongoBucketStorage;
1173
+ const streamDocBefore = (await mongoFactory.db.sync_rules.findOne({
1174
+ _id: first.replicationStreamId
1175
+ })) as ReplicationStreamDocumentV3;
1176
+ // The stream-level head was advanced durably to (at least) the first config's checkpoint.
1177
+ expect(streamDocBefore.last_persisted_op).not.toBeNull();
1178
+ expect(BigInt(streamDocBefore.last_persisted_op!)).toBeGreaterThanOrEqual(firstCheckpoint);
1179
+
1180
+ // Append a compatible second config that replicates nothing new.
1181
+ const second = await factory.updateSyncRules(
1182
+ updateSyncRulesFromYaml(
1183
+ `
1184
+ config:
1185
+ edition: 3
1186
+
1187
+ streams:
1188
+ by_project:
1189
+ query: SELECT * FROM todos WHERE project_id = subscription.parameter('project_id')
1190
+ `,
1191
+ { storageVersion }
1192
+ )
1193
+ );
1194
+ expect(second.replicationStreamId).toEqual(first.replicationStreamId);
1195
+
1196
+ const replicatingStreams = await factory.getReplicatingReplicationStreams();
1197
+ expect(replicatingStreams).toHaveLength(1);
1198
+ const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage;
1199
+ await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
1200
+ // No new data replicated - just complete the snapshot and commit.
1201
+ await secondWriter.markAllSnapshotDone('2/1');
1202
+ await secondWriter.commit('2/1');
1203
+
1204
+ // The appended config's checkpoint must adopt the stream head, not regress to 0.
1205
+ const streamDocAfter = (await mongoFactory.db.sync_rules.findOne({
1206
+ _id: first.replicationStreamId
1207
+ })) as ReplicationStreamDocumentV3;
1208
+ const head = BigInt(streamDocAfter.last_persisted_op!);
1209
+ for (const config of streamDocAfter.sync_configs) {
1210
+ expect(config.last_checkpoint).not.toBeNull();
1211
+ expect(BigInt(config.last_checkpoint!)).toEqual(head);
1212
+ }
1213
+ // All configs share the same checkpoint, equal to the stream head.
1214
+ expect(head).toBeGreaterThanOrEqual(firstCheckpoint);
1215
+
1216
+ // After activation, the active config's checkpoint does not regress.
1217
+ const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage;
1218
+ const activeCheckpoint = (await activeStorage.getCheckpoint()).checkpoint;
1219
+ expect(activeCheckpoint).toBeGreaterThanOrEqual(firstCheckpoint);
1220
+ }
1221
+ );
1222
+
1223
+ test.runIf(storageVersion >= 3)('creates a new replication stream when compatibility options differ', async () => {
1224
+ await using factory = await storageConfig.factory();
1225
+
1226
+ const first = await factory.updateSyncRules(
1227
+ updateSyncRulesFromYaml(
1228
+ `
1229
+ config:
1230
+ edition: 3
1231
+
1232
+ streams:
1233
+ by_owner:
1234
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1235
+ `,
1236
+ { storageVersion }
1237
+ )
1238
+ );
1239
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1240
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1241
+ await firstWriter.markAllSnapshotDone('1/1');
1242
+ await firstWriter.commit('1/1');
1243
+
1244
+ // Same streams, but a different compatibility edition - must not append to the active stream.
1245
+ const second = await factory.updateSyncRules(
1246
+ updateSyncRulesFromYaml(
1247
+ `
1248
+ config:
1249
+ edition: 2
1250
+
1251
+ streams:
1252
+ by_owner:
1253
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1254
+ `,
1255
+ { storageVersion }
1256
+ )
1257
+ );
1258
+ expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
1259
+
1260
+ // The first stream stays active until the replacement has replicated.
1261
+ const active = await factory.getActiveSyncConfig();
1262
+ expect(active?.replicationStream.replicationStreamId).toEqual(first.replicationStreamId);
1263
+ expect((await factory.getDeployingSyncConfig())?.replicationStream.replicationStreamId).toEqual(
1264
+ second.replicationStreamId
1265
+ );
1266
+ });
1267
+
1268
+ test.runIf(storageVersion >= 3)(
1269
+ 'creates a new replication stream for legacy sync rules without a serialized plan',
1270
+ async () => {
1271
+ await using factory = await storageConfig.factory();
1272
+
1273
+ const first = await factory.updateSyncRules(
1274
+ updateSyncRulesFromYaml(
1275
+ `
1276
+ config:
1277
+ edition: 3
1278
+
1279
+ streams:
1280
+ by_owner:
1281
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1282
+ `,
1283
+ { storageVersion }
1284
+ )
1285
+ );
1286
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1287
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1288
+ await firstWriter.markAllSnapshotDone('1/1');
1289
+ await firstWriter.commit('1/1');
1290
+
1291
+ // Legacy sync rules have no serialized plan - must not append to the active stream.
1292
+ const second = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
1293
+ expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
1294
+ }
1295
+ );
1296
+
1297
+ test.runIf(storageVersion >= 3)('does not append to an active legacy sync config', async () => {
1298
+ await using factory = await storageConfig.factory();
1299
+
1300
+ const first = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }));
1301
+ const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage;
1302
+ await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
1303
+ await firstWriter.markAllSnapshotDone('1/1');
1304
+ await firstWriter.commit('1/1');
1305
+
1306
+ // The active config has no serialized plan, so a new streams config must not be appended to it.
1307
+ const second = await factory.updateSyncRules(
1308
+ updateSyncRulesFromYaml(
1309
+ `
1310
+ config:
1311
+ edition: 3
1312
+
1313
+ streams:
1314
+ by_owner:
1315
+ query: SELECT * FROM todos WHERE owner_id = subscription.parameter('owner_id')
1316
+ `,
1317
+ { storageVersion }
1318
+ )
1319
+ );
1320
+ expect(second.replicationStreamId).not.toEqual(first.replicationStreamId);
1321
+ });
1322
+
1323
+ test.runIf(storageVersion < 3)('can replace processing legacy sync rules', async () => {
1324
+ await using factory = await storageConfig.factory();
1325
+
1326
+ const firstSyncRules = await factory.updateSyncRules(
1327
+ updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion })
1328
+ );
1329
+
1330
+ await expect(
1331
+ factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion }))
1332
+ ).resolves.toBeDefined();
1333
+
1334
+ const mongoFactory = factory as MongoBucketStorage;
1335
+ expect((await mongoFactory.db.sync_rules.findOne({ _id: firstSyncRules.replicationStreamId }))?.state).toBe(
1336
+ storage.SyncRuleState.STOP
1337
+ );
1338
+ });
1339
+
1340
+ test('can lock newly-created sync rules', async () => {
1341
+ await using factory = await storageConfig.factory();
1342
+
1343
+ const syncRules = await factory.updateSyncRules(
1344
+ updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion, lock: true })
1345
+ );
1346
+
1347
+ expect(syncRules.current_lock?.sync_rules_id).toBe(syncRules.replicationStreamId);
1348
+ await syncRules.current_lock?.release();
1349
+ });
1350
+
232
1351
  test.runIf(storageVersion < 3)('uses a single current_data collection for v1 source records', async () => {
233
1352
  await using factory = await storageConfig.factory();
234
1353
  const syncRules = await factory.updateSyncRules(
@@ -260,10 +1379,10 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
260
1379
  await writer.commit('1/1');
261
1380
 
262
1381
  const mongoFactory = factory as MongoBucketStorage;
263
- expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1);
1382
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
264
1383
 
265
1384
  const sourceRecordCollections = await mongoFactory.db.db
266
- .listCollections({ name: new RegExp(`^source_records_${syncRules.id}_`) }, { nameOnly: true })
1385
+ .listCollections({ name: new RegExp(`^source_records_${syncRules.replicationStreamId}_`) }, { nameOnly: true })
267
1386
  .toArray();
268
1387
  expect(sourceRecordCollections).toEqual([]);
269
1388
  });
@@ -299,11 +1418,11 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
299
1418
  await writer.commit('1/1');
300
1419
 
301
1420
  const mongoFactory = factory as MongoBucketStorage;
302
- expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1);
1421
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
303
1422
 
304
1423
  await bucketStorage.clear();
305
1424
 
306
- expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(0);
1425
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(0);
307
1426
  });
308
1427
 
309
1428
  test.runIf(storageVersion < 3)('storage metrics include v1 current_data', async () => {
@@ -338,7 +1457,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
338
1457
  await writer.commit('1/1');
339
1458
 
340
1459
  const mongoFactory = factory as MongoBucketStorage;
341
- expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.id })).toBe(1);
1460
+ expect(await mongoFactory.db.current_data.countDocuments({ '_id.g': syncRules.replicationStreamId })).toBe(1);
342
1461
 
343
1462
  const metricsAfter = await factory.getStorageMetrics();
344
1463
  expect(metricsAfter.replication_size_bytes).toBeGreaterThan(metricsBefore.replication_size_bytes);
@@ -396,7 +1515,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
396
1515
  }
397
1516
  );
398
1517
 
399
- test.runIf(storageVersion >= 3)('cleans pending deletes only for tracked v3 source tables', async () => {
1518
+ test.runIf(storageVersion == 3)('cleans pending deletes only for tracked v3 source tables', async () => {
400
1519
  await using factory = await storageConfig.factory();
401
1520
  const syncRules = await factory.updateSyncRules(
402
1521
  updateSyncRulesFromYaml(
@@ -411,21 +1530,20 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
411
1530
  );
412
1531
 
413
1532
  const mongoFactory = factory as MongoBucketStorage;
414
- const bucketStorage = mongoFactory.getInstance(syncRules) as any;
415
- const db = bucketStorage.db;
416
- await db.initializeStreamStorage(syncRules.id);
1533
+ const bucketStorage = mongoFactory.getInstance(syncRules);
1534
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
1535
+ await db.initializeStreamStorage(syncRules.replicationStreamId);
417
1536
 
418
1537
  const sourceTableA = new bson.ObjectId();
419
1538
  const sourceTableB = new bson.ObjectId();
420
- await db.sourceTablesV3(syncRules.id).insertMany([
1539
+ await db.sourceTables(syncRules.replicationStreamId).insertMany([
421
1540
  {
422
1541
  _id: sourceTableA,
423
1542
  connection_id: 1,
424
1543
  relation_id: 'a',
425
1544
  schema_name: 'public',
426
1545
  table_name: 'table_a',
427
- replica_id_columns: null,
428
- replica_id_columns2: [],
1546
+ replica_id_columns: [],
429
1547
  snapshot_done: true,
430
1548
  snapshot_status: undefined,
431
1549
  bucket_data_source_ids: [],
@@ -438,8 +1556,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
438
1556
  relation_id: 'b',
439
1557
  schema_name: 'public',
440
1558
  table_name: 'table_b',
441
- replica_id_columns: null,
442
- replica_id_columns2: [],
1559
+ replica_id_columns: [],
443
1560
  snapshot_done: true,
444
1561
  snapshot_status: undefined,
445
1562
  bucket_data_source_ids: [],
@@ -448,42 +1565,312 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor
448
1565
  }
449
1566
  ]);
450
1567
 
451
- await db.sourceRecordsV3(syncRules.id, sourceTableA).insertMany([
1568
+ await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).insertMany([
452
1569
  { _id: 'deleted-1', data: null, buckets: [], lookups: [], pending_delete: 5n },
453
1570
  { _id: 'deleted-2', data: null, buckets: [], lookups: [], pending_delete: 9n },
454
1571
  { _id: 'active', data: null, buckets: [], lookups: [] }
455
1572
  ]);
456
1573
  await db
457
- .sourceRecordsV3(syncRules.id, sourceTableB)
1574
+ .sourceRecords(syncRules.replicationStreamId, sourceTableB)
458
1575
  .insertMany([{ _id: 'later-delete', data: null, buckets: [], lookups: [], pending_delete: 12n }]);
459
1576
 
460
- const store = new SourceRecordStoreV3(db, syncRules.id, bucketStorage.sync_rules.mapping);
1577
+ const store = new SourceRecordStoreV3(
1578
+ db,
1579
+ syncRules.replicationStreamId,
1580
+ bucketStorage.replicationStream.syncConfigContent[0].mapping
1581
+ );
461
1582
  const logger = { info() {} } as any;
462
1583
 
463
1584
  await store.postCommitCleanup(6n, logger);
464
1585
 
465
- expect(await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: 5n })).toBe(0);
466
- expect(await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: 9n })).toBe(1);
467
- expect(await db.sourceRecordsV3(syncRules.id, sourceTableB).countDocuments({ pending_delete: 12n })).toBe(1);
468
- expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableA }))?.latest_pending_delete).toBe(9n);
469
- expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableB }))?.latest_pending_delete).toBe(12n);
1586
+ expect(
1587
+ await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).countDocuments({ pending_delete: 5n })
1588
+ ).toBe(0);
1589
+ expect(
1590
+ await db.sourceRecords(syncRules.replicationStreamId, sourceTableA).countDocuments({ pending_delete: 9n })
1591
+ ).toBe(1);
1592
+ expect(
1593
+ await db.sourceRecords(syncRules.replicationStreamId, sourceTableB).countDocuments({ pending_delete: 12n })
1594
+ ).toBe(1);
1595
+ expect(
1596
+ (await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableA }))?.latest_pending_delete
1597
+ ).toBe(9n);
1598
+ expect(
1599
+ (await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableB }))?.latest_pending_delete
1600
+ ).toBe(12n);
470
1601
 
471
1602
  await store.postCommitCleanup(10n, logger);
472
1603
 
473
1604
  expect(
474
- await db.sourceRecordsV3(syncRules.id, sourceTableA).countDocuments({ pending_delete: { $exists: true } })
1605
+ await db
1606
+ .sourceRecords(syncRules.replicationStreamId, sourceTableA)
1607
+ .countDocuments({ pending_delete: { $exists: true } })
475
1608
  ).toBe(0);
476
1609
  expect(
477
- (await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableA }))?.latest_pending_delete
1610
+ (await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableA }))?.latest_pending_delete
478
1611
  ).toBeUndefined();
479
- expect((await db.sourceTablesV3(syncRules.id).findOne({ _id: sourceTableB }))?.latest_pending_delete).toBe(12n);
1612
+ expect(
1613
+ (await db.sourceTables(syncRules.replicationStreamId).findOne({ _id: sourceTableB }))?.latest_pending_delete
1614
+ ).toBe(12n);
480
1615
  });
481
1616
  }
482
1617
 
483
1618
  describe('sync - mongodb', () => {
1619
+ test('v3 activation stops legacy active sync rules', async () => {
1620
+ await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
1621
+ const mongoFactory = factory as MongoBucketStorage;
1622
+
1623
+ const legacySyncRules = await factory.updateSyncRules(
1624
+ updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion: storage.LEGACY_STORAGE_VERSION })
1625
+ );
1626
+ const legacyStorage = factory.getInstance(legacySyncRules);
1627
+ await using legacyWriter = await legacyStorage.createWriter(test_utils.BATCH_OPTIONS);
1628
+ await legacyWriter.markAllSnapshotDone('1/1');
1629
+ await legacyWriter.commit('1/1');
1630
+
1631
+ expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.replicationStreamId }))?.state).toBe(
1632
+ storage.SyncRuleState.ACTIVE
1633
+ );
1634
+
1635
+ const v3SyncRules = await factory.updateSyncRules(
1636
+ updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion: storage.STORAGE_VERSION_3 })
1637
+ );
1638
+ const v3Storage = factory.getInstance(v3SyncRules);
1639
+ await using v3Writer = await v3Storage.createWriter(test_utils.BATCH_OPTIONS);
1640
+ await v3Writer.markAllSnapshotDone('2/1');
1641
+ await v3Writer.commit('2/1');
1642
+
1643
+ expect((await mongoFactory.db.sync_rules.findOne({ _id: legacySyncRules.replicationStreamId }))?.state).toBe(
1644
+ storage.SyncRuleState.STOP
1645
+ );
1646
+ });
1647
+
484
1648
  for (const storageVersion of TEST_STORAGE_VERSIONS) {
485
1649
  describe(`storage v${storageVersion}`, () => {
486
1650
  registerSyncStorageTests(INITIALIZED_MONGO_STORAGE_FACTORY, storageVersion);
1651
+
1652
+ describe.runIf(storageVersion == 3)('V3 read filtering boundaries', () => {
1653
+ async function setupFilteringTest() {
1654
+ await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
1655
+ const syncRules = await factory.updateSyncRules(
1656
+ updateSyncRulesFromYaml(
1657
+ `
1658
+ bucket_definitions:
1659
+ global:
1660
+ data:
1661
+ - SELECT id, description FROM test
1662
+ `,
1663
+ { storageVersion }
1664
+ )
1665
+ );
1666
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
1667
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
1668
+ const mapping = syncRules.syncConfigContent[0].mapping;
1669
+
1670
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
1671
+ const definitionId = mapping.bucketSourceId(request.source);
1672
+ const collection = db.bucketData(syncRules.replicationStreamId, definitionId);
1673
+
1674
+ const bucketName = request.bucket;
1675
+ const sourceTable = new bson.ObjectId();
1676
+ const bucketKey: BucketKey = {
1677
+ replicationStreamId: syncRules.replicationStreamId,
1678
+ definitionId,
1679
+ bucket: bucketName
1680
+ };
1681
+
1682
+ function makeOps(opIds: bigint[]): BucketDataDoc[] {
1683
+ return opIds.map((opId) => ({
1684
+ bucketKey,
1685
+ o: opId,
1686
+ op: 'PUT' as const,
1687
+ source_table: sourceTable,
1688
+ source_key: test_utils.rid(`row-${opId}`),
1689
+ table: 'items',
1690
+ row_id: `row-${opId}`,
1691
+ checksum: BigInt(opId) * 10n,
1692
+ data: `{"id":"row-${opId}"}`
1693
+ }));
1694
+ }
1695
+
1696
+ const docA = serializeBucketData(bucketName, makeOps([10n, 20n, 30n]));
1697
+ const docB = serializeBucketData(bucketName, makeOps([40n, 50n, 60n]));
1698
+ const docC = serializeBucketData(bucketName, makeOps([70n, 80n, 90n]));
1699
+
1700
+ await collection.insertMany([docA, docB, docC]);
1701
+
1702
+ return { factory, syncRules, bucketStorage, bucketName };
1703
+ }
1704
+
1705
+ async function getFilteredOps(start: number, checkpoint: number): Promise<bigint[]> {
1706
+ const { syncRules, bucketStorage } = await setupFilteringTest();
1707
+ const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', BigInt(start));
1708
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(BigInt(checkpoint), [request]));
1709
+ const ops = batch.flatMap((b) => b.chunkData.data.map((d) => BigInt(d.op_id)));
1710
+ return ops;
1711
+ }
1712
+
1713
+ test('case 1: start=5, checkpoint=95 → all ops', async () => {
1714
+ const ops = await getFilteredOps(5, 95);
1715
+ expect(ops).toEqual([10n, 20n, 30n, 40n, 50n, 60n, 70n, 80n, 90n]);
1716
+ });
1717
+
1718
+ test('case 2: start=10, checkpoint=90 → ops in (10,90]', async () => {
1719
+ const ops = await getFilteredOps(10, 90);
1720
+ expect(ops).toEqual([20n, 30n, 40n, 50n, 60n, 70n, 80n, 90n]);
1721
+ });
1722
+
1723
+ test('case 3: start=15, checkpoint=85 → partial doc boundaries', async () => {
1724
+ const ops = await getFilteredOps(15, 85);
1725
+ expect(ops).toEqual([20n, 30n, 40n, 50n, 60n, 70n, 80n]);
1726
+ });
1727
+
1728
+ test('case 4: start=25, checkpoint=55 → spans two docs', async () => {
1729
+ const ops = await getFilteredOps(25, 55);
1730
+ expect(ops).toEqual([30n, 40n, 50n]);
1731
+ });
1732
+
1733
+ test('case 5: start=35, checkpoint=45 → single op within doc', async () => {
1734
+ const ops = await getFilteredOps(35, 45);
1735
+ expect(ops).toEqual([40n]);
1736
+ });
1737
+
1738
+ test('case 6: start=35, checkpoint=65 → full doc B', async () => {
1739
+ const ops = await getFilteredOps(35, 65);
1740
+ expect(ops).toEqual([40n, 50n, 60n]);
1741
+ });
1742
+
1743
+ test('case 7: start=25, checkpoint=35 → single op from doc A', async () => {
1744
+ const ops = await getFilteredOps(25, 35);
1745
+ expect(ops).toEqual([30n]);
1746
+ });
1747
+
1748
+ test('case 8: start=30, checkpoint=40 → op at checkpoint from next doc', async () => {
1749
+ const ops = await getFilteredOps(30, 40);
1750
+ expect(ops).toEqual([40n]);
1751
+ });
1752
+
1753
+ test('case 9: start=100, checkpoint=200 → beyond all docs', async () => {
1754
+ const ops = await getFilteredOps(100, 200);
1755
+ expect(ops).toEqual([]);
1756
+ });
1757
+
1758
+ test('case 10: start=0, checkpoint=5 → before all docs', async () => {
1759
+ const ops = await getFilteredOps(0, 5);
1760
+ expect(ops).toEqual([]);
1761
+ });
1762
+
1763
+ test('case 11: start=50, checkpoint=50 → zero-width range', async () => {
1764
+ const ops = await getFilteredOps(50, 50);
1765
+ expect(ops).toEqual([]);
1766
+ });
1767
+
1768
+ test('case 12: start=45, checkpoint=50 → op at checkpoint boundary', async () => {
1769
+ const ops = await getFilteredOps(45, 50);
1770
+ expect(ops).toEqual([50n]);
1771
+ });
1772
+
1773
+ test('case 13: start=50, checkpoint=55 → no ops strictly after start', async () => {
1774
+ const ops = await getFilteredOps(50, 55);
1775
+ expect(ops).toEqual([]);
1776
+ });
1777
+
1778
+ test('all-filtered first batch still returns data behind the batch boundary', async () => {
1779
+ // Documents straddling the requested (start, end] window are matched by the
1780
+ // query, but contribute no rows after filtering. If an entire server batch
1781
+ // (~101 documents) consists of such straddlers, the remaining documents in
1782
+ // the cursor must still be reachable. Storage reports the straddler buckets
1783
+ // as complete via empty chunks, and the caller re-requests the rest.
1784
+ await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
1785
+ const syncRules = await factory.updateSyncRules(
1786
+ updateSyncRulesFromYaml(
1787
+ `
1788
+ bucket_definitions:
1789
+ by_user:
1790
+ parameters: select request.user_id() as user_id
1791
+ data: [select * from test where owner_id = bucket.user_id]
1792
+ `,
1793
+ { storageVersion }
1794
+ )
1795
+ );
1796
+ const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorageV3;
1797
+ const db = bucketStorage.db as VersionedPowerSyncMongoV3;
1798
+
1799
+ const start = 5n;
1800
+ const end = 50n;
1801
+
1802
+ // 150 buckets sorted before the data bucket, each with a single document
1803
+ // containing ops at 1 and 100: matched (_id.o=100 > start, min_op=1 <= end),
1804
+ // but no op in (5, 50].
1805
+ const straddlerNames = Array.from({ length: 150 }, (_, i) => `b${`${i}`.padStart(3, '0')}`);
1806
+ const requests = [...straddlerNames, 'zzz'].map((id) =>
1807
+ bucketRequest(syncRules.syncConfigContent[0], `by_user["${id}"]`, start)
1808
+ );
1809
+ const mapping = syncRules.syncConfigContent[0].mapping;
1810
+ const definitionId = mapping.bucketSourceId(requests[0].source);
1811
+ const collection = db.bucketData(syncRules.replicationStreamId, definitionId);
1812
+ const sourceTable = new bson.ObjectId();
1813
+
1814
+ function makeOps(bucket: string, opIds: bigint[]): BucketDataDoc[] {
1815
+ const bucketKey: BucketKey = {
1816
+ replicationStreamId: syncRules.replicationStreamId,
1817
+ definitionId,
1818
+ bucket
1819
+ };
1820
+ return opIds.map((opId) => ({
1821
+ bucketKey,
1822
+ o: opId,
1823
+ op: 'PUT' as const,
1824
+ source_table: sourceTable,
1825
+ source_key: test_utils.rid(`row-${opId}`),
1826
+ table: 'test',
1827
+ row_id: `row-${opId}`,
1828
+ checksum: BigInt(opId) * 10n,
1829
+ data: `{"id":"row-${opId}"}`
1830
+ }));
1831
+ }
1832
+
1833
+ const straddlerDocs = requests
1834
+ .slice(0, -1)
1835
+ .map((request) => serializeBucketData(request.bucket, makeOps(request.bucket, [1n, 100n])));
1836
+ const dataBucket = requests[requests.length - 1].bucket;
1837
+ const dataDoc = serializeBucketData(dataBucket, makeOps(dataBucket, [10n]));
1838
+ await collection.insertMany([...straddlerDocs, dataDoc]);
1839
+
1840
+ // Emulate the caller loop in sync.ts / BucketChecksumState: advance bucket
1841
+ // positions from each chunk, drop completed buckets, and re-request while
1842
+ // any chunk reported has_more.
1843
+ const positions = new Map(requests.map((request) => [request.bucket, request.start]));
1844
+ const pending = new Set(positions.keys());
1845
+ const receivedOps: bigint[] = [];
1846
+ let rounds = 0;
1847
+
1848
+ while (rounds < 10) {
1849
+ rounds++;
1850
+ const roundRequests = requests
1851
+ .filter((request) => pending.has(request.bucket))
1852
+ .map((request) => ({ ...request, start: positions.get(request.bucket)! }));
1853
+ const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(end, roundRequests));
1854
+ let anyHasMore = false;
1855
+ for (const { chunkData } of batch) {
1856
+ positions.set(chunkData.bucket, BigInt(chunkData.next_after));
1857
+ if (chunkData.has_more) {
1858
+ anyHasMore = true;
1859
+ } else {
1860
+ pending.delete(chunkData.bucket);
1861
+ }
1862
+ receivedOps.push(...chunkData.data.map((entry) => BigInt(entry.op_id)));
1863
+ }
1864
+ if (!anyHasMore) {
1865
+ break;
1866
+ }
1867
+ }
1868
+
1869
+ // The op behind the all-straddler first batch must be returned.
1870
+ expect(receivedOps).toEqual([10n]);
1871
+ expect(rounds).toBeLessThan(10);
1872
+ });
1873
+ });
487
1874
  });
488
1875
  }
489
1876
  });