@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/CHANGELOG.md +101 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +41 -16
  6. package/dist/storage/MongoBucketStorage.js +226 -71
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
  9. package/dist/storage/implementation/MongoBucketBatch.js +110 -62
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
  15. package/dist/storage/implementation/MongoCompactor.js +80 -263
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
  21. package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +39 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  46. package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
  47. package/dist/storage/implementation/common/PersistedBatch.js +12 -0
  48. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  49. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  51. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  52. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  53. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  54. package/dist/storage/implementation/db.js +17 -15
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +20 -13
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  62. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  63. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
  65. package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
  66. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  71. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  72. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +10 -1
  77. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  78. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  79. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  81. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
  82. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  84. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  85. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  86. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
  87. package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
  88. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
  90. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
  91. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
  93. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
  94. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
  96. package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +77 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +30 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +101 -21
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
  115. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  126. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  128. package/dist/storage/storage-index.d.ts +4 -4
  129. package/dist/storage/storage-index.js +4 -4
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +49 -3
  132. package/dist/types/types.js +48 -1
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +1 -2
  135. package/dist/utils/test-utils.js +7 -1
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.d.ts +9 -2
  138. package/dist/utils/util.js +128 -4
  139. package/dist/utils/util.js.map +1 -1
  140. package/package.json +12 -19
  141. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  142. package/src/storage/MongoBucketStorage.ts +347 -88
  143. package/src/storage/implementation/MongoBucketBatch.ts +147 -82
  144. package/src/storage/implementation/MongoChecksums.ts +93 -153
  145. package/src/storage/implementation/MongoCompactor.ts +102 -316
  146. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  147. package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
  148. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  149. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  150. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  151. package/src/storage/implementation/MongoStorageProvider.ts +46 -4
  152. package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
  153. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  154. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
  156. package/src/storage/implementation/common/PersistedBatch.ts +19 -3
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  158. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  159. package/src/storage/implementation/db.ts +19 -20
  160. package/src/storage/implementation/models.ts +20 -14
  161. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
  162. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  163. package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
  164. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
  165. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  166. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  167. package/src/storage/implementation/v1/models.ts +11 -1
  168. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  169. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
  170. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  171. package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
  172. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
  173. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
  174. package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
  175. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  176. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
  177. package/src/storage/implementation/v3/bucket-format.ts +97 -0
  178. package/src/storage/implementation/v3/chunking.ts +40 -0
  179. package/src/storage/implementation/v3/models.ts +132 -55
  180. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
  181. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
  184. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  185. package/src/storage/storage-index.ts +12 -4
  186. package/src/types/types.ts +58 -1
  187. package/src/utils/test-utils.ts +8 -3
  188. package/src/utils/util.ts +185 -6
  189. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
  191. package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
  192. package/test/src/helpers/MemoryObjectStorage.ts +53 -0
  193. package/test/src/helpers/s3TestFactory.ts +69 -0
  194. package/test/src/object_storage_lifecycle.test.ts +59 -0
  195. package/test/src/setup.ts +6 -1
  196. package/test/src/storage.test.ts +227 -13
  197. package/test/src/storage_compacting.test.ts +1951 -12
  198. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  199. package/test/src/storage_s3_checksums.test.ts +191 -0
  200. package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
  201. package/test/src/storage_s3_reading.test.ts +526 -0
  202. package/test/src/storage_s3_writing.test.ts +135 -0
  203. package/test/src/storage_sync.test.ts +1180 -73
  204. package/test/src/storeCurrentData.test.ts +227 -0
  205. package/test/src/util.ts +2 -1
  206. package/tsconfig.json +1 -4
  207. package/tsconfig.tsbuildinfo +1 -1
  208. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  209. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  210. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  211. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  212. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  213. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  214. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  215. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  216. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  217. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  218. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  219. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  220. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  221. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  222. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  223. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  224. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  225. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  226. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  227. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  228. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  229. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  230. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  231. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  232. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  233. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  234. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  235. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  236. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  237. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  238. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  239. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -3,39 +3,88 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework';
3
3
  import { storage } from '@powersync/service-core';
4
4
  import * as bson from 'bson';
5
5
  import { mongoTableId } from '../../../utils/util.js';
6
- import { calculateCheckpointState } from '../CheckpointState.js';
6
+ import { canCheckpointState } from '../CheckpointState.js';
7
7
  import { MongoBucketBatch } from '../MongoBucketBatch.js';
8
8
  import { syncRuleStateUpdatePipeline } from '../SyncRuleStateUpdate.js';
9
9
  import { PersistedBatchV3 } from './PersistedBatchV3.js';
10
10
  import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js';
11
- function sameStringArray(left, right) {
12
- return left.length == right.length && left.every((value, index) => value == right[index]);
13
- }
11
+ import { createNewSourceTable, designateEventCarrier, overlappingSourceTableFilter, planSourceTableReconciliation, sourceTableDesiredResolution, sourceTableFromDocument } from './source-table-utils.js';
14
12
  export class MongoBucketBatchV3 extends MongoBucketBatch {
15
13
  store;
16
- syncConfigId;
14
+ syncConfigIds;
17
15
  needsActivationV3 = true;
18
16
  lastWaitingLogThrottledV3 = 0;
19
17
  constructor(options) {
20
18
  super(options);
21
- if (options.syncConfigId == null) {
19
+ const syncConfigIds = options.syncConfigIds ?? [];
20
+ if (syncConfigIds.length == 0) {
22
21
  throw new ReplicationAssertionError('Missing sync config id for v3 batch');
23
22
  }
24
- this.syncConfigId = options.syncConfigId;
25
- this.store = new SourceRecordStoreV3(this.db, this.group_id, this.mapping);
23
+ this.syncConfigIds = syncConfigIds;
24
+ this.store = new SourceRecordStoreV3(this.db, this.replicationStreamId, this.mapping);
26
25
  }
27
26
  createPersistedBatch(writtenSize) {
28
- return new PersistedBatchV3(this.db, this.group_id, this.mapping, writtenSize, {
29
- logger: this.logger
27
+ return new PersistedBatchV3(this.db, this.replicationStreamId, this.mapping, writtenSize, {
28
+ logger: this.logger,
29
+ objectStorage: this.options.objectStorage,
30
+ inlineThresholdBytes: this.options.inlineThresholdBytes
30
31
  });
31
32
  }
32
33
  get sourceRecordStore() {
33
34
  return this.store;
34
35
  }
36
+ async onReplicationTransactionFlush(session, lastOp) {
37
+ // Durably advance the stream-level head of persisted ops within the flush transaction.
38
+ // This ensures a checkpoint created later (even by an empty commit, or by a freshly-appended
39
+ // config that replicates nothing) covers all ops persisted before a potential crash.
40
+ await this.db.sync_rules.updateOne({
41
+ _id: this.replicationStreamId
42
+ }, {
43
+ $max: {
44
+ last_persisted_op: lastOp
45
+ }
46
+ }, { session });
47
+ }
48
+ selectedSyncConfigObjectIds(syncConfigIds) {
49
+ const selectedIds = new Set(this.syncConfigIds.map((id) => id.toHexString()));
50
+ return syncConfigIds
51
+ .filter((id) => selectedIds.has(id) && bson.ObjectId.isValid(id))
52
+ .map((id) => new bson.ObjectId(id));
53
+ }
54
+ relevantSyncConfigIds(table) {
55
+ const bucketDataSourceIds = table.bucketDataSources.map((source) => this.mapping.bucketSourceId(source));
56
+ const parameterLookupSourceIds = table.parameterLookupSources.map((source) => this.mapping.parameterLookupId(source));
57
+ return this.selectedSyncConfigObjectIds(this.mapping.syncConfigIdsForSourceTable(this.syncConfigIds.map((id) => id.toHexString()), table.ref, bucketDataSourceIds, parameterLookupSourceIds));
58
+ }
59
+ relevantSyncConfigIdsForTables(tables) {
60
+ const ids = new Map();
61
+ for (const table of tables) {
62
+ for (const id of this.relevantSyncConfigIds(table)) {
63
+ ids.set(id.toHexString(), id);
64
+ }
65
+ }
66
+ return [...ids.values()];
67
+ }
68
+ snapshotBlockingSourceTablesFilter() {
69
+ const clauses = this.syncConfigIds.flatMap((syncConfigId) => {
70
+ const filter = this.mapping.snapshotBlockingSourceTablesFilter(syncConfigId.toHexString());
71
+ return filter.$or ?? [];
72
+ });
73
+ if (clauses.length == 0) {
74
+ return {
75
+ snapshot_done: false,
76
+ _id: { $exists: false }
77
+ };
78
+ }
79
+ return {
80
+ snapshot_done: false,
81
+ $or: clauses
82
+ };
83
+ }
35
84
  async cleanupDroppedSourceTables(sourceTables) {
36
85
  for (const table of sourceTables) {
37
86
  await this.db
38
- .sourceRecordsV3(this.group_id, mongoTableId(table.id))
87
+ .sourceRecords(this.replicationStreamId, mongoTableId(table.id))
39
88
  .drop()
40
89
  .catch((error) => {
41
90
  if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
@@ -46,179 +95,91 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
46
95
  }
47
96
  }
48
97
  async resolveTables(options) {
49
- const ref = options.source;
50
- const syncRules = options.syncRules ?? this.sync_rules;
51
- const matchingSources = syncRules.getMatchingSources(ref);
98
+ // The test-only override is a whole parsed set, so the sync rules and the mapping
99
+ // used below always come from the same parse.
100
+ const parsedOverride = options.parsedSyncConfig;
101
+ const syncConfig = parsedOverride?.hydratedSyncConfig ?? this.sync_rules;
102
+ const mapping = parsedOverride?.mapping ?? this.mapping;
52
103
  const { connection_id, source } = options;
53
- const { schema, name, objectId, replicaIdColumns, connectionTag } = source;
54
- const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
55
- name: column.name,
56
- type: column.type,
57
- type_oid: column.typeId
58
- }));
104
+ const context = {
105
+ connectionId: connection_id,
106
+ connectionTag: source.connectionTag,
107
+ identity: {
108
+ schema: source.schema,
109
+ name: source.name,
110
+ objectId: source.objectId,
111
+ replicaIdColumns: source.replicaIdColumns.map((column) => ({
112
+ name: column.name,
113
+ type: column.type,
114
+ type_oid: column.typeId
115
+ }))
116
+ },
117
+ storeCurrentData: source.sendsCompleteRows !== true,
118
+ syncConfig,
119
+ mapping,
120
+ desired: sourceTableDesiredResolution(syncConfig, source, mapping)
121
+ };
59
122
  let result = null;
60
- const initializeSourceRecordsFor = [];
61
- await this.db.client.withSession(async (session) => {
62
- const col = this.db.commonSourceTables(this.group_id);
63
- const exactFilter = {
64
- connection_id,
65
- schema_name: schema,
66
- table_name: name,
67
- replica_id_columns2: normalizedReplicaIdColumns
68
- };
69
- if (objectId != null) {
70
- exactFilter.relation_id = objectId;
71
- }
72
- const exactDocs = (await col.find(exactFilter, { session }).toArray());
73
- const bucketSourceById = new Map(matchingSources.bucketDataSources.map((source) => [this.mapping.bucketSourceId(source), source]));
74
- const parameterLookupSourceById = new Map(matchingSources.parameterLookupSources.map((source) => [this.mapping.parameterLookupId(source), source]));
75
- const desiredBucketIds = new Set(bucketSourceById.keys());
76
- const desiredLookupIds = new Set(parameterLookupSourceById.keys());
77
- const desiredHasMembership = desiredBucketIds.size > 0 || desiredLookupIds.size > 0;
78
- const triggersEvent = syncRules.tableTriggersEvent(ref);
79
- const coveredBucketIds = new Set();
80
- const coveredLookupIds = new Set();
81
- const retainedDocIds = [];
82
- const tables = [];
83
- let retainedEventOnlyTable = false;
84
- for (const doc of exactDocs) {
85
- const bucketDataSourceIds = doc.bucket_data_source_ids.filter((id) => desiredBucketIds.has(id));
86
- const parameterLookupSourceIds = doc.parameter_lookup_source_ids.filter((id) => desiredLookupIds.has(id));
87
- const coversDesiredMembership = bucketDataSourceIds.length > 0 || parameterLookupSourceIds.length > 0;
88
- const coversEventOnlyTable = !desiredHasMembership && triggersEvent && !retainedEventOnlyTable;
89
- for (const id of bucketDataSourceIds) {
90
- coveredBucketIds.add(id);
91
- }
92
- for (const id of parameterLookupSourceIds) {
93
- coveredLookupIds.add(id);
94
- }
95
- if (!sameStringArray(doc.bucket_data_source_ids, bucketDataSourceIds) ||
96
- !sameStringArray(doc.parameter_lookup_source_ids, parameterLookupSourceIds)) {
97
- await col.updateOne({ _id: doc._id }, {
98
- $set: {
99
- bucket_data_source_ids: bucketDataSourceIds,
100
- parameter_lookup_source_ids: parameterLookupSourceIds
101
- }
102
- }, { session });
103
- }
104
- if (coversDesiredMembership || coversEventOnlyTable) {
105
- if (coversEventOnlyTable) {
106
- retainedEventOnlyTable = true;
123
+ const session = this.db.client.startSession();
124
+ await using _ = { [Symbol.asyncDispose]: () => session.endSession() };
125
+ await session.withTransaction(async () => {
126
+ const col = this.db.sourceTables(this.replicationStreamId);
127
+ // Fetch every persisted source-table doc that can overlap this physical table.
128
+ // Exact-identity docs are candidates for reuse; non-exact overlaps are possible drops.
129
+ const candidateDocs = await col
130
+ .find(overlappingSourceTableFilter(connection_id, context.identity), { session })
131
+ .toArray();
132
+ // Pure planning: which docs to retain (and narrow), whether a new doc is needed for
133
+ // uncovered memberships, and which docs conflict with the current identity.
134
+ const plan = planSourceTableReconciliation(candidateDocs, context);
135
+ // Persist narrowing for incomplete snapshots only. Snapshot-complete docs keep stale
136
+ // coverage ids so compatible future configs can reuse already-snapshotted data.
137
+ // Narrowing occurs after removing a sync config, meaning we don't process those
138
+ // definitions anymore.
139
+ for (const update of plan.narrowingUpdates) {
140
+ await col.updateOne({ _id: update.id }, {
141
+ $set: {
142
+ bucket_data_source_ids: update.memberships.bucketDataSourceIds,
143
+ parameter_lookup_source_ids: update.memberships.parameterLookupSourceIds
107
144
  }
108
- retainedDocIds.push(doc._id);
109
- tables.push(this.sourceTableFromDocument({
110
- ...doc,
111
- bucket_data_source_ids: bucketDataSourceIds,
112
- parameter_lookup_source_ids: parameterLookupSourceIds
113
- }, connectionTag, syncRules, {
114
- bucketDataSources: bucketDataSourceIds.map((id) => bucketSourceById.get(id)),
115
- parameterLookupSources: parameterLookupSourceIds.map((id) => parameterLookupSourceById.get(id))
116
- }));
117
- }
145
+ }, { session });
118
146
  }
119
- const uncoveredBucketIds = [...desiredBucketIds].filter((id) => !coveredBucketIds.has(id));
120
- const uncoveredLookupIds = [...desiredLookupIds].filter((id) => !coveredLookupIds.has(id));
121
- if (uncoveredBucketIds.length > 0 || uncoveredLookupIds.length > 0 || (triggersEvent && tables.length == 0)) {
147
+ // Any desired membership not covered by an existing doc gets a new source table.
148
+ // That table snapshots only the uncovered memberships.
149
+ if (plan.newTableMemberships != null) {
122
150
  const id = options.idGenerator ? options.idGenerator() : new bson.ObjectId();
123
- const sourceTable = new storage.SourceTable({
124
- id,
125
- ref,
126
- objectId,
127
- replicaIdColumns,
128
- snapshotComplete: false,
129
- bucketDataSources: uncoveredBucketIds.map((id) => bucketSourceById.get(id)),
130
- parameterLookupSources: uncoveredLookupIds.map((id) => parameterLookupSourceById.get(id))
131
- });
132
- sourceTable.syncData = uncoveredBucketIds.length > 0;
133
- sourceTable.syncParameters = uncoveredLookupIds.length > 0;
134
- sourceTable.syncEvent = triggersEvent;
135
- const createDoc = {
136
- _id: id,
137
- connection_id,
138
- relation_id: objectId,
139
- schema_name: schema,
140
- table_name: name,
141
- replica_id_columns: null,
142
- replica_id_columns2: normalizedReplicaIdColumns,
143
- snapshot_done: false,
144
- snapshot_status: undefined,
145
- bucket_data_source_ids: uncoveredBucketIds,
146
- parameter_lookup_source_ids: uncoveredLookupIds
147
- };
148
- await col.insertOne(createDoc, { session });
149
- initializeSourceRecordsFor.push(createDoc._id);
150
- retainedDocIds.push(createDoc._id);
151
- tables.push(sourceTable);
152
- }
153
- const conflictFilter = [{ schema_name: schema, table_name: name }];
154
- if (objectId != null) {
155
- conflictFilter.push({ relation_id: objectId });
151
+ const { doc, table } = createNewSourceTable(id, plan.newTableMemberships, context);
152
+ await col.insertOne(doc, { session });
153
+ await this.db.initializeSourceRecordsCollection(this.replicationStreamId, doc._id, session);
154
+ plan.tables.push(table);
156
155
  }
157
- const dropTables = await col
158
- .find({
159
- connection_id,
160
- _id: { $nin: retainedDocIds },
161
- $or: conflictFilter
162
- }, { session })
163
- .toArray();
156
+ // If memberships are split across multiple source tables, only one may fire events.
157
+ designateEventCarrier(plan.tables, context.desired.triggersEvent);
164
158
  result = {
165
- tables,
166
- dropTables: dropTables.map((doc) => this.sourceTableFromDocument(doc, connectionTag, syncRules))
159
+ tables: plan.tables,
160
+ dropTables: plan.dropDocs.map((doc) => sourceTableFromDocument(doc, context.connectionTag, syncConfig, mapping))
167
161
  };
168
162
  });
169
- for (const sourceTableId of initializeSourceRecordsFor) {
170
- await this.db.initializeSourceRecordsCollection(this.group_id, sourceTableId);
171
- }
172
163
  return result;
173
164
  }
174
- sourceTableFromDocument(doc, connectionTag, syncRules, memberships) {
175
- const resolvedMemberships = memberships ?? this.sourceTableMembershipsFromDocument(doc, syncRules);
176
- const table = new storage.SourceTable({
177
- id: doc._id,
178
- ref: {
179
- connectionTag,
180
- schema: doc.schema_name,
181
- name: doc.table_name
182
- },
183
- objectId: doc.relation_id,
184
- replicaIdColumns: doc.replica_id_columns2.map((c) => ({ name: c.name, typeId: c.type_oid, type: c.type })),
185
- snapshotComplete: doc.snapshot_done ?? true,
186
- bucketDataSources: resolvedMemberships.bucketDataSources,
187
- parameterLookupSources: resolvedMemberships.parameterLookupSources
188
- });
189
- table.syncData = table.bucketDataSources.length > 0;
190
- table.syncParameters = table.parameterLookupSources.length > 0;
191
- table.syncEvent = syncRules.tableTriggersEvent(table.ref);
192
- table.snapshotStatus =
193
- doc.snapshot_status == null
194
- ? undefined
195
- : {
196
- lastKey: doc.snapshot_status.last_key?.buffer ?? null,
197
- totalEstimatedCount: doc.snapshot_status.total_estimated_count,
198
- replicatedCount: doc.snapshot_status.replicated_count
199
- };
200
- return table;
201
- }
202
- sourceTableMembershipsFromDocument(doc, syncRules) {
203
- const bucketDataSourceIds = new Set(doc.bucket_data_source_ids);
204
- const parameterLookupSourceIds = new Set(doc.parameter_lookup_source_ids);
205
- return {
206
- bucketDataSources: syncRules.bucketDataSources.filter((source) => bucketDataSourceIds.has(this.mapping.bucketSourceId(source))),
207
- parameterLookupSources: syncRules.bucketParameterLookupSources.filter((source) => parameterLookupSourceIds.has(this.mapping.parameterLookupId(source)))
208
- };
209
- }
210
165
  async getSourceTableStatus(table) {
211
166
  const doc = (await this.db
212
- .commonSourceTables(this.group_id)
167
+ .sourceTables(this.replicationStreamId)
213
168
  .findOne({ _id: mongoTableId(table.id) }, { session: this.session }));
214
169
  if (doc == null) {
215
170
  return null;
216
171
  }
217
- return this.sourceTableFromDocument(doc, table.ref.connectionTag, this.sync_rules);
172
+ const refreshed = sourceTableFromDocument(doc, table.ref.connectionTag, this.sync_rules, this.mapping);
173
+ // The event-carrier designation is decided per resolveTables result and not persisted -
174
+ // preserve the caller's designation instead of recomputing it from the ref, so that
175
+ // refreshing a non-carrier table does not make it fire events.
176
+ refreshed.syncEvent = table.syncEvent;
177
+ return refreshed;
218
178
  }
219
179
  async commit(lsn, options) {
220
180
  const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
221
181
  await this.flush(options);
182
+ using _ = this.tracer.span('storage', 'commit');
222
183
  const now = new Date();
223
184
  await this.db.write_checkpoints.updateMany({
224
185
  processed_at_lsn: null,
@@ -230,94 +191,145 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
230
191
  }, {
231
192
  session: this.session
232
193
  });
233
- const preUpdateDocument = await this.db.sync_rules.findOne({
234
- _id: this.group_id,
235
- 'sync_configs._id': this.syncConfigId
194
+ const preUpdateDocument = (await this.db.sync_rules.findOne({
195
+ _id: this.replicationStreamId,
196
+ 'sync_configs._id': { $in: this.syncConfigIds }
236
197
  }, {
237
198
  session: this.session,
238
199
  projection: {
239
- snapshot_lsn: 1,
240
- sync_configs: {
241
- $elemMatch: {
242
- _id: this.syncConfigId
243
- }
244
- }
200
+ sync_configs: 1,
201
+ last_persisted_op: 1
245
202
  }
246
- });
247
- const state = preUpdateDocument?.sync_configs?.[0];
248
- if (state == null) {
249
- throw new ReplicationAssertionError(`Failed to update checkpoint - no matching sync_config for _id: ${this.group_id}/${this.syncConfigId.toHexString()}`);
203
+ }));
204
+ const states = preUpdateDocument?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ?? [];
205
+ if (states.length == 0) {
206
+ throw new ReplicationAssertionError(`Failed to update checkpoint - no matching sync_config for _id: ${this.replicationStreamId}/${this.syncConfigIds
207
+ .map((id) => id.toHexString())
208
+ .join(',')}`);
250
209
  }
251
- const checkpointState = calculateCheckpointState({
252
- lsn,
253
- snapshotDone: state.snapshot_done === true,
254
- lastCheckpointLsn: state.last_checkpoint_lsn,
255
- noCheckpointBefore: state.no_checkpoint_before,
256
- keepaliveOp: state.keepalive_op == null ? null : BigInt(state.keepalive_op),
257
- lastCheckpoint: state.last_checkpoint,
258
- persistedOp: this.persisted_op,
259
- createEmptyCheckpoints
260
- });
261
- const updateSet = {
262
- last_keepalive_ts: now,
263
- last_fatal_error: null,
264
- last_fatal_error_ts: null,
265
- 'sync_configs.$[config].keepalive_op': checkpointState.newKeepaliveOp,
266
- 'sync_configs.$[config].last_checkpoint': checkpointState.newLastCheckpoint
267
- };
268
- if (checkpointState.checkpointCreated) {
269
- updateSet['sync_configs.$[config].last_checkpoint_lsn'] = lsn;
270
- updateSet['snapshot_lsn'] = null;
271
- updateSet['last_checkpoint_ts'] = now;
210
+ // The replication job / MongoBucketBatch must be constructed with all replicating (PROCESSING / ACTIVE)
211
+ // sync configs in the stream, otherwise we'll get inconsistencies. Configs in other states (e.g. STOP)
212
+ // remain embedded in the document and are ignored here.
213
+ const missingSyncConfig = preUpdateDocument.sync_configs.find((config) => [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE].includes(config.state) &&
214
+ !this.syncConfigIds.some((id) => id.equals(config._id)));
215
+ if (missingSyncConfig != null) {
216
+ throw new ReplicationAssertionError(`Replication job not configured for sync config ${missingSyncConfig._id}`);
272
217
  }
273
- await this.db.sync_rules.updateOne({
274
- _id: this.group_id,
275
- 'sync_configs._id': this.syncConfigId
276
- }, {
277
- $set: updateSet
278
- }, {
279
- session: this.session,
280
- arrayFilters: [{ 'config._id': this.syncConfigId }]
281
- });
282
- if (checkpointState.checkpointBlocked) {
283
- if (Date.now() - this.lastWaitingLogThrottledV3 > 5_000) {
284
- this.logger.info(`Waiting before creating checkpoint, currently at ${lsn} / ${checkpointState.newKeepaliveOp}. Current state: ${JSON.stringify({
218
+ // Effective head of the stream's op sequence.
219
+ // last_persisted_op is $max-advanced durably in the same transaction as every flush, and this
220
+ // read uses the same session, so it covers all ops persisted by this batch.
221
+ const newCheckpoint = preUpdateDocument?.last_persisted_op == null ? 0n : BigInt(preUpdateDocument.last_persisted_op);
222
+ let checkpointBlocked = false;
223
+ let checkpointCreated = false;
224
+ let checkpointLogState = null;
225
+ const unblockedConfigIds = [];
226
+ for (const state of states) {
227
+ if (state.last_checkpoint != null && state.last_checkpoint > newCheckpoint) {
228
+ // last_persisted_op is $max-advanced durably in the same transaction as every flush, and
229
+ // checkpoints are only ever created at that head, so a checkpoint past the head means the
230
+ // op sequence or the stored state is corrupt.
231
+ throw new ReplicationAssertionError(`Invariant violation: sync config ${state._id} has last_checkpoint ${state.last_checkpoint} > stream head ${newCheckpoint}`);
232
+ }
233
+ const canCheckpoint = canCheckpointState(lsn, {
234
+ snapshotDone: state.snapshot_done === true,
235
+ lastCheckpointLsn: state.last_checkpoint_lsn,
236
+ noCheckpointBefore: state.no_checkpoint_before
237
+ });
238
+ if (!canCheckpoint) {
239
+ checkpointBlocked = true;
240
+ // Log the first blocked config's state.
241
+ checkpointLogState ??= {
285
242
  snapshot_done: state.snapshot_done,
286
243
  last_checkpoint_lsn: state.last_checkpoint_lsn,
287
244
  no_checkpoint_before: state.no_checkpoint_before
288
- })}`);
245
+ };
246
+ continue;
247
+ }
248
+ checkpointCreated ||= createEmptyCheckpoints || state.last_checkpoint !== newCheckpoint;
249
+ unblockedConfigIds.push(state._id);
250
+ }
251
+ // Every commit advances the stream's resume position: commit() flushes first, so all
252
+ // source changes up to this lsn have been persisted, even when checkpoints are blocked.
253
+ // In the future we could also advance this on flush, when the connector provides the
254
+ // current position (see setResumeLsn, which connectors may already call after flushing).
255
+ const resumeLsnUpdate = { resume_lsn: lsn };
256
+ if (unblockedConfigIds.length > 0) {
257
+ // All unblocked configs get the SAME new value, so we apply it with a single updateOne
258
+ // (single-document atomicity).
259
+ const updateSet = {
260
+ last_keepalive_ts: now,
261
+ last_fatal_error: null,
262
+ last_fatal_error_ts: null
263
+ };
264
+ // Only advance checkpoint fields when an actual (non-empty) checkpoint is created, matching
265
+ // the previous per-config / v1 behaviour.
266
+ if (checkpointCreated) {
267
+ updateSet['sync_configs.$[config].last_checkpoint'] = newCheckpoint;
268
+ updateSet['sync_configs.$[config].last_checkpoint_lsn'] = lsn;
269
+ updateSet['last_checkpoint_ts'] = now;
270
+ }
271
+ await this.db.sync_rules.updateOne({
272
+ _id: this.replicationStreamId,
273
+ 'sync_configs._id': { $in: unblockedConfigIds }
274
+ }, { $set: updateSet, $max: resumeLsnUpdate }, {
275
+ session: this.session,
276
+ arrayFilters: checkpointCreated ? [{ 'config._id': { $in: unblockedConfigIds } }] : undefined
277
+ });
278
+ }
279
+ else {
280
+ // All selected configs are blocked - only update keepalive/error tracking and the
281
+ // resume position.
282
+ await this.db.sync_rules.updateOne({
283
+ _id: this.replicationStreamId
284
+ }, {
285
+ $set: {
286
+ last_keepalive_ts: now,
287
+ last_fatal_error: null,
288
+ last_fatal_error_ts: null
289
+ },
290
+ $max: resumeLsnUpdate
291
+ }, { session: this.session });
292
+ }
293
+ if (checkpointBlocked) {
294
+ if (Date.now() - this.lastWaitingLogThrottledV3 > 5_000) {
295
+ this.logger.info(`Waiting before creating checkpoint, currently at ${lsn}. Current state: ${JSON.stringify(checkpointLogState)}`);
289
296
  this.lastWaitingLogThrottledV3 = Date.now();
290
297
  }
291
298
  }
292
299
  else {
293
- if (checkpointState.checkpointCreated) {
294
- this.logger.debug(`Created checkpoint at ${lsn} / ${checkpointState.newLastCheckpoint}`);
300
+ if (checkpointCreated) {
301
+ this.logger.debug(`Created checkpoint at ${lsn}`);
295
302
  }
296
303
  await this.autoActivateV3(lsn);
304
+ // All configs are now checkpointed at newCheckpoint (the stream head).
305
+ await this.sourceRecordStore.postCommitCleanup(newCheckpoint, this.logger);
306
+ }
307
+ if (checkpointCreated) {
297
308
  await this.db.notifyCheckpoint();
298
- this.persisted_op = null;
299
- this.last_checkpoint_lsn = lsn;
300
- if (checkpointState.newLastCheckpoint != null) {
301
- await this.sourceRecordStore.postCommitCleanup(checkpointState.newLastCheckpoint, this.logger);
302
- }
303
309
  }
304
310
  return {
305
- checkpointBlocked: checkpointState.checkpointBlocked,
306
- checkpointCreated: checkpointState.checkpointCreated
311
+ checkpointBlocked,
312
+ checkpointCreated
307
313
  };
308
314
  }
309
315
  async keepalive(lsn) {
310
316
  return await this.commit(lsn, { createEmptyCheckpoints: true });
311
317
  }
312
318
  async setResumeLsn(lsn) {
319
+ using _ = this.tracer.span('storage', 'set_resume_lsn');
313
320
  await this.db.sync_rules.updateOne({
314
- _id: this.group_id,
315
- 'sync_configs._id': this.syncConfigId
321
+ _id: this.replicationStreamId
316
322
  }, {
317
323
  $set: {
318
- snapshot_lsn: lsn
324
+ resume_lsn: lsn
319
325
  }
320
- }, { session: this.session });
326
+ }, {
327
+ session: this.session,
328
+ // Losing occasional resume LSN is fine. That may mean reprocessing
329
+ // some source changes in some edge cases, which is not an issue since
330
+ // changes are processed in an idempotent way.
331
+ writeConcern: { w: 1 }
332
+ });
321
333
  }
322
334
  async autoActivateV3(lsn) {
323
335
  if (!this.needsActivationV3) {
@@ -325,30 +337,32 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
325
337
  }
326
338
  const session = this.session;
327
339
  let activated = false;
340
+ let needsFutureActivationCheck = true;
328
341
  await session.withTransaction(async () => {
342
+ // Reset on transaction retries.
343
+ needsFutureActivationCheck = true;
344
+ activated = false;
329
345
  const doc = await this.db.sync_rules.findOne({
330
- _id: this.group_id,
331
- 'sync_configs._id': this.syncConfigId
346
+ _id: this.replicationStreamId,
347
+ 'sync_configs._id': { $in: this.syncConfigIds }
332
348
  }, {
333
349
  session,
334
350
  projection: {
335
351
  state: 1,
336
- sync_configs: {
337
- $elemMatch: {
338
- _id: this.syncConfigId
339
- }
340
- }
352
+ sync_configs: 1
341
353
  }
342
354
  });
343
- const state = doc?.sync_configs?.[0];
344
- if (doc &&
345
- doc.state == storage.SyncRuleState.PROCESSING &&
346
- state?.state == storage.SyncRuleState.PROCESSING &&
347
- state.snapshot_done &&
348
- state.last_checkpoint != null) {
355
+ const states = doc?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ?? [];
356
+ if (doc == null || states.length == 0) {
357
+ return;
358
+ }
359
+ const processingStates = states.filter((state) => state.state == storage.SyncRuleState.PROCESSING);
360
+ if (doc.state == storage.SyncRuleState.PROCESSING &&
361
+ processingStates.length == states.length &&
362
+ states.every((state) => state.snapshot_done && state.last_checkpoint != null)) {
349
363
  await this.db.sync_rules.updateOne({
350
- _id: this.group_id,
351
- 'sync_configs._id': this.syncConfigId
364
+ _id: this.replicationStreamId,
365
+ 'sync_configs._id': { $in: this.syncConfigIds }
352
366
  }, {
353
367
  $set: {
354
368
  state: storage.SyncRuleState.ACTIVE,
@@ -356,16 +370,39 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
356
370
  }
357
371
  }, {
358
372
  session,
359
- arrayFilters: [{ 'config._id': this.syncConfigId }]
373
+ arrayFilters: [{ 'config._id': { $in: this.syncConfigIds } }]
360
374
  });
361
375
  await this.db.sync_rules.updateMany({
362
- _id: { $ne: this.group_id },
376
+ _id: { $ne: this.replicationStreamId },
363
377
  state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
364
378
  }, syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP), { session });
365
379
  activated = true;
366
380
  }
367
- else if (doc?.state != storage.SyncRuleState.PROCESSING) {
368
- this.needsActivationV3 = false;
381
+ else if (doc.state == storage.SyncRuleState.ACTIVE &&
382
+ processingStates.length > 0 &&
383
+ processingStates.every((state) => state.snapshot_done && state.last_checkpoint != null)) {
384
+ await this.db.sync_rules.updateOne({
385
+ _id: this.replicationStreamId,
386
+ 'sync_configs._id': { $in: processingStates.map((state) => state._id) }
387
+ }, {
388
+ $set: {
389
+ 'sync_configs.$[activeConfig].state': storage.SyncRuleState.STOP,
390
+ 'sync_configs.$[processingConfig].state': storage.SyncRuleState.ACTIVE
391
+ }
392
+ }, {
393
+ session,
394
+ arrayFilters: [
395
+ { 'activeConfig.state': storage.SyncRuleState.ACTIVE },
396
+ { 'processingConfig._id': { $in: processingStates.map((state) => state._id) } }
397
+ ]
398
+ });
399
+ activated = true;
400
+ }
401
+ else if (doc.state != storage.SyncRuleState.PROCESSING && doc.state != storage.SyncRuleState.ACTIVE) {
402
+ needsFutureActivationCheck = false;
403
+ }
404
+ else if (doc.state == storage.SyncRuleState.ACTIVE && processingStates.length == 0) {
405
+ needsFutureActivationCheck = false;
369
406
  }
370
407
  });
371
408
  if (activated) {
@@ -373,11 +410,14 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
373
410
  await this.db.notifyCheckpoint();
374
411
  this.needsActivationV3 = false;
375
412
  }
413
+ else if (!needsFutureActivationCheck) {
414
+ this.needsActivationV3 = false;
415
+ }
376
416
  }
377
417
  async markAllSnapshotDone(no_checkpoint_before_lsn) {
378
418
  await this.db.sync_rules.updateOne({
379
- _id: this.group_id,
380
- 'sync_configs._id': this.syncConfigId
419
+ _id: this.replicationStreamId,
420
+ 'sync_configs._id': { $in: this.syncConfigIds }
381
421
  }, {
382
422
  $set: {
383
423
  'sync_configs.$[config].snapshot_done': true,
@@ -388,18 +428,22 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
388
428
  }
389
429
  }, {
390
430
  session: this.session,
391
- arrayFilters: [{ 'config._id': this.syncConfigId }]
431
+ arrayFilters: [{ 'config._id': { $in: this.syncConfigIds } }]
392
432
  });
393
433
  }
394
434
  async markSnapshotDone(no_checkpoint_before_lsn, options) {
395
435
  await this.withTransaction(async () => {
396
436
  // Protect against race conditions
397
- const count = await this.db.sourceTablesV3(this.group_id).countDocuments({
398
- snapshot_done: false
399
- }, { session: this.session });
400
- if (count > 0) {
437
+ const blockingTables = await this.db
438
+ .sourceTables(this.replicationStreamId)
439
+ .find(this.snapshotBlockingSourceTablesFilter(), {
440
+ session: this.session,
441
+ projection: { schema_name: 1, table_name: 1 }
442
+ })
443
+ .toArray();
444
+ if (blockingTables.length > 0) {
401
445
  if (options?.throwOnConflict ?? true) {
402
- throw new ReplicationAssertionError(`Cannot mark snapshot done while ${count} source table${count == 1 ? '' : 's'} still require snapshotting`);
446
+ throw new ReplicationAssertionError(`Cannot mark snapshot done while source tables still require snapshotting. Tables: ${blockingTables.map((t) => `${t.schema_name}.${t.table_name}`).join(', ')}`);
403
447
  }
404
448
  else {
405
449
  return;
@@ -408,24 +452,29 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
408
452
  await this.markAllSnapshotDone(no_checkpoint_before_lsn);
409
453
  });
410
454
  }
411
- async markTableSnapshotRequired(_table) {
455
+ async markTableSnapshotRequired(table) {
456
+ const syncConfigIds = this.relevantSyncConfigIds(table);
457
+ if (syncConfigIds.length == 0) {
458
+ return;
459
+ }
412
460
  await this.db.sync_rules.updateOne({
413
- _id: this.group_id,
414
- 'sync_configs._id': this.syncConfigId
461
+ _id: this.replicationStreamId,
462
+ 'sync_configs._id': { $in: syncConfigIds }
415
463
  }, {
416
464
  $set: {
417
465
  'sync_configs.$[config].snapshot_done': false
418
466
  }
419
467
  }, {
420
468
  session: this.session,
421
- arrayFilters: [{ 'config._id': this.syncConfigId }]
469
+ arrayFilters: [{ 'config._id': { $in: syncConfigIds } }]
422
470
  });
423
471
  }
424
472
  async markTableSnapshotDone(tables, no_checkpoint_before_lsn) {
425
473
  const session = this.session;
426
474
  const ids = tables.map((table) => mongoTableId(table.id));
475
+ const syncConfigIds = this.relevantSyncConfigIdsForTables(tables);
427
476
  await this.withTransaction(async () => {
428
- await this.db.commonSourceTables(this.group_id).updateMany({ _id: { $in: ids } }, {
477
+ await this.db.sourceTables(this.replicationStreamId).updateMany({ _id: { $in: ids } }, {
429
478
  $set: {
430
479
  snapshot_done: true
431
480
  },
@@ -433,10 +482,10 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
433
482
  snapshot_status: 1
434
483
  }
435
484
  }, { session });
436
- if (no_checkpoint_before_lsn != null) {
485
+ if (no_checkpoint_before_lsn != null && syncConfigIds.length > 0) {
437
486
  await this.db.sync_rules.updateOne({
438
- _id: this.group_id,
439
- 'sync_configs._id': this.syncConfigId
487
+ _id: this.replicationStreamId,
488
+ 'sync_configs._id': { $in: syncConfigIds }
440
489
  }, {
441
490
  $set: {
442
491
  last_keepalive_ts: new Date()
@@ -446,7 +495,8 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
446
495
  }
447
496
  }, {
448
497
  session: this.session,
449
- arrayFilters: [{ 'config._id': this.syncConfigId }]
498
+ // Only set for sync configs that use this table
499
+ arrayFilters: [{ 'config._id': { $in: syncConfigIds } }]
450
500
  });
451
501
  }
452
502
  });