@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/CHANGELOG.md +129 -7
  2. package/dist/storage/MongoBucketStorage.d.ts +42 -5
  3. package/dist/storage/MongoBucketStorage.js +404 -215
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
  6. package/dist/storage/implementation/MongoBucketBatch.js +169 -98
  7. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  8. package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
  9. package/dist/storage/implementation/MongoCompactor.js +27 -238
  10. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  11. package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
  12. package/dist/storage/implementation/MongoIdSequence.js +14 -1
  13. package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
  14. package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
  15. package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
  16. package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
  17. package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
  18. package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
  19. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  20. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
  21. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
  22. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
  23. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
  24. package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
  25. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
  26. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
  27. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
  28. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
  29. package/dist/storage/implementation/MongoStorageProvider.js +8 -4
  30. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  31. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
  32. package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
  33. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
  35. package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
  36. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  37. package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
  38. package/dist/storage/implementation/MongoWriteBatch.js +149 -0
  39. package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  44. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  45. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
  47. package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
  48. package/dist/storage/implementation/common/PersistedBatch.js +57 -14
  49. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
  51. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  53. package/dist/storage/implementation/db.d.ts +9 -1
  54. package/dist/storage/implementation/db.js +28 -1
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +35 -0
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
  62. package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
  63. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  65. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  66. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
  71. package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
  72. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/models.js +1 -0
  74. package/dist/storage/implementation/v1/models.js.map +1 -1
  75. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  76. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  77. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  78. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
  79. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  81. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
  82. package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
  83. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  84. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
  85. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
  86. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
  87. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
  88. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
  90. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
  91. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  93. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  94. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  95. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  96. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
  98. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  99. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
  100. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  102. package/dist/storage/implementation/v3/bucket-format.js +2 -0
  103. package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
  104. package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
  105. package/dist/storage/implementation/v3/compact-utils.js +223 -0
  106. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  107. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  108. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  109. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +90 -6
  111. package/dist/storage/implementation/v3/models.js.map +1 -1
  112. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
  115. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
  126. package/dist/storage/implementation/v3/source-table-utils.js +49 -57
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
  128. package/dist/storage/storage-index.d.ts +2 -1
  129. package/dist/storage/storage-index.js +1 -0
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +28 -1
  132. package/dist/types/types.js +28 -7
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +4 -2
  135. package/dist/utils/test-utils.js +7 -4
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.js +3 -1
  138. package/dist/utils/util.js.map +1 -1
  139. package/package.json +10 -9
  140. package/src/storage/MongoBucketStorage.ts +545 -277
  141. package/src/storage/implementation/MongoBucketBatch.ts +258 -124
  142. package/src/storage/implementation/MongoCompactor.ts +37 -346
  143. package/src/storage/implementation/MongoIdSequence.ts +21 -1
  144. package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
  145. package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
  146. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
  147. package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
  148. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
  149. package/src/storage/implementation/MongoStorageProvider.ts +16 -4
  150. package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
  151. package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
  152. package/src/storage/implementation/MongoWriteBatch.ts +172 -0
  153. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
  154. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
  156. package/src/storage/implementation/common/PersistedBatch.ts +76 -20
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
  158. package/src/storage/implementation/db.ts +36 -2
  159. package/src/storage/implementation/models.ts +39 -0
  160. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
  161. package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
  162. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  163. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
  164. package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
  165. package/src/storage/implementation/v1/models.ts +1 -0
  166. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  167. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
  168. package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
  169. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
  170. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
  171. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
  172. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  173. package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
  174. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
  175. package/src/storage/implementation/v3/bucket-format.ts +2 -0
  176. package/src/storage/implementation/v3/compact-utils.ts +329 -0
  177. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  178. package/src/storage/implementation/v3/models.ts +97 -4
  179. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
  180. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
  181. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
  184. package/src/storage/implementation/v3/source-table-utils.ts +82 -76
  185. package/src/storage/storage-index.ts +2 -0
  186. package/src/types/types.ts +42 -7
  187. package/src/utils/test-utils.ts +9 -5
  188. package/src/utils/util.ts +3 -1
  189. package/test/src/__snapshots__/storage.test.ts.snap +25 -0
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
  191. package/test/src/chunk_compaction_config.test.ts +38 -0
  192. package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
  193. package/test/src/compact-utils.test.ts +251 -0
  194. package/test/src/concurrent_writers.test.ts +599 -0
  195. package/test/src/connection_config.test.ts +159 -0
  196. package/test/src/helpers/MemoryObjectStorage.ts +14 -6
  197. package/test/src/helpers/concurrentWriter.mjs +29 -0
  198. package/test/src/helpers/s3TestFactory.ts +12 -12
  199. package/test/src/object_storage_usage.test.ts +268 -0
  200. package/test/src/parameter_compacting_v1.test.ts +120 -0
  201. package/test/src/parameter_compacting_v3.test.ts +192 -0
  202. package/test/src/parameter_compaction_fence.test.ts +276 -0
  203. package/test/src/storage.test.ts +280 -24
  204. package/test/src/storage_compacting.test.ts +1097 -253
  205. package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
  206. package/test/src/storage_s3_checksums.test.ts +6 -7
  207. package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
  208. package/test/src/storage_s3_reading.test.ts +294 -6
  209. package/test/src/storage_s3_writing.test.ts +4 -3
  210. package/test/src/storage_sync.test.ts +412 -80
  211. package/test/src/storeCurrentData.test.ts +5 -5
  212. package/test/src/util.ts +1 -1
  213. package/tsconfig.tsbuildinfo +1 -1
@@ -1,16 +1,20 @@
1
1
  import * as lib_mongo from '@powersync/lib-service-mongodb';
2
- import { ReplicationAssertionError } from '@powersync/lib-services-framework';
2
+ import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
3
3
  import { storage } from '@powersync/service-core';
4
4
  import * as bson from 'bson';
5
5
  import { mongoTableId } from '../../../utils/util.js';
6
6
  import { canCheckpointState } from '../CheckpointState.js';
7
7
  import { MongoBucketBatch } from '../MongoBucketBatch.js';
8
- import { syncRuleStateUpdatePipeline } from '../SyncRuleStateUpdate.js';
8
+ import { stopReplicationStreamPipeline } from '../SyncRuleStateUpdate.js';
9
9
  import { PersistedBatchV3 } from './PersistedBatchV3.js';
10
10
  import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js';
11
- import { createNewSourceTable, designateEventCarrier, overlappingSourceTableFilter, planSourceTableReconciliation, sourceTableDesiredResolution, sourceTableFromDocument } from './source-table-utils.js';
11
+ import { createNewSourceTable, overlappingSourceTableFilter, planSourceTableReconciliation, sourceTableDesiredResolution, sourceTableFromDocument } from './source-table-utils.js';
12
12
  export class MongoBucketBatchV3 extends MongoBucketBatch {
13
13
  store;
14
+ /** Avoid repeating idempotent createIndexes calls for every checkpoint flush. */
15
+ initializedCustomCheckpointEventIds = new Set();
16
+ // Used only to validate supplied checkpoint ids; it does not initialize collections.
17
+ knownEventIds;
14
18
  syncConfigIds;
15
19
  needsActivationV3 = true;
16
20
  lastWaitingLogThrottledV3 = 0;
@@ -21,29 +25,35 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
21
25
  throw new ReplicationAssertionError('Missing sync config id for v3 batch');
22
26
  }
23
27
  this.syncConfigIds = syncConfigIds;
28
+ this.knownEventIds = new Set(this.mapping.allEventDefinitionIds());
24
29
  this.store = new SourceRecordStoreV3(this.db, this.replicationStreamId, this.mapping);
25
30
  }
26
31
  createPersistedBatch(writtenSize) {
27
32
  return new PersistedBatchV3(this.db, this.replicationStreamId, this.mapping, writtenSize, {
28
33
  logger: this.logger,
29
34
  objectStorage: this.options.objectStorage,
30
- inlineThresholdBytes: this.options.inlineThresholdBytes
35
+ inlineThresholdBytes: this.options.inlineThresholdBytes,
36
+ signal: this.options.signal,
37
+ objectStorageUsageWriterId: this.objectStorageUsageWriterId
31
38
  });
32
39
  }
33
40
  get sourceRecordStore() {
34
41
  return this.store;
35
42
  }
36
- async onReplicationTransactionFlush(session, lastOp) {
43
+ persistedOpHead(stream) {
44
+ return stream.last_persisted_op ?? 0n;
45
+ }
46
+ onReplicationTransactionFlush(writes, lastOp) {
37
47
  // Durably advance the stream-level head of persisted ops within the flush transaction.
38
48
  // This ensures a checkpoint created later (even by an empty commit, or by a freshly-appended
39
49
  // config that replicates nothing) covers all ops persisted before a potential crash.
40
- await this.db.sync_rules.updateOne({
50
+ writes.updateOne(this.db.sync_rules, {
41
51
  _id: this.replicationStreamId
42
52
  }, {
43
53
  $max: {
44
54
  last_persisted_op: lastOp
45
55
  }
46
- }, { session });
56
+ });
47
57
  }
48
58
  selectedSyncConfigObjectIds(syncConfigIds) {
49
59
  const selectedIds = new Set(this.syncConfigIds.map((id) => id.toHexString()));
@@ -54,7 +64,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
54
64
  relevantSyncConfigIds(table) {
55
65
  const bucketDataSourceIds = table.bucketDataSources.map((source) => this.mapping.bucketSourceId(source));
56
66
  const parameterLookupSourceIds = table.parameterLookupSources.map((source) => this.mapping.parameterLookupId(source));
57
- return this.selectedSyncConfigObjectIds(this.mapping.syncConfigIdsForSourceTable(this.syncConfigIds.map((id) => id.toHexString()), table.ref, bucketDataSourceIds, parameterLookupSourceIds));
67
+ return this.selectedSyncConfigObjectIds(this.mapping.syncConfigIdsForSourceTable(this.syncConfigIds.map((id) => id.toHexString()), table.ref, bucketDataSourceIds, parameterLookupSourceIds, [...table.eventDefinitionIds]));
58
68
  }
59
69
  relevantSyncConfigIdsForTables(tables) {
60
70
  const ids = new Map();
@@ -87,7 +97,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
87
97
  .sourceRecords(this.replicationStreamId, mongoTableId(table.id))
88
98
  .drop()
89
99
  .catch((error) => {
90
- if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
100
+ if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
91
101
  return;
92
102
  }
93
103
  throw error;
@@ -100,37 +110,48 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
100
110
  const parsedOverride = options.parsedSyncConfig;
101
111
  const syncConfig = parsedOverride?.hydratedSyncConfig ?? this.sync_rules;
102
112
  const mapping = parsedOverride?.mapping ?? this.mapping;
113
+ const eventById = (parsedOverride ?? this.options.parsedSyncConfig).eventById;
103
114
  const { connection_id, source } = options;
104
- const context = {
105
- connectionId: connection_id,
106
- connectionTag: source.connectionTag,
107
- identity: {
108
- schema: source.schema,
109
- name: source.name,
110
- objectId: source.objectId,
111
- replicaIdColumns: source.replicaIdColumns.map((column) => ({
112
- name: column.name,
113
- type: column.type,
114
- type_oid: column.typeId
115
- }))
116
- },
117
- storeCurrentData: source.sendsCompleteRows !== true,
118
- syncConfig,
119
- mapping,
120
- desired: sourceTableDesiredResolution(syncConfig, source, mapping)
115
+ const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
116
+ const identity = {
117
+ schema: source.schema,
118
+ name: source.name,
119
+ objectId: source.objectId,
120
+ replicaIdColumns: source.replicaIdColumns.map((column) => ({
121
+ name: column.name,
122
+ type: column.type,
123
+ type_oid: column.typeId
124
+ }))
121
125
  };
122
126
  let result = null;
123
127
  const session = this.db.client.startSession();
124
128
  await using _ = { [Symbol.asyncDispose]: () => session.endSession() };
125
- await session.withTransaction(async () => {
129
+ await this.withTransaction(async () => {
126
130
  const col = this.db.sourceTables(this.replicationStreamId);
127
- // Fetch every persisted source-table doc that can overlap this physical table.
128
- // Exact-identity docs are candidates for reuse; non-exact overlaps are possible drops.
131
+ // Find records that overlap by name or relation id.
129
132
  const candidateDocs = await col
130
- .find(overlappingSourceTableFilter(connection_id, context.identity), { session })
133
+ .find(overlappingSourceTableFilter(connection_id, identity), { session })
131
134
  .toArray();
132
- // Pure planning: which docs to retain (and narrow), whether a new doc is needed for
133
- // uncovered memberships, and which docs conflict with the current identity.
135
+ const candidateTables = candidateDocs.map((doc) => sourceTableFromDocument(doc, source.connectionTag, syncConfig, mapping, eventById));
136
+ const candidates = candidateTables.map((table) => table.clone());
137
+ const resolution = await reconcile({ source, candidates });
138
+ storage.validateSourceTableCandidateResolution(candidates, resolution);
139
+ // Persist metadata from the reconciler without mutating the queried documents.
140
+ for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
141
+ await col.updateOne({ _id: mongoTableId(id) }, { $set: { source_metadata: sourceMetadata } }, { session });
142
+ }
143
+ const context = {
144
+ connectionId: connection_id,
145
+ connectionTag: source.connectionTag,
146
+ identity,
147
+ storeCurrentData: source.sendsCompleteRows !== true,
148
+ syncConfig,
149
+ mapping,
150
+ desired: sourceTableDesiredResolution(syncConfig, source, mapping, eventById),
151
+ sourceCompatibleTables: resolution.compatibleTables,
152
+ newTableSourceMetadata: resolution.newTableValues.sourceMetadata
153
+ };
154
+ // Plan record reuse, membership changes, creation, and removal.
134
155
  const plan = planSourceTableReconciliation(candidateDocs, context);
135
156
  // Persist narrowing for incomplete snapshots only. Snapshot-complete docs keep stale
136
157
  // coverage ids so compatible future configs can reuse already-snapshotted data.
@@ -140,7 +161,8 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
140
161
  await col.updateOne({ _id: update.id }, {
141
162
  $set: {
142
163
  bucket_data_source_ids: update.memberships.bucketDataSourceIds,
143
- parameter_lookup_source_ids: update.memberships.parameterLookupSourceIds
164
+ parameter_lookup_source_ids: update.memberships.parameterLookupSourceIds,
165
+ event_definition_ids: update.memberships.eventDefinitionIds
144
166
  }
145
167
  }, { session });
146
168
  }
@@ -153,13 +175,11 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
153
175
  await this.db.initializeSourceRecordsCollection(this.replicationStreamId, doc._id, session);
154
176
  plan.tables.push(table);
155
177
  }
156
- // If memberships are split across multiple source tables, only one may fire events.
157
- designateEventCarrier(plan.tables, context.desired.triggersEvent);
158
178
  result = {
159
179
  tables: plan.tables,
160
- dropTables: plan.dropDocs.map((doc) => sourceTableFromDocument(doc, context.connectionTag, syncConfig, mapping))
180
+ dropTables: plan.dropDocs.map((doc) => sourceTableFromDocument(doc, context.connectionTag, syncConfig, mapping, eventById))
161
181
  };
162
- });
182
+ }, session);
163
183
  return result;
164
184
  }
165
185
  async getSourceTableStatus(table) {
@@ -169,127 +189,110 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
169
189
  if (doc == null) {
170
190
  return null;
171
191
  }
172
- const refreshed = sourceTableFromDocument(doc, table.ref.connectionTag, this.sync_rules, this.mapping);
173
- // The event-carrier designation is decided per resolveTables result and not persisted -
174
- // preserve the caller's designation instead of recomputing it from the ref, so that
175
- // refreshing a non-carrier table does not make it fire events.
176
- refreshed.syncEvent = table.syncEvent;
177
- return refreshed;
192
+ return sourceTableFromDocument(doc, table.ref.connectionTag, this.sync_rules, this.mapping, this.options.parsedSyncConfig.eventById);
178
193
  }
179
194
  async commit(lsn, options) {
180
195
  const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
181
- await this.flush(options);
182
196
  using _ = this.tracer.span('storage', 'commit');
183
- const now = new Date();
184
- await this.db.write_checkpoints.updateMany({
185
- processed_at_lsn: null,
186
- 'lsns.1': { $lte: lsn }
187
- }, {
188
- $set: {
189
- processed_at_lsn: lsn
190
- }
191
- }, {
192
- session: this.session
193
- });
194
- const preUpdateDocument = (await this.db.sync_rules.findOne({
195
- _id: this.replicationStreamId,
196
- 'sync_configs._id': { $in: this.syncConfigIds }
197
- }, {
198
- session: this.session,
199
- projection: {
200
- sync_configs: 1,
201
- last_persisted_op: 1
197
+ const checkpoint = async (stream, lastOp) => {
198
+ const now = new Date();
199
+ const preUpdateDocument = stream;
200
+ const writes = this.db.createWriteBatch(this.session, { ordered: false });
201
+ writes.updateMany(this.db.write_checkpoints, { processed_at_lsn: null, 'lsns.1': { $lte: lsn } }, { $set: { processed_at_lsn: lsn } });
202
+ const states = preUpdateDocument?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ??
203
+ [];
204
+ if (states.length == 0) {
205
+ throw new ReplicationAssertionError(`Failed to update checkpoint - no matching sync_config for _id: ${this.replicationStreamId}/${this.syncConfigIds
206
+ .map((id) => id.toHexString())
207
+ .join(',')}`);
202
208
  }
203
- }));
204
- const states = preUpdateDocument?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ?? [];
205
- if (states.length == 0) {
206
- throw new ReplicationAssertionError(`Failed to update checkpoint - no matching sync_config for _id: ${this.replicationStreamId}/${this.syncConfigIds
207
- .map((id) => id.toHexString())
208
- .join(',')}`);
209
- }
210
- // The replication job / MongoBucketBatch must be constructed with all replicating (PROCESSING / ACTIVE)
211
- // sync configs in the stream, otherwise we'll get inconsistencies. Configs in other states (e.g. STOP)
212
- // remain embedded in the document and are ignored here.
213
- const missingSyncConfig = preUpdateDocument.sync_configs.find((config) => [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE].includes(config.state) &&
214
- !this.syncConfigIds.some((id) => id.equals(config._id)));
215
- if (missingSyncConfig != null) {
216
- throw new ReplicationAssertionError(`Replication job not configured for sync config ${missingSyncConfig._id}`);
217
- }
218
- // Effective head of the stream's op sequence.
219
- // last_persisted_op is $max-advanced durably in the same transaction as every flush, and this
220
- // read uses the same session, so it covers all ops persisted by this batch.
221
- const newCheckpoint = preUpdateDocument?.last_persisted_op == null ? 0n : BigInt(preUpdateDocument.last_persisted_op);
222
- let checkpointBlocked = false;
223
- let checkpointCreated = false;
224
- let checkpointLogState = null;
225
- const unblockedConfigIds = [];
226
- for (const state of states) {
227
- if (state.last_checkpoint != null && state.last_checkpoint > newCheckpoint) {
228
- // last_persisted_op is $max-advanced durably in the same transaction as every flush, and
229
- // checkpoints are only ever created at that head, so a checkpoint past the head means the
230
- // op sequence or the stored state is corrupt.
231
- throw new ReplicationAssertionError(`Invariant violation: sync config ${state._id} has last_checkpoint ${state.last_checkpoint} > stream head ${newCheckpoint}`);
209
+ // The replication job / MongoBucketBatch must be constructed with all replicating (PROCESSING / ACTIVE)
210
+ // sync configs in the stream, otherwise we'll get inconsistencies. Configs in other states (e.g. STOP)
211
+ // remain embedded in the document and are ignored here.
212
+ const missingSyncConfig = preUpdateDocument.sync_configs.find((config) => [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE].includes(config.state) &&
213
+ !this.syncConfigIds.some((id) => id.equals(config._id)));
214
+ if (missingSyncConfig != null) {
215
+ throw new ReplicationAssertionError(`Replication job not configured for sync config ${missingSyncConfig._id}`);
232
216
  }
233
- const canCheckpoint = canCheckpointState(lsn, {
234
- snapshotDone: state.snapshot_done === true,
235
- lastCheckpointLsn: state.last_checkpoint_lsn,
236
- noCheckpointBefore: state.no_checkpoint_before
237
- });
238
- if (!canCheckpoint) {
239
- checkpointBlocked = true;
240
- // Log the first blocked config's state.
241
- checkpointLogState ??= {
242
- snapshot_done: state.snapshot_done,
243
- last_checkpoint_lsn: state.last_checkpoint_lsn,
244
- no_checkpoint_before: state.no_checkpoint_before
245
- };
246
- continue;
217
+ // Effective head of the stream's op sequence.
218
+ // A combined flush supplies its new head; an empty commit uses the fenced persisted head.
219
+ const newCheckpoint = lastOp ?? this.persistedOpHead(preUpdateDocument);
220
+ let checkpointBlocked = false;
221
+ let checkpointCreated = false;
222
+ let checkpointLogState = null;
223
+ const unblockedConfigIds = [];
224
+ for (const state of states) {
225
+ if (state.last_checkpoint != null && state.last_checkpoint > newCheckpoint) {
226
+ // last_persisted_op is $max-advanced durably in the same transaction as every flush, and
227
+ // checkpoints are only ever created at that head, so a checkpoint past the head means the
228
+ // op sequence or the stored state is corrupt.
229
+ throw new ReplicationAssertionError(`Invariant violation: sync config ${state._id} has last_checkpoint ${state.last_checkpoint} > stream head ${newCheckpoint}`);
230
+ }
231
+ const canCheckpoint = canCheckpointState(lsn, {
232
+ snapshotDone: state.snapshot_done === true,
233
+ lastCheckpointLsn: state.last_checkpoint_lsn,
234
+ noCheckpointBefore: state.no_checkpoint_before
235
+ });
236
+ if (!canCheckpoint) {
237
+ checkpointBlocked = true;
238
+ // Log the first blocked config's state.
239
+ checkpointLogState ??= {
240
+ snapshot_done: state.snapshot_done,
241
+ last_checkpoint_lsn: state.last_checkpoint_lsn,
242
+ no_checkpoint_before: state.no_checkpoint_before
243
+ };
244
+ continue;
245
+ }
246
+ checkpointCreated ||= createEmptyCheckpoints || state.last_checkpoint !== newCheckpoint;
247
+ unblockedConfigIds.push(state._id);
247
248
  }
248
- checkpointCreated ||= createEmptyCheckpoints || state.last_checkpoint !== newCheckpoint;
249
- unblockedConfigIds.push(state._id);
250
- }
251
- // Every commit advances the stream's resume position: commit() flushes first, so all
252
- // source changes up to this lsn have been persisted, even when checkpoints are blocked.
253
- // In the future we could also advance this on flush, when the connector provides the
254
- // current position (see setResumeLsn, which connectors may already call after flushing).
255
- const resumeLsnUpdate = { resume_lsn: lsn };
256
- if (unblockedConfigIds.length > 0) {
257
- // All unblocked configs get the SAME new value, so we apply it with a single updateOne
258
- // (single-document atomicity).
259
- const updateSet = {
260
- last_keepalive_ts: now,
261
- last_fatal_error: null,
262
- last_fatal_error_ts: null
249
+ // Every commit advances the stream's resume position: commit() flushes first, so all
250
+ // source changes up to this lsn have been persisted, even when checkpoints are blocked.
251
+ // In the future we could also advance this on flush, when the connector provides the
252
+ // current position (see setResumeLsn, which connectors may already call after flushing).
253
+ const resumeLsnUpdate = {
254
+ resume_lsn: lsn,
255
+ ...(lastOp == null ? {} : { last_persisted_op: lastOp })
263
256
  };
264
- // Only advance checkpoint fields when an actual (non-empty) checkpoint is created, matching
265
- // the previous per-config / v1 behaviour.
266
- if (checkpointCreated) {
267
- updateSet['sync_configs.$[config].last_checkpoint'] = newCheckpoint;
268
- updateSet['sync_configs.$[config].last_checkpoint_lsn'] = lsn;
269
- updateSet['last_checkpoint_ts'] = now;
270
- }
271
- await this.db.sync_rules.updateOne({
272
- _id: this.replicationStreamId,
273
- 'sync_configs._id': { $in: unblockedConfigIds }
274
- }, { $set: updateSet, $max: resumeLsnUpdate }, {
275
- session: this.session,
276
- arrayFilters: checkpointCreated ? [{ 'config._id': { $in: unblockedConfigIds } }] : undefined
277
- });
278
- }
279
- else {
280
- // All selected configs are blocked - only update keepalive/error tracking and the
281
- // resume position.
282
- await this.db.sync_rules.updateOne({
283
- _id: this.replicationStreamId
284
- }, {
285
- $set: {
286
- last_keepalive_ts: now,
257
+ if (unblockedConfigIds.length > 0) {
258
+ // All unblocked configs get the SAME new value, so we apply it with a single updateOne
259
+ // (single-document atomicity).
260
+ const updateSet = {
287
261
  last_fatal_error: null,
288
262
  last_fatal_error_ts: null
289
- },
290
- $max: resumeLsnUpdate
291
- }, { session: this.session });
292
- }
263
+ };
264
+ // Only advance checkpoint fields when an actual (non-empty) checkpoint is created, matching
265
+ // the previous per-config / v1 behaviour.
266
+ if (checkpointCreated) {
267
+ updateSet['sync_configs.$[config].last_checkpoint'] = newCheckpoint;
268
+ updateSet['sync_configs.$[config].last_checkpoint_lsn'] = lsn;
269
+ updateSet['last_checkpoint_ts'] = now;
270
+ }
271
+ writes.updateOne(this.db.sync_rules, {
272
+ _id: this.replicationStreamId,
273
+ 'sync_configs._id': { $in: unblockedConfigIds }
274
+ }, { $set: updateSet, $max: resumeLsnUpdate }, {
275
+ arrayFilters: checkpointCreated ? [{ 'config._id': { $in: unblockedConfigIds } }] : undefined
276
+ });
277
+ }
278
+ else {
279
+ // All selected configs are blocked - only update keepalive/error tracking and the
280
+ // resume position.
281
+ writes.updateOne(this.db.sync_rules, {
282
+ _id: this.replicationStreamId
283
+ }, {
284
+ $set: {
285
+ last_fatal_error: null,
286
+ last_fatal_error_ts: null
287
+ },
288
+ $max: resumeLsnUpdate
289
+ });
290
+ }
291
+ await writes.execute();
292
+ return { checkpointBlocked, checkpointCreated, checkpointLogState, newCheckpoint };
293
+ };
294
+ const projection = { sync_configs: 1, last_persisted_op: 1 };
295
+ const { checkpointBlocked, checkpointCreated, checkpointLogState, newCheckpoint } = await this.flushAndCommit(checkpoint, () => this.withFencedTransaction(() => this.fence(this.session, projection), checkpoint), options, projection);
293
296
  if (checkpointBlocked) {
294
297
  if (Date.now() - this.lastWaitingLogThrottledV3 > 5_000) {
295
298
  this.logger.info(`Waiting before creating checkpoint, currently at ${lsn}. Current state: ${JSON.stringify(checkpointLogState)}`);
@@ -317,19 +320,9 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
317
320
  }
318
321
  async setResumeLsn(lsn) {
319
322
  using _ = this.tracer.span('storage', 'set_resume_lsn');
320
- await this.db.sync_rules.updateOne({
321
- _id: this.replicationStreamId
322
- }, {
323
- $set: {
324
- resume_lsn: lsn
325
- }
326
- }, {
327
- session: this.session,
328
- // Losing occasional resume LSN is fine. That may mean reprocessing
329
- // some source changes in some edge cases, which is not an issue since
330
- // changes are processed in an idempotent way.
331
- writeConcern: { w: 1 }
332
- });
323
+ // Losing occasional resume LSN would only reprocess source changes.
324
+ // Keep the lease check atomic, but do not wait for majority replication of this resume hint.
325
+ await this.updateStreamMetadata({ resume_lsn: { $literal: lsn } }, {}, { w: 1 });
333
326
  }
334
327
  async autoActivateV3(lsn) {
335
328
  if (!this.needsActivationV3) {
@@ -338,7 +331,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
338
331
  const session = this.session;
339
332
  let activated = false;
340
333
  let needsFutureActivationCheck = true;
341
- await session.withTransaction(async () => {
334
+ await this.withTransaction(async () => {
342
335
  // Reset on transaction retries.
343
336
  needsFutureActivationCheck = true;
344
337
  activated = false;
@@ -375,7 +368,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
375
368
  await this.db.sync_rules.updateMany({
376
369
  _id: { $ne: this.replicationStreamId },
377
370
  state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
378
- }, syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP), { session });
371
+ }, stopReplicationStreamPipeline(), { session });
379
372
  activated = true;
380
373
  }
381
374
  else if (doc.state == storage.SyncRuleState.ACTIVE &&
@@ -404,7 +397,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
404
397
  else if (doc.state == storage.SyncRuleState.ACTIVE && processingStates.length == 0) {
405
398
  needsFutureActivationCheck = false;
406
399
  }
407
- });
400
+ }, session);
408
401
  if (activated) {
409
402
  this.logger.info(`Activated new replication stream at ${lsn}`);
410
403
  await this.db.notifyCheckpoint();
@@ -414,21 +407,32 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
414
407
  this.needsActivationV3 = false;
415
408
  }
416
409
  }
410
+ syncConfigMetadataUpdate(syncConfigIds, set) {
411
+ return {
412
+ sync_configs: {
413
+ $map: {
414
+ input: '$sync_configs',
415
+ as: 'config',
416
+ in: {
417
+ $cond: [
418
+ { $in: ['$$config._id', { $literal: syncConfigIds }] },
419
+ { $mergeObjects: ['$$config', set] },
420
+ '$$config'
421
+ ]
422
+ }
423
+ }
424
+ }
425
+ };
426
+ }
427
+ snapshotDoneUpdate(no_checkpoint_before_lsn) {
428
+ return this.syncConfigMetadataUpdate(this.syncConfigIds, {
429
+ snapshot_done: true,
430
+ no_checkpoint_before: { $max: ['$$config.no_checkpoint_before', { $literal: no_checkpoint_before_lsn }] }
431
+ });
432
+ }
417
433
  async markAllSnapshotDone(no_checkpoint_before_lsn) {
418
- await this.db.sync_rules.updateOne({
419
- _id: this.replicationStreamId,
434
+ await this.updateStreamMetadata(this.snapshotDoneUpdate(no_checkpoint_before_lsn), {
420
435
  'sync_configs._id': { $in: this.syncConfigIds }
421
- }, {
422
- $set: {
423
- 'sync_configs.$[config].snapshot_done': true,
424
- last_keepalive_ts: new Date()
425
- },
426
- $max: {
427
- 'sync_configs.$[config].no_checkpoint_before': no_checkpoint_before_lsn
428
- }
429
- }, {
430
- session: this.session,
431
- arrayFilters: [{ 'config._id': { $in: this.syncConfigIds } }]
432
436
  });
433
437
  }
434
438
  async markSnapshotDone(no_checkpoint_before_lsn, options) {
@@ -449,7 +453,9 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
449
453
  return;
450
454
  }
451
455
  }
452
- await this.markAllSnapshotDone(no_checkpoint_before_lsn);
456
+ await this.updateStreamMetadataInTransaction(this.snapshotDoneUpdate(no_checkpoint_before_lsn), {
457
+ 'sync_configs._id': { $in: this.syncConfigIds }
458
+ });
453
459
  });
454
460
  }
455
461
  async markTableSnapshotRequired(table) {
@@ -457,16 +463,8 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
457
463
  if (syncConfigIds.length == 0) {
458
464
  return;
459
465
  }
460
- await this.db.sync_rules.updateOne({
461
- _id: this.replicationStreamId,
466
+ await this.updateStreamMetadata(this.syncConfigMetadataUpdate(syncConfigIds, { snapshot_done: false }), {
462
467
  'sync_configs._id': { $in: syncConfigIds }
463
- }, {
464
- $set: {
465
- 'sync_configs.$[config].snapshot_done': false
466
- }
467
- }, {
468
- session: this.session,
469
- arrayFilters: [{ 'config._id': { $in: syncConfigIds } }]
470
468
  });
471
469
  }
472
470
  async markTableSnapshotDone(tables, no_checkpoint_before_lsn) {
@@ -474,31 +472,29 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
474
472
  const ids = tables.map((table) => mongoTableId(table.id));
475
473
  const syncConfigIds = this.relevantSyncConfigIdsForTables(tables);
476
474
  await this.withTransaction(async () => {
477
- await this.db.sourceTables(this.replicationStreamId).updateMany({ _id: { $in: ids } }, {
475
+ const writes = this.db.createWriteBatch(session, { ordered: false });
476
+ writes.updateMany(this.db.sourceTables(this.replicationStreamId), { _id: { $in: ids } }, {
478
477
  $set: {
479
478
  snapshot_done: true
480
479
  },
481
480
  $unset: {
482
481
  snapshot_status: 1
483
482
  }
484
- }, { session });
483
+ });
485
484
  if (no_checkpoint_before_lsn != null && syncConfigIds.length > 0) {
486
- await this.db.sync_rules.updateOne({
485
+ writes.updateOne(this.db.sync_rules, {
487
486
  _id: this.replicationStreamId,
488
487
  'sync_configs._id': { $in: syncConfigIds }
489
488
  }, {
490
- $set: {
491
- last_keepalive_ts: new Date()
492
- },
493
489
  $max: {
494
490
  'sync_configs.$[config].no_checkpoint_before': no_checkpoint_before_lsn
495
491
  }
496
492
  }, {
497
- session: this.session,
498
493
  // Only set for sync configs that use this table
499
494
  arrayFilters: [{ 'config._id': { $in: syncConfigIds } }]
500
495
  });
501
496
  }
497
+ await writes.execute();
502
498
  });
503
499
  return tables.map((table) => {
504
500
  const copy = table.clone();
@@ -506,5 +502,76 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
506
502
  return copy;
507
503
  });
508
504
  }
505
+ async batchCreateCustomWriteCheckpoints(session, opId) {
506
+ if (this.write_checkpoint_batch.length == 0) {
507
+ return;
508
+ }
509
+ const checkpointsByEvent = Map.groupBy(this.write_checkpoint_batch, (checkpoint) => this.validateCustomCheckpointEventId(checkpoint.event_id));
510
+ const writes = this.db.createWriteBatch(session, { ordered: false });
511
+ for (const [eventId, checkpoints] of checkpointsByEvent) {
512
+ this.batchCreateEventCustomWriteCheckpoints(writes, opId, eventId, checkpoints);
513
+ }
514
+ await writes.execute();
515
+ }
516
+ async prepareCustomWriteCheckpoints() {
517
+ // Most events never produce custom checkpoints, so create collections lazily
518
+ // only for the event ids present in this batch. This hook runs before the
519
+ // replication transaction because MongoDB cannot create indexes within it.
520
+ const eventIds = new Set(this.write_checkpoint_batch.map((checkpoint) => this.validateCustomCheckpointEventId(checkpoint.event_id)));
521
+ for (const eventId of eventIds) {
522
+ if (this.initializedCustomCheckpointEventIds.has(eventId)) {
523
+ continue;
524
+ }
525
+ await this.db.initializeCustomCheckpointRequestsCollection({
526
+ replicationStreamId: this.replicationStreamId,
527
+ eventId
528
+ });
529
+ this.initializedCustomCheckpointEventIds.add(eventId);
530
+ }
531
+ }
532
+ validateCustomCheckpointEventId(eventId) {
533
+ if (eventId == null) {
534
+ throw new ServiceAssertionError('V3 custom checkpoints require an event definition id');
535
+ }
536
+ if (!this.knownEventIds.has(eventId)) {
537
+ throw new ServiceAssertionError(`Unknown custom checkpoint event definition ${eventId}`);
538
+ }
539
+ return eventId;
540
+ }
541
+ batchCreateEventCustomWriteCheckpoints(writes, opId, eventId, checkpoints) {
542
+ // A repeated user within an event replaces the complete checkpoint state.
543
+ const uniqueCheckpoints = new Map(checkpoints.map((checkpoint) => [checkpoint.user_id, checkpoint]));
544
+ writes.bulkWriteUnordered(this.db.customCheckpointRequests({
545
+ eventId,
546
+ replicationStreamId: this.replicationStreamId
547
+ }), [...uniqueCheckpoints.values()].map((checkpointOptions) => {
548
+ const set = {
549
+ user_id: checkpointOptions.user_id,
550
+ checkpoint: checkpointOptions.checkpoint,
551
+ op_id: opId
552
+ };
553
+ if (checkpointOptions.checkpoint_requested_at != null) {
554
+ set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
555
+ }
556
+ return {
557
+ updateOne: {
558
+ filter: {
559
+ user_id: checkpointOptions.user_id
560
+ },
561
+ update: {
562
+ $set: set,
563
+ ...(checkpointOptions.checkpoint_requested_at == null
564
+ ? {
565
+ $unset: {
566
+ checkpoint_requested_at: 1
567
+ }
568
+ }
569
+ : {})
570
+ },
571
+ upsert: true
572
+ }
573
+ };
574
+ }));
575
+ }
509
576
  }
510
577
  //# sourceMappingURL=MongoBucketBatchV3.js.map