@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/CHANGELOG.md +129 -7
  2. package/dist/storage/MongoBucketStorage.d.ts +42 -5
  3. package/dist/storage/MongoBucketStorage.js +404 -215
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
  6. package/dist/storage/implementation/MongoBucketBatch.js +169 -98
  7. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  8. package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
  9. package/dist/storage/implementation/MongoCompactor.js +27 -238
  10. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  11. package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
  12. package/dist/storage/implementation/MongoIdSequence.js +14 -1
  13. package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
  14. package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
  15. package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
  16. package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
  17. package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
  18. package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
  19. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  20. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
  21. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
  22. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
  23. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
  24. package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
  25. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
  26. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
  27. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
  28. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
  29. package/dist/storage/implementation/MongoStorageProvider.js +8 -4
  30. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  31. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
  32. package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
  33. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
  35. package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
  36. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  37. package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
  38. package/dist/storage/implementation/MongoWriteBatch.js +149 -0
  39. package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  44. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  45. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
  47. package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
  48. package/dist/storage/implementation/common/PersistedBatch.js +57 -14
  49. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
  51. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  53. package/dist/storage/implementation/db.d.ts +9 -1
  54. package/dist/storage/implementation/db.js +28 -1
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +35 -0
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
  62. package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
  63. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  65. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  66. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
  71. package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
  72. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/models.js +1 -0
  74. package/dist/storage/implementation/v1/models.js.map +1 -1
  75. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  76. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  77. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  78. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
  79. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  81. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
  82. package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
  83. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  84. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
  85. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
  86. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
  87. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
  88. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
  90. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
  91. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  93. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  94. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  95. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  96. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
  98. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  99. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
  100. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  102. package/dist/storage/implementation/v3/bucket-format.js +2 -0
  103. package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
  104. package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
  105. package/dist/storage/implementation/v3/compact-utils.js +223 -0
  106. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  107. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  108. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  109. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +90 -6
  111. package/dist/storage/implementation/v3/models.js.map +1 -1
  112. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
  115. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
  126. package/dist/storage/implementation/v3/source-table-utils.js +49 -57
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
  128. package/dist/storage/storage-index.d.ts +2 -1
  129. package/dist/storage/storage-index.js +1 -0
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +28 -1
  132. package/dist/types/types.js +28 -7
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +4 -2
  135. package/dist/utils/test-utils.js +7 -4
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.js +3 -1
  138. package/dist/utils/util.js.map +1 -1
  139. package/package.json +10 -9
  140. package/src/storage/MongoBucketStorage.ts +545 -277
  141. package/src/storage/implementation/MongoBucketBatch.ts +258 -124
  142. package/src/storage/implementation/MongoCompactor.ts +37 -346
  143. package/src/storage/implementation/MongoIdSequence.ts +21 -1
  144. package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
  145. package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
  146. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
  147. package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
  148. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
  149. package/src/storage/implementation/MongoStorageProvider.ts +16 -4
  150. package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
  151. package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
  152. package/src/storage/implementation/MongoWriteBatch.ts +172 -0
  153. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
  154. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
  156. package/src/storage/implementation/common/PersistedBatch.ts +76 -20
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
  158. package/src/storage/implementation/db.ts +36 -2
  159. package/src/storage/implementation/models.ts +39 -0
  160. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
  161. package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
  162. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  163. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
  164. package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
  165. package/src/storage/implementation/v1/models.ts +1 -0
  166. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  167. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
  168. package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
  169. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
  170. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
  171. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
  172. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  173. package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
  174. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
  175. package/src/storage/implementation/v3/bucket-format.ts +2 -0
  176. package/src/storage/implementation/v3/compact-utils.ts +329 -0
  177. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  178. package/src/storage/implementation/v3/models.ts +97 -4
  179. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
  180. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
  181. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
  184. package/src/storage/implementation/v3/source-table-utils.ts +82 -76
  185. package/src/storage/storage-index.ts +2 -0
  186. package/src/types/types.ts +42 -7
  187. package/src/utils/test-utils.ts +9 -5
  188. package/src/utils/util.ts +3 -1
  189. package/test/src/__snapshots__/storage.test.ts.snap +25 -0
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
  191. package/test/src/chunk_compaction_config.test.ts +38 -0
  192. package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
  193. package/test/src/compact-utils.test.ts +251 -0
  194. package/test/src/concurrent_writers.test.ts +599 -0
  195. package/test/src/connection_config.test.ts +159 -0
  196. package/test/src/helpers/MemoryObjectStorage.ts +14 -6
  197. package/test/src/helpers/concurrentWriter.mjs +29 -0
  198. package/test/src/helpers/s3TestFactory.ts +12 -12
  199. package/test/src/object_storage_usage.test.ts +268 -0
  200. package/test/src/parameter_compacting_v1.test.ts +120 -0
  201. package/test/src/parameter_compacting_v3.test.ts +192 -0
  202. package/test/src/parameter_compaction_fence.test.ts +276 -0
  203. package/test/src/storage.test.ts +280 -24
  204. package/test/src/storage_compacting.test.ts +1097 -253
  205. package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
  206. package/test/src/storage_s3_checksums.test.ts +6 -7
  207. package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
  208. package/test/src/storage_s3_reading.test.ts +294 -6
  209. package/test/src/storage_s3_writing.test.ts +4 -3
  210. package/test/src/storage_sync.test.ts +412 -80
  211. package/test/src/storeCurrentData.test.ts +5 -5
  212. package/test/src/util.ts +1 -1
  213. package/tsconfig.tsbuildinfo +1 -1
@@ -1,15 +1,20 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
1
2
  import { ReplicationAssertionError } from '@powersync/lib-services-framework';
2
3
  import { ColumnDescriptor, InternalOpId, SourceTable, storage } from '@powersync/service-core';
4
+ import { HydratedSyncConfig } from '@powersync/service-sync-rules';
3
5
  import * as bson from 'bson';
4
6
  import { mongoTableId } from '../../../utils/util.js';
5
7
  import { calculateCheckpointState } from '../CheckpointState.js';
6
8
  import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
9
+ import { MongoSyncRulesLock } from '../MongoSyncRulesLock.js';
10
+ import { MongoWriteBatch } from '../MongoWriteBatch.js';
7
11
  import { PersistedBatch } from '../common/PersistedBatch.js';
8
12
  import { SourceRecordStore } from '../common/SourceRecordStore.js';
13
+ import { SyncRuleDocumentBase } from '../models.js';
9
14
  import { PersistedBatchV1 } from './PersistedBatchV1.js';
10
15
  import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
11
16
  import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
12
- import { SyncRuleDocumentV1 } from './models.js';
17
+ import { SourceTableDocumentV1, SyncRuleDocumentV1 } from './models.js';
13
18
 
14
19
  export class MongoBucketBatchV1 extends MongoBucketBatch {
15
20
  declare readonly db: VersionedPowerSyncMongoV1;
@@ -32,6 +37,27 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
32
37
  this.store = new SourceRecordStoreV1(this.db, this.replicationStreamId);
33
38
  }
34
39
 
40
+ protected override persistedOpHead(stream: SyncRuleDocumentBase): InternalOpId {
41
+ const legacy = stream as SyncRuleDocumentV1;
42
+ const checkpoint = legacy.last_checkpoint ?? 0n;
43
+ const keepalive = legacy.keepalive_op == null ? 0n : BigInt(legacy.keepalive_op);
44
+ return checkpoint > keepalive ? checkpoint : keepalive;
45
+ }
46
+
47
+ protected override onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: InternalOpId): void {
48
+ // Keep flushed operations recoverable before a checkpoint can be published.
49
+ // keepalive_op remains a decimal string for compatibility with legacy readers.
50
+ writes.updateOne(this.db.sync_rules, { _id: this.replicationStreamId }, [
51
+ {
52
+ $set: {
53
+ keepalive_op: {
54
+ $toString: { $max: [{ $toLong: '$keepalive_op' }, '$last_checkpoint', lastOp] }
55
+ }
56
+ }
57
+ }
58
+ ]);
59
+ }
60
+
35
61
  protected override recordPersistedOp(lastOp: InternalOpId): void {
36
62
  this.persisted_op = lastOp;
37
63
  }
@@ -49,8 +75,46 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
49
75
  // No-op for V1: source records live in a shared collection.
50
76
  }
51
77
 
78
+ protected override async batchCreateCustomWriteCheckpoints(
79
+ session: mongo.ClientSession,
80
+ opId: InternalOpId
81
+ ): Promise<void> {
82
+ // Only the final checkpoint per user/stream is visible after this transaction.
83
+ const checkpoints = new Map(
84
+ this.write_checkpoint_batch.map((checkpoint) => [
85
+ JSON.stringify([checkpoint.sync_rules_id, checkpoint.user_id]),
86
+ checkpoint
87
+ ])
88
+ );
89
+ await this.db.custom_write_checkpoints.bulkWrite(
90
+ [...checkpoints.values()].map((checkpointOptions) => {
91
+ const set: Record<string, unknown> = {
92
+ checkpoint: checkpointOptions.checkpoint,
93
+ sync_rules_id: checkpointOptions.sync_rules_id,
94
+ op_id: opId
95
+ };
96
+ if (checkpointOptions.checkpoint_requested_at != null) {
97
+ set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
98
+ }
99
+
100
+ return {
101
+ updateOne: {
102
+ filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
103
+ update: {
104
+ $set: set,
105
+ ...(checkpointOptions.checkpoint_requested_at == null ? { $unset: { checkpoint_requested_at: 1 } } : {})
106
+ },
107
+ upsert: true
108
+ }
109
+ };
110
+ }),
111
+ { session, ordered: false }
112
+ );
113
+ }
114
+
52
115
  async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
53
116
  const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
117
+ const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
54
118
  const { connection_id, source } = options;
55
119
  const { schema, name, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
56
120
 
@@ -61,22 +125,41 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
61
125
  }));
62
126
 
63
127
  let result: storage.ResolveTablesResult | null = null;
64
- await this.db.client.withSession(async (session) => {
128
+ const session = this.session;
129
+ await this.withTransaction(async () => {
65
130
  const col = this.db.sourceTablesV1(this.replicationStreamId);
66
- const filter: any = {
67
- group_id: this.replicationStreamId,
68
- connection_id,
69
- schema_name: schema,
70
- table_name: name,
71
- replica_id_columns2: normalizedReplicaIdColumns
72
- };
131
+
132
+ // Find records that overlap by name or relation id.
133
+ const orClauses: Record<string, unknown>[] = [{ schema_name: schema, table_name: name }];
73
134
  if (objectId != null) {
74
- filter.relation_id = objectId;
135
+ orClauses.push({ relation_id: objectId });
75
136
  }
137
+ const candidateDocs = await col
138
+ .find({ group_id: this.replicationStreamId, connection_id, $or: orClauses }, { session })
139
+ .toArray();
76
140
 
77
- let doc = await col.findOne(filter, { session });
78
- if (doc == null) {
79
- doc = {
141
+ const candidateTables = candidateDocs.map((doc) =>
142
+ this.hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows)
143
+ );
144
+ const candidates = candidateTables.map((table) => table.clone());
145
+ const resolution = await reconcile({ source, candidates });
146
+ storage.validateSourceTableCandidateResolution(candidates, resolution);
147
+
148
+ for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
149
+ await col.updateOne({ _id: mongoTableId(id) }, { $set: { source_metadata: sourceMetadata } }, { session });
150
+ }
151
+
152
+ const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
153
+ const compatibleTables = materializedResolution.compatibleTables;
154
+
155
+ // V1 keeps one record, preferring one that has already been snapshotted.
156
+ const reuse = compatibleTables.find((table) => table.snapshotComplete) ?? compatibleTables[0] ?? null;
157
+
158
+ let sourceTable: storage.SourceTable;
159
+ if (reuse != null) {
160
+ sourceTable = reuse;
161
+ } else {
162
+ const doc: SourceTableDocumentV1 = {
80
163
  _id: options.idGenerator ? (options.idGenerator() as bson.ObjectId) : new bson.ObjectId(),
81
164
  group_id: this.replicationStreamId,
82
165
  connection_id,
@@ -86,69 +169,18 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
86
169
  replica_id_columns: null,
87
170
  replica_id_columns2: normalizedReplicaIdColumns,
88
171
  snapshot_done: false,
89
- snapshot_status: undefined
172
+ snapshot_status: undefined,
173
+ source_metadata: resolution.newTableValues.sourceMetadata
90
174
  };
91
175
  await col.insertOne(doc, { session });
176
+ sourceTable = this.hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows);
92
177
  }
93
178
 
94
- const sourceTable = new storage.SourceTable({
95
- id: doc._id,
96
- ref: source,
97
- objectId,
98
- replicaIdColumns,
99
- snapshotComplete: doc.snapshot_done ?? true,
100
- ...syncRules.getMatchingSources(source)
101
- });
102
- sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
103
- sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
104
- sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
105
- sourceTable.storeCurrentData = sendsCompleteRows !== true;
106
- sourceTable.snapshotStatus =
107
- doc.snapshot_status == null
108
- ? undefined
109
- : {
110
- lastKey: doc.snapshot_status.last_key?.buffer ?? null,
111
- totalEstimatedCount: doc.snapshot_status.total_estimated_count,
112
- replicatedCount: doc.snapshot_status.replicated_count
113
- };
114
-
115
- const truncateFilter = [{ schema_name: schema, table_name: name }] as any[];
116
- if (objectId != null) {
117
- truncateFilter.push({ relation_id: objectId });
118
- }
119
- const truncate = await col
120
- .find(
121
- {
122
- group_id: this.replicationStreamId,
123
- connection_id,
124
- _id: { $ne: doc._id },
125
- $or: truncateFilter
126
- },
127
- { session }
128
- )
129
- .toArray();
130
- const dropTables = truncate.map((dropDoc) => {
131
- const ref = {
132
- connectionTag,
133
- schema: dropDoc.schema_name,
134
- name: dropDoc.table_name
135
- };
136
- const table = new storage.SourceTable({
137
- id: dropDoc._id,
138
- ref,
139
- objectId: dropDoc.relation_id,
140
- replicaIdColumns:
141
- dropDoc.replica_id_columns2?.map(
142
- (c) => ({ name: c.name, typeId: c.type_oid, type: c.type }) satisfies ColumnDescriptor
143
- ) ?? [],
144
- snapshotComplete: dropDoc.snapshot_done ?? true,
145
- ...syncRules.getMatchingSources(ref)
146
- });
147
- table.syncEvent = syncRules.tableTriggersEvent(ref);
148
- table.syncData = table.bucketDataSources.length > 0;
149
- table.syncParameters = table.parameterLookupSources.length > 0;
150
- return table;
151
- });
179
+ const dropTables = [
180
+ ...materializedResolution.incompatibleTables,
181
+ // V1 only keeps one SourceTable per physical table.
182
+ ...compatibleTables.filter((table) => !storage.sourceTableIdEquals(table.id, sourceTable.id))
183
+ ];
152
184
 
153
185
  result = { tables: [sourceTable], dropTables };
154
186
  });
@@ -156,38 +188,33 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
156
188
  return result!;
157
189
  }
158
190
 
159
- async getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null> {
160
- const doc = await this.db.sourceTablesV1(this.replicationStreamId).findOne(
161
- {
162
- group_id: this.replicationStreamId,
163
- _id: mongoTableId(table.id)
164
- },
165
- { session: this.session }
166
- );
167
- if (doc == null) {
168
- return null;
169
- }
170
-
171
- const ref = {
172
- connectionTag: table.ref.connectionTag,
173
- schema: doc.schema_name,
174
- name: doc.table_name
175
- };
176
- const sourceTable = new storage.SourceTable({
191
+ /**
192
+ * Build a SourceTable from its persisted v1 record.
193
+ */
194
+ private hydrateSourceTable(
195
+ doc: SourceTableDocumentV1,
196
+ connectionTag: string,
197
+ syncRules: HydratedSyncConfig,
198
+ sendsCompleteRows: boolean | undefined
199
+ ): storage.SourceTable {
200
+ const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
201
+ const table = new storage.SourceTable({
177
202
  id: doc._id,
178
203
  ref,
179
204
  objectId: doc.relation_id,
180
205
  replicaIdColumns:
181
206
  doc.replica_id_columns2?.map(
182
- (c) => ({ name: c.name, typeId: c.type_oid, type: c.type }) satisfies ColumnDescriptor
207
+ (column) => ({ name: column.name, typeId: column.type_oid, type: column.type }) satisfies ColumnDescriptor
183
208
  ) ?? [],
184
209
  snapshotComplete: doc.snapshot_done ?? true,
185
- ...this.sync_rules.getMatchingSources(ref)
210
+ sourceMetadata: doc.source_metadata ?? null,
211
+ ...syncRules.getMatchingSources(ref)
186
212
  });
187
- sourceTable.syncEvent = this.sync_rules.tableTriggersEvent(ref);
188
- sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
189
- sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
190
- sourceTable.snapshotStatus =
213
+ table.syncEvent = syncRules.tableTriggersEvent(ref);
214
+ table.syncData = table.bucketDataSources.length > 0;
215
+ table.syncParameters = table.parameterLookupSources.length > 0;
216
+ table.storeCurrentData = sendsCompleteRows !== true;
217
+ table.snapshotStatus =
191
218
  doc.snapshot_status == null
192
219
  ? undefined
193
220
  : {
@@ -195,135 +222,158 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
195
222
  totalEstimatedCount: doc.snapshot_status.total_estimated_count,
196
223
  replicatedCount: doc.snapshot_status.replicated_count
197
224
  };
198
- return sourceTable;
225
+ return table;
226
+ }
227
+
228
+ async getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null> {
229
+ const doc = await this.db.sourceTablesV1(this.replicationStreamId).findOne(
230
+ {
231
+ group_id: this.replicationStreamId,
232
+ _id: mongoTableId(table.id)
233
+ },
234
+ { session: this.session }
235
+ );
236
+ if (doc == null) {
237
+ return null;
238
+ }
239
+
240
+ // Use the default storeCurrentData value when reading outside resolution.
241
+ return this.hydrateSourceTable(doc, table.ref.connectionTag, this.sync_rules, undefined);
199
242
  }
200
243
 
201
244
  async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult> {
202
245
  const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
203
246
 
204
- await this.flush(options);
205
-
206
247
  using _ = this.tracer.span('storage', 'commit');
207
248
 
208
- const now = new Date();
249
+ const updateCheckpoint = async (persistedOp = this.persisted_op) => {
250
+ const now = new Date();
209
251
 
210
- await this.db.write_checkpoints.updateMany(
211
- {
212
- processed_at_lsn: null,
213
- 'lsns.1': { $lte: lsn }
214
- },
215
- {
216
- $set: {
217
- processed_at_lsn: lsn
218
- }
219
- },
220
- {
221
- session: this.session
222
- }
223
- );
252
+ const can_checkpoint = {
253
+ $and: [
254
+ { $eq: ['$snapshot_done', true] },
255
+ {
256
+ $or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
257
+ },
258
+ {
259
+ $or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
260
+ }
261
+ ]
262
+ };
224
263
 
225
- const can_checkpoint = {
226
- $and: [
227
- { $eq: ['$snapshot_done', true] },
228
- {
229
- $or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
230
- },
231
- {
232
- $or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
233
- }
234
- ]
235
- };
264
+ const new_keepalive_op = {
265
+ $cond: [
266
+ can_checkpoint,
267
+ { $literal: null },
268
+ {
269
+ $toString: {
270
+ $max: [{ $toLong: '$keepalive_op' }, { $literal: persistedOp }, 0n]
271
+ }
272
+ }
273
+ ]
274
+ };
236
275
 
237
- const new_keepalive_op = {
238
- $cond: [
239
- can_checkpoint,
240
- { $literal: null },
276
+ const new_last_checkpoint = {
277
+ $cond: [
278
+ can_checkpoint,
279
+ {
280
+ $max: ['$last_checkpoint', { $literal: persistedOp }, { $toLong: '$keepalive_op' }, 0n]
281
+ },
282
+ '$last_checkpoint'
283
+ ]
284
+ };
285
+
286
+ const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate(
287
+ MongoSyncRulesLock.ownerFilter(this.replicationStreamId, this.options.replicationLock),
288
+ [
289
+ {
290
+ $set: {
291
+ _can_checkpoint: can_checkpoint,
292
+ _not_empty: createEmptyCheckpoints
293
+ ? true
294
+ : {
295
+ $or: [
296
+ { $literal: createEmptyCheckpoints },
297
+ { $ne: ['$keepalive_op', new_keepalive_op] },
298
+ { $ne: ['$last_checkpoint', new_last_checkpoint] }
299
+ ]
300
+ }
301
+ }
302
+ },
303
+ {
304
+ $set: {
305
+ ...MongoSyncRulesLock.heartbeatUpdate(),
306
+ last_checkpoint_lsn: {
307
+ $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
308
+ },
309
+ last_checkpoint_ts: {
310
+ $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
311
+ },
312
+
313
+ last_fatal_error: { $literal: null },
314
+ last_fatal_error_ts: { $literal: null },
315
+ keepalive_op: new_keepalive_op,
316
+ last_checkpoint: new_last_checkpoint,
317
+ snapshot_lsn: {
318
+ $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
319
+ }
320
+ }
321
+ },
322
+ {
323
+ $unset: ['_can_checkpoint', '_not_empty']
324
+ }
325
+ ],
241
326
  {
242
- $toString: {
243
- $max: [{ $toLong: '$keepalive_op' }, { $literal: this.persisted_op }, 0n]
327
+ session: this.session,
328
+ returnDocument: 'before',
329
+ projection: {
330
+ snapshot_done: 1,
331
+ last_checkpoint_lsn: 1,
332
+ no_checkpoint_before: 1,
333
+ keepalive_op: 1,
334
+ last_checkpoint: 1
244
335
  }
245
336
  }
246
- ]
247
- };
337
+ )) as SyncRuleDocumentV1;
248
338
 
249
- const new_last_checkpoint = {
250
- $cond: [
251
- can_checkpoint,
252
- {
253
- $max: ['$last_checkpoint', { $literal: this.persisted_op }, { $toLong: '$keepalive_op' }, 0n]
254
- },
255
- '$last_checkpoint'
256
- ]
339
+ return MongoSyncRulesLock.assertOwned(preUpdateDocument);
257
340
  };
258
341
 
259
- const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate(
260
- { _id: this.replicationStreamId },
261
- [
342
+ const finishCheckpoint = async (preUpdateDocument: SyncRuleDocumentV1, lastOp?: InternalOpId) => {
343
+ await this.db.write_checkpoints.updateMany(
262
344
  {
263
- $set: {
264
- _can_checkpoint: can_checkpoint,
265
- _not_empty: createEmptyCheckpoints
266
- ? true
267
- : {
268
- $or: [
269
- { $literal: createEmptyCheckpoints },
270
- { $ne: ['$keepalive_op', new_keepalive_op] },
271
- { $ne: ['$last_checkpoint', new_last_checkpoint] }
272
- ]
273
- }
274
- }
345
+ processed_at_lsn: null,
346
+ 'lsns.1': { $lte: lsn }
275
347
  },
276
348
  {
277
349
  $set: {
278
- last_checkpoint_lsn: {
279
- $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
280
- },
281
- last_checkpoint_ts: {
282
- $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
283
- },
284
- last_keepalive_ts: { $literal: now },
285
- last_fatal_error: { $literal: null },
286
- last_fatal_error_ts: { $literal: null },
287
- keepalive_op: new_keepalive_op,
288
- last_checkpoint: new_last_checkpoint,
289
- snapshot_lsn: {
290
- $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
291
- }
350
+ processed_at_lsn: lsn
292
351
  }
293
352
  },
294
353
  {
295
- $unset: ['_can_checkpoint', '_not_empty']
354
+ session: this.session
296
355
  }
297
- ],
298
- {
299
- session: this.session,
300
- returnDocument: 'before',
301
- projection: {
302
- snapshot_done: 1,
303
- last_checkpoint_lsn: 1,
304
- no_checkpoint_before: 1,
305
- keepalive_op: 1,
306
- last_checkpoint: 1
307
- }
308
- }
309
- )) as SyncRuleDocumentV1;
310
-
311
- if (preUpdateDocument == null) {
312
- throw new ReplicationAssertionError(
313
- 'Failed to update checkpoint - no matching sync_rules document for _id: ' + this.replicationStreamId
314
356
  );
315
- }
316
357
 
317
- const checkpointState = calculateCheckpointState({
318
- lsn,
319
- snapshotDone: preUpdateDocument.snapshot_done === true,
320
- lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
321
- noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
322
- keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
323
- lastCheckpoint: preUpdateDocument.last_checkpoint,
324
- persistedOp: this.persisted_op,
325
- createEmptyCheckpoints
326
- });
358
+ const checkpointState = calculateCheckpointState({
359
+ lsn,
360
+ snapshotDone: preUpdateDocument.snapshot_done === true,
361
+ lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
362
+ noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
363
+ keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
364
+ lastCheckpoint: preUpdateDocument.last_checkpoint,
365
+ persistedOp: lastOp ?? this.persistedOpHead(preUpdateDocument),
366
+ createEmptyCheckpoints
367
+ });
368
+ return { checkpointState, preUpdateDocument };
369
+ };
370
+
371
+ const { checkpointState, preUpdateDocument } = await this.flushAndCommit(
372
+ async (_stream, lastOp) => finishCheckpoint(await updateCheckpoint(lastOp), lastOp),
373
+ // Without a flush, the checkpoint update itself fences the transaction.
374
+ () => this.withFencedTransaction(updateCheckpoint, finishCheckpoint),
375
+ options
376
+ );
327
377
  if (checkpointState.checkpointBlocked) {
328
378
  if (Date.now() - this.lastWaitingLogThrottled > 5_000) {
329
379
  this.logger.info(
@@ -361,41 +411,20 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
361
411
 
362
412
  async setResumeLsn(lsn: string): Promise<void> {
363
413
  using _ = this.tracer.span('storage', 'set_resume_lsn');
364
- await this.db.sync_rules.updateOne(
365
- {
366
- _id: this.replicationStreamId
367
- },
368
- {
369
- $set: {
370
- snapshot_lsn: lsn
371
- }
372
- },
373
- {
374
- session: this.session,
375
- // Losing occasional resume LSN is fine. That may mean reprocessing
376
- // some source changes in some edge cases, which is not an issue since
377
- // changes are processed in an idempotent way.
378
- writeConcern: { w: 1 }
379
- }
380
- );
414
+ // Losing occasional resume LSN would only reprocess source changes.
415
+ // Keep the lease check atomic, but do not wait for majority replication of this resume hint.
416
+ await this.updateStreamMetadata({ snapshot_lsn: { $literal: lsn } }, {}, { w: 1 });
381
417
  }
382
418
 
383
419
  async markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void> {
384
- await this.db.sync_rules.updateOne(
385
- {
386
- _id: this.replicationStreamId
387
- },
388
- {
389
- $set: {
390
- snapshot_done: true,
391
- last_keepalive_ts: new Date()
392
- },
393
- $max: {
394
- no_checkpoint_before: no_checkpoint_before_lsn
395
- }
396
- },
397
- { session: this.session }
398
- );
420
+ await this.updateStreamMetadata(this.snapshotDoneUpdate(no_checkpoint_before_lsn));
421
+ }
422
+
423
+ private snapshotDoneUpdate(no_checkpoint_before_lsn: string) {
424
+ return {
425
+ snapshot_done: true,
426
+ no_checkpoint_before: { $max: ['$no_checkpoint_before', { $literal: no_checkpoint_before_lsn }] }
427
+ };
399
428
  }
400
429
 
401
430
  async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void> {
@@ -425,22 +454,12 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
425
454
  }
426
455
  }
427
456
 
428
- await this.markAllSnapshotDone(no_checkpoint_before_lsn);
457
+ await this.updateStreamMetadataInTransaction(this.snapshotDoneUpdate(no_checkpoint_before_lsn));
429
458
  });
430
459
  }
431
460
 
432
461
  async markTableSnapshotRequired(_table: storage.SourceTable): Promise<void> {
433
- await this.db.sync_rules.updateOne(
434
- {
435
- _id: this.replicationStreamId
436
- },
437
- {
438
- $set: {
439
- snapshot_done: false
440
- }
441
- },
442
- { session: this.session }
443
- );
462
+ await this.updateStreamMetadata({ snapshot_done: false });
444
463
  }
445
464
 
446
465
  async markTableSnapshotDone(
@@ -451,7 +470,9 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
451
470
  const ids = tables.map((table) => mongoTableId(table.id));
452
471
 
453
472
  await this.withTransaction(async () => {
454
- await this.db.sourceTablesV1(this.replicationStreamId).updateMany(
473
+ const writes = this.db.createWriteBatch(session, { ordered: false });
474
+ writes.updateMany(
475
+ this.db.sourceTablesV1(this.replicationStreamId),
455
476
  { _id: { $in: ids } },
456
477
  {
457
478
  $set: {
@@ -460,26 +481,23 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
460
481
  $unset: {
461
482
  snapshot_status: 1
462
483
  }
463
- },
464
- { session }
484
+ }
465
485
  );
466
486
 
467
487
  if (no_checkpoint_before_lsn != null) {
468
- await this.db.sync_rules.updateOne(
488
+ writes.updateOne(
489
+ this.db.sync_rules,
469
490
  {
470
491
  _id: this.replicationStreamId
471
492
  },
472
493
  {
473
- $set: {
474
- last_keepalive_ts: new Date()
475
- },
476
494
  $max: {
477
495
  no_checkpoint_before: no_checkpoint_before_lsn
478
496
  }
479
- },
480
- { session: this.session }
497
+ }
481
498
  );
482
499
  }
500
+ await writes.execute();
483
501
  });
484
502
  return tables.map((table) => {
485
503
  const copy = table.clone();
@@ -496,7 +514,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
496
514
  const session = this.session;
497
515
  let activated = false;
498
516
  let needsFutureActivationCheck = true;
499
- await session.withTransaction(async () => {
517
+ await this.withTransaction(async () => {
500
518
  // Reset on transaction retries.
501
519
  activated = false;
502
520
  needsFutureActivationCheck = true;
@@ -534,7 +552,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
534
552
  } else if (doc?.state != storage.SyncRuleState.PROCESSING) {
535
553
  needsFutureActivationCheck = false;
536
554
  }
537
- });
555
+ }, session);
538
556
  if (activated) {
539
557
  this.logger.info(`Activated new replication stream at ${lsn}`);
540
558
  await this.db.notifyCheckpoint();