@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/CHANGELOG.md +129 -7
  2. package/dist/storage/MongoBucketStorage.d.ts +42 -5
  3. package/dist/storage/MongoBucketStorage.js +404 -215
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
  6. package/dist/storage/implementation/MongoBucketBatch.js +169 -98
  7. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  8. package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
  9. package/dist/storage/implementation/MongoCompactor.js +27 -238
  10. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  11. package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
  12. package/dist/storage/implementation/MongoIdSequence.js +14 -1
  13. package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
  14. package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
  15. package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
  16. package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
  17. package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
  18. package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
  19. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  20. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
  21. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
  22. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
  23. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
  24. package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
  25. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
  26. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
  27. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
  28. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
  29. package/dist/storage/implementation/MongoStorageProvider.js +8 -4
  30. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  31. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
  32. package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
  33. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
  35. package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
  36. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  37. package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
  38. package/dist/storage/implementation/MongoWriteBatch.js +149 -0
  39. package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  44. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  45. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
  47. package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
  48. package/dist/storage/implementation/common/PersistedBatch.js +57 -14
  49. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
  51. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  53. package/dist/storage/implementation/db.d.ts +9 -1
  54. package/dist/storage/implementation/db.js +28 -1
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +35 -0
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
  62. package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
  63. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  65. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  66. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
  71. package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
  72. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/models.js +1 -0
  74. package/dist/storage/implementation/v1/models.js.map +1 -1
  75. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  76. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  77. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  78. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
  79. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  81. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
  82. package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
  83. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  84. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
  85. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
  86. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
  87. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
  88. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
  90. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
  91. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  93. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  94. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  95. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  96. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
  98. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  99. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
  100. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  102. package/dist/storage/implementation/v3/bucket-format.js +2 -0
  103. package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
  104. package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
  105. package/dist/storage/implementation/v3/compact-utils.js +223 -0
  106. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  107. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  108. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  109. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +90 -6
  111. package/dist/storage/implementation/v3/models.js.map +1 -1
  112. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
  115. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
  126. package/dist/storage/implementation/v3/source-table-utils.js +49 -57
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
  128. package/dist/storage/storage-index.d.ts +2 -1
  129. package/dist/storage/storage-index.js +1 -0
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +28 -1
  132. package/dist/types/types.js +28 -7
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +4 -2
  135. package/dist/utils/test-utils.js +7 -4
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.js +3 -1
  138. package/dist/utils/util.js.map +1 -1
  139. package/package.json +10 -9
  140. package/src/storage/MongoBucketStorage.ts +545 -277
  141. package/src/storage/implementation/MongoBucketBatch.ts +258 -124
  142. package/src/storage/implementation/MongoCompactor.ts +37 -346
  143. package/src/storage/implementation/MongoIdSequence.ts +21 -1
  144. package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
  145. package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
  146. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
  147. package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
  148. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
  149. package/src/storage/implementation/MongoStorageProvider.ts +16 -4
  150. package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
  151. package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
  152. package/src/storage/implementation/MongoWriteBatch.ts +172 -0
  153. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
  154. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
  156. package/src/storage/implementation/common/PersistedBatch.ts +76 -20
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
  158. package/src/storage/implementation/db.ts +36 -2
  159. package/src/storage/implementation/models.ts +39 -0
  160. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
  161. package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
  162. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  163. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
  164. package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
  165. package/src/storage/implementation/v1/models.ts +1 -0
  166. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  167. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
  168. package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
  169. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
  170. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
  171. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
  172. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  173. package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
  174. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
  175. package/src/storage/implementation/v3/bucket-format.ts +2 -0
  176. package/src/storage/implementation/v3/compact-utils.ts +329 -0
  177. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  178. package/src/storage/implementation/v3/models.ts +97 -4
  179. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
  180. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
  181. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
  184. package/src/storage/implementation/v3/source-table-utils.ts +82 -76
  185. package/src/storage/storage-index.ts +2 -0
  186. package/src/types/types.ts +42 -7
  187. package/src/utils/test-utils.ts +9 -5
  188. package/src/utils/util.ts +3 -1
  189. package/test/src/__snapshots__/storage.test.ts.snap +25 -0
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
  191. package/test/src/chunk_compaction_config.test.ts +38 -0
  192. package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
  193. package/test/src/compact-utils.test.ts +251 -0
  194. package/test/src/concurrent_writers.test.ts +599 -0
  195. package/test/src/connection_config.test.ts +159 -0
  196. package/test/src/helpers/MemoryObjectStorage.ts +14 -6
  197. package/test/src/helpers/concurrentWriter.mjs +29 -0
  198. package/test/src/helpers/s3TestFactory.ts +12 -12
  199. package/test/src/object_storage_usage.test.ts +268 -0
  200. package/test/src/parameter_compacting_v1.test.ts +120 -0
  201. package/test/src/parameter_compacting_v3.test.ts +192 -0
  202. package/test/src/parameter_compaction_fence.test.ts +276 -0
  203. package/test/src/storage.test.ts +280 -24
  204. package/test/src/storage_compacting.test.ts +1097 -253
  205. package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
  206. package/test/src/storage_s3_checksums.test.ts +6 -7
  207. package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
  208. package/test/src/storage_s3_reading.test.ts +294 -6
  209. package/test/src/storage_s3_writing.test.ts +4 -3
  210. package/test/src/storage_sync.test.ts +412 -80
  211. package/test/src/storeCurrentData.test.ts +5 -5
  212. package/test/src/util.ts +1 -1
  213. package/tsconfig.tsbuildinfo +1 -1
@@ -1,7 +1,10 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
1
2
  import { InternalOpId, SourceTable, storage } from '@powersync/service-core';
2
3
  import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
4
+ import { MongoWriteBatch } from '../MongoWriteBatch.js';
3
5
  import { PersistedBatch } from '../common/PersistedBatch.js';
4
6
  import { SourceRecordStore } from '../common/SourceRecordStore.js';
7
+ import { SyncRuleDocumentBase } from '../models.js';
5
8
  import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
6
9
  export declare class MongoBucketBatchV1 extends MongoBucketBatch {
7
10
  readonly db: VersionedPowerSyncMongoV1;
@@ -16,16 +19,24 @@ export declare class MongoBucketBatchV1 extends MongoBucketBatch {
16
19
  */
17
20
  private persisted_op;
18
21
  constructor(options: MongoBucketBatchOptions);
22
+ protected persistedOpHead(stream: SyncRuleDocumentBase): InternalOpId;
23
+ protected onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: InternalOpId): void;
19
24
  protected recordPersistedOp(lastOp: InternalOpId): void;
20
25
  protected createPersistedBatch(writtenSize: number): PersistedBatch;
21
26
  protected get sourceRecordStore(): SourceRecordStore;
22
27
  protected cleanupDroppedSourceTables(_tables: SourceTable[]): Promise<void>;
28
+ protected batchCreateCustomWriteCheckpoints(session: mongo.ClientSession, opId: InternalOpId): Promise<void>;
23
29
  resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult>;
30
+ /**
31
+ * Build a SourceTable from its persisted v1 record.
32
+ */
33
+ private hydrateSourceTable;
24
34
  getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null>;
25
35
  commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult>;
26
36
  keepalive(lsn: string): Promise<storage.CheckpointResult>;
27
37
  setResumeLsn(lsn: string): Promise<void>;
28
38
  markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
39
+ private snapshotDoneUpdate;
29
40
  markSnapshotDone(no_checkpoint_before_lsn: string, options?: {
30
41
  throwOnConflict?: boolean;
31
42
  }): Promise<void>;
@@ -4,6 +4,7 @@ import * as bson from 'bson';
4
4
  import { mongoTableId } from '../../../utils/util.js';
5
5
  import { calculateCheckpointState } from '../CheckpointState.js';
6
6
  import { MongoBucketBatch } from '../MongoBucketBatch.js';
7
+ import { MongoSyncRulesLock } from '../MongoSyncRulesLock.js';
7
8
  import { PersistedBatchV1 } from './PersistedBatchV1.js';
8
9
  import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
9
10
  export class MongoBucketBatchV1 extends MongoBucketBatch {
@@ -22,6 +23,25 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
22
23
  this.persisted_op = options.keepaliveOp ?? null;
23
24
  this.store = new SourceRecordStoreV1(this.db, this.replicationStreamId);
24
25
  }
26
+ persistedOpHead(stream) {
27
+ const legacy = stream;
28
+ const checkpoint = legacy.last_checkpoint ?? 0n;
29
+ const keepalive = legacy.keepalive_op == null ? 0n : BigInt(legacy.keepalive_op);
30
+ return checkpoint > keepalive ? checkpoint : keepalive;
31
+ }
32
+ onReplicationTransactionFlush(writes, lastOp) {
33
+ // Keep flushed operations recoverable before a checkpoint can be published.
34
+ // keepalive_op remains a decimal string for compatibility with legacy readers.
35
+ writes.updateOne(this.db.sync_rules, { _id: this.replicationStreamId }, [
36
+ {
37
+ $set: {
38
+ keepalive_op: {
39
+ $toString: { $max: [{ $toLong: '$keepalive_op' }, '$last_checkpoint', lastOp] }
40
+ }
41
+ }
42
+ }
43
+ ]);
44
+ }
25
45
  recordPersistedOp(lastOp) {
26
46
  this.persisted_op = lastOp;
27
47
  }
@@ -36,8 +56,36 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
36
56
  async cleanupDroppedSourceTables(_tables) {
37
57
  // No-op for V1: source records live in a shared collection.
38
58
  }
59
+ async batchCreateCustomWriteCheckpoints(session, opId) {
60
+ // Only the final checkpoint per user/stream is visible after this transaction.
61
+ const checkpoints = new Map(this.write_checkpoint_batch.map((checkpoint) => [
62
+ JSON.stringify([checkpoint.sync_rules_id, checkpoint.user_id]),
63
+ checkpoint
64
+ ]));
65
+ await this.db.custom_write_checkpoints.bulkWrite([...checkpoints.values()].map((checkpointOptions) => {
66
+ const set = {
67
+ checkpoint: checkpointOptions.checkpoint,
68
+ sync_rules_id: checkpointOptions.sync_rules_id,
69
+ op_id: opId
70
+ };
71
+ if (checkpointOptions.checkpoint_requested_at != null) {
72
+ set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
73
+ }
74
+ return {
75
+ updateOne: {
76
+ filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
77
+ update: {
78
+ $set: set,
79
+ ...(checkpointOptions.checkpoint_requested_at == null ? { $unset: { checkpoint_requested_at: 1 } } : {})
80
+ },
81
+ upsert: true
82
+ }
83
+ };
84
+ }), { session, ordered: false });
85
+ }
39
86
  async resolveTables(options) {
40
87
  const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
88
+ const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
41
89
  const { connection_id, source } = options;
42
90
  const { schema, name, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
43
91
  const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
@@ -46,21 +94,34 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
46
94
  type_oid: column.typeId
47
95
  }));
48
96
  let result = null;
49
- await this.db.client.withSession(async (session) => {
97
+ const session = this.session;
98
+ await this.withTransaction(async () => {
50
99
  const col = this.db.sourceTablesV1(this.replicationStreamId);
51
- const filter = {
52
- group_id: this.replicationStreamId,
53
- connection_id,
54
- schema_name: schema,
55
- table_name: name,
56
- replica_id_columns2: normalizedReplicaIdColumns
57
- };
100
+ // Find records that overlap by name or relation id.
101
+ const orClauses = [{ schema_name: schema, table_name: name }];
58
102
  if (objectId != null) {
59
- filter.relation_id = objectId;
103
+ orClauses.push({ relation_id: objectId });
60
104
  }
61
- let doc = await col.findOne(filter, { session });
62
- if (doc == null) {
63
- doc = {
105
+ const candidateDocs = await col
106
+ .find({ group_id: this.replicationStreamId, connection_id, $or: orClauses }, { session })
107
+ .toArray();
108
+ const candidateTables = candidateDocs.map((doc) => this.hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows));
109
+ const candidates = candidateTables.map((table) => table.clone());
110
+ const resolution = await reconcile({ source, candidates });
111
+ storage.validateSourceTableCandidateResolution(candidates, resolution);
112
+ for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
113
+ await col.updateOne({ _id: mongoTableId(id) }, { $set: { source_metadata: sourceMetadata } }, { session });
114
+ }
115
+ const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
116
+ const compatibleTables = materializedResolution.compatibleTables;
117
+ // V1 keeps one record, preferring one that has already been snapshotted.
118
+ const reuse = compatibleTables.find((table) => table.snapshotComplete) ?? compatibleTables[0] ?? null;
119
+ let sourceTable;
120
+ if (reuse != null) {
121
+ sourceTable = reuse;
122
+ }
123
+ else {
124
+ const doc = {
64
125
  _id: options.idGenerator ? options.idGenerator() : new bson.ObjectId(),
65
126
  group_id: this.replicationStreamId,
66
127
  connection_id,
@@ -70,90 +131,40 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
70
131
  replica_id_columns: null,
71
132
  replica_id_columns2: normalizedReplicaIdColumns,
72
133
  snapshot_done: false,
73
- snapshot_status: undefined
134
+ snapshot_status: undefined,
135
+ source_metadata: resolution.newTableValues.sourceMetadata
74
136
  };
75
137
  await col.insertOne(doc, { session });
138
+ sourceTable = this.hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows);
76
139
  }
77
- const sourceTable = new storage.SourceTable({
78
- id: doc._id,
79
- ref: source,
80
- objectId,
81
- replicaIdColumns,
82
- snapshotComplete: doc.snapshot_done ?? true,
83
- ...syncRules.getMatchingSources(source)
84
- });
85
- sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
86
- sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
87
- sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
88
- sourceTable.storeCurrentData = sendsCompleteRows !== true;
89
- sourceTable.snapshotStatus =
90
- doc.snapshot_status == null
91
- ? undefined
92
- : {
93
- lastKey: doc.snapshot_status.last_key?.buffer ?? null,
94
- totalEstimatedCount: doc.snapshot_status.total_estimated_count,
95
- replicatedCount: doc.snapshot_status.replicated_count
96
- };
97
- const truncateFilter = [{ schema_name: schema, table_name: name }];
98
- if (objectId != null) {
99
- truncateFilter.push({ relation_id: objectId });
100
- }
101
- const truncate = await col
102
- .find({
103
- group_id: this.replicationStreamId,
104
- connection_id,
105
- _id: { $ne: doc._id },
106
- $or: truncateFilter
107
- }, { session })
108
- .toArray();
109
- const dropTables = truncate.map((dropDoc) => {
110
- const ref = {
111
- connectionTag,
112
- schema: dropDoc.schema_name,
113
- name: dropDoc.table_name
114
- };
115
- const table = new storage.SourceTable({
116
- id: dropDoc._id,
117
- ref,
118
- objectId: dropDoc.relation_id,
119
- replicaIdColumns: dropDoc.replica_id_columns2?.map((c) => ({ name: c.name, typeId: c.type_oid, type: c.type })) ?? [],
120
- snapshotComplete: dropDoc.snapshot_done ?? true,
121
- ...syncRules.getMatchingSources(ref)
122
- });
123
- table.syncEvent = syncRules.tableTriggersEvent(ref);
124
- table.syncData = table.bucketDataSources.length > 0;
125
- table.syncParameters = table.parameterLookupSources.length > 0;
126
- return table;
127
- });
140
+ const dropTables = [
141
+ ...materializedResolution.incompatibleTables,
142
+ // V1 only keeps one SourceTable per physical table.
143
+ ...compatibleTables.filter((table) => !storage.sourceTableIdEquals(table.id, sourceTable.id))
144
+ ];
128
145
  result = { tables: [sourceTable], dropTables };
129
146
  });
130
147
  return result;
131
148
  }
132
- async getSourceTableStatus(table) {
133
- const doc = await this.db.sourceTablesV1(this.replicationStreamId).findOne({
134
- group_id: this.replicationStreamId,
135
- _id: mongoTableId(table.id)
136
- }, { session: this.session });
137
- if (doc == null) {
138
- return null;
139
- }
140
- const ref = {
141
- connectionTag: table.ref.connectionTag,
142
- schema: doc.schema_name,
143
- name: doc.table_name
144
- };
145
- const sourceTable = new storage.SourceTable({
149
+ /**
150
+ * Build a SourceTable from its persisted v1 record.
151
+ */
152
+ hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows) {
153
+ const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
154
+ const table = new storage.SourceTable({
146
155
  id: doc._id,
147
156
  ref,
148
157
  objectId: doc.relation_id,
149
- replicaIdColumns: doc.replica_id_columns2?.map((c) => ({ name: c.name, typeId: c.type_oid, type: c.type })) ?? [],
158
+ replicaIdColumns: doc.replica_id_columns2?.map((column) => ({ name: column.name, typeId: column.type_oid, type: column.type })) ?? [],
150
159
  snapshotComplete: doc.snapshot_done ?? true,
151
- ...this.sync_rules.getMatchingSources(ref)
160
+ sourceMetadata: doc.source_metadata ?? null,
161
+ ...syncRules.getMatchingSources(ref)
152
162
  });
153
- sourceTable.syncEvent = this.sync_rules.tableTriggersEvent(ref);
154
- sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
155
- sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
156
- sourceTable.snapshotStatus =
163
+ table.syncEvent = syncRules.tableTriggersEvent(ref);
164
+ table.syncData = table.bucketDataSources.length > 0;
165
+ table.syncParameters = table.parameterLookupSources.length > 0;
166
+ table.storeCurrentData = sendsCompleteRows !== true;
167
+ table.snapshotStatus =
157
168
  doc.snapshot_status == null
158
169
  ? undefined
159
170
  : {
@@ -161,114 +172,130 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
161
172
  totalEstimatedCount: doc.snapshot_status.total_estimated_count,
162
173
  replicatedCount: doc.snapshot_status.replicated_count
163
174
  };
164
- return sourceTable;
175
+ return table;
176
+ }
177
+ async getSourceTableStatus(table) {
178
+ const doc = await this.db.sourceTablesV1(this.replicationStreamId).findOne({
179
+ group_id: this.replicationStreamId,
180
+ _id: mongoTableId(table.id)
181
+ }, { session: this.session });
182
+ if (doc == null) {
183
+ return null;
184
+ }
185
+ // Use the default storeCurrentData value when reading outside resolution.
186
+ return this.hydrateSourceTable(doc, table.ref.connectionTag, this.sync_rules, undefined);
165
187
  }
166
188
  async commit(lsn, options) {
167
189
  const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
168
- await this.flush(options);
169
190
  using _ = this.tracer.span('storage', 'commit');
170
- const now = new Date();
171
- await this.db.write_checkpoints.updateMany({
172
- processed_at_lsn: null,
173
- 'lsns.1': { $lte: lsn }
174
- }, {
175
- $set: {
176
- processed_at_lsn: lsn
177
- }
178
- }, {
179
- session: this.session
180
- });
181
- const can_checkpoint = {
182
- $and: [
183
- { $eq: ['$snapshot_done', true] },
191
+ const updateCheckpoint = async (persistedOp = this.persisted_op) => {
192
+ const now = new Date();
193
+ const can_checkpoint = {
194
+ $and: [
195
+ { $eq: ['$snapshot_done', true] },
196
+ {
197
+ $or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
198
+ },
199
+ {
200
+ $or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
201
+ }
202
+ ]
203
+ };
204
+ const new_keepalive_op = {
205
+ $cond: [
206
+ can_checkpoint,
207
+ { $literal: null },
208
+ {
209
+ $toString: {
210
+ $max: [{ $toLong: '$keepalive_op' }, { $literal: persistedOp }, 0n]
211
+ }
212
+ }
213
+ ]
214
+ };
215
+ const new_last_checkpoint = {
216
+ $cond: [
217
+ can_checkpoint,
218
+ {
219
+ $max: ['$last_checkpoint', { $literal: persistedOp }, { $toLong: '$keepalive_op' }, 0n]
220
+ },
221
+ '$last_checkpoint'
222
+ ]
223
+ };
224
+ const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate(MongoSyncRulesLock.ownerFilter(this.replicationStreamId, this.options.replicationLock), [
184
225
  {
185
- $or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
226
+ $set: {
227
+ _can_checkpoint: can_checkpoint,
228
+ _not_empty: createEmptyCheckpoints
229
+ ? true
230
+ : {
231
+ $or: [
232
+ { $literal: createEmptyCheckpoints },
233
+ { $ne: ['$keepalive_op', new_keepalive_op] },
234
+ { $ne: ['$last_checkpoint', new_last_checkpoint] }
235
+ ]
236
+ }
237
+ }
186
238
  },
187
239
  {
188
- $or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
189
- }
190
- ]
191
- };
192
- const new_keepalive_op = {
193
- $cond: [
194
- can_checkpoint,
195
- { $literal: null },
196
- {
197
- $toString: {
198
- $max: [{ $toLong: '$keepalive_op' }, { $literal: this.persisted_op }, 0n]
240
+ $set: {
241
+ ...MongoSyncRulesLock.heartbeatUpdate(),
242
+ last_checkpoint_lsn: {
243
+ $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
244
+ },
245
+ last_checkpoint_ts: {
246
+ $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
247
+ },
248
+ last_fatal_error: { $literal: null },
249
+ last_fatal_error_ts: { $literal: null },
250
+ keepalive_op: new_keepalive_op,
251
+ last_checkpoint: new_last_checkpoint,
252
+ snapshot_lsn: {
253
+ $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
254
+ }
199
255
  }
200
- }
201
- ]
202
- };
203
- const new_last_checkpoint = {
204
- $cond: [
205
- can_checkpoint,
206
- {
207
- $max: ['$last_checkpoint', { $literal: this.persisted_op }, { $toLong: '$keepalive_op' }, 0n]
208
256
  },
209
- '$last_checkpoint'
210
- ]
211
- };
212
- const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate({ _id: this.replicationStreamId }, [
213
- {
214
- $set: {
215
- _can_checkpoint: can_checkpoint,
216
- _not_empty: createEmptyCheckpoints
217
- ? true
218
- : {
219
- $or: [
220
- { $literal: createEmptyCheckpoints },
221
- { $ne: ['$keepalive_op', new_keepalive_op] },
222
- { $ne: ['$last_checkpoint', new_last_checkpoint] }
223
- ]
224
- }
257
+ {
258
+ $unset: ['_can_checkpoint', '_not_empty']
225
259
  }
226
- },
227
- {
260
+ ], {
261
+ session: this.session,
262
+ returnDocument: 'before',
263
+ projection: {
264
+ snapshot_done: 1,
265
+ last_checkpoint_lsn: 1,
266
+ no_checkpoint_before: 1,
267
+ keepalive_op: 1,
268
+ last_checkpoint: 1
269
+ }
270
+ }));
271
+ return MongoSyncRulesLock.assertOwned(preUpdateDocument);
272
+ };
273
+ const finishCheckpoint = async (preUpdateDocument, lastOp) => {
274
+ await this.db.write_checkpoints.updateMany({
275
+ processed_at_lsn: null,
276
+ 'lsns.1': { $lte: lsn }
277
+ }, {
228
278
  $set: {
229
- last_checkpoint_lsn: {
230
- $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
231
- },
232
- last_checkpoint_ts: {
233
- $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
234
- },
235
- last_keepalive_ts: { $literal: now },
236
- last_fatal_error: { $literal: null },
237
- last_fatal_error_ts: { $literal: null },
238
- keepalive_op: new_keepalive_op,
239
- last_checkpoint: new_last_checkpoint,
240
- snapshot_lsn: {
241
- $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
242
- }
279
+ processed_at_lsn: lsn
243
280
  }
244
- },
245
- {
246
- $unset: ['_can_checkpoint', '_not_empty']
247
- }
248
- ], {
249
- session: this.session,
250
- returnDocument: 'before',
251
- projection: {
252
- snapshot_done: 1,
253
- last_checkpoint_lsn: 1,
254
- no_checkpoint_before: 1,
255
- keepalive_op: 1,
256
- last_checkpoint: 1
257
- }
258
- }));
259
- if (preUpdateDocument == null) {
260
- throw new ReplicationAssertionError('Failed to update checkpoint - no matching sync_rules document for _id: ' + this.replicationStreamId);
261
- }
262
- const checkpointState = calculateCheckpointState({
263
- lsn,
264
- snapshotDone: preUpdateDocument.snapshot_done === true,
265
- lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
266
- noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
267
- keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
268
- lastCheckpoint: preUpdateDocument.last_checkpoint,
269
- persistedOp: this.persisted_op,
270
- createEmptyCheckpoints
271
- });
281
+ }, {
282
+ session: this.session
283
+ });
284
+ const checkpointState = calculateCheckpointState({
285
+ lsn,
286
+ snapshotDone: preUpdateDocument.snapshot_done === true,
287
+ lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
288
+ noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
289
+ keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
290
+ lastCheckpoint: preUpdateDocument.last_checkpoint,
291
+ persistedOp: lastOp ?? this.persistedOpHead(preUpdateDocument),
292
+ createEmptyCheckpoints
293
+ });
294
+ return { checkpointState, preUpdateDocument };
295
+ };
296
+ const { checkpointState, preUpdateDocument } = await this.flushAndCommit(async (_stream, lastOp) => finishCheckpoint(await updateCheckpoint(lastOp), lastOp),
297
+ // Without a flush, the checkpoint update itself fences the transaction.
298
+ () => this.withFencedTransaction(updateCheckpoint, finishCheckpoint), options);
272
299
  if (checkpointState.checkpointBlocked) {
273
300
  if (Date.now() - this.lastWaitingLogThrottled > 5_000) {
274
301
  this.logger.info(`Waiting before creating checkpoint, currently at ${lsn} / ${checkpointState.newKeepaliveOp}. Current state: ${JSON.stringify({
@@ -300,32 +327,18 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
300
327
  }
301
328
  async setResumeLsn(lsn) {
302
329
  using _ = this.tracer.span('storage', 'set_resume_lsn');
303
- await this.db.sync_rules.updateOne({
304
- _id: this.replicationStreamId
305
- }, {
306
- $set: {
307
- snapshot_lsn: lsn
308
- }
309
- }, {
310
- session: this.session,
311
- // Losing occasional resume LSN is fine. That may mean reprocessing
312
- // some source changes in some edge cases, which is not an issue since
313
- // changes are processed in an idempotent way.
314
- writeConcern: { w: 1 }
315
- });
330
+ // Losing occasional resume LSN would only reprocess source changes.
331
+ // Keep the lease check atomic, but do not wait for majority replication of this resume hint.
332
+ await this.updateStreamMetadata({ snapshot_lsn: { $literal: lsn } }, {}, { w: 1 });
316
333
  }
317
334
  async markAllSnapshotDone(no_checkpoint_before_lsn) {
318
- await this.db.sync_rules.updateOne({
319
- _id: this.replicationStreamId
320
- }, {
321
- $set: {
322
- snapshot_done: true,
323
- last_keepalive_ts: new Date()
324
- },
325
- $max: {
326
- no_checkpoint_before: no_checkpoint_before_lsn
327
- }
328
- }, { session: this.session });
335
+ await this.updateStreamMetadata(this.snapshotDoneUpdate(no_checkpoint_before_lsn));
336
+ }
337
+ snapshotDoneUpdate(no_checkpoint_before_lsn) {
338
+ return {
339
+ snapshot_done: true,
340
+ no_checkpoint_before: { $max: ['$no_checkpoint_before', { $literal: no_checkpoint_before_lsn }] }
341
+ };
329
342
  }
330
343
  async markSnapshotDone(no_checkpoint_before_lsn, options) {
331
344
  await this.withTransaction(async () => {
@@ -348,42 +361,35 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
348
361
  return;
349
362
  }
350
363
  }
351
- await this.markAllSnapshotDone(no_checkpoint_before_lsn);
364
+ await this.updateStreamMetadataInTransaction(this.snapshotDoneUpdate(no_checkpoint_before_lsn));
352
365
  });
353
366
  }
354
367
  async markTableSnapshotRequired(_table) {
355
- await this.db.sync_rules.updateOne({
356
- _id: this.replicationStreamId
357
- }, {
358
- $set: {
359
- snapshot_done: false
360
- }
361
- }, { session: this.session });
368
+ await this.updateStreamMetadata({ snapshot_done: false });
362
369
  }
363
370
  async markTableSnapshotDone(tables, no_checkpoint_before_lsn) {
364
371
  const session = this.session;
365
372
  const ids = tables.map((table) => mongoTableId(table.id));
366
373
  await this.withTransaction(async () => {
367
- await this.db.sourceTablesV1(this.replicationStreamId).updateMany({ _id: { $in: ids } }, {
374
+ const writes = this.db.createWriteBatch(session, { ordered: false });
375
+ writes.updateMany(this.db.sourceTablesV1(this.replicationStreamId), { _id: { $in: ids } }, {
368
376
  $set: {
369
377
  snapshot_done: true
370
378
  },
371
379
  $unset: {
372
380
  snapshot_status: 1
373
381
  }
374
- }, { session });
382
+ });
375
383
  if (no_checkpoint_before_lsn != null) {
376
- await this.db.sync_rules.updateOne({
384
+ writes.updateOne(this.db.sync_rules, {
377
385
  _id: this.replicationStreamId
378
386
  }, {
379
- $set: {
380
- last_keepalive_ts: new Date()
381
- },
382
387
  $max: {
383
388
  no_checkpoint_before: no_checkpoint_before_lsn
384
389
  }
385
- }, { session: this.session });
390
+ });
386
391
  }
392
+ await writes.execute();
387
393
  });
388
394
  return tables.map((table) => {
389
395
  const copy = table.clone();
@@ -398,7 +404,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
398
404
  const session = this.session;
399
405
  let activated = false;
400
406
  let needsFutureActivationCheck = true;
401
- await session.withTransaction(async () => {
407
+ await this.withTransaction(async () => {
402
408
  // Reset on transaction retries.
403
409
  activated = false;
404
410
  needsFutureActivationCheck = true;
@@ -424,7 +430,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
424
430
  else if (doc?.state != storage.SyncRuleState.PROCESSING) {
425
431
  needsFutureActivationCheck = false;
426
432
  }
427
- });
433
+ }, session);
428
434
  if (activated) {
429
435
  this.logger.info(`Activated new replication stream at ${lsn}`);
430
436
  await this.db.notifyCheckpoint();