@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/CHANGELOG.md +101 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +41 -16
  6. package/dist/storage/MongoBucketStorage.js +226 -71
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
  9. package/dist/storage/implementation/MongoBucketBatch.js +110 -62
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
  15. package/dist/storage/implementation/MongoCompactor.js +80 -263
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
  21. package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +39 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  46. package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
  47. package/dist/storage/implementation/common/PersistedBatch.js +12 -0
  48. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  49. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  51. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  52. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  53. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  54. package/dist/storage/implementation/db.js +17 -15
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +20 -13
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  62. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  63. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
  65. package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
  66. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  71. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  72. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +10 -1
  77. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  78. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  79. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  81. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
  82. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  84. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  85. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  86. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
  87. package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
  88. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
  90. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
  91. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
  93. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
  94. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
  96. package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +77 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +30 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +101 -21
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
  115. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  126. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  128. package/dist/storage/storage-index.d.ts +4 -4
  129. package/dist/storage/storage-index.js +4 -4
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +49 -3
  132. package/dist/types/types.js +48 -1
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +1 -2
  135. package/dist/utils/test-utils.js +7 -1
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.d.ts +9 -2
  138. package/dist/utils/util.js +128 -4
  139. package/dist/utils/util.js.map +1 -1
  140. package/package.json +12 -19
  141. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  142. package/src/storage/MongoBucketStorage.ts +347 -88
  143. package/src/storage/implementation/MongoBucketBatch.ts +147 -82
  144. package/src/storage/implementation/MongoChecksums.ts +93 -153
  145. package/src/storage/implementation/MongoCompactor.ts +102 -316
  146. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  147. package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
  148. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  149. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  150. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  151. package/src/storage/implementation/MongoStorageProvider.ts +46 -4
  152. package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
  153. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  154. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
  156. package/src/storage/implementation/common/PersistedBatch.ts +19 -3
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  158. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  159. package/src/storage/implementation/db.ts +19 -20
  160. package/src/storage/implementation/models.ts +20 -14
  161. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
  162. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  163. package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
  164. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
  165. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  166. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  167. package/src/storage/implementation/v1/models.ts +11 -1
  168. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  169. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
  170. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  171. package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
  172. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
  173. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
  174. package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
  175. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  176. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
  177. package/src/storage/implementation/v3/bucket-format.ts +97 -0
  178. package/src/storage/implementation/v3/chunking.ts +40 -0
  179. package/src/storage/implementation/v3/models.ts +132 -55
  180. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
  181. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
  184. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  185. package/src/storage/storage-index.ts +12 -4
  186. package/src/types/types.ts +58 -1
  187. package/src/utils/test-utils.ts +8 -3
  188. package/src/utils/util.ts +185 -6
  189. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
  191. package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
  192. package/test/src/helpers/MemoryObjectStorage.ts +53 -0
  193. package/test/src/helpers/s3TestFactory.ts +69 -0
  194. package/test/src/object_storage_lifecycle.test.ts +59 -0
  195. package/test/src/setup.ts +6 -1
  196. package/test/src/storage.test.ts +227 -13
  197. package/test/src/storage_compacting.test.ts +1951 -12
  198. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  199. package/test/src/storage_s3_checksums.test.ts +191 -0
  200. package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
  201. package/test/src/storage_s3_reading.test.ts +526 -0
  202. package/test/src/storage_s3_writing.test.ts +135 -0
  203. package/test/src/storage_sync.test.ts +1180 -73
  204. package/test/src/storeCurrentData.test.ts +227 -0
  205. package/test/src/util.ts +2 -1
  206. package/tsconfig.json +1 -4
  207. package/tsconfig.tsbuildinfo +1 -1
  208. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  209. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  210. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  211. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  212. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  213. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  214. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  215. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  216. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  217. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  218. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  219. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  220. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  221. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  222. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  223. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  224. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  225. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  226. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  227. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  228. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  229. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  230. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  231. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  232. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  233. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  234. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  235. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  236. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  237. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  238. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  239. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -12,6 +12,7 @@ import {
12
12
  ServiceError
13
13
  } from '@powersync/lib-services-framework';
14
14
  import {
15
+ BucketDefinitionMapping,
15
16
  BucketStorageMarkRecordUnavailable,
16
17
  deserializeBson,
17
18
  InternalOpId,
@@ -23,14 +24,15 @@ import {
23
24
  } from '@powersync/service-core';
24
25
  import * as timers from 'node:timers/promises';
25
26
  import { mongoTableId } from '../../utils/util.js';
26
- import { BucketDefinitionMapping } from './BucketDefinitionMapping.js';
27
27
  import { PersistedBatch } from './common/PersistedBatch.js';
28
28
  import { LoadedSourceRecord, SourceRecordStore } from './common/SourceRecordStore.js';
29
29
  import type { VersionedPowerSyncMongo } from './db.js';
30
30
  import { MAX_ROW_SIZE } from './MongoBucketBatchShared.js';
31
31
  import { MongoIdSequence } from './MongoIdSequence.js';
32
+ import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
32
33
  import { batchCreateCustomWriteCheckpoints } from './MongoWriteCheckpointAPI.js';
33
34
  import { OperationBatch, RecordOperation } from './OperationBatch.js';
35
+ import { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
34
36
 
35
37
  // Currently, we can only have a single flush() at a time, since it locks the op_id sequence.
36
38
  // While the MongoDB transaction retry mechanism handles this okay, using an in-process Mutex
@@ -41,15 +43,25 @@ const replicationMutex = new utils.Mutex();
41
43
 
42
44
  export interface MongoBucketBatchOptions {
43
45
  db: VersionedPowerSyncMongo;
44
- syncRules: HydratedSyncConfig;
45
- groupId: number;
46
- slotName: string;
47
- syncConfigId?: bson.ObjectId | null;
48
- lastCheckpointLsn: string | null;
49
- keepaliveOp: InternalOpId | null;
46
+ /**
47
+ * The parsed sync config set for this batch.
48
+ *
49
+ * The batch derives both the hydrated sync rules and the bucket definition mapping from
50
+ * this single set, so they always come from the same parse. Do not add separate
51
+ * syncRules/mapping options - pairing values from different parses is exactly the bug
52
+ * this shape prevents.
53
+ */
54
+ parsedSyncConfig: MongoParsedSyncConfigSet;
55
+ replicationStreamId: number;
56
+ replicationStreamName: string;
57
+ syncConfigIds?: bson.ObjectId[];
58
+ /**
59
+ * Seeds the in-memory persisted-op tracking for v1 storage. Not used by v3 storage, which
60
+ * tracks the persisted-op head durably on the replication stream document instead.
61
+ */
62
+ keepaliveOp?: InternalOpId | null;
50
63
  resumeFromLsn: string | null;
51
64
  storeCurrentData: boolean;
52
- mapping: BucketDefinitionMapping;
53
65
  /**
54
66
  * Set to true for initial replication.
55
67
  */
@@ -60,6 +72,9 @@ export interface MongoBucketBatchOptions {
60
72
 
61
73
  logger: Logger;
62
74
  tracer?: PerformanceTracer<'storage' | 'evaluate'>;
75
+
76
+ objectStorage?: ObjectStorage;
77
+ inlineThresholdBytes?: number;
63
78
  }
64
79
 
65
80
  export abstract class MongoBucketBatch
@@ -70,13 +85,22 @@ export abstract class MongoBucketBatch
70
85
  protected logger: Logger;
71
86
 
72
87
  private readonly client: mongo.MongoClient;
73
- public readonly db: VersionedPowerSyncMongo;
88
+ readonly db: VersionedPowerSyncMongo;
74
89
  public readonly session: mongo.ClientSession;
75
90
  protected readonly sync_rules: HydratedSyncConfig;
76
91
 
77
- protected readonly group_id: number;
92
+ protected readonly replicationStreamId: number;
78
93
 
79
- private readonly slot_name: string;
94
+ private readonly replicationStreamName: string;
95
+ /**
96
+ * Source-level setting for whether raw row data should be stored in current_data.
97
+ *
98
+ * Some sources always send complete rows (MongoDB, MySQL with binlog_row_image=full),
99
+ * in which case this is false for the whole batch. For sources where it depends on the
100
+ * table (Postgres REPLICA IDENTITY), this is true and the decision is refined per-table
101
+ * via SourceTable.storeCurrentData. The effective per-record value is the conjunction of
102
+ * the two.
103
+ */
80
104
  private readonly storeCurrentData: boolean;
81
105
  public readonly skipExistingRows: boolean;
82
106
  protected readonly mapping: BucketDefinitionMapping;
@@ -87,18 +111,7 @@ export abstract class MongoBucketBatch
87
111
  private hooks: storage.StorageHooks | undefined;
88
112
  private clearedError = false;
89
113
 
90
- private tracer: PerformanceTracer<'storage' | 'evaluate'>;
91
-
92
- /**
93
- * Last LSN received associated with a checkpoint.
94
- *
95
- * This could be either:
96
- * 1. A commit LSN.
97
- * 2. A keepalive message LSN.
98
- */
99
- protected last_checkpoint_lsn: string | null = null;
100
-
101
- protected persisted_op: InternalOpId | null = null;
114
+ protected tracer: PerformanceTracer<'storage' | 'evaluate'>;
102
115
 
103
116
  /**
104
117
  * Last written op, if any. This may not reflect a consistent checkpoint.
@@ -106,9 +119,9 @@ export abstract class MongoBucketBatch
106
119
  public last_flushed_op: InternalOpId | null = null;
107
120
 
108
121
  /**
109
- * lastCheckpointLsn is the last consistent commit.
122
+ * LSN to resume replication from.
110
123
  *
111
- * While that is generally a "safe" point to resume from, there are cases where we may want to resume from a different point:
124
+ * This is typically the last commit LSN, but there are cases where it differs:
112
125
  * 1. After an initial snapshot, we don't have a consistent commit yet, but need to resume from the snapshot LSN.
113
126
  * 2. If "no_checkpoint_before_lsn" is set far in advance, it may take a while to reach that point. We
114
127
  * may want to resume at incremental points before that.
@@ -123,34 +136,28 @@ export abstract class MongoBucketBatch
123
136
  this.options = options;
124
137
  this.client = options.db.client;
125
138
  this.db = options.db;
126
- this.group_id = options.groupId;
127
- this.last_checkpoint_lsn = options.lastCheckpointLsn;
139
+ this.replicationStreamId = options.replicationStreamId;
128
140
  this.resumeFromLsn = options.resumeFromLsn;
129
141
  this.session = this.client.startSession();
130
- this.slot_name = options.slotName;
131
- this.sync_rules = options.syncRules;
142
+ this.replicationStreamName = options.replicationStreamName;
143
+ this.sync_rules = options.parsedSyncConfig.hydratedSyncConfig;
132
144
  this.storeCurrentData = options.storeCurrentData;
133
- this.mapping = options.mapping;
145
+ this.mapping = options.parsedSyncConfig.mapping;
134
146
  this.skipExistingRows = options.skipExistingRows;
135
147
  this.markRecordUnavailable = options.markRecordUnavailable;
136
148
  this.hooks = options.hooks;
137
149
  this.batch = new OperationBatch();
138
150
 
139
- this.persisted_op = options.keepaliveOp ?? null;
140
151
  this.tracer = options.tracer ?? new PerformanceTracer('MongoDB storage');
141
152
  }
142
153
 
143
154
  addCustomWriteCheckpoint(checkpoint: storage.BatchedCustomWriteCheckpointOptions): void {
144
155
  this.write_checkpoint_batch.push({
145
156
  ...checkpoint,
146
- sync_rules_id: this.group_id
157
+ sync_rules_id: this.replicationStreamId
147
158
  });
148
159
  }
149
160
 
150
- get lastCheckpointLsn() {
151
- return this.last_checkpoint_lsn;
152
- }
153
-
154
161
  abstract resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult>;
155
162
 
156
163
  protected abstract createPersistedBatch(writtenSize: number): PersistedBatch;
@@ -195,14 +202,18 @@ export abstract class MongoBucketBatch
195
202
  const batch = this.batch;
196
203
  let last_op: InternalOpId | null = null;
197
204
  let resumeBatch: OperationBatch | null = null;
205
+ let clearedError = false;
198
206
 
199
207
  using _ = this.tracer.span('storage', 'flush');
200
208
 
201
209
  await this.hooks?.beforeBatchFlush?.(this);
202
210
 
203
211
  await this.withReplicationTransaction(`Flushing ${batch?.length ?? 0} ops`, async (session, opSeq) => {
212
+ clearedError = false;
204
213
  if (batch != null) {
205
- resumeBatch = await this.replicateBatch(session, batch, opSeq, options);
214
+ const result = await this.replicateBatch(session, batch, opSeq, options);
215
+ resumeBatch = result.resumeBatch;
216
+ clearedError ||= result.clearedError;
206
217
  }
207
218
 
208
219
  if (this.write_checkpoint_batch.length > 0) {
@@ -214,6 +225,10 @@ export abstract class MongoBucketBatch
214
225
  last_op = opSeq.last();
215
226
  });
216
227
 
228
+ if (clearedError) {
229
+ this.clearedError = true;
230
+ }
231
+
217
232
  // null if done, set if we need another flush
218
233
  this.batch = resumeBatch;
219
234
 
@@ -221,7 +236,7 @@ export abstract class MongoBucketBatch
221
236
  throw new ReplicationAssertionError('Unexpected last_op == null');
222
237
  }
223
238
 
224
- this.persisted_op = last_op;
239
+ this.recordPersistedOp(last_op);
225
240
  this.last_flushed_op = last_op;
226
241
  await this.hooks?.afterBatchFlush?.(this);
227
242
  return { flushed_op: last_op };
@@ -232,11 +247,15 @@ export abstract class MongoBucketBatch
232
247
  batch: OperationBatch,
233
248
  op_seq: MongoIdSequence,
234
249
  options?: storage.BucketBatchCommitOptions
235
- ): Promise<OperationBatch | null> {
250
+ ): Promise<{ resumeBatch: OperationBatch | null; clearedError: boolean }> {
236
251
  let sizes: Map<string, number> | undefined = undefined;
237
252
  using _ = this.tracer.span('storage', 'replicate_batch');
238
- if (this.storeCurrentData && !this.skipExistingRows) {
239
- // We skip this step if we don't store current_data, since the sizes will
253
+ // Only look up current_data sizes if the batch stores current_data and at least one
254
+ // table in it does too (per-table can disable it, e.g. Postgres REPLICA IDENTITY FULL).
255
+ const anyTableStoresCurrentData =
256
+ this.storeCurrentData && batch.batch.some((r) => r.record.sourceTable.storeCurrentData);
257
+ if (anyTableStoresCurrentData && !this.skipExistingRows) {
258
+ // We skip this step if no tables store current_data, since the sizes will
240
259
  // always be small in that case.
241
260
 
242
261
  // With skipExistingRows, we don't load the full documents into memory,
@@ -249,10 +268,14 @@ export abstract class MongoBucketBatch
249
268
  // (automatically limited to 48MB(?) per batch by MongoDB). The issue is that it changes
250
269
  // the order of processing, which then becomes really tricky to manage.
251
270
  // This now takes 2+ queries, but doesn't have any issues with order of operations.
252
- const sizeLookups = batch.batch.map((r) => ({
253
- sourceTableId: mongoTableId(r.record.sourceTable.id),
254
- replicaId: r.beforeId
255
- }));
271
+ // Within this branch this.storeCurrentData is true, so the per-table flag is the
272
+ // effective value - only look up sizes for tables that actually store current_data.
273
+ const sizeLookups = batch.batch
274
+ .filter((r) => r.record.sourceTable.storeCurrentData)
275
+ .map((r) => ({
276
+ sourceTableId: mongoTableId(r.record.sourceTable.id),
277
+ replicaId: r.beforeId
278
+ }));
256
279
 
257
280
  sizes = await this.sourceRecordStore.loadSizes(session, sizeLookups);
258
281
  }
@@ -329,12 +352,17 @@ export abstract class MongoBucketBatch
329
352
  }
330
353
  }
331
354
 
332
- if (didFlush) {
355
+ const clearedError = didFlush && !this.clearedError;
356
+ if (clearedError) {
357
+ // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
333
358
  using _ = this.tracer.span('storage', 'clear_error');
334
- await this.clearError();
359
+ await this.clearError(this.session);
335
360
  }
336
361
 
337
- return resumeBatch?.hasData() ? resumeBatch : null;
362
+ return {
363
+ resumeBatch: resumeBatch?.hasData() ? resumeBatch : null,
364
+ clearedError
365
+ };
338
366
  }
339
367
 
340
368
  private saveOperation(
@@ -348,6 +376,9 @@ export abstract class MongoBucketBatch
348
376
  const afterId = operation.afterId;
349
377
  let after = record.after;
350
378
  const sourceTable = record.sourceTable;
379
+ // Effective per-record flag: store current_data only if both the batch (source-level,
380
+ // e.g. Postgres) and the table (e.g. non-FULL replica identity) require it.
381
+ const storeCurrentData = this.storeCurrentData && sourceTable.storeCurrentData;
351
382
 
352
383
  let existing_buckets: LoadedSourceRecord['buckets'] = [];
353
384
  let new_buckets: LoadedSourceRecord['buckets'] = [];
@@ -376,7 +407,7 @@ export abstract class MongoBucketBatch
376
407
  // Not an error if we re-apply a transaction
377
408
  existing_buckets = [];
378
409
  existing_lookups = [];
379
- if (!isCompleteRow(this.storeCurrentData, after!)) {
410
+ if (!isCompleteRow(storeCurrentData, after!)) {
380
411
  if (this.markRecordUnavailable != null) {
381
412
  // This will trigger a "resnapshot" of the record.
382
413
  // This is not relevant if storeCurrentData is false, since we'll get the full row
@@ -392,7 +423,7 @@ export abstract class MongoBucketBatch
392
423
  } else {
393
424
  existing_buckets = result.buckets;
394
425
  existing_lookups = result.lookups;
395
- if (this.storeCurrentData && result.data != null) {
426
+ if (storeCurrentData && result.data != null) {
396
427
  const data = deserializeBson(result.data.buffer) as SqliteRow;
397
428
  after = storage.mergeToast<SqliteValue>(after!, data);
398
429
  }
@@ -403,7 +434,9 @@ export abstract class MongoBucketBatch
403
434
  // Not an error if we re-apply a transaction
404
435
  existing_buckets = [];
405
436
  existing_lookups = [];
406
- // Log to help with debugging if there was a consistency issue
437
+ // Log to help with debugging if there was a consistency issue.
438
+ // Gate on the batch-level flag: FULL tables (per-record flag false) still get a
439
+ // current_data entry, so a missing record on DELETE is meaningful for them too.
407
440
  if (this.storeCurrentData && this.markRecordUnavailable == null) {
408
441
  this.logger.warn(
409
442
  `Cannot find previous record for delete on ${record.sourceTable.qualifiedName}: ${beforeId} / ${record.before?.id}`
@@ -416,7 +449,7 @@ export abstract class MongoBucketBatch
416
449
  }
417
450
 
418
451
  let afterData: bson.Binary | null = null;
419
- if (afterId != null && !this.storeCurrentData) {
452
+ if (afterId != null && !storeCurrentData) {
420
453
  afterData = null;
421
454
  } else if (afterId != null) {
422
455
  try {
@@ -440,7 +473,7 @@ export abstract class MongoBucketBatch
440
473
  {
441
474
  level: errors.ErrorSeverity.WARNING,
442
475
  metadata: {
443
- replication_slot: this.slot_name,
476
+ replication_slot: this.replicationStreamName,
444
477
  table: record.sourceTable.qualifiedName
445
478
  }
446
479
  }
@@ -483,7 +516,7 @@ export abstract class MongoBucketBatch
483
516
  // However, it will be valid by the end of the transaction.
484
517
  //
485
518
  // In this case, we don't save the op, but we do save the current data.
486
- if (afterId && after && utils.isCompleteRow(this.storeCurrentData, after)) {
519
+ if (afterId && after && utils.isCompleteRow(storeCurrentData, after)) {
487
520
  // Insert or update
488
521
  if (sourceTable.syncData) {
489
522
  const { results, errors: syncErrors } = this.sync_rules.evaluateRowWithErrors({
@@ -499,7 +532,7 @@ export abstract class MongoBucketBatch
499
532
  {
500
533
  level: errors.ErrorSeverity.WARNING,
501
534
  metadata: {
502
- replication_slot: this.slot_name,
535
+ replication_slot: this.replicationStreamName,
503
536
  table: record.sourceTable.qualifiedName
504
537
  }
505
538
  }
@@ -534,7 +567,7 @@ export abstract class MongoBucketBatch
534
567
  {
535
568
  level: errors.ErrorSeverity.WARNING,
536
569
  metadata: {
537
- replication_slot: this.slot_name,
570
+ replication_slot: this.replicationStreamName,
538
571
  table: record.sourceTable.qualifiedName
539
572
  }
540
573
  }
@@ -669,7 +702,7 @@ export abstract class MongoBucketBatch
669
702
 
670
703
  await this.db.sync_rules.updateOne(
671
704
  {
672
- _id: this.group_id
705
+ _id: this.replicationStreamId
673
706
  },
674
707
  {
675
708
  $set: {
@@ -678,12 +711,41 @@ export abstract class MongoBucketBatch
678
711
  },
679
712
  { session }
680
713
  );
714
+
715
+ // Allow subclasses to persist additional flush-time state in the same transaction
716
+ // (e.g. v3 advances the stream-level last_persisted_op).
717
+ await this.onReplicationTransactionFlush(session, opSeq.last());
681
718
  // We don't notify checkpoint here - we don't make any checkpoint updates directly
682
719
  });
683
720
  }
684
721
 
722
+ /**
723
+ * Hook called inside the replication flush transaction, after ops have been persisted.
724
+ *
725
+ * The base implementation does nothing; v3 storage overrides this to `$max` the stream-level
726
+ * `last_persisted_op` durably within the same transaction.
727
+ */
728
+ protected async onReplicationTransactionFlush(_session: mongo.ClientSession, _lastOp: InternalOpId): Promise<void> {
729
+ // No-op by default (v1 behaviour unchanged).
730
+ }
731
+
732
+ /**
733
+ * Called after a replication transaction has successfully committed, with the last persisted op id.
734
+ *
735
+ * v1 storage tracks this in memory to fold into the next checkpoint. v3 storage does not need it:
736
+ * the stream-level `last_persisted_op` is already advanced durably by
737
+ * {@link onReplicationTransactionFlush} within the same transaction, and checkpoints read it
738
+ * from the document.
739
+ *
740
+ * Keep calls to this adjacent to {@link withReplicationTransaction} usage - both must observe
741
+ * every path that persists ops.
742
+ */
743
+ protected recordPersistedOp(_lastOp: InternalOpId): void {
744
+ // No-op by default.
745
+ }
746
+
685
747
  async [Symbol.asyncDispose]() {
686
- if (this.batch != null || this.write_checkpoint_batch.length > 0) {
748
+ if (this.batch?.hasData() || this.write_checkpoint_batch.length > 0) {
687
749
  // We don't error here, since:
688
750
  // 1. In error states, this is expected (we can't distinguish between disposing after success or error).
689
751
  // 2. SuppressedError is messy to deal with.
@@ -699,19 +761,25 @@ export abstract class MongoBucketBatch
699
761
 
700
762
  async save(record: storage.SaveOptions): Promise<storage.FlushedResult | null> {
701
763
  const { after, before, sourceTable, tag } = record;
702
- for (const event of this.getTableEvents(sourceTable)) {
703
- this.iterateListeners((cb) =>
704
- cb.replicationEvent?.({
705
- batch: this,
706
- table: sourceTable,
707
- data: {
708
- op: tag,
709
- after: after && utils.isCompleteRow(this.storeCurrentData, after) ? after : undefined,
710
- before: before && utils.isCompleteRow(this.storeCurrentData, before) ? before : undefined
711
- },
712
- event
713
- })
714
- );
764
+ const storeCurrentData = this.storeCurrentData && sourceTable.storeCurrentData;
765
+ // syncEvent is the per-table designation from resolveTables. With v3 storage, multiple
766
+ // SourceTables can exist for the same ref, with a row change saved once per table -
767
+ // only the designated event carrier may fire events, so each event fires once per row.
768
+ if (sourceTable.syncEvent) {
769
+ for (const event of this.getTableEvents(sourceTable)) {
770
+ this.iterateListeners((cb) =>
771
+ cb.replicationEvent?.({
772
+ batch: this,
773
+ table: sourceTable,
774
+ data: {
775
+ op: tag,
776
+ after: after && utils.isCompleteRow(storeCurrentData, after) ? after : undefined,
777
+ before: before && utils.isCompleteRow(storeCurrentData, before) ? before : undefined
778
+ },
779
+ event
780
+ })
781
+ );
782
+ }
715
783
  }
716
784
 
717
785
  /**
@@ -744,7 +812,7 @@ export abstract class MongoBucketBatch
744
812
 
745
813
  await this.withTransaction(async () => {
746
814
  for (let table of sourceTables) {
747
- await this.db.commonSourceTables(this.group_id).deleteOne({ _id: mongoTableId(table.id) });
815
+ await this.db.commonSourceTables(this.replicationStreamId).deleteOne({ _id: mongoTableId(table.id) });
748
816
  }
749
817
  });
750
818
 
@@ -761,7 +829,7 @@ export abstract class MongoBucketBatch
761
829
  }
762
830
 
763
831
  if (last_op) {
764
- this.persisted_op = last_op;
832
+ this.recordPersistedOp(last_op);
765
833
  return {
766
834
  flushed_op: last_op
767
835
  };
@@ -830,7 +898,7 @@ export abstract class MongoBucketBatch
830
898
  copy.snapshotStatus = snapshotStatus;
831
899
 
832
900
  await this.withTransaction(async () => {
833
- await this.db.commonSourceTables(this.group_id).updateOne(
901
+ await this.db.commonSourceTables(this.replicationStreamId).updateOne(
834
902
  { _id: mongoTableId(table.id) },
835
903
  {
836
904
  $set: {
@@ -848,24 +916,21 @@ export abstract class MongoBucketBatch
848
916
  return copy;
849
917
  }
850
918
 
851
- protected async clearError(): Promise<void> {
852
- // No need to clear an error more than once per batch, since an error would always result in restarting the batch.
853
- if (this.clearedError) {
854
- return;
855
- }
856
-
919
+ protected async clearError(session: mongo.ClientSession): Promise<void> {
857
920
  await this.db.sync_rules.updateOne(
858
921
  {
859
- _id: this.group_id
922
+ _id: this.replicationStreamId
860
923
  },
861
924
  {
862
925
  $set: {
863
926
  last_fatal_error: null,
864
927
  last_fatal_error_ts: null
865
928
  }
929
+ },
930
+ {
931
+ session
866
932
  }
867
933
  );
868
- this.clearedError = true;
869
934
  }
870
935
 
871
936
  /**