@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/CHANGELOG.md +101 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +41 -16
  6. package/dist/storage/MongoBucketStorage.js +226 -71
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
  9. package/dist/storage/implementation/MongoBucketBatch.js +110 -62
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
  15. package/dist/storage/implementation/MongoCompactor.js +80 -263
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
  21. package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +39 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  46. package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
  47. package/dist/storage/implementation/common/PersistedBatch.js +12 -0
  48. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  49. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  51. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  52. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  53. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  54. package/dist/storage/implementation/db.js +17 -15
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +20 -13
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  62. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  63. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
  65. package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
  66. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  71. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  72. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +10 -1
  77. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  78. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  79. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  81. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
  82. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  84. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  85. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  86. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
  87. package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
  88. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
  90. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
  91. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
  93. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
  94. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
  96. package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +77 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +30 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +101 -21
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
  115. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  126. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  128. package/dist/storage/storage-index.d.ts +4 -4
  129. package/dist/storage/storage-index.js +4 -4
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +49 -3
  132. package/dist/types/types.js +48 -1
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +1 -2
  135. package/dist/utils/test-utils.js +7 -1
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.d.ts +9 -2
  138. package/dist/utils/util.js +128 -4
  139. package/dist/utils/util.js.map +1 -1
  140. package/package.json +12 -19
  141. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  142. package/src/storage/MongoBucketStorage.ts +347 -88
  143. package/src/storage/implementation/MongoBucketBatch.ts +147 -82
  144. package/src/storage/implementation/MongoChecksums.ts +93 -153
  145. package/src/storage/implementation/MongoCompactor.ts +102 -316
  146. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  147. package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
  148. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  149. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  150. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  151. package/src/storage/implementation/MongoStorageProvider.ts +46 -4
  152. package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
  153. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  154. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
  156. package/src/storage/implementation/common/PersistedBatch.ts +19 -3
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  158. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  159. package/src/storage/implementation/db.ts +19 -20
  160. package/src/storage/implementation/models.ts +20 -14
  161. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
  162. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  163. package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
  164. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
  165. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  166. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  167. package/src/storage/implementation/v1/models.ts +11 -1
  168. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  169. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
  170. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  171. package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
  172. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
  173. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
  174. package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
  175. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  176. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
  177. package/src/storage/implementation/v3/bucket-format.ts +97 -0
  178. package/src/storage/implementation/v3/chunking.ts +40 -0
  179. package/src/storage/implementation/v3/models.ts +132 -55
  180. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
  181. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
  184. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  185. package/src/storage/storage-index.ts +12 -4
  186. package/src/types/types.ts +58 -1
  187. package/src/utils/test-utils.ts +8 -3
  188. package/src/utils/util.ts +185 -6
  189. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
  191. package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
  192. package/test/src/helpers/MemoryObjectStorage.ts +53 -0
  193. package/test/src/helpers/s3TestFactory.ts +69 -0
  194. package/test/src/object_storage_lifecycle.test.ts +59 -0
  195. package/test/src/setup.ts +6 -1
  196. package/test/src/storage.test.ts +227 -13
  197. package/test/src/storage_compacting.test.ts +1951 -12
  198. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  199. package/test/src/storage_s3_checksums.test.ts +191 -0
  200. package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
  201. package/test/src/storage_s3_reading.test.ts +526 -0
  202. package/test/src/storage_s3_writing.test.ts +135 -0
  203. package/test/src/storage_sync.test.ts +1180 -73
  204. package/test/src/storeCurrentData.test.ts +227 -0
  205. package/test/src/util.ts +2 -1
  206. package/tsconfig.json +1 -4
  207. package/tsconfig.tsbuildinfo +1 -1
  208. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  209. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  210. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  211. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  212. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  213. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  214. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  215. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  216. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  217. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  218. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  219. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  220. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  221. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  222. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  223. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  224. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  225. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  226. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  227. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  228. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  229. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  230. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  231. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  232. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  233. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  234. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  235. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  236. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  237. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  238. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  239. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -17,6 +17,7 @@ import {
17
17
  PopulateChecksumCacheOptions,
18
18
  PopulateChecksumCacheResults,
19
19
  ReplicationCheckpoint,
20
+ ReplicationStreamStorageIds,
20
21
  storage,
21
22
  utils,
22
23
  WatchWriteCheckpointOptions
@@ -25,21 +26,28 @@ import { HydratedSyncConfig, ParameterLookupRows, ScopedParameterLookup } from '
25
26
  import * as bson from 'bson';
26
27
  import { LRUCache } from 'lru-cache';
27
28
  import * as timers from 'timers/promises';
28
- import { retryOnMongoMaxTimeMSExpired } from '../../utils/util.js';
29
+ import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../../types/types.js';
29
30
  import { MongoBucketStorage } from '../MongoBucketStorage.js';
30
- import { MongoSyncBucketStorageContext } from './common/MongoSyncBucketStorageContext.js';
31
+ import { DEFAULT_INLINE_THRESHOLD_BYTES } from './common/PersistedBatch.js';
31
32
  import type { VersionedPowerSyncMongo } from './db.js';
32
33
  import { StorageConfig } from './models.js';
33
34
  import { MongoBucketBatchOptions } from './MongoBucketBatch.js';
34
35
  import { MongoChecksumOptions, MongoChecksums } from './MongoChecksums.js';
35
36
  import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js';
36
37
  import { MongoParameterCompactor } from './MongoParameterCompactor.js';
37
- import { MongoPersistedSyncRulesContentV1 } from './MongoPersistedSyncRulesContent.js';
38
+ import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
39
+ import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js';
38
40
  import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
41
+ import { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
39
42
 
40
43
  export interface MongoSyncBucketStorageOptions {
41
- checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig' | 'mapping'>;
44
+ checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
45
+ readPreference?: mongo.ReadPreference;
46
+ checksumCacheTtlMs?: number;
47
+ clearBatchThrottleRate?: number;
42
48
  storageConfig: StorageConfig;
49
+ objectStorage?: ObjectStorage;
50
+ inlineThresholdBytes?: number;
43
51
  }
44
52
 
45
53
  interface InternalCheckpointChanges extends CheckpointChanges {
@@ -47,13 +55,6 @@ interface InternalCheckpointChanges extends CheckpointChanges {
47
55
  invalidateWriteCheckpoints: boolean;
48
56
  }
49
57
 
50
- interface WriterSyncState {
51
- lastCheckpointLsn: string | null;
52
- resumeFromLsn: string | null;
53
- keepaliveOp: InternalOpId | null;
54
- syncConfigId?: bson.ObjectId | null;
55
- }
56
-
57
58
  /**
58
59
  * Only keep checkpoints around for a minute, before fetching a fresh one.
59
60
  *
@@ -70,34 +71,53 @@ export abstract class MongoSyncBucketStorage
70
71
  implements storage.SyncRulesBucketStorage
71
72
  {
72
73
  readonly db: VersionedPowerSyncMongo;
74
+
73
75
  [DO_NOT_LOG] = true;
74
76
 
75
77
  readonly checksums: MongoChecksums;
76
78
 
77
- private parsedSyncRulesCache: { parsed: HydratedSyncConfig; options: storage.ParseSyncRulesOptions } | undefined;
79
+ readonly objectStorage?: ObjectStorage;
80
+ readonly inlineThresholdBytes: number;
81
+
82
+ /**
83
+ * Canonical parsed sync config sets, keyed by defaultSchema.
84
+ *
85
+ * Entries are never evicted: each parse options value maps to exactly one parsed set for
86
+ * the lifetime of this storage instance, so parsed source objects and mappings always
87
+ * stay associated.
88
+ */
89
+ private readonly parsedSyncConfigSets = new Map<string, MongoParsedSyncConfigSet>();
78
90
  private writeCheckpointAPI: MongoWriteCheckpointAPI;
79
91
  public readonly logger: Logger;
80
92
  public readonly storageConfig: StorageConfig;
93
+ public readonly readPreference: mongo.ReadPreference | undefined;
94
+ public readonly clearBatchThrottleRate: number;
81
95
  #storageInitialized = false;
82
96
 
83
97
  constructor(
84
98
  public readonly factory: MongoBucketStorage,
85
- public readonly group_id: number,
86
- protected readonly sync_rules: MongoPersistedSyncRulesContentV1,
87
- public readonly slot_name: string,
99
+ public readonly replicationStreamId: number,
100
+ public readonly replicationStream: MongoPersistedReplicationStream,
101
+ public readonly replicationStreamName: string,
88
102
  writeCheckpointMode: storage.WriteCheckpointMode | undefined,
89
103
  options: MongoSyncBucketStorageOptions
90
104
  ) {
91
105
  super();
92
106
  this.storageConfig = options.storageConfig;
107
+ this.objectStorage = options.objectStorage;
108
+ // Keep small chunks inline in MongoDB rather than offloading them to S3.
109
+ // Configurable via object_storage.inline_threshold_bytes.
110
+ this.inlineThresholdBytes = options.inlineThresholdBytes ?? DEFAULT_INLINE_THRESHOLD_BYTES;
111
+ this.readPreference = options.readPreference;
112
+ this.clearBatchThrottleRate = options.clearBatchThrottleRate ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;
93
113
  this.db = factory.db.versioned(this.storageConfig);
94
114
  this.checksums = this.createMongoChecksums(options);
95
115
  this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
96
116
  db: this.db,
97
117
  mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED,
98
- sync_rules_id: group_id
118
+ sync_rules_id: replicationStreamId
99
119
  });
100
- this.logger = sync_rules.logger;
120
+ this.logger = replicationStream.logger;
101
121
  }
102
122
 
103
123
  /**
@@ -117,40 +137,41 @@ export abstract class MongoSyncBucketStorage
117
137
  return this.writeCheckpointAPI.writeCheckpointMode;
118
138
  }
119
139
 
120
- get mapping() {
121
- return this.sync_rules.mapping;
122
- }
123
-
124
- protected get versionContext(): MongoSyncBucketStorageContext {
125
- return {
126
- db: this.db,
127
- group_id: this.group_id,
128
- mapping: this.mapping
129
- };
140
+ /**
141
+ * Persisted storage ids of all sync configs in this replication stream. Parse-free.
142
+ */
143
+ get storageIds(): ReplicationStreamStorageIds {
144
+ return this.replicationStream.storageIds;
130
145
  }
131
146
 
132
147
  setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
133
148
  this.writeCheckpointAPI.setWriteCheckpointMode(mode);
134
149
  }
135
150
 
136
- createManagedWriteCheckpoint(checkpoint: storage.ManagedWriteCheckpointOptions): Promise<bigint> {
137
- return this.writeCheckpointAPI.createManagedWriteCheckpoint(checkpoint);
151
+ createManagedWriteCheckpoints(
152
+ checkpoints: storage.ManagedWriteCheckpointOptions[]
153
+ ): Promise<storage.CreateManagedWriteCheckpointsResult> {
154
+ return this.writeCheckpointAPI.createManagedWriteCheckpoints(checkpoints);
138
155
  }
139
156
 
140
157
  lastWriteCheckpoint(filters: storage.SyncStorageLastWriteCheckpointFilters): Promise<bigint | null> {
141
158
  return this.writeCheckpointAPI.lastWriteCheckpoint({
142
159
  ...filters,
143
- sync_rules_id: this.group_id
160
+ sync_rules_id: this.replicationStreamId
144
161
  });
145
162
  }
146
163
 
147
- getParsedSyncRules(options: storage.ParseSyncRulesOptions): HydratedSyncConfig {
148
- const { parsed, options: cachedOptions } = this.parsedSyncRulesCache ?? {};
149
- if (!parsed || options.defaultSchema != cachedOptions?.defaultSchema) {
150
- this.parsedSyncRulesCache = { parsed: this.sync_rules.parsed(options).hydratedSyncConfig(), options };
164
+ getParsedSyncConfigSet(options: storage.ParseSyncConfigOptions): MongoParsedSyncConfigSet {
165
+ let parsed = this.parsedSyncConfigSets.get(options.defaultSchema);
166
+ if (parsed == null) {
167
+ parsed = this.replicationStream.parsed(options);
168
+ this.parsedSyncConfigSets.set(options.defaultSchema, parsed);
151
169
  }
170
+ return parsed;
171
+ }
152
172
 
153
- return this.parsedSyncRulesCache!.parsed;
173
+ getParsedSyncRules(options: storage.ParseSyncConfigOptions): HydratedSyncConfig {
174
+ return this.getParsedSyncConfigSet(options).hydratedSyncConfig;
154
175
  }
155
176
 
156
177
  async getCheckpoint(): Promise<storage.ReplicationCheckpoint> {
@@ -169,10 +190,14 @@ export abstract class MongoSyncBucketStorage
169
190
  }
170
191
 
171
192
  const snapshotTime = (session as any).snapshotTime as bson.Timestamp | undefined;
193
+ const clusterTime = session.clusterTime;
172
194
  if (snapshotTime == null) {
173
195
  throw new ServiceAssertionError('Missing snapshotTime in getCheckpoint()');
174
196
  }
175
- return new MongoReplicationCheckpoint(this, state.checkpoint, state.lsn, snapshotTime);
197
+ if (clusterTime == null) {
198
+ throw new ServiceAssertionError('Missing clusterTime in getCheckpoint()');
199
+ }
200
+ return new MongoReplicationCheckpoint(this, state.checkpoint, state.lsn, snapshotTime, clusterTime);
176
201
  });
177
202
  }
178
203
 
@@ -183,37 +208,42 @@ export abstract class MongoSyncBucketStorage
183
208
  return;
184
209
  }
185
210
 
186
- await this.db.initializeStreamStorage(this.group_id);
211
+ await this.db.initializeStreamStorage(this.replicationStreamId);
187
212
  await this.initializeVersionStorage();
188
213
  this.#storageInitialized = true;
189
214
  }
190
215
 
191
- protected abstract createWriterImpl(batchOptions: MongoBucketBatchOptions): storage.BucketStorageBatch;
192
- protected abstract getWriterSyncState(): Promise<WriterSyncState>;
193
-
194
- async createWriter(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
195
- await this.initializeStorage();
196
-
197
- const state = await this.getWriterSyncState();
216
+ /**
217
+ * Create the version-specific writer. Implementations fetch their own resume state
218
+ * (e.g. resume LSN, v1 keepalive op) and construct the batch from
219
+ * {@link writerBatchOptions} plus the version-specific fields.
220
+ */
221
+ protected abstract createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch>;
198
222
 
199
- const batchOptions: MongoBucketBatchOptions = {
223
+ /**
224
+ * The version-independent part of the batch options.
225
+ */
226
+ protected writerBatchOptions(options: storage.CreateWriterOptions): Omit<MongoBucketBatchOptions, 'resumeFromLsn'> {
227
+ return {
200
228
  logger: options.logger ?? this.logger,
201
229
  db: this.db,
202
- syncRules: this.sync_rules.parsed(options).hydratedSyncConfig(),
203
- mapping: this.sync_rules.mapping,
204
- groupId: this.group_id,
205
- slotName: this.slot_name,
206
- lastCheckpointLsn: state.lastCheckpointLsn,
207
- resumeFromLsn: state.resumeFromLsn,
208
- keepaliveOp: state.keepaliveOp,
230
+ parsedSyncConfig: this.getParsedSyncConfigSet(options),
231
+ replicationStreamId: this.replicationStreamId,
232
+ replicationStreamName: this.replicationStreamName,
209
233
  storeCurrentData: options.storeCurrentData,
210
234
  skipExistingRows: options.skipExistingRows ?? false,
211
235
  markRecordUnavailable: options.markRecordUnavailable,
212
236
  hooks: options.hooks,
213
- syncConfigId: state.syncConfigId,
214
- tracer: options.tracer
237
+ tracer: options.tracer,
238
+ objectStorage: this.objectStorage,
239
+ inlineThresholdBytes: this.inlineThresholdBytes
215
240
  };
216
- const writer = this.createWriterImpl(batchOptions);
241
+ }
242
+
243
+ async createWriter(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
244
+ await this.initializeStorage();
245
+
246
+ const writer = await this.createWriterImpl(options);
217
247
  this.iterateListeners((cb) => cb.batchStarted?.(writer));
218
248
  return writer;
219
249
  }
@@ -243,24 +273,31 @@ export abstract class MongoSyncBucketStorage
243
273
  }
244
274
 
245
275
  protected abstract getBucketDataBatchImpl(
246
- checkpoint: utils.InternalOpId,
276
+ checkpoint: MongoReplicationCheckpoint,
247
277
  dataBuckets: storage.BucketDataRequest[],
248
278
  options?: storage.BucketDataBatchOptions
249
- ): AsyncIterable<storage.SyncBucketDataChunk>;
279
+ ): AsyncIterable<storage.SyncBucketDataChunk | storage.SyncBucketDataBatchEnd>;
250
280
 
251
281
  async *getBucketDataBatch(
252
- checkpoint: utils.InternalOpId,
282
+ checkpoint: storage.ReplicationCheckpoint,
253
283
  dataBuckets: storage.BucketDataRequest[],
254
284
  options?: storage.BucketDataBatchOptions
255
- ): AsyncIterable<storage.SyncBucketDataChunk> {
256
- yield* this.getBucketDataBatchImpl(checkpoint, dataBuckets, options);
285
+ ): AsyncIterable<storage.SyncBucketDataChunk | storage.SyncBucketDataBatchEnd> {
286
+ yield* this.getBucketDataBatchImpl(checkpoint as MongoReplicationCheckpoint, dataBuckets, options);
257
287
  }
258
288
 
259
289
  async getChecksums(
260
- checkpoint: utils.InternalOpId,
261
- buckets: storage.BucketChecksumRequest[]
290
+ checkpoint: storage.ReplicationCheckpoint,
291
+ buckets: storage.BucketChecksumRequest[],
292
+ options?: storage.BucketChecksumOptions
262
293
  ): Promise<utils.ChecksumMap> {
263
- return this.checksums.getChecksums(checkpoint, buckets);
294
+ const mongoCheckpoint = checkpoint as MongoReplicationCheckpoint;
295
+ const snapshotTime = mongoCheckpoint.snapshotTime; // May be undefined in tests
296
+ return this.checksums.getChecksums(checkpoint.checkpoint, buckets, {
297
+ snapshotTime,
298
+ clusterTime: mongoCheckpoint.clusterTime,
299
+ readPreference: options?.requestHint == 'bulk' ? this.readPreference : undefined
300
+ });
264
301
  }
265
302
 
266
303
  clearChecksumCache() {
@@ -277,9 +314,9 @@ export abstract class MongoSyncBucketStorage
277
314
  await this.db.notifyCheckpoint();
278
315
  }
279
316
 
280
- protected abstract getStatusImpl(): Promise<storage.SyncRuleStatus>;
317
+ protected abstract getStatusImpl(): Promise<storage.ReplicationStreamStatus>;
281
318
 
282
- async getStatus(): Promise<storage.SyncRuleStatus> {
319
+ async getStatus(): Promise<storage.ReplicationStreamStatus> {
283
320
  return this.getStatusImpl();
284
321
  }
285
322
 
@@ -312,28 +349,11 @@ export abstract class MongoSyncBucketStorage
312
349
  this.#storageInitialized = false;
313
350
  }
314
351
 
315
- protected async clearDeleteMany(
316
- label: string,
317
- operation: () => Promise<mongo.DeleteResult>,
318
- signal?: AbortSignal
319
- ): Promise<void> {
320
- await retryOnMongoMaxTimeMSExpired(operation, {
321
- signal,
322
- abortMessage: 'Aborted clearing data',
323
- retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS / 5,
324
- onRetry: () => {
325
- this.logger.info(
326
- `Cleared batch of ${label} in ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, continuing...`
327
- );
328
- }
329
- });
330
- }
331
-
332
352
  async reportError(e: any): Promise<void> {
333
353
  const message = String(e.message ?? 'Replication failure');
334
354
  await this.db.sync_rules.updateOne(
335
355
  {
336
- _id: this.group_id
356
+ _id: this.replicationStreamId
337
357
  },
338
358
  {
339
359
  $set: {
@@ -357,11 +377,21 @@ export abstract class MongoSyncBucketStorage
357
377
  }
358
378
  }
359
379
 
380
+ /**
381
+ * The highest op id persisted for this stream, whether or not covered by a checkpoint.
382
+ *
383
+ * Used as the default `maxOpId` for {@link populatePersistentChecksumCache}, which runs after
384
+ * initial replication but before the first checkpoint exists.
385
+ */
386
+ protected abstract fetchPersistedOpHead(): Promise<InternalOpId | null>;
387
+
360
388
  async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
361
389
  this.logger.info(`Populating persistent checksum cache...`);
362
390
  const start = Date.now();
391
+ const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined;
363
392
  const compactor = this.createMongoCompactor({
364
393
  ...options,
394
+ maxOpId,
365
395
  memoryLimitMB: 0,
366
396
  logger: this.logger
367
397
  });
@@ -413,7 +443,7 @@ export abstract class MongoSyncBucketStorage
413
443
  for await (const nextCheckpoint of iter) {
414
444
  if (nextCheckpoint.lsn != null && !queriedInitialWriteCheckpoint) {
415
445
  writeCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({
416
- sync_rules_id: this.group_id,
446
+ sync_rules_id: this.replicationStreamId,
417
447
  user_id: options.user_id,
418
448
  heads: {
419
449
  '1': nextCheckpoint.lsn
@@ -446,7 +476,7 @@ export abstract class MongoSyncBucketStorage
446
476
  let updatedWriteCheckpoint = updates.updatedWriteCheckpoints.get(options.user_id) ?? null;
447
477
  if (updates.invalidateWriteCheckpoints) {
448
478
  updatedWriteCheckpoint = await this.writeCheckpointAPI.lastWriteCheckpoint({
449
- sync_rules_id: this.group_id,
479
+ sync_rules_id: this.replicationStreamId,
450
480
  user_id: options.user_id,
451
481
  heads: {
452
482
  '1': nextCheckpoint.lsn!
@@ -566,6 +596,9 @@ export abstract class MongoSyncBucketStorage
566
596
  >({
567
597
  max: 50,
568
598
  maxSize: 12 * 1024 * 1024,
599
+ // When we have more fetches than the cache size, complete the fetches instead
600
+ // of failing with Error('evicted').
601
+ ignoreFetchAbort: true,
569
602
  sizeCalculation: (value: InternalCheckpointChanges) => {
570
603
  const paramSize = [...value.updatedParameterLookups].reduce<number>((a, b) => a + b.length, 0);
571
604
  const bucketSize = [...value.updatedDataBuckets].reduce<number>((a, b) => a + b.length, 0);
@@ -603,7 +636,8 @@ class MongoReplicationCheckpoint implements ReplicationCheckpoint {
603
636
  storage: MongoSyncBucketStorage,
604
637
  public readonly checkpoint: InternalOpId,
605
638
  public readonly lsn: string | null,
606
- public snapshotTime: mongo.Timestamp
639
+ public snapshotTime: mongo.Timestamp,
640
+ public clusterTime: mongo.ClusterTime
607
641
  ) {
608
642
  this.#storage = storage;
609
643
  }
@@ -617,7 +651,7 @@ class EmptyReplicationCheckpoint implements ReplicationCheckpoint {
617
651
  readonly checkpoint: InternalOpId = 0n;
618
652
  readonly lsn: string | null = null;
619
653
 
620
- async getParameterSets(_lookups: ScopedParameterLookup[]): Promise<ParameterLookupRows[]> {
654
+ async getParameterSets(_lookups: ScopedParameterLookup[], _limit: number): Promise<ParameterLookupRows[]> {
621
655
  return [];
622
656
  }
623
657
  }
@@ -17,12 +17,12 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
17
17
  */
18
18
  static async createLock(
19
19
  db: VersionedPowerSyncMongo,
20
- sync_rules: storage.PersistedSyncRulesContent,
20
+ sync_rules: storage.PersistedReplicationStream,
21
21
  session?: mongo.ClientSession
22
22
  ): Promise<MongoSyncRulesLock> {
23
23
  const lockId = crypto.randomBytes(8).toString('hex');
24
24
  const doc = await db.sync_rules.findOneAndUpdate(
25
- { _id: sync_rules.id, $or: [{ lock: null }, { 'lock.expires_at': { $lt: new Date() } }] },
25
+ { _id: sync_rules.replicationStreamId, $or: [{ lock: null }, { 'lock.expires_at': { $lt: new Date() } }] },
26
26
  {
27
27
  $set: {
28
28
  lock: {
@@ -40,7 +40,10 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
40
40
 
41
41
  if (doc == null) {
42
42
  // Query the existing lock to get the expiration time (best effort - it may have been released in the meantime).
43
- const heldLock = await db.sync_rules.findOne({ _id: sync_rules.id }, { projection: { lock: 1 }, session });
43
+ const heldLock = await db.sync_rules.findOne(
44
+ { _id: sync_rules.replicationStreamId },
45
+ { projection: { lock: 1 }, session }
46
+ );
44
47
  if (heldLock?.lock?.expires_at) {
45
48
  throw new ServiceError(
46
49
  ErrorCode.PSYNC_S1003,
@@ -51,7 +54,7 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
51
54
  }
52
55
  }
53
56
  sync_rules.logger.info(`Locked replication stream for processing`);
54
- return new MongoSyncRulesLock(db, sync_rules.id, lockId, sync_rules.logger);
57
+ return new MongoSyncRulesLock(db, sync_rules.replicationStreamId, lockId, sync_rules.logger);
55
58
  }
56
59
 
57
60
  constructor(