@powersync/service-module-mongodb-storage 0.20.0 → 0.22.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (192) hide show
  1. package/CHANGELOG.md +62 -0
  2. package/dist/storage/MongoBucketStorage.d.ts +36 -3
  3. package/dist/storage/MongoBucketStorage.js +372 -207
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/MongoBucketBatch.d.ts +17 -5
  6. package/dist/storage/implementation/MongoBucketBatch.js +44 -17
  7. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  8. package/dist/storage/implementation/MongoCompactor.d.ts +19 -79
  9. package/dist/storage/implementation/MongoCompactor.js +27 -238
  10. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  11. package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
  12. package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
  13. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  14. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
  15. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +29 -0
  16. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
  17. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +4 -2
  18. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
  19. package/dist/storage/implementation/MongoStorageProvider.js +7 -3
  20. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  21. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +27 -11
  22. package/dist/storage/implementation/MongoSyncBucketStorage.js +78 -31
  23. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  24. package/dist/storage/implementation/MongoSyncRulesLock.js +21 -9
  25. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  26. package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
  27. package/dist/storage/implementation/MongoWriteBatch.js +149 -0
  28. package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
  29. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
  30. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
  31. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  32. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  33. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  34. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  35. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
  36. package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
  37. package/dist/storage/implementation/common/PersistedBatch.js +49 -6
  38. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  39. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
  40. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
  41. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  42. package/dist/storage/implementation/db.d.ts +9 -1
  43. package/dist/storage/implementation/db.js +27 -1
  44. package/dist/storage/implementation/db.js.map +1 -1
  45. package/dist/storage/implementation/models.d.ts +31 -0
  46. package/dist/storage/implementation/models.js.map +1 -1
  47. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +2 -0
  48. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +33 -4
  49. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  50. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
  51. package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
  52. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  53. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  54. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  55. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  56. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
  57. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +65 -16
  58. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  59. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
  60. package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
  61. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
  62. package/dist/storage/implementation/v1/models.js +1 -0
  63. package/dist/storage/implementation/v1/models.js.map +1 -1
  64. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  65. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  66. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  67. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +10 -2
  68. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +100 -26
  69. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  70. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +82 -25
  71. package/dist/storage/implementation/v3/MongoCompactorV3.js +698 -159
  72. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  73. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
  74. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
  75. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
  76. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
  77. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
  78. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
  79. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
  80. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +91 -31
  81. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  82. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  83. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  84. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  85. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
  86. package/dist/storage/implementation/v3/PersistedBatchV3.js +76 -42
  87. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  88. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
  89. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
  90. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  91. package/dist/storage/implementation/v3/bucket-format.js +2 -0
  92. package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
  93. package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
  94. package/dist/storage/implementation/v3/compact-utils.js +216 -0
  95. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  96. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  97. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  98. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  99. package/dist/storage/implementation/v3/models.d.ts +84 -4
  100. package/dist/storage/implementation/v3/models.js.map +1 -1
  101. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
  102. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
  103. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
  104. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
  105. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
  106. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
  107. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
  108. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
  109. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
  110. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  111. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
  112. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
  113. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
  114. package/dist/storage/implementation/v3/source-table-utils.d.ts +6 -6
  115. package/dist/storage/implementation/v3/source-table-utils.js +30 -37
  116. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
  117. package/dist/storage/storage-index.d.ts +2 -1
  118. package/dist/storage/storage-index.js +1 -0
  119. package/dist/storage/storage-index.js.map +1 -1
  120. package/dist/types/types.d.ts +28 -1
  121. package/dist/types/types.js +28 -7
  122. package/dist/types/types.js.map +1 -1
  123. package/dist/utils/test-utils.js +1 -0
  124. package/dist/utils/test-utils.js.map +1 -1
  125. package/dist/utils/util.js +3 -1
  126. package/dist/utils/util.js.map +1 -1
  127. package/package.json +10 -9
  128. package/src/storage/MongoBucketStorage.ts +507 -269
  129. package/src/storage/implementation/MongoBucketBatch.ts +67 -24
  130. package/src/storage/implementation/MongoCompactor.ts +35 -346
  131. package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
  132. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +34 -0
  133. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +4 -2
  134. package/src/storage/implementation/MongoStorageProvider.ts +15 -3
  135. package/src/storage/implementation/MongoSyncBucketStorage.ts +114 -40
  136. package/src/storage/implementation/MongoSyncRulesLock.ts +22 -9
  137. package/src/storage/implementation/MongoWriteBatch.ts +172 -0
  138. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
  139. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  140. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
  141. package/src/storage/implementation/common/PersistedBatch.ts +66 -10
  142. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
  143. package/src/storage/implementation/db.ts +35 -2
  144. package/src/storage/implementation/models.ts +34 -0
  145. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +46 -6
  146. package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
  147. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  148. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +87 -27
  149. package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
  150. package/src/storage/implementation/v1/models.ts +1 -0
  151. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  152. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +138 -34
  153. package/src/storage/implementation/v3/MongoCompactorV3.ts +958 -225
  154. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
  155. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
  156. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +111 -41
  157. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  158. package/src/storage/implementation/v3/PersistedBatchV3.ts +93 -48
  159. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
  160. package/src/storage/implementation/v3/bucket-format.ts +2 -0
  161. package/src/storage/implementation/v3/compact-utils.ts +320 -0
  162. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  163. package/src/storage/implementation/v3/models.ts +91 -3
  164. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
  165. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
  166. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
  167. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
  168. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
  169. package/src/storage/implementation/v3/source-table-utils.ts +39 -41
  170. package/src/storage/storage-index.ts +2 -0
  171. package/src/types/types.ts +42 -7
  172. package/src/utils/test-utils.ts +1 -0
  173. package/src/utils/util.ts +3 -1
  174. package/test/src/__snapshots__/storage.test.ts.snap +25 -0
  175. package/test/src/__snapshots__/storage_sync.test.ts.snap +605 -16
  176. package/test/src/chunk_compaction_config.test.ts +38 -0
  177. package/test/src/cleanup-stopped-sync-configs.test.ts +40 -13
  178. package/test/src/compact-utils.test.ts +226 -0
  179. package/test/src/helpers/MemoryObjectStorage.ts +14 -6
  180. package/test/src/object_storage_usage.test.ts +267 -0
  181. package/test/src/parameter_compacting_v1.test.ts +120 -0
  182. package/test/src/parameter_compacting_v3.test.ts +192 -0
  183. package/test/src/parameter_compaction_fence.test.ts +276 -0
  184. package/test/src/storage.test.ts +273 -22
  185. package/test/src/storage_compacting.test.ts +968 -246
  186. package/test/src/storage_s3_checksums.test.ts +3 -4
  187. package/test/src/storage_s3_compaction_lifecycle.test.ts +138 -12
  188. package/test/src/storage_s3_reading.test.ts +290 -2
  189. package/test/src/storage_s3_writing.test.ts +1 -0
  190. package/test/src/storage_sync.test.ts +336 -41
  191. package/test/src/util.ts +1 -1
  192. package/tsconfig.tsbuildinfo +1 -1
@@ -1,7 +1,11 @@
1
1
  import * as lib_mongo from '@powersync/lib-service-mongodb';
2
2
  import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
3
3
  import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
4
- import { MongoStorageConfig, normalizeClearBatchThrottleRate } from '../../types/types.js';
4
+ import {
5
+ MongoStorageConfig,
6
+ normalizeChunkCompactionConcurrency,
7
+ normalizeClearBatchThrottleRate
8
+ } from '../../types/types.js';
5
9
  import { MongoBucketStorage } from '../MongoBucketStorage.js';
6
10
  import { MongoReportStorage } from '../MongoReportStorage.js';
7
11
  import { PowerSyncMongo } from './db.js';
@@ -24,6 +28,10 @@ export class MongoStorageProvider implements storage.StorageProvider {
24
28
  }
25
29
 
26
30
  const decodedConfig = MongoStorageConfig.decode(storage as any);
31
+ const chunkCompactionConcurrency = normalizeChunkCompactionConcurrency(
32
+ decodedConfig.chunk_compaction_concurrency,
33
+ decodedConfig.object_storage != null
34
+ );
27
35
 
28
36
  let objectStorage: ObjectStorage | undefined;
29
37
  if (decodedConfig.object_storage?.type === 's3') {
@@ -37,7 +45,8 @@ export class MongoStorageProvider implements storage.StorageProvider {
37
45
  forcePathStyle: decodedConfig.object_storage.force_path_style,
38
46
  accessKeyId: decodedConfig.object_storage.access_key_id,
39
47
  secretAccessKey: decodedConfig.object_storage.secret_access_key,
40
- concurrencyLimit: decodedConfig.object_storage.concurrency_limit
48
+ concurrencyLimit: decodedConfig.object_storage.concurrency_limit,
49
+ defaultsMode: decodedConfig.object_storage.defaults_mode
41
50
  });
42
51
  }
43
52
 
@@ -46,6 +55,8 @@ export class MongoStorageProvider implements storage.StorageProvider {
46
55
  maxPoolSize: resolvedConfig.storage.max_pool_size ?? 8
47
56
  });
48
57
 
58
+ const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
59
+
49
60
  let shuttingDown = false;
50
61
 
51
62
  // Explicitly connect on startup.
@@ -55,7 +66,6 @@ export class MongoStorageProvider implements storage.StorageProvider {
55
66
  // Errors here will cause the process to exit.
56
67
  await client.connect();
57
68
 
58
- const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
59
69
  const readPreference =
60
70
  decodedConfig.bulk_read_preference == null
61
71
  ? undefined
@@ -65,10 +75,12 @@ export class MongoStorageProvider implements storage.StorageProvider {
65
75
  maxStalenessSeconds: decodedConfig.bulk_read_preference == 'primary' ? undefined : 90
66
76
  });
67
77
  const syncStorageFactory = new MongoBucketStorage(database, {
78
+ chunkCompactionConcurrency,
68
79
  replicationStreamNamePrefix: resolvedConfig.slot_name_prefix,
69
80
  readPreference,
70
81
  clearBatchThrottleRate: normalizeClearBatchThrottleRate(decodedConfig.clear_batch_throttle_rate),
71
82
  checksumCacheTtlMs: resolvedConfig.api_parameters.bucket_count_cache_ttl_minutes * 60_000,
83
+ defaultStorageVersion: decodedConfig.default_storage_version,
72
84
  // Right now, only MongoDB source databases supports incremental reprocessing.
73
85
  // Remove this filter when we support it for other source databases.
74
86
  // This assumes a single source connection - revisit if we ever support multiple connections.
@@ -2,6 +2,7 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
2
2
  import { mongo } from '@powersync/lib-service-mongodb';
3
3
  import {
4
4
  BaseObserver,
5
+ logger as defaultLogger,
5
6
  DO_NOT_LOG,
6
7
  Logger,
7
8
  ReplicationAbortedError,
@@ -11,14 +12,15 @@ import {
11
12
  BroadcastIterable,
12
13
  CHECKPOINT_INVALIDATE_ALL,
13
14
  CheckpointChanges,
15
+ CompactInitialReplicationOptions,
16
+ CompactInitialReplicationResults,
14
17
  GetCheckpointChangesOptions,
15
18
  InternalOpId,
16
19
  mergeAsyncIterables,
17
- PopulateChecksumCacheOptions,
18
- PopulateChecksumCacheResults,
19
20
  ReplicationCheckpoint,
20
21
  ReplicationStreamStorageIds,
21
22
  storage,
23
+ SyncRuleState,
22
24
  utils,
23
25
  WatchWriteCheckpointOptions
24
26
  } from '@powersync/service-core';
@@ -28,6 +30,10 @@ import { LRUCache } from 'lru-cache';
28
30
  import * as timers from 'timers/promises';
29
31
  import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../../types/types.js';
30
32
  import { MongoBucketStorage } from '../MongoBucketStorage.js';
33
+ import {
34
+ MongoGetCheckpointChangesOptions,
35
+ MongoSyncBucketStorageCheckpoint
36
+ } from './common/MongoSyncBucketStorageCheckpoint.js';
31
37
  import { DEFAULT_INLINE_THRESHOLD_BYTES } from './common/PersistedBatch.js';
32
38
  import type { VersionedPowerSyncMongo } from './db.js';
33
39
  import { StorageConfig } from './models.js';
@@ -37,7 +43,7 @@ import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js';
37
43
  import { MongoParameterCompactor } from './MongoParameterCompactor.js';
38
44
  import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
39
45
  import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js';
40
- import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
46
+ import { MongoCheckpointAPIOptions, MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
41
47
  import { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
42
48
 
43
49
  export interface MongoSyncBucketStorageOptions {
@@ -50,6 +56,16 @@ export interface MongoSyncBucketStorageOptions {
50
56
  inlineThresholdBytes?: number;
51
57
  }
52
58
 
59
+ /**
60
+ * The stream state read for a checkpoint. All fields must come from the same snapshot.
61
+ */
62
+ export interface MongoCheckpointState {
63
+ checkpoint: InternalOpId;
64
+ lsn: string | null;
65
+ /** See {@link MongoSyncBucketStorageCheckpoint.parameterChangesInvalidBefore}. */
66
+ parameterChangesInvalidBefore: InternalOpId;
67
+ }
68
+
53
69
  interface InternalCheckpointChanges extends CheckpointChanges {
54
70
  updatedWriteCheckpoints: Map<string, bigint>;
55
71
  invalidateWriteCheckpoints: boolean;
@@ -112,12 +128,27 @@ export abstract class MongoSyncBucketStorage
112
128
  this.clearBatchThrottleRate = options.clearBatchThrottleRate ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;
113
129
  this.db = factory.db.versioned(this.storageConfig);
114
130
  this.checksums = this.createMongoChecksums(options);
115
- this.writeCheckpointAPI = new MongoWriteCheckpointAPI({
131
+ this.writeCheckpointAPI = this.createWriteCheckpointAPI({
116
132
  db: this.db,
117
- mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED,
118
- sync_rules_id: replicationStreamId
133
+ writeCheckpointMode: { mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED },
134
+ replicationStreamId
119
135
  });
120
- this.logger = replicationStream.logger;
136
+
137
+ // Include both replication stream name and sync config version labels in log prefix.
138
+ // Examples:
139
+ // No sync config with a label: [stream]
140
+ // All sync configs have a label: [stream][v5,v6]
141
+ // Mixed: [stream][?,v6]
142
+ const hasVersionLabel = replicationStream.syncConfigContent.some((config) => config.version_label != null);
143
+ const versionLabels = hasVersionLabel
144
+ ? replicationStream.syncConfigContent.map((config) => config.version_label ?? '?')
145
+ : [];
146
+ this.logger =
147
+ versionLabels.length == 0
148
+ ? replicationStream.logger
149
+ : defaultLogger.child({
150
+ prefix: `[${replicationStreamName}][${versionLabels.join(',')}] `
151
+ });
121
152
  }
122
153
 
123
154
  /**
@@ -128,11 +159,20 @@ export abstract class MongoSyncBucketStorage
128
159
  abstract createMongoCompactor(options: MongoCompactOptions): MongoCompactor;
129
160
 
130
161
  protected abstract createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums;
162
+ protected createWriteCheckpointAPI(options: MongoCheckpointAPIOptions): MongoWriteCheckpointAPI {
163
+ return new MongoWriteCheckpointAPI(options);
164
+ }
165
+
131
166
  protected abstract createMongoParameterCompactor(
132
167
  checkpoint: InternalOpId,
133
168
  options: storage.CompactOptions
134
169
  ): MongoParameterCompactor;
135
170
 
171
+ /** MongoDB parameter compaction uses a persisted operation-id cursor. */
172
+ public supportsIncrementalParameterCompaction(): boolean {
173
+ return true;
174
+ }
175
+
136
176
  get writeCheckpointMode() {
137
177
  return this.writeCheckpointAPI.writeCheckpointMode;
138
178
  }
@@ -144,8 +184,8 @@ export abstract class MongoSyncBucketStorage
144
184
  return this.replicationStream.storageIds;
145
185
  }
146
186
 
147
- setWriteCheckpointMode(mode: storage.WriteCheckpointMode): void {
148
- this.writeCheckpointAPI.setWriteCheckpointMode(mode);
187
+ setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
188
+ this.writeCheckpointAPI.setWriteCheckpointMode(config);
149
189
  }
150
190
 
151
191
  createManagedWriteCheckpoints(
@@ -178,9 +218,7 @@ export abstract class MongoSyncBucketStorage
178
218
  return (await this.getCheckpointInternal()) ?? new EmptyReplicationCheckpoint();
179
219
  }
180
220
 
181
- protected abstract fetchCheckpointState(
182
- session: mongo.ClientSession
183
- ): Promise<{ checkpoint: bigint; lsn: string | null } | null>;
221
+ protected abstract fetchCheckpointState(session: mongo.ClientSession): Promise<MongoCheckpointState | null>;
184
222
 
185
223
  async getCheckpointInternal(): Promise<storage.ReplicationCheckpoint | null> {
186
224
  return await this.db.client.withSession({ snapshot: true }, async (session) => {
@@ -197,7 +235,14 @@ export abstract class MongoSyncBucketStorage
197
235
  if (clusterTime == null) {
198
236
  throw new ServiceAssertionError('Missing clusterTime in getCheckpoint()');
199
237
  }
200
- return new MongoReplicationCheckpoint(this, state.checkpoint, state.lsn, snapshotTime, clusterTime);
238
+ return new MongoReplicationCheckpoint(
239
+ this,
240
+ state.checkpoint,
241
+ state.lsn,
242
+ snapshotTime,
243
+ clusterTime,
244
+ state.parameterChangesInvalidBefore
245
+ );
201
246
  });
202
247
  }
203
248
 
@@ -235,6 +280,7 @@ export abstract class MongoSyncBucketStorage
235
280
  markRecordUnavailable: options.markRecordUnavailable,
236
281
  hooks: options.hooks,
237
282
  tracer: options.tracer,
283
+ signal: options.signal,
238
284
  objectStorage: this.objectStorage,
239
285
  inlineThresholdBytes: this.inlineThresholdBytes
240
286
  };
@@ -329,6 +375,7 @@ export abstract class MongoSyncBucketStorage
329
375
  protected abstract clearBucketState(signal?: AbortSignal): Promise<void>;
330
376
 
331
377
  protected abstract clearSourceTables(signal?: AbortSignal): Promise<void>;
378
+ protected async clearCustomCheckpointRequests(_signal?: AbortSignal): Promise<void> {}
332
379
  protected abstract clearSyncRuleState(): Promise<void>;
333
380
 
334
381
  async clear(options?: storage.ClearStorageOptions): Promise<void> {
@@ -345,6 +392,7 @@ export abstract class MongoSyncBucketStorage
345
392
  await this.clearSourceRecords(signal);
346
393
  await this.clearBucketState(signal);
347
394
  await this.clearSourceTables(signal);
395
+ await this.clearCustomCheckpointRequests(signal);
348
396
 
349
397
  this.#storageInitialized = false;
350
398
  }
@@ -367,43 +415,30 @@ export abstract class MongoSyncBucketStorage
367
415
  async compact(options?: storage.CompactOptions) {
368
416
  let maxOpId = options?.maxOpId;
369
417
  if (maxOpId == null) {
418
+ // For PROCESSING streams, this will be undefined.
370
419
  const checkpoint = await this.getCheckpointInternal();
371
420
  maxOpId = checkpoint?.checkpoint ?? undefined;
372
421
  }
373
422
  await this.createMongoCompactor({ ...options, maxOpId, logger: this.logger }).compact();
374
423
 
375
- if (maxOpId != null && options?.compactParameterData) {
376
- await this.createMongoParameterCompactor(maxOpId, options).compact();
424
+ if (maxOpId != null && options?.compactParameterData && this.replicationStream.state == SyncRuleState.ACTIVE) {
425
+ // Use the stream-scoped logger, matching bucket compaction above.
426
+ await this.createMongoParameterCompactor(maxOpId, { ...options, logger: this.logger }).compact();
377
427
  }
378
428
  }
379
429
 
430
+ abstract compactInitialReplication(
431
+ options: CompactInitialReplicationOptions
432
+ ): Promise<CompactInitialReplicationResults>;
433
+
380
434
  /**
381
435
  * The highest op id persisted for this stream, whether or not covered by a checkpoint.
382
436
  *
383
- * Used as the default `maxOpId` for {@link populatePersistentChecksumCache}, which runs after
437
+ * Used as the default `maxOpId` for {@link compactInitialReplication}, which runs after
384
438
  * initial replication but before the first checkpoint exists.
385
439
  */
386
440
  protected abstract fetchPersistedOpHead(): Promise<InternalOpId | null>;
387
441
 
388
- async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
389
- this.logger.info(`Populating persistent checksum cache...`);
390
- const start = Date.now();
391
- const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined;
392
- const compactor = this.createMongoCompactor({
393
- ...options,
394
- maxOpId,
395
- memoryLimitMB: 0,
396
- logger: this.logger
397
- });
398
-
399
- const result = await compactor.populateChecksums({
400
- minBucketChanges: options.minBucketChanges ?? 10
401
- });
402
- const duration = Date.now() - start;
403
- this.logger.info(`Populated persistent checksum cache in ${(duration / 1000).toFixed(1)}s`);
404
- return result;
405
- }
406
-
407
442
  private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ReplicationCheckpoint> {
408
443
  if (signal.aborted) {
409
444
  return;
@@ -580,13 +615,31 @@ export abstract class MongoSyncBucketStorage
580
615
  }
581
616
 
582
617
  protected abstract getParameterBucketChangesImpl(
583
- options: GetCheckpointChangesOptions
618
+ options: MongoGetCheckpointChangesOptions
584
619
  ): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;
585
620
 
586
621
  private async getParameterBucketChanges(
587
622
  options: GetCheckpointChangesOptions
588
623
  ): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
589
- return this.getParameterBucketChangesImpl(options);
624
+ const nextCheckpoint = requireMongoCheckpoint(options.nextCheckpoint);
625
+ if (options.lastCheckpoint.checkpoint < nextCheckpoint.parameterChangesInvalidBefore) {
626
+ // Parameter compaction may have deleted parameter entries in the range we'd have to query
627
+ // to find the individual changed lookups. Invalidate all parameter buckets instead.
628
+ //
629
+ // The fence is committed before the first delete of a compaction pass, and captured in the
630
+ // same snapshot as the checkpoint, so a checkpoint that could miss a deleted entry always
631
+ // observes the fence as well.
632
+ return {
633
+ invalidateParameterBuckets: true,
634
+ updatedParameterLookups: new Set<string>()
635
+ };
636
+ }
637
+ // The query below runs at the checkpoint snapshot, which still sees entries deleted by a
638
+ // compaction pass that started after the snapshot.
639
+ return this.getParameterBucketChangesImpl({
640
+ lastCheckpoint: options.lastCheckpoint,
641
+ nextCheckpoint
642
+ });
590
643
  }
591
644
 
592
645
  private checkpointChangesCache = new LRUCache<
@@ -611,7 +664,10 @@ export abstract class MongoSyncBucketStorage
611
664
  });
612
665
 
613
666
  async getCheckpointChanges(options: GetCheckpointChangesOptions): Promise<InternalCheckpointChanges> {
614
- const key = `${options.lastCheckpoint.checkpoint}_${options.lastCheckpoint.lsn}__${options.nextCheckpoint.checkpoint}_${options.nextCheckpoint.lsn}`;
667
+ // The invalidation fence is part of the identity: the same checkpoint pair read before and
668
+ // after a compaction pass produces different results (specific lookups vs. invalidate-all).
669
+ const fence = requireMongoCheckpoint(options.nextCheckpoint).parameterChangesInvalidBefore;
670
+ const key = `${options.lastCheckpoint.checkpoint}_${options.lastCheckpoint.lsn}__${options.nextCheckpoint.checkpoint}_${options.nextCheckpoint.lsn}_${fence}`;
615
671
  const result = await this.checkpointChangesCache.fetch(key, { context: { options } });
616
672
  return result!;
617
673
  }
@@ -629,7 +685,19 @@ export abstract class MongoSyncBucketStorage
629
685
  }
630
686
  }
631
687
 
632
- class MongoReplicationCheckpoint implements ReplicationCheckpoint {
688
+ /**
689
+ * We don't support any other constructions of ReplicationCheckpoint.
690
+ */
691
+ function requireMongoCheckpoint(checkpoint: ReplicationCheckpoint): MongoReplicationCheckpoint {
692
+ if (!(checkpoint instanceof MongoReplicationCheckpoint)) {
693
+ throw new ServiceAssertionError(
694
+ `Checkpoint changes require a checkpoint from getCheckpointInternal(), got ${checkpoint.constructor.name}`
695
+ );
696
+ }
697
+ return checkpoint;
698
+ }
699
+
700
+ class MongoReplicationCheckpoint implements MongoSyncBucketStorageCheckpoint {
633
701
  #storage: MongoSyncBucketStorage;
634
702
 
635
703
  constructor(
@@ -637,7 +705,9 @@ class MongoReplicationCheckpoint implements ReplicationCheckpoint {
637
705
  public readonly checkpoint: InternalOpId,
638
706
  public readonly lsn: string | null,
639
707
  public snapshotTime: mongo.Timestamp,
640
- public clusterTime: mongo.ClusterTime
708
+ public clusterTime: mongo.ClusterTime,
709
+ /** Captured in the same snapshot as the checkpoint. */
710
+ public readonly parameterChangesInvalidBefore: InternalOpId
641
711
  ) {
642
712
  this.#storage = storage;
643
713
  }
@@ -647,6 +717,10 @@ class MongoReplicationCheckpoint implements ReplicationCheckpoint {
647
717
  }
648
718
  }
649
719
 
720
+ /**
721
+ * Used when no checkpoint has been persisted yet. This has no snapshot or invalidation fence, so
722
+ * it cannot be used for checkpoint change detection - see {@link requireMongoCheckpoint}.
723
+ */
650
724
  class EmptyReplicationCheckpoint implements ReplicationCheckpoint {
651
725
  readonly checkpoint: InternalOpId = 0n;
652
726
  readonly lsn: string | null = null;
@@ -5,6 +5,8 @@ import { ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framewo
5
5
  import { storage } from '@powersync/service-core';
6
6
  import { VersionedPowerSyncMongo } from './db.js';
7
7
 
8
+ const LOCK_DURATION_MS = 60 * 1000;
9
+
8
10
  /**
9
11
  * Manages a lock on a replication stream document, so that only one process
10
12
  * processes that replication stream at a time.
@@ -22,15 +24,20 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
22
24
  ): Promise<MongoSyncRulesLock> {
23
25
  const lockId = crypto.randomBytes(8).toString('hex');
24
26
  const doc = await db.sync_rules.findOneAndUpdate(
25
- { _id: sync_rules.replicationStreamId, $or: [{ lock: null }, { 'lock.expires_at': { $lt: new Date() } }] },
26
27
  {
27
- $set: {
28
- lock: {
29
- id: lockId,
30
- expires_at: new Date(Date.now() + 60 * 1000)
28
+ _id: sync_rules.replicationStreamId,
29
+ $or: [{ lock: null }, { $expr: { $lt: ['$lock.expires_at', '$$NOW'] } }]
30
+ },
31
+ [
32
+ {
33
+ $set: {
34
+ lock: {
35
+ id: lockId,
36
+ expires_at: { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: LOCK_DURATION_MS } }
37
+ }
31
38
  }
32
39
  }
33
- },
40
+ ],
34
41
  {
35
42
  projection: { lock: 1 },
36
43
  returnDocument: 'before',
@@ -96,9 +103,15 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
96
103
  _id: this.sync_rules_id,
97
104
  'lock.id': this.lock_id
98
105
  },
99
- {
100
- $set: { 'lock.expires_at': new Date(Date.now() + 60 * 1000) }
101
- },
106
+ [
107
+ {
108
+ $set: {
109
+ 'lock.expires_at': {
110
+ $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: LOCK_DURATION_MS }
111
+ }
112
+ }
113
+ }
114
+ ],
102
115
  { returnDocument: 'after' }
103
116
  );
104
117
  if (result == null) {
@@ -0,0 +1,172 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+
3
+ /**
4
+ * Explicitly groups writes with no intervening reads or result dependencies.
5
+ * Callers retain responsibility for transaction and ordering boundaries.
6
+ *
7
+ * On MongoDB 8.0+, this uses the client-level bulkWrite API.
8
+ *
9
+ * On MongoDB earlier versions, this uses the the collection-level bulkWrite API, or individual operations.
10
+ * This is less efficient, but produces the same results.
11
+ */
12
+ export class MongoWriteBatch {
13
+ private readonly operations: mongo.AnyClientBulkWriteModel<mongo.Document>[] = [];
14
+ private readonly deleteChecks: { index: number; check: (count: number) => void }[] = [];
15
+ private readonly fallback: (() => Promise<unknown>)[] = [];
16
+
17
+ private readonly ordered: boolean;
18
+
19
+ constructor(
20
+ private readonly client: mongo.MongoClient,
21
+ private readonly supportsClientBulkWrite: boolean,
22
+ private readonly session: mongo.ClientSession | undefined,
23
+ options: { ordered: boolean }
24
+ ) {
25
+ this.ordered = options.ordered;
26
+ }
27
+
28
+ /**
29
+ * Perform a collection-level bulkWrite.
30
+ *
31
+ * On MongoDB 8.0+, these operations join the client-level bulkWrite with the batch's ordering.
32
+ * The driver may split commands to respect server batch limits.
33
+ *
34
+ * On earlier MongoDB versions, this always uses an unordered collection-level bulkWrite.
35
+ * Operations within this collection bulk must therefore be independent of their order.
36
+ */
37
+ bulkWriteUnordered<T extends mongo.Document>(
38
+ collection: mongo.Collection<T>,
39
+ operations: mongo.AnyBulkWriteOperation<T>[]
40
+ ): void {
41
+ if (operations.length === 0) return;
42
+ if (!this.supportsClientBulkWrite) {
43
+ this.fallback.push(() => collection.bulkWrite(operations, { ordered: false, session: this.session }));
44
+ return;
45
+ }
46
+ for (const operation of operations) {
47
+ // Collection and client models contain the same operation fields, with a
48
+ // different envelope. Keep the schema checked at the collection boundary.
49
+ const [name, model] = Object.entries(operation)[0];
50
+ this.operations.push({ ...model, name, namespace: collection.namespace });
51
+ }
52
+ }
53
+
54
+ insertOne<T extends mongo.Document>(collection: mongo.Collection<T>, document: mongo.OptionalUnlessRequiredId<T>) {
55
+ if (!this.supportsClientBulkWrite) {
56
+ this.fallback.push(() => collection.insertOne(document, { session: this.session }));
57
+ return;
58
+ }
59
+ this.addModel<T>({ name: 'insertOne', namespace: collection.namespace, document: document as mongo.OptionalId<T> });
60
+ }
61
+
62
+ insertMany<T extends mongo.Document>(
63
+ collection: mongo.Collection<T>,
64
+ documents: mongo.OptionalUnlessRequiredId<T>[]
65
+ ) {
66
+ if (documents.length === 0) return;
67
+ if (!this.supportsClientBulkWrite) {
68
+ this.fallback.push(() => collection.insertMany(documents, { session: this.session }));
69
+ return;
70
+ }
71
+ for (const document of documents) {
72
+ this.addModel<T>({
73
+ name: 'insertOne',
74
+ namespace: collection.namespace,
75
+ document: document as mongo.OptionalId<T>
76
+ });
77
+ }
78
+ }
79
+
80
+ deleteMany<T extends mongo.Document>(
81
+ collection: mongo.Collection<T>,
82
+ filter: mongo.Filter<T>,
83
+ checkDeletedCount?: (count: number) => void
84
+ ) {
85
+ if (checkDeletedCount) {
86
+ // Client-bulk results are checked after all writes execute. A failed
87
+ // invariant must roll back the entire batch, not leave later writes visible.
88
+ if (!this.session?.inTransaction()) throw new Error('Checked deletes require a transaction');
89
+ }
90
+ if (!this.supportsClientBulkWrite) {
91
+ this.fallback.push(async () => {
92
+ const result = await collection.deleteMany(filter, { session: this.session });
93
+ checkDeletedCount?.(result.deletedCount);
94
+ });
95
+ return;
96
+ }
97
+ if (checkDeletedCount) {
98
+ this.deleteChecks.push({ index: this.operations.length, check: checkDeletedCount });
99
+ }
100
+ this.addModel<T>({ name: 'deleteMany', namespace: collection.namespace, filter });
101
+ }
102
+
103
+ updateOne<T extends mongo.Document>(
104
+ collection: mongo.Collection<T>,
105
+ filter: mongo.Filter<T>,
106
+ update: mongo.UpdateFilter<T> | mongo.Document[],
107
+ options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
108
+ ) {
109
+ if (!this.supportsClientBulkWrite) {
110
+ this.fallback.push(() => collection.updateOne(filter, update, { ...options, session: this.session }));
111
+ return;
112
+ }
113
+ this.addModel({ name: 'updateOne', namespace: collection.namespace, filter, update, ...options });
114
+ }
115
+
116
+ updateMany<T extends mongo.Document>(
117
+ collection: mongo.Collection<T>,
118
+ filter: mongo.Filter<T>,
119
+ update: mongo.UpdateFilter<T> | mongo.Document[],
120
+ options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
121
+ ) {
122
+ if (!this.supportsClientBulkWrite) {
123
+ this.fallback.push(() => collection.updateMany(filter, update, { ...options, session: this.session }));
124
+ return;
125
+ }
126
+ this.addModel({ name: 'updateMany', namespace: collection.namespace, filter, update, ...options });
127
+ }
128
+
129
+ private addModel<T extends mongo.Document>(operation: mongo.AnyClientBulkWriteModel<T>) {
130
+ // The collection-specific methods validate the schema before erasing it here.
131
+ this.operations.push(operation as unknown as mongo.AnyClientBulkWriteModel<mongo.Document>);
132
+ }
133
+
134
+ /**
135
+ * Execute the queued writes.
136
+ *
137
+ * On MongoDB 8.0+, this uses the client-level bulkWrite API, typically resulting in a single command.
138
+ * The command may be split if there are a large number of operations.
139
+ *
140
+ * On earlier MongoDB versions, this executes each queued operation individually.
141
+ *
142
+ * If no operations were queued, this is a no-op.
143
+ */
144
+ async execute(): Promise<void> {
145
+ if (!this.supportsClientBulkWrite) {
146
+ // Sessions cannot execute concurrent operations within a transaction.
147
+ for (const write of this.fallback) await write();
148
+ return;
149
+ }
150
+ if (this.operations.length === 0) return;
151
+
152
+ try {
153
+ const result = await this.client.bulkWrite(this.operations, {
154
+ session: this.session,
155
+ ordered: this.ordered,
156
+ verboseResults: this.deleteChecks.length > 0
157
+ });
158
+ for (const { index, check } of this.deleteChecks) {
159
+ const deleted = result.deleteResults?.get(index);
160
+ if (deleted == null) throw new Error('Missing client bulk delete result');
161
+ check(deleted.deletedCount);
162
+ }
163
+ } catch (error) {
164
+ // The driver wraps command failures without copying their error labels.
165
+ // Preserve withTransaction's retry handling for e.g. write conflicts.
166
+ if (error instanceof mongo.MongoClientBulkWriteError && error.cause instanceof mongo.MongoError) {
167
+ throw error.cause;
168
+ }
169
+ throw error;
170
+ }
171
+ }
172
+ }