@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/CHANGELOG.md +129 -7
  2. package/dist/storage/MongoBucketStorage.d.ts +42 -5
  3. package/dist/storage/MongoBucketStorage.js +404 -215
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
  6. package/dist/storage/implementation/MongoBucketBatch.js +169 -98
  7. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  8. package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
  9. package/dist/storage/implementation/MongoCompactor.js +27 -238
  10. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  11. package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
  12. package/dist/storage/implementation/MongoIdSequence.js +14 -1
  13. package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
  14. package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
  15. package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
  16. package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
  17. package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
  18. package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
  19. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  20. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
  21. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
  22. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
  23. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
  24. package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
  25. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
  26. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
  27. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
  28. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
  29. package/dist/storage/implementation/MongoStorageProvider.js +8 -4
  30. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  31. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
  32. package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
  33. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
  35. package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
  36. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  37. package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
  38. package/dist/storage/implementation/MongoWriteBatch.js +149 -0
  39. package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
  44. package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
  45. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
  46. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
  47. package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
  48. package/dist/storage/implementation/common/PersistedBatch.js +57 -14
  49. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
  51. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
  52. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  53. package/dist/storage/implementation/db.d.ts +9 -1
  54. package/dist/storage/implementation/db.js +28 -1
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +35 -0
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
  62. package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
  63. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
  65. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
  66. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
  71. package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
  72. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/models.js +1 -0
  74. package/dist/storage/implementation/v1/models.js.map +1 -1
  75. package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
  76. package/dist/storage/implementation/v3/CompactionLease.js +131 -0
  77. package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
  78. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
  79. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  81. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
  82. package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
  83. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  84. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
  85. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
  86. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
  87. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
  88. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
  90. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
  91. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  93. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
  94. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
  95. package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
  96. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
  98. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  99. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
  100. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  102. package/dist/storage/implementation/v3/bucket-format.js +2 -0
  103. package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
  104. package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
  105. package/dist/storage/implementation/v3/compact-utils.js +223 -0
  106. package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
  107. package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
  108. package/dist/storage/implementation/v3/compaction-constants.js +6 -0
  109. package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +90 -6
  111. package/dist/storage/implementation/v3/models.js.map +1 -1
  112. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
  115. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
  126. package/dist/storage/implementation/v3/source-table-utils.js +49 -57
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
  128. package/dist/storage/storage-index.d.ts +2 -1
  129. package/dist/storage/storage-index.js +1 -0
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +28 -1
  132. package/dist/types/types.js +28 -7
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +4 -2
  135. package/dist/utils/test-utils.js +7 -4
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.js +3 -1
  138. package/dist/utils/util.js.map +1 -1
  139. package/package.json +10 -9
  140. package/src/storage/MongoBucketStorage.ts +545 -277
  141. package/src/storage/implementation/MongoBucketBatch.ts +258 -124
  142. package/src/storage/implementation/MongoCompactor.ts +37 -346
  143. package/src/storage/implementation/MongoIdSequence.ts +21 -1
  144. package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
  145. package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
  146. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
  147. package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
  148. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
  149. package/src/storage/implementation/MongoStorageProvider.ts +16 -4
  150. package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
  151. package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
  152. package/src/storage/implementation/MongoWriteBatch.ts +172 -0
  153. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
  154. package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
  156. package/src/storage/implementation/common/PersistedBatch.ts +76 -20
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
  158. package/src/storage/implementation/db.ts +36 -2
  159. package/src/storage/implementation/models.ts +39 -0
  160. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
  161. package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
  162. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
  163. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
  164. package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
  165. package/src/storage/implementation/v1/models.ts +1 -0
  166. package/src/storage/implementation/v3/CompactionLease.ts +152 -0
  167. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
  168. package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
  169. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
  170. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
  171. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
  172. package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
  173. package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
  174. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
  175. package/src/storage/implementation/v3/bucket-format.ts +2 -0
  176. package/src/storage/implementation/v3/compact-utils.ts +329 -0
  177. package/src/storage/implementation/v3/compaction-constants.ts +5 -0
  178. package/src/storage/implementation/v3/models.ts +97 -4
  179. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
  180. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
  181. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
  184. package/src/storage/implementation/v3/source-table-utils.ts +82 -76
  185. package/src/storage/storage-index.ts +2 -0
  186. package/src/types/types.ts +42 -7
  187. package/src/utils/test-utils.ts +9 -5
  188. package/src/utils/util.ts +3 -1
  189. package/test/src/__snapshots__/storage.test.ts.snap +25 -0
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
  191. package/test/src/chunk_compaction_config.test.ts +38 -0
  192. package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
  193. package/test/src/compact-utils.test.ts +251 -0
  194. package/test/src/concurrent_writers.test.ts +599 -0
  195. package/test/src/connection_config.test.ts +159 -0
  196. package/test/src/helpers/MemoryObjectStorage.ts +14 -6
  197. package/test/src/helpers/concurrentWriter.mjs +29 -0
  198. package/test/src/helpers/s3TestFactory.ts +12 -12
  199. package/test/src/object_storage_usage.test.ts +268 -0
  200. package/test/src/parameter_compacting_v1.test.ts +120 -0
  201. package/test/src/parameter_compacting_v3.test.ts +192 -0
  202. package/test/src/parameter_compaction_fence.test.ts +276 -0
  203. package/test/src/storage.test.ts +280 -24
  204. package/test/src/storage_compacting.test.ts +1097 -253
  205. package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
  206. package/test/src/storage_s3_checksums.test.ts +6 -7
  207. package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
  208. package/test/src/storage_s3_reading.test.ts +294 -6
  209. package/test/src/storage_s3_writing.test.ts +4 -3
  210. package/test/src/storage_sync.test.ts +412 -80
  211. package/test/src/storeCurrentData.test.ts +5 -5
  212. package/test/src/util.ts +1 -1
  213. package/tsconfig.tsbuildinfo +1 -1
@@ -0,0 +1,38 @@
1
+ import { MongoStorageConfig, normalizeChunkCompactionConcurrency } from '@module/types/types.js';
2
+ import { describe, expect, test } from 'vitest';
3
+
4
+ describe('chunk compaction concurrency configuration', () => {
5
+ test('defaults to two workers', () => {
6
+ expect(normalizeChunkCompactionConcurrency(undefined)).toBe(2);
7
+ });
8
+
9
+ test('defaults to four workers with object storage', () => {
10
+ expect(normalizeChunkCompactionConcurrency(undefined, true)).toBe(4);
11
+ });
12
+
13
+ test.each([false, true])('preserves an explicit override with object storage %s', (hasObjectStorage) => {
14
+ expect(normalizeChunkCompactionConcurrency(3, hasObjectStorage)).toBe(3);
15
+ });
16
+
17
+ test('decodes a configured worker count', () => {
18
+ const config = MongoStorageConfig.decode({
19
+ type: 'mongodb',
20
+ uri: 'mongodb://localhost:27017/powersync',
21
+ chunk_compaction_concurrency: 4
22
+ });
23
+ expect(normalizeChunkCompactionConcurrency(config.chunk_compaction_concurrency)).toBe(4);
24
+ });
25
+
26
+ test.each([1, 64])('accepts boundary worker count %s', (value) => {
27
+ expect(normalizeChunkCompactionConcurrency(value)).toBe(value);
28
+ });
29
+
30
+ test.each([0, -1, 1.5, 65, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1])(
31
+ 'rejects invalid worker count %s',
32
+ (value) => {
33
+ expect(() => normalizeChunkCompactionConcurrency(value)).toThrow(
34
+ 'storage.chunk_compaction_concurrency must be an integer between 1 and 64'
35
+ );
36
+ }
37
+ );
38
+ });
@@ -2,7 +2,7 @@ import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/Mong
2
2
  import { ObjectStorageLifecycle } from '@module/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js';
3
3
  import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
4
4
  import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
5
- import { test_utils } from '@powersync/service-core-tests';
5
+ import { getTestStorage, test_utils } from '@powersync/service-core-tests';
6
6
  import * as bson from 'bson';
7
7
  import { describe, expect, test } from 'vitest';
8
8
  import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
@@ -78,7 +78,7 @@ streams:
78
78
  { storageVersion: 3 }
79
79
  )
80
80
  );
81
- const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3;
81
+ const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3;
82
82
  const db = firstStorage.db as VersionedPowerSyncMongoV3;
83
83
  await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
84
84
 
@@ -155,7 +155,7 @@ streams:
155
155
  );
156
156
  expect(second.replicationStreamId).toBe(first.replicationStreamId);
157
157
  const replicatingStreams = await factory.getReplicatingReplicationStreams();
158
- const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3;
158
+ const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3;
159
159
  await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
160
160
  await secondWriter.markAllSnapshotDone('2/1');
161
161
  await secondWriter.commit('2/1');
@@ -218,7 +218,7 @@ streams:
218
218
  { storageVersion: 3 }
219
219
  )
220
220
  );
221
- const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3;
221
+ const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3;
222
222
  const db = firstStorage.db as VersionedPowerSyncMongoV3;
223
223
  await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
224
224
  const resolved = await firstWriter.resolveTables({
@@ -266,7 +266,7 @@ streams:
266
266
  expect(secondDefinitionId).toBe(projectDefinitionId);
267
267
 
268
268
  const replicatingStreams = await factory.getReplicatingReplicationStreams();
269
- const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3;
269
+ const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3;
270
270
  await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
271
271
  await secondWriter.markAllSnapshotDone('2/1');
272
272
  await secondWriter.commit('2/1');
@@ -289,7 +289,10 @@ streams:
289
289
  expect(await collectionExists(db, ownerBucketDataCollection)).toBe(false);
290
290
  expect(await collectionExists(db, sourceRecordsCollection)).toBe(true);
291
291
 
292
- const activeStorage = (await factory.getActiveSyncConfig())!.storage as MongoSyncBucketStorageV3;
292
+ const activeStorage = (await getTestStorage(
293
+ factory,
294
+ (await factory.getActiveSyncConfig())!.replicationStream
295
+ )) as MongoSyncBucketStorageV3;
293
296
  await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
294
297
  const activeTable = (
295
298
  await activeWriter.resolveTables({
@@ -324,7 +327,7 @@ streams:
324
327
  test('drops current_data when a table becomes event-only but is kept by a live event', async () => {
325
328
  await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
326
329
 
327
- // The first config syncs `todos` data and also fires events for it.
330
+ // The first config syncs `todos` data and also evaluates the audit event for its rows.
328
331
  const first = await factory.updateSyncRules(
329
332
  updateSyncRulesFromYaml(
330
333
  `
@@ -343,7 +346,7 @@ event_definitions:
343
346
  { storageVersion: 3 }
344
347
  )
345
348
  );
346
- const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3;
349
+ const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3;
347
350
  const db = firstStorage.db as VersionedPowerSyncMongoV3;
348
351
  await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
349
352
  const resolved = await firstWriter.resolveTables({
@@ -360,15 +363,22 @@ event_definitions:
360
363
  },
361
364
  afterReplicaId: test_utils.rid('todo-1')
362
365
  });
366
+ const eventDefinitionId = [...resolved.tables[0].eventDefinitionIds!][0];
367
+ firstWriter.addCustomWriteCheckpoint({
368
+ user_id: 'user-1',
369
+ checkpoint: 1n,
370
+ event_id: eventDefinitionId
371
+ });
363
372
  await firstWriter.markAllSnapshotDone('1/1');
364
373
  await firstWriter.commit('1/1');
365
374
 
366
375
  const sourceTableId = resolved.tables[0].id as bson.ObjectId;
367
376
  const sourceRecordsCollection = db.sourceRecords(first.replicationStreamId, sourceTableId).collectionName;
368
377
  const ownerDefinitionId = first.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
369
- expect(
370
- (await db.sourceTables(first.replicationStreamId).findOne({ _id: sourceTableId }))?.bucket_data_source_ids
371
- ).toEqual([ownerDefinitionId]);
378
+ const initialSourceTable = await db.sourceTables(first.replicationStreamId).findOne({ _id: sourceTableId });
379
+ expect(initialSourceTable?.bucket_data_source_ids).toEqual([ownerDefinitionId]);
380
+ expect(initialSourceTable?.event_definition_ids).toHaveLength(1);
381
+ const eventDefinitionIds = initialSourceTable!.event_definition_ids;
372
382
  // The prior snapshot is persisted in current_data.
373
383
  expect(await collectionExists(db, sourceRecordsCollection)).toBe(true);
374
384
 
@@ -395,7 +405,7 @@ event_definitions:
395
405
  expect(second.replicationStreamId).toBe(first.replicationStreamId);
396
406
 
397
407
  const replicatingStreams = await factory.getReplicatingReplicationStreams();
398
- const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3;
408
+ const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3;
399
409
  await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
400
410
  await secondWriter.markAllSnapshotDone('2/1');
401
411
  await secondWriter.commit('2/1');
@@ -414,13 +424,22 @@ event_definitions:
414
424
  sourceTablesDeleted: 0
415
425
  });
416
426
 
417
- // The source table row survives (the live event still needs it), narrowed to empty
418
- // memberships, but its now-unused current_data collection is dropped.
427
+ // The source table row survives (the live event still needs it), narrowed to empty data and
428
+ // parameter memberships, but its event membership remains and current_data is dropped.
419
429
  const sourceTable = await db.sourceTables(first.replicationStreamId).findOne({ _id: sourceTableId });
420
430
  expect(sourceTable).not.toBeNull();
421
431
  expect(sourceTable?.bucket_data_source_ids).toEqual([]);
422
432
  expect(sourceTable?.parameter_lookup_source_ids).toEqual([]);
433
+ expect(sourceTable?.event_definition_ids).toEqual(eventDefinitionIds);
423
434
  expect(await collectionExists(db, sourceRecordsCollection)).toBe(false);
435
+ await expect(
436
+ db
437
+ .customCheckpointRequests({
438
+ replicationStreamId: first.replicationStreamId,
439
+ eventId: eventDefinitionId
440
+ })
441
+ .findOne({ user_id: 'user-1' })
442
+ ).resolves.not.toBeNull();
424
443
 
425
444
  const streamDoc = (await db.sync_rules.findOne({ _id: first.replicationStreamId })) as ReplicationStreamDocumentV3;
426
445
  expect(streamDoc.sync_configs.map((config) => config.state)).toEqual([storage.SyncRuleState.ACTIVE]);
@@ -429,9 +448,9 @@ event_definitions:
429
448
  test('cleans up event-only source tables no longer triggered by a live sync config', async () => {
430
449
  await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
431
450
 
432
- // The first config syncs `todos` and additionally fires events for `audit_log`. The
433
- // `audit_log` table is referenced only by the event trigger, so its source table carries
434
- // empty membership arrays (an event-only table).
451
+ // The first config syncs `todos` and additionally evaluates the audit event for `audit_log`. The
452
+ // `audit_log` table is referenced only by the event trigger, so its source table has only an
453
+ // event membership (an event-only table).
435
454
  const first = await factory.updateSyncRules(
436
455
  updateSyncRulesFromYaml(
437
456
  `
@@ -450,7 +469,7 @@ event_definitions:
450
469
  { storageVersion: 3 }
451
470
  )
452
471
  );
453
- const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3;
472
+ const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3;
454
473
  const db = firstStorage.db as VersionedPowerSyncMongoV3;
455
474
  await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
456
475
 
@@ -485,21 +504,27 @@ event_definitions:
485
504
  },
486
505
  afterReplicaId: test_utils.rid('audit-1')
487
506
  });
507
+ const auditEventDefinitionId = [...auditTable.eventDefinitionIds!][0];
508
+ firstWriter.addCustomWriteCheckpoint({
509
+ user_id: 'audit-user',
510
+ checkpoint: 1n,
511
+ event_id: auditEventDefinitionId
512
+ });
488
513
  await firstWriter.markAllSnapshotDone('1/1');
489
514
  await firstWriter.commit('1/1');
490
515
 
491
516
  const todosTableId = todosTable.id as bson.ObjectId;
492
517
  const auditTableId = auditTable.id as bson.ObjectId;
493
- // The event-only table is persisted with empty membership arrays, so the membership filter
494
- // never selects it.
518
+ // The event-only table has empty bucket and parameter memberships, plus its event id.
495
519
  const auditSourceTable = await db.sourceTables(first.replicationStreamId).findOne({ _id: auditTableId });
496
520
  expect(auditSourceTable?.bucket_data_source_ids).toEqual([]);
497
521
  expect(auditSourceTable?.parameter_lookup_source_ids).toEqual([]);
522
+ expect(auditSourceTable?.event_definition_ids).toHaveLength(1);
498
523
  const auditRecordsCollection = db.sourceRecords(first.replicationStreamId, auditTableId).collectionName;
499
524
 
500
525
  // The second (active) config keeps the same `by_owner` stream but drops the audit event.
501
- // The shared bucket definition stays in use, so no bucket/parameter ids become unused and
502
- // the membership-cleanup block is skipped entirely.
526
+ // The shared bucket definition stays in use, while the unused event id selects this table for
527
+ // membership cleanup.
503
528
  const second = await factory.updateSyncRules(
504
529
  updateSyncRulesFromYaml(
505
530
  `
@@ -516,7 +541,7 @@ streams:
516
541
  expect(second.replicationStreamId).toBe(first.replicationStreamId);
517
542
 
518
543
  const replicatingStreams = await factory.getReplicatingReplicationStreams();
519
- const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3;
544
+ const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3;
520
545
  await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
521
546
  await secondWriter.markAllSnapshotDone('2/1');
522
547
  await secondWriter.commit('2/1');
@@ -537,6 +562,11 @@ streams:
537
562
  // The orphaned event-only table and its source records are removed...
538
563
  expect(await db.sourceTables(first.replicationStreamId).countDocuments({ _id: auditTableId })).toBe(0);
539
564
  expect(await collectionExists(db, auditRecordsCollection)).toBe(false);
565
+ const auditCheckpointCollection = db.customCheckpointRequests({
566
+ replicationStreamId: first.replicationStreamId,
567
+ eventId: auditEventDefinitionId
568
+ }).collectionName;
569
+ expect(await collectionExists(db, auditCheckpointCollection)).toBe(false);
540
570
  // ...while the data table still used by the live config is retained.
541
571
  expect(await db.sourceTables(first.replicationStreamId).countDocuments({ _id: todosTableId })).toBe(1);
542
572
 
@@ -572,7 +602,7 @@ streams:
572
602
  { storageVersion: 3 }
573
603
  )
574
604
  );
575
- const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3;
605
+ const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3;
576
606
  const db = firstStorage.db as VersionedPowerSyncMongoV3;
577
607
  await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS);
578
608
  const resolved = await firstWriter.resolveTables({
@@ -625,7 +655,7 @@ streams:
625
655
  expect(secondIndexId).toBe(roleIndexId);
626
656
 
627
657
  const replicatingStreams = await factory.getReplicatingReplicationStreams();
628
- const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3;
658
+ const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3;
629
659
  await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS);
630
660
  await secondWriter.markAllSnapshotDone('2/1');
631
661
  await secondWriter.commit('2/1');
@@ -637,7 +667,10 @@ streams:
637
667
  expect(await collectionExists(db, orgParameterIndexCollection)).toBe(false);
638
668
  expect(await collectionExists(db, sourceRecordsCollection)).toBe(true);
639
669
 
640
- const activeStorage = (await factory.getActiveSyncConfig())!.storage as MongoSyncBucketStorageV3;
670
+ const activeStorage = (await getTestStorage(
671
+ factory,
672
+ (await factory.getActiveSyncConfig())!.replicationStream
673
+ )) as MongoSyncBucketStorageV3;
641
674
  await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS);
642
675
  const activeTable = (
643
676
  await activeWriter.resolveTables({
@@ -0,0 +1,251 @@
1
+ import {
2
+ applyCompactionDelta,
3
+ applyStatsReplacement,
4
+ bucketStats,
5
+ chooseCompactionKind,
6
+ chooseRequestedCompactionKind,
7
+ combineAdjacentStats,
8
+ combineChunkStats,
9
+ CompactIntervalConfig,
10
+ CompactionKind,
11
+ emptyBucketStats,
12
+ fullCompactionCheckAt,
13
+ statsForDocument,
14
+ statsForDocuments
15
+ } from '@module/storage/implementation/v3/compact-utils.js';
16
+ import { BucketStateDocumentV3 } from '@module/storage/implementation/v3/models.js';
17
+ import { describe, expect, test } from 'vitest';
18
+
19
+ const INTERVAL_CONFIG: CompactIntervalConfig = {
20
+ minCompactChunkIntervalMs: 1_000,
21
+ minCompactFullIntervalMs: 10_000,
22
+ maxCompactFullIntervalMs: 100_000
23
+ };
24
+
25
+ function bucketState(overrides: Partial<BucketStateDocumentV3> = {}): BucketStateDocumentV3 {
26
+ return {
27
+ _id: { d: 'definition', b: 'bucket[]' },
28
+ last_op: 10n,
29
+ next_compact_check: new Date(0),
30
+ first_uncompacted_write: new Date(0),
31
+ bucket_stats: { count: 10, bytes: 100n, chunks: 1 },
32
+ ...overrides
33
+ };
34
+ }
35
+
36
+ function compactedState(overrides: Partial<NonNullable<BucketStateDocumentV3['compacted_state']>> = {}) {
37
+ return {
38
+ op_id: 5n,
39
+ checksum: 50n,
40
+ count: 5,
41
+ bytes: 50n,
42
+ chunks: 1,
43
+ at: new Date(0),
44
+ ...overrides
45
+ };
46
+ }
47
+
48
+ describe('V3 compact utilities', () => {
49
+ test.each([
50
+ {
51
+ name: 'uses the minimum interval before the first full compact',
52
+ state: bucketState(),
53
+ expected: new Date(10_000)
54
+ },
55
+ {
56
+ name: 'scales the interval by the uncompacted row ratio',
57
+ state: bucketState({
58
+ bucket_stats: { count: 100, bytes: 100n, chunks: 1 },
59
+ last_full_compact: { op_id: 90n, count: 90, puts: 90, at: new Date(0) }
60
+ }),
61
+ expected: new Date(100_000)
62
+ },
63
+ {
64
+ name: 'uses the maximum interval when there are no uncompacted rows',
65
+ state: bucketState({
66
+ bucket_stats: { count: 100, bytes: 100n, chunks: 1 },
67
+ last_full_compact: { op_id: 10n, count: 100, puts: 100, at: new Date(0) }
68
+ }),
69
+ expected: new Date(100_000)
70
+ }
71
+ ])('$name', ({ state, expected }) => {
72
+ expect(fullCompactionCheckAt(state, INTERVAL_CONFIG)).toEqual(expected);
73
+ });
74
+
75
+ test('rejects a scheduled bucket without an uncompacted write', () => {
76
+ expect(() => fullCompactionCheckAt(bucketState({ first_uncompacted_write: undefined }), INTERVAL_CONFIG)).toThrow(
77
+ 'Scheduled V3 bucket bucket[] has no first uncompacted write'
78
+ );
79
+ });
80
+
81
+ test('chooses a full compact when its check is due', () => {
82
+ const decision = chooseCompactionKind(
83
+ bucketState({ bucket_stats: { count: 10, bytes: 100n, chunks: 10 } }),
84
+ new Date(10_000),
85
+ INTERVAL_CONFIG
86
+ );
87
+
88
+ expect(decision.kind).toBe(CompactionKind.Full);
89
+ expect(decision.nextCompactCheck).toEqual({
90
+ $min: [new Date(70_000), { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: 1_000 } }]
91
+ });
92
+ });
93
+
94
+ test('chooses a chunk compact after enough chunks and the minimum interval', () => {
95
+ const decision = chooseCompactionKind(
96
+ bucketState({
97
+ bucket_stats: { count: 10, bytes: 100n, chunks: 9 },
98
+ compacted_state: compactedState({ chunks: 1 })
99
+ }),
100
+ new Date(5_000),
101
+ INTERVAL_CONFIG
102
+ );
103
+
104
+ expect(decision.kind).toBe(CompactionKind.Chunks);
105
+ });
106
+
107
+ test('waits when the chunk interval has not elapsed', () => {
108
+ const decision = chooseCompactionKind(
109
+ bucketState({
110
+ bucket_stats: { count: 10, bytes: 100n, chunks: 9 },
111
+ compacted_state: compactedState({ chunks: 1, at: new Date(4_500) })
112
+ }),
113
+ new Date(5_000),
114
+ INTERVAL_CONFIG
115
+ );
116
+
117
+ expect(decision.kind).toBeNull();
118
+ });
119
+
120
+ test('schedules only the full check when too few chunks were added', () => {
121
+ const decision = chooseCompactionKind(
122
+ bucketState({ bucket_stats: { count: 7, bytes: 70n, chunks: 7 } }),
123
+ new Date(5_000),
124
+ INTERVAL_CONFIG
125
+ );
126
+
127
+ expect(decision).toEqual({ kind: null, nextCompactCheck: new Date(70_000) });
128
+ });
129
+
130
+ test.each([
131
+ {
132
+ name: 'does not force an unspecified kind',
133
+ requestedKind: undefined,
134
+ compactedOpId: undefined,
135
+ maxOpIdCap: undefined,
136
+ expected: null
137
+ },
138
+ {
139
+ name: 'forces work without previous compacted state',
140
+ requestedKind: CompactionKind.Chunks,
141
+ compactedOpId: undefined,
142
+ maxOpIdCap: undefined,
143
+ expected: CompactionKind.Chunks
144
+ },
145
+ {
146
+ name: 'skips work already compacted through the cap',
147
+ requestedKind: CompactionKind.Chunks,
148
+ compactedOpId: 5n,
149
+ maxOpIdCap: 5n,
150
+ expected: null
151
+ },
152
+ {
153
+ name: 'skips work already compacted through the bucket head',
154
+ requestedKind: CompactionKind.Chunks,
155
+ compactedOpId: 10n,
156
+ maxOpIdCap: undefined,
157
+ expected: null
158
+ },
159
+ {
160
+ name: 'forces work beyond the compacted state',
161
+ requestedKind: CompactionKind.Chunks,
162
+ compactedOpId: 5n,
163
+ maxOpIdCap: 6n,
164
+ expected: CompactionKind.Chunks
165
+ }
166
+ ])('$name', ({ requestedKind, compactedOpId, maxOpIdCap, expected }) => {
167
+ const state = bucketState({
168
+ bucket_stats: { count: 10, bytes: 100n, chunks: 9 },
169
+ compacted_state: compactedOpId == null ? undefined : compactedState({ op_id: compactedOpId })
170
+ });
171
+ expect(chooseRequestedCompactionKind(state, requestedKind, { maxOpIdCap })).toBe(expected);
172
+ });
173
+
174
+ test.each([undefined, compactedState({ at: new Date() })])(
175
+ 'initial chunk compaction requires eight new chunks, regardless of the interval: %o',
176
+ (compacted) => {
177
+ for (const newChunks of [1, 7, 8]) {
178
+ const state = bucketState({
179
+ bucket_stats: { count: 10, bytes: 100n, chunks: (compacted?.chunks ?? 0) + newChunks },
180
+ compacted_state: compacted
181
+ });
182
+ expect(chooseRequestedCompactionKind(state, CompactionKind.Chunks, { maxOpIdCap: undefined })).toBe(
183
+ newChunks < 8 ? null : CompactionKind.Chunks
184
+ );
185
+ }
186
+ }
187
+ );
188
+
189
+ test('the testing override bypasses the chunk threshold but preserves the compaction watermark', () => {
190
+ const state = bucketState();
191
+ const config = { maxOpIdCap: undefined, forceChunkCompaction: true };
192
+ expect(chooseRequestedCompactionKind(state, CompactionKind.Chunks, config)).toBe(CompactionKind.Chunks);
193
+ expect(chooseRequestedCompactionKind(state, undefined, config)).toBeNull();
194
+ state.compacted_state = compactedState({ op_id: state.last_op });
195
+ expect(chooseRequestedCompactionKind(state, CompactionKind.Chunks, config)).toBeNull();
196
+ });
197
+
198
+ test('derives and combines bucket statistics', () => {
199
+ const state = bucketState({ bucket_stats: { count: 12, bytes: 120n, chunks: 3 } });
200
+ expect(bucketStats(state)).toEqual({ count: 12, bytes: 120n, chunks: 3 });
201
+ expect(emptyBucketStats()).toEqual({ count: 0, bytes: 0n, chunks: 0, checksum: 0 });
202
+ expect(statsForDocument({ count: 2, size: 20, checksum: 10n })).toEqual({
203
+ count: 2,
204
+ bytes: 20n,
205
+ chunks: 1,
206
+ checksum: 10
207
+ });
208
+ expect(
209
+ statsForDocuments([
210
+ { count: 2, size: 20, checksum: 10n },
211
+ { count: 3, size: 30, checksum: 20n }
212
+ ])
213
+ ).toEqual({ count: 5, bytes: 50n, chunks: 2, checksum: 30 });
214
+ expect(
215
+ combineAdjacentStats(
216
+ { count: 2, bytes: 20n, chunks: 1, checksum: 10 },
217
+ { count: 3, bytes: 30n, chunks: 2, checksum: 20 }
218
+ )
219
+ ).toEqual({ count: 5, bytes: 50n, chunks: 3, checksum: 30 });
220
+ });
221
+
222
+ test('replaces an overlapping chunk in cached statistics', () => {
223
+ expect(
224
+ combineChunkStats(
225
+ compactedState({ count: 10, bytes: 100n, chunks: 4, checksum: 100n }),
226
+ { count: 6, bytes: 60n, chunks: 2, checksum: 50 },
227
+ { count: 3, bytes: 30n, chunks: 1, checksum: 30 }
228
+ )
229
+ ).toEqual({ count: 13, bytes: 130n, chunks: 5, checksum: 120 });
230
+ });
231
+
232
+ test('applies compacted statistics as a delta to total bucket statistics', () => {
233
+ expect(
234
+ applyCompactionDelta(
235
+ { count: 20, bytes: 200n, chunks: 8 },
236
+ { count: 5, bytes: 50n, chunks: 3, checksum: 10 },
237
+ { count: 3, bytes: 30n, chunks: 1, checksum: 10 }
238
+ )
239
+ ).toEqual({ count: 18, bytes: 180n, chunks: 6 });
240
+ });
241
+
242
+ test('applies a stored range replacement to statistics including its checksum', () => {
243
+ expect(
244
+ applyStatsReplacement(
245
+ { count: 20, bytes: 200n, chunks: 8, checksum: 100 },
246
+ { count: 5, bytes: 50n, chunks: 3, checksum: 40 },
247
+ { count: 3, bytes: 30n, chunks: 1, checksum: 20 }
248
+ )
249
+ ).toEqual({ count: 18, bytes: 180n, chunks: 6, checksum: 80 });
250
+ });
251
+ });