@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/CHANGELOG.md +101 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +41 -16
  6. package/dist/storage/MongoBucketStorage.js +226 -71
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
  9. package/dist/storage/implementation/MongoBucketBatch.js +110 -62
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
  15. package/dist/storage/implementation/MongoCompactor.js +80 -263
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
  21. package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +39 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  46. package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
  47. package/dist/storage/implementation/common/PersistedBatch.js +12 -0
  48. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  49. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  51. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  52. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  53. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  54. package/dist/storage/implementation/db.js +17 -15
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +20 -13
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  62. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  63. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
  65. package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
  66. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  71. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  72. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +10 -1
  77. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  78. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  79. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  81. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
  82. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  84. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  85. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  86. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
  87. package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
  88. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
  90. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
  91. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
  93. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
  94. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
  96. package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +77 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +30 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +101 -21
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
  115. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  126. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  128. package/dist/storage/storage-index.d.ts +4 -4
  129. package/dist/storage/storage-index.js +4 -4
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +49 -3
  132. package/dist/types/types.js +48 -1
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +1 -2
  135. package/dist/utils/test-utils.js +7 -1
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.d.ts +9 -2
  138. package/dist/utils/util.js +128 -4
  139. package/dist/utils/util.js.map +1 -1
  140. package/package.json +12 -19
  141. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  142. package/src/storage/MongoBucketStorage.ts +347 -88
  143. package/src/storage/implementation/MongoBucketBatch.ts +147 -82
  144. package/src/storage/implementation/MongoChecksums.ts +93 -153
  145. package/src/storage/implementation/MongoCompactor.ts +102 -316
  146. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  147. package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
  148. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  149. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  150. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  151. package/src/storage/implementation/MongoStorageProvider.ts +46 -4
  152. package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
  153. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  154. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
  156. package/src/storage/implementation/common/PersistedBatch.ts +19 -3
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  158. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  159. package/src/storage/implementation/db.ts +19 -20
  160. package/src/storage/implementation/models.ts +20 -14
  161. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
  162. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  163. package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
  164. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
  165. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  166. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  167. package/src/storage/implementation/v1/models.ts +11 -1
  168. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  169. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
  170. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  171. package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
  172. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
  173. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
  174. package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
  175. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  176. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
  177. package/src/storage/implementation/v3/bucket-format.ts +97 -0
  178. package/src/storage/implementation/v3/chunking.ts +40 -0
  179. package/src/storage/implementation/v3/models.ts +132 -55
  180. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
  181. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
  184. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  185. package/src/storage/storage-index.ts +12 -4
  186. package/src/types/types.ts +58 -1
  187. package/src/utils/test-utils.ts +8 -3
  188. package/src/utils/util.ts +185 -6
  189. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
  191. package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
  192. package/test/src/helpers/MemoryObjectStorage.ts +53 -0
  193. package/test/src/helpers/s3TestFactory.ts +69 -0
  194. package/test/src/object_storage_lifecycle.test.ts +59 -0
  195. package/test/src/setup.ts +6 -1
  196. package/test/src/storage.test.ts +227 -13
  197. package/test/src/storage_compacting.test.ts +1951 -12
  198. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  199. package/test/src/storage_s3_checksums.test.ts +191 -0
  200. package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
  201. package/test/src/storage_s3_reading.test.ts +526 -0
  202. package/test/src/storage_s3_writing.test.ts +135 -0
  203. package/test/src/storage_sync.test.ts +1180 -73
  204. package/test/src/storeCurrentData.test.ts +227 -0
  205. package/test/src/util.ts +2 -1
  206. package/tsconfig.json +1 -4
  207. package/tsconfig.tsbuildinfo +1 -1
  208. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  209. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  210. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  211. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  212. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  213. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  214. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  215. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  216. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  217. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  218. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  219. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  220. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  221. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  222. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  223. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  224. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  225. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  226. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  227. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  228. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  229. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  230. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  231. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  232. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  233. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  234. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  235. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  236. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  237. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  238. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  239. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -1,69 +1,95 @@
1
1
  import { mongo } from '@powersync/lib-service-mongodb';
2
2
  import { ReplicationAssertionError } from '@powersync/lib-services-framework';
3
- import { InternalOpId, storage } from '@powersync/service-core';
3
+ import { BucketDefinitionMapping, InternalOpId, storage } from '@powersync/service-core';
4
4
  import { BucketDataSource, BucketDefinitionId } from '@powersync/service-sync-rules';
5
5
  import * as bson from 'bson';
6
6
  import { mongoTableId } from '../../../utils/util.js';
7
+ import { BucketDataDoc } from '../common/BucketDataDoc.js';
7
8
  import {
8
9
  BucketStateUpdate,
9
10
  PersistedBatch,
11
+ PersistedBatchOptions,
10
12
  SaveParameterDataOptions,
11
13
  UpsertCurrentDataOptions
12
14
  } from '../common/PersistedBatch.js';
13
- import { SourceTableKey } from '../models.js';
15
+ import { SourceRecordLookupState } from '../common/SourceRecordStore.js';
16
+ import { serializeBucketData } from './bucket-format.js';
17
+ import { chunkBucketData } from './chunking.js';
14
18
  import {
15
- BucketParameterDocumentV3,
19
+ BucketDataDocumentV3,
16
20
  BucketStateDocumentV3,
17
21
  CurrentDataDocumentV3,
18
- serializeBucketDataV3,
22
+ serializeParameterLookup,
19
23
  SourceTableDocumentV3,
20
- taggedBucketParameterDocumentToV3
24
+ taggedBucketParameterDocumentToTagged
21
25
  } from './models.js';
22
- import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js';
26
+ import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
23
27
  import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
24
28
 
25
29
  export class PersistedBatchV3 extends PersistedBatch {
26
- declare protected readonly db: VersionedPowerSyncMongoV3;
27
-
28
30
  currentData: { sourceTableId: bson.ObjectId; operation: mongo.AnyBulkWriteOperation<CurrentDataDocumentV3> }[] = [];
29
31
  sourceTablePendingDeletes = new Map<string, InternalOpId>();
32
+ protected readonly objectStorageLifecycle?: ObjectStorageLifecycle;
33
+
34
+ declare protected readonly db: VersionedPowerSyncMongoV3;
35
+
36
+ constructor(
37
+ db: VersionedPowerSyncMongoV3,
38
+ group_id: number,
39
+ mapping: BucketDefinitionMapping,
40
+ writtenSize: number,
41
+ options?: PersistedBatchOptions
42
+ ) {
43
+ super(db, group_id, mapping, writtenSize, options);
44
+ if (this.objectStorage) {
45
+ this.objectStorageLifecycle = new ObjectStorageLifecycle(this.db, this.group_id, this.objectStorage);
46
+ }
47
+ }
30
48
 
31
- protected checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId {
49
+ // Abstract override from PersistedBatch (V3-specific error message)
50
+
51
+ protected override checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId {
32
52
  if (definitionId == null) {
33
- // This is required for V3 storage.
34
53
  throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled');
35
54
  }
36
55
  return definitionId;
37
56
  }
38
57
 
58
+ // Concrete implementations from PersistedBatchShared
59
+
39
60
  protected getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId {
40
61
  return this.mapping.bucketSourceId(bucketSource);
41
62
  }
42
63
 
43
64
  saveParameterData(data: SaveParameterDataOptions) {
44
65
  const { sourceTable, sourceKey, evaluated } = data;
45
- const remaining_lookups = new Map<string, SaveParameterDataOptions['existing_lookups'][number]>();
66
+ const remaining_lookups = new Map<string, SourceRecordLookupState>();
46
67
 
47
68
  for (let lookup of data.existing_lookups) {
48
69
  if (lookup.indexId == null) {
49
- throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled');
70
+ throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
71
+ }
72
+ if (sourceTable.parameterLookupSourceIds != null && !sourceTable.parameterLookupSourceIds.has(lookup.indexId)) {
73
+ // The parameter index is not active anymore.
74
+ // We don't cleanup these references upfront, but do need to ignore them after the definition is removed.
75
+ continue;
50
76
  }
51
77
  remaining_lookups.set(`${lookup.indexId}.${lookup.lookup.toString('base64')}`, lookup);
52
78
  }
53
79
 
54
80
  for (let result of evaluated) {
55
81
  const sourceDefinitionId = this.mapping.parameterLookupId(result.lookup.source);
56
- const binLookup = serializeParameterLookupV3(result.lookup);
82
+ const binLookup = serializeParameterLookup(result.lookup);
57
83
  remaining_lookups.delete(`${sourceDefinitionId}.${binLookup.toString('base64')}`);
58
84
 
59
85
  const op_id = data.op_seq.next();
60
86
  this.debugLastOpId = op_id;
61
- const values: BucketParameterDocumentV3 = {
87
+ const values = {
62
88
  _id: op_id,
63
89
  key: {
64
90
  t: mongoTableId(sourceTable.id),
65
91
  k: sourceKey
66
- } satisfies SourceTableKey,
92
+ },
67
93
  lookup: binLookup,
68
94
  bucket_parameters: result.bucketParameters
69
95
  };
@@ -80,14 +106,14 @@ export class PersistedBatchV3 extends PersistedBatch {
80
106
  this.debugLastOpId = op_id;
81
107
  const indexId = lookup.indexId;
82
108
  if (indexId == null) {
83
- throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled');
109
+ throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
84
110
  }
85
- const values: BucketParameterDocumentV3 = {
111
+ const values = {
86
112
  _id: op_id,
87
113
  key: {
88
114
  t: mongoTableId(sourceTable.id),
89
115
  k: sourceKey
90
- } satisfies SourceTableKey,
116
+ },
91
117
  lookup: lookup.lookup,
92
118
  bucket_parameters: []
93
119
  };
@@ -125,8 +151,8 @@ export class PersistedBatchV3 extends PersistedBatch {
125
151
  update: {
126
152
  $set: {
127
153
  data: null,
128
- buckets: [] as CurrentDataDocumentV3['buckets'],
129
- lookups: [] as CurrentDataDocumentV3['lookups'],
154
+ buckets: [],
155
+ lookups: [],
130
156
  pending_delete: checkpointGreaterThan
131
157
  }
132
158
  },
@@ -146,7 +172,7 @@ export class PersistedBatchV3 extends PersistedBatch {
146
172
  upsertCurrentData(values: UpsertCurrentDataOptions) {
147
173
  const buckets = values.buckets.map((bucket) => {
148
174
  if (bucket.definitionId == null) {
149
- throw new ReplicationAssertionError('Expected v3 bucket when incrementalReprocessing is enabled');
175
+ throw new ReplicationAssertionError('Expected bucket when incrementalReprocessing is enabled');
150
176
  }
151
177
  return {
152
178
  def: bucket.definitionId,
@@ -157,7 +183,7 @@ export class PersistedBatchV3 extends PersistedBatch {
157
183
  });
158
184
  const lookups = values.lookups.map((lookup) => {
159
185
  if (lookup.indexId == null) {
160
- throw new ReplicationAssertionError('Expected v3 lookup when incrementalReprocessing is enabled');
186
+ throw new ReplicationAssertionError('Expected lookup when incrementalReprocessing is enabled');
161
187
  }
162
188
  return {
163
189
  i: lookup.indexId,
@@ -189,26 +215,81 @@ export class PersistedBatchV3 extends PersistedBatch {
189
215
  return this.currentData.length;
190
216
  }
191
217
 
218
+ // Flush methods
219
+
192
220
  protected async flushBucketData(session: mongo.ClientSession) {
193
- const operationsByDefinition = new Map<BucketDefinitionId, typeof this.bucketData>();
221
+ const operationsByDefinition = new Map<BucketDefinitionId, BucketDataDoc[]>();
194
222
  for (const document of this.bucketData) {
195
223
  const existing = operationsByDefinition.get(document.bucketKey.definitionId) ?? [];
196
224
  existing.push(document);
197
225
  operationsByDefinition.set(document.bucketKey.definitionId, existing);
198
226
  }
199
227
 
200
- for (const [definitionId, documents] of operationsByDefinition.entries()) {
201
- await this.db.bucketDataV3(this.group_id, definitionId).bulkWrite(
202
- documents.map((document) => ({
203
- insertOne: {
204
- document: serializeBucketDataV3(document)
228
+ let uploadCount = 0;
229
+ const plans = Array.from(operationsByDefinition, ([definitionId, documents]) => {
230
+ const operationsByBucket = Map.groupBy(documents, (document) => document.bucketKey.bucket);
231
+ const lifecycle = this.objectStorage
232
+ ? new ObjectStorageLifecycle(this.db, this.group_id, this.objectStorage)
233
+ : undefined;
234
+ const createInserts: (() => Promise<mongo.AnyBulkWriteOperation<BucketDataDocumentV3>>)[] = [];
235
+
236
+ for (const [bucket, ops] of operationsByBucket) {
237
+ for (const chunk of chunkBucketData(ops)) {
238
+ const serialized = serializeBucketData(bucket, chunk);
239
+ if (lifecycle == null || serialized.size <= this.inlineThresholdBytes) {
240
+ createInserts.push(async () => ({
241
+ insertOne: {
242
+ document: serialized
243
+ }
244
+ }));
245
+ continue;
205
246
  }
206
- })),
207
- {
208
- session,
209
- ordered: false
247
+
248
+ uploadCount += 1;
249
+ createInserts.push(async () => {
250
+ const minOp = chunk[0].o;
251
+ const maxOp = chunk[chunk.length - 1].o;
252
+ const { ops: bucketOps, ...metadata } = serialized;
253
+ const path = lifecycle.allocatePath(definitionId, bucket, minOp, maxOp);
254
+ const { fileSize } = await lifecycle.bucketData.store(path, bucketOps!);
255
+ return {
256
+ insertOne: {
257
+ document: {
258
+ ...metadata,
259
+ storage_ref: {
260
+ path,
261
+ file_size: fileSize
262
+ }
263
+ }
264
+ }
265
+ };
266
+ });
210
267
  }
268
+ }
269
+
270
+ return { definitionId, createInserts };
271
+ });
272
+
273
+ const createAllInserts = () =>
274
+ Promise.all(
275
+ plans.map(async ({ definitionId, createInserts }) => ({
276
+ definitionId,
277
+ inserts: await Promise.all(createInserts.map((createInsert) => createInsert()))
278
+ }))
211
279
  );
280
+
281
+ // S3ObjectStorage applies one shared concurrency limit across all callers,
282
+ // so replication can schedule its uploads together without creating a
283
+ // separate limiter here.
284
+ const writes = await createAllInserts();
285
+
286
+ for (const { definitionId, inserts } of writes) {
287
+ if (inserts.length > 0) {
288
+ await this.db.bucketData(this.group_id, definitionId).bulkWrite(inserts, {
289
+ session,
290
+ ordered: false
291
+ });
292
+ }
212
293
  }
213
294
  }
214
295
 
@@ -221,10 +302,10 @@ export class PersistedBatchV3 extends PersistedBatch {
221
302
  }
222
303
 
223
304
  for (const [indexId, documents] of operationsByIndex.entries()) {
224
- await this.db.parameterIndexV3(this.group_id, indexId).bulkWrite(
305
+ await this.db.parameterIndex(this.group_id, indexId).bulkWrite(
225
306
  documents.map((document) => ({
226
307
  insertOne: {
227
- document: taggedBucketParameterDocumentToV3(document)
308
+ document: taggedBucketParameterDocumentToTagged(document)
228
309
  }
229
310
  })),
230
311
  {
@@ -260,12 +341,12 @@ export class PersistedBatchV3 extends PersistedBatch {
260
341
  });
261
342
 
262
343
  if (sourceTableUpdates.length > 0) {
263
- await this.db.sourceTablesV3(this.group_id).bulkWrite(sourceTableUpdates, { session, ordered: false });
344
+ await this.db.sourceTables(this.group_id).bulkWrite(sourceTableUpdates, { session, ordered: false });
264
345
  }
265
346
 
266
347
  for (const operations of operationsBySourceTable.values()) {
267
348
  const sourceTableId = operations[0]!.sourceTableId;
268
- await this.db.sourceRecordsV3(this.group_id, sourceTableId).bulkWrite(
349
+ await this.db.sourceRecords(this.group_id, sourceTableId).bulkWrite(
269
350
  operations.map((entry) => entry.operation),
270
351
  {
271
352
  session,
@@ -276,7 +357,7 @@ export class PersistedBatchV3 extends PersistedBatch {
276
357
  }
277
358
 
278
359
  protected async flushBucketStates(session: mongo.ClientSession) {
279
- await this.db.bucketStateV3(this.group_id).bulkWrite(this.getBucketStateUpdates(), {
360
+ await this.db.bucketState(this.group_id).bulkWrite(this.getBucketStateUpdates(), {
280
361
  session,
281
362
  ordered: false
282
363
  });
@@ -1,21 +1,19 @@
1
1
  import * as lib_mongo from '@powersync/lib-service-mongodb';
2
2
  import { mongo } from '@powersync/lib-service-mongodb';
3
3
  import { Logger } from '@powersync/lib-services-framework';
4
- import { storage } from '@powersync/service-core';
4
+ import { BucketDefinitionMapping, storage } from '@powersync/service-core';
5
5
  import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
6
6
  import * as bson from 'bson';
7
7
  import { retryOnMongoMaxTimeMSExpired } from '../../../utils/util.js';
8
- import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js';
9
8
  import { cacheKey } from '../OperationBatch.js';
10
9
  import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordStore } from '../common/SourceRecordStore.js';
11
- import { serializeParameterLookupV3 } from './MongoParameterLookupV3.js';
12
10
  import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
13
- import { CurrentDataDocumentV3, SourceTableDocumentV3 } from './models.js';
11
+ import { CurrentDataDocumentV3, serializeParameterLookup, SourceTableDocumentV3 } from './models.js';
14
12
 
15
13
  export class SourceRecordStoreV3 implements SourceRecordStore {
16
14
  constructor(
17
15
  private readonly db: VersionedPowerSyncMongoV3,
18
- private readonly groupId: number,
16
+ private readonly replicationStreamId: number,
19
17
  private readonly mapping: BucketDefinitionMapping
20
18
  ) {}
21
19
 
@@ -31,7 +29,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
31
29
  mapParameterLookups(paramEvaluated: EvaluatedParameters[]): LoadedSourceRecord['lookups'] {
32
30
  return paramEvaluated.map((entry) => ({
33
31
  indexId: this.mapping.parameterLookupId(entry.lookup.source),
34
- lookup: serializeParameterLookupV3(entry.lookup)
32
+ lookup: serializeParameterLookup(entry.lookup)
35
33
  }));
36
34
  }
37
35
 
@@ -67,7 +65,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
67
65
  _id: { $in: replicaIds as any[] }
68
66
  } as unknown as mongo.Filter<CurrentDataDocumentV3>;
69
67
  const sizeCursor: mongo.AggregationCursor<CurrentDataDocumentV3 & { size: number }> = this.db
70
- .sourceRecordsV3(this.groupId, sourceTableId)
68
+ .sourceRecords(this.replicationStreamId, sourceTableId)
71
69
  .aggregate(
72
70
  [
73
71
  {
@@ -100,7 +98,9 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
100
98
  const filter = {
101
99
  _id: { $in: replicaIds as any[] }
102
100
  } as unknown as mongo.Filter<CurrentDataDocumentV3>;
103
- const cursor = this.db.sourceRecordsV3(this.groupId, sourceTableId).find(filter, { session, projection });
101
+ const cursor = this.db
102
+ .sourceRecords(this.replicationStreamId, sourceTableId)
103
+ .find(filter, { session, projection });
104
104
  for await (const doc of cursor.stream()) {
105
105
  const loaded = this.createLoadedDocument(
106
106
  sourceTableId,
@@ -120,7 +120,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
120
120
  sourceTableId: bson.ObjectId,
121
121
  limit: number
122
122
  ): Promise<LoadedSourceRecord[]> {
123
- const cursor = this.db.sourceRecordsV3(this.groupId, sourceTableId).find(
123
+ const cursor = this.db.sourceRecords(this.replicationStreamId, sourceTableId).find(
124
124
  {
125
125
  pending_delete: { $exists: false }
126
126
  },
@@ -145,7 +145,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
145
145
  // ones have dirty deletes in source_tables.
146
146
 
147
147
  const dirtySourceTables = await this.db
148
- .sourceTablesV3(this.groupId)
148
+ .sourceTables(this.replicationStreamId)
149
149
  .find(
150
150
  {
151
151
  latest_pending_delete: { $exists: true }
@@ -159,7 +159,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
159
159
  let deletedCount = 0;
160
160
  const sourceTableUpdates: mongo.AnyBulkWriteOperation<SourceTableDocumentV3>[] = [];
161
161
  for (const sourceTable of dirtySourceTables) {
162
- const collection = this.db.sourceRecordsV3(this.groupId, sourceTable._id);
162
+ const collection = this.db.sourceRecords(this.replicationStreamId, sourceTable._id);
163
163
  const result = await this.deletePendingDeletes(collection, sourceTable._id, lastCheckpoint, logger);
164
164
  deletedCount += result.deletedCount;
165
165
 
@@ -182,7 +182,7 @@ export class SourceRecordStoreV3 implements SourceRecordStore {
182
182
  }
183
183
 
184
184
  if (sourceTableUpdates.length > 0) {
185
- await this.db.sourceTablesV3(this.groupId).bulkWrite(sourceTableUpdates, { ordered: false });
185
+ await this.db.sourceTables(this.replicationStreamId).bulkWrite(sourceTableUpdates, { ordered: false });
186
186
  }
187
187
  if (deletedCount > 0) {
188
188
  logger.info(`Cleaned up ${deletedCount} pending delete current_data records for checkpoint ${lastCheckpoint}`);
@@ -7,58 +7,90 @@ import {
7
7
  BucketParameterDocumentV3,
8
8
  BucketStateDocumentV3,
9
9
  CurrentDataDocumentV3,
10
+ ObjectStorageDeletionMarker,
10
11
  SourceTableDocumentV3,
11
12
  SyncConfigDefinition
12
13
  } from './models.js';
13
14
 
14
15
  export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
16
+ constructor(
17
+ upstream: ConstructorParameters<typeof BaseVersionedPowerSyncMongo>[0],
18
+ storageConfig: ConstructorParameters<typeof BaseVersionedPowerSyncMongo>[1]
19
+ ) {
20
+ super(upstream, storageConfig);
21
+ }
22
+
15
23
  get syncConfigDefinitions() {
16
24
  return this.db.collection<SyncConfigDefinition>('sync_config');
17
25
  }
18
26
 
19
- sourceRecordsV3(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection<CurrentDataDocumentV3> {
27
+ sourceRecords(replicationStreamId: number, sourceTableId: mongo.ObjectId): mongo.Collection<CurrentDataDocumentV3> {
20
28
  const collectionName = this.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
21
29
  return this.db.collection<CurrentDataDocumentV3>(collectionName);
22
30
  }
23
31
 
24
- async listSourceRecordCollectionsV3(replicationStreamId: number): Promise<mongo.Collection<CurrentDataDocumentV3>[]> {
32
+ async listSourceRecordCollections(replicationStreamId: number): Promise<mongo.Collection<CurrentDataDocumentV3>[]> {
25
33
  return this.listCollectionsByPrefix<CurrentDataDocumentV3>(`source_records_${replicationStreamId}_`);
26
34
  }
27
35
 
28
- async initializeSourceRecordsCollection(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
29
- await this.sourceRecordsV3(replicationStreamId, sourceTableId).createIndex(
36
+ async initializeSourceRecordsCollection(
37
+ replicationStreamId: number,
38
+ sourceTableId: mongo.ObjectId,
39
+ session?: mongo.ClientSession
40
+ ) {
41
+ await this.sourceRecords(replicationStreamId, sourceTableId).createIndex(
30
42
  {
31
43
  pending_delete: 1
32
44
  },
33
45
  {
34
46
  partialFilterExpression: { pending_delete: { $exists: true } },
35
- name: 'pending_delete'
47
+ name: 'pending_delete',
48
+ session
36
49
  }
37
50
  );
38
51
  }
39
52
 
40
53
  commonSourceTables(replicationStreamId: number): mongo.Collection<CommonSourceTableDocument> {
41
- return this.sourceTablesV3(replicationStreamId) as mongo.Collection<CommonSourceTableDocument>;
54
+ return this.sourceTables(replicationStreamId) as any as mongo.Collection<CommonSourceTableDocument>;
42
55
  }
43
56
 
44
- bucketStateV3(replicationStreamId: number): mongo.Collection<BucketStateDocumentV3> {
57
+ bucketState(replicationStreamId: number): mongo.Collection<BucketStateDocumentV3> {
45
58
  return this.db.collection(`bucket_state_${replicationStreamId}`);
46
59
  }
47
60
 
48
- parameterIndexV3(
49
- replicationStreamId: number,
50
- indexId: ParameterIndexId
51
- ): mongo.Collection<BucketParameterDocumentV3> {
61
+ parameterIndex(replicationStreamId: number, indexId: ParameterIndexId): mongo.Collection<BucketParameterDocumentV3> {
52
62
  return this.db.collection(`parameter_index_${replicationStreamId}_${indexId}`);
53
63
  }
54
64
 
55
- sourceTablesV3(replicationStreamId: number): mongo.Collection<SourceTableDocumentV3> {
65
+ sourceTables(replicationStreamId: number): mongo.Collection<SourceTableDocumentV3> {
56
66
  return this.db.collection<SourceTableDocumentV3>(this.sourceTableCollectionName(replicationStreamId));
57
67
  }
58
68
 
69
+ bucketData(replicationStreamId: number, definitionId: BucketDefinitionId): mongo.Collection<BucketDataDocumentV3> {
70
+ return this.db.collection<BucketDataDocumentV3>(`bucket_data_${replicationStreamId}_${definitionId}`);
71
+ }
72
+
73
+ pendingObjectStorageDeletes(replicationStreamId: number): mongo.Collection<ObjectStorageDeletionMarker> {
74
+ return this.db.collection<ObjectStorageDeletionMarker>(`pending_object_storage_deletes_${replicationStreamId}`);
75
+ }
76
+
77
+ listBucketDataCollections(replicationStreamId: number) {
78
+ return this.listCollectionsByPrefix(`bucket_data_${replicationStreamId}_`);
79
+ }
80
+
81
+ async listParameterIndexCollections(replicationStreamId: number) {
82
+ const prefix = `parameter_index_${replicationStreamId}_`;
83
+ const collections = await this.listCollectionsByPrefix(prefix);
84
+
85
+ return collections.map((collection) => ({
86
+ collection,
87
+ indexId: collection.collectionName.slice(prefix.length)
88
+ }));
89
+ }
90
+
59
91
  async initializeStreamStorage(replicationStreamId: number) {
60
- const sourceTables = this.sourceTablesV3(replicationStreamId);
61
- const bucketState = this.bucketStateV3(replicationStreamId);
92
+ const sourceTables = this.sourceTables(replicationStreamId);
93
+ const bucketState = this.bucketState(replicationStreamId);
62
94
  await sourceTables.createIndex(
63
95
  {
64
96
  connection_id: 1,
@@ -92,26 +124,4 @@ export class VersionedPowerSyncMongoV3 extends BaseVersionedPowerSyncMongo {
92
124
  { name: 'dirty_count' }
93
125
  );
94
126
  }
95
-
96
- bucketDataV3(replicationStreamId: number, definitionId: BucketDefinitionId) {
97
- return this.db.collection<BucketDataDocumentV3>(`bucket_data_${replicationStreamId}_${definitionId}`);
98
- }
99
-
100
- listBucketDataCollectionsV3(replicationStreamId: number) {
101
- return this.upstream.listBucketDataCollectionsV3(replicationStreamId);
102
- }
103
-
104
- async listParameterIndexCollectionsV3(
105
- replicationStreamId: number
106
- ): Promise<{ collection: mongo.Collection<BucketParameterDocumentV3>; indexId: ParameterIndexId }[]> {
107
- const prefix = `parameter_index_${replicationStreamId}_`;
108
- const collections = await this.db.listCollections({ name: new RegExp(`^${prefix}`) }, { nameOnly: true }).toArray();
109
-
110
- return collections
111
- .filter((collection) => collection.name.startsWith(prefix))
112
- .map((collection) => ({
113
- collection: this.db.collection<BucketParameterDocumentV3>(collection.name),
114
- indexId: collection.name.slice(prefix.length)
115
- }));
116
- }
117
127
  }
@@ -0,0 +1,97 @@
1
+ import { ServiceAssertionError } from '@powersync/lib-services-framework';
2
+ import { bson, InternalOpId } from '@powersync/service-core';
3
+ import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
4
+ import { BucketDataDocumentV3, BucketOperation } from './models.js';
5
+
6
+ /**
7
+ * Serialize a non-empty list of operations in strictly ascending `o` order.
8
+ * Preserving that order establishes the {@link BucketDataDocumentV3} range
9
+ * invariants used by metadata-only queries.
10
+ */
11
+ export function serializeBucketData(
12
+ bucket: string,
13
+ operations: BucketDataDoc[],
14
+ options?: { targetOp?: InternalOpId | null }
15
+ ): BucketDataDocumentV3 {
16
+ const minOp = operations[0].o;
17
+ const maxOp = operations[operations.length - 1].o;
18
+
19
+ let totalChecksum = 0n;
20
+ let maxTargetOp: InternalOpId | null = options?.targetOp ?? null;
21
+ let hasClearOp = false;
22
+
23
+ const ops: BucketOperation[] = operations.map((op) => {
24
+ totalChecksum += op.checksum;
25
+
26
+ if (op.op == 'CLEAR') {
27
+ hasClearOp = true;
28
+ }
29
+
30
+ return {
31
+ o: op.o,
32
+ op: op.op,
33
+ source_table: op.source_table,
34
+ source_key: op.source_key,
35
+ table: op.table,
36
+ row_id: op.row_id,
37
+ checksum: op.checksum,
38
+ data: op.data
39
+ };
40
+ });
41
+
42
+ const size = bson.calculateObjectSize(ops);
43
+
44
+ return {
45
+ _id: {
46
+ b: bucket,
47
+ o: maxOp
48
+ },
49
+ min_op: minOp,
50
+ checksum: totalChecksum,
51
+ count: operations.length,
52
+ size,
53
+ target_op: maxTargetOp,
54
+ ...(hasClearOp ? { has_clear_op: true as const } : {}),
55
+ ops
56
+ };
57
+ }
58
+
59
+ export function* loadBucketDataDocument(
60
+ context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>,
61
+ doc: BucketDataDocumentV3
62
+ ): Generator<BucketDataDoc> {
63
+ const { _id, ops } = doc;
64
+ if (!ops) {
65
+ throw new ServiceAssertionError(
66
+ `Missing ops array on BucketDataDocumentV3 at _id.o=${_id.o}. Callers must patch doc.ops from S3 before calling this function.`
67
+ );
68
+ }
69
+ const bucketKey = {
70
+ ...context,
71
+ bucket: _id.b
72
+ };
73
+
74
+ for (const op of ops) {
75
+ yield {
76
+ bucketKey,
77
+ o: op.o,
78
+ op: op.op,
79
+ source_table: op.source_table,
80
+ source_key: op.source_key,
81
+ table: op.table,
82
+ row_id: op.row_id,
83
+ checksum: op.checksum,
84
+ data: op.data
85
+ };
86
+ }
87
+ }
88
+
89
+ export function maxOpId(a: InternalOpId | null | undefined, b: InternalOpId | null | undefined): InternalOpId | null {
90
+ if (a == null) {
91
+ return b ?? null;
92
+ }
93
+ if (b == null) {
94
+ return a ?? null;
95
+ }
96
+ return a > b ? a : b;
97
+ }
@@ -0,0 +1,40 @@
1
+ import { bson } from '@powersync/service-core';
2
+ import { BucketDataDoc } from '../common/BucketDataDoc.js';
3
+
4
+ export const DEFAULT_MAX_DOC_SIZE_BYTES = 1024 * 1024; // 1MB
5
+
6
+ /**
7
+ * Split an array of bucket data operations into chunks where each chunk's
8
+ * total data size stays close to `maxDocSizeBytes`.
9
+ *
10
+ * The threshold is a target, not a hard ceiling — a single operation whose
11
+ * `data` field exceeds the limit is placed in its own chunk and is NOT split.
12
+ */
13
+ export function chunkBucketData(
14
+ operations: BucketDataDoc[],
15
+ maxDocSizeBytes = DEFAULT_MAX_DOC_SIZE_BYTES
16
+ ): BucketDataDoc[][] {
17
+ const chunks: BucketDataDoc[][] = [];
18
+ let currentChunk: BucketDataDoc[] = [];
19
+ let currentSize = 0;
20
+
21
+ for (const op of operations) {
22
+ // Add 5 to account for array index up to 4 digits (9999) plus null byte.
23
+ const opSize = bson.calculateObjectSize(op) + 5;
24
+
25
+ if (currentSize + opSize > maxDocSizeBytes && currentChunk.length > 0) {
26
+ chunks.push(currentChunk);
27
+ currentChunk = [];
28
+ currentSize = 0;
29
+ }
30
+
31
+ currentChunk.push(op);
32
+ currentSize += opSize;
33
+ }
34
+
35
+ if (currentChunk.length > 0) {
36
+ chunks.push(currentChunk);
37
+ }
38
+
39
+ return chunks;
40
+ }