@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/CHANGELOG.md +101 -5
  2. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
  3. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
  4. package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
  5. package/dist/storage/MongoBucketStorage.d.ts +41 -16
  6. package/dist/storage/MongoBucketStorage.js +226 -71
  7. package/dist/storage/MongoBucketStorage.js.map +1 -1
  8. package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
  9. package/dist/storage/implementation/MongoBucketBatch.js +110 -62
  10. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  11. package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
  12. package/dist/storage/implementation/MongoChecksums.js +36 -121
  13. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  14. package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
  15. package/dist/storage/implementation/MongoCompactor.js +80 -263
  16. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  17. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  18. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  19. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  20. package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
  21. package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
  22. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  23. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  24. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  25. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  27. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  28. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  29. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  30. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  31. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  32. package/dist/storage/implementation/MongoStorageProvider.js +39 -4
  33. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
  35. package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
  36. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  38. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  39. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  40. package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
  41. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  43. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
  44. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  45. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  46. package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
  47. package/dist/storage/implementation/common/PersistedBatch.js +12 -0
  48. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  49. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  50. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  51. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  52. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  53. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  54. package/dist/storage/implementation/db.js +17 -15
  55. package/dist/storage/implementation/db.js.map +1 -1
  56. package/dist/storage/implementation/models.d.ts +20 -13
  57. package/dist/storage/implementation/models.js.map +1 -1
  58. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  59. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
  60. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  61. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
  62. package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
  63. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  64. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
  65. package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
  66. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
  67. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
  68. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
  69. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  70. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
  71. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
  72. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
  73. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  74. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  75. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  76. package/dist/storage/implementation/v1/models.d.ts +10 -1
  77. package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
  78. package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
  79. package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
  80. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  81. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
  82. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  83. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
  84. package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
  85. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  86. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
  87. package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
  88. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  89. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
  90. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
  91. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  92. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
  93. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
  94. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  95. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
  96. package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
  97. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  98. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  99. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  100. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  101. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
  102. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
  103. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  104. package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
  105. package/dist/storage/implementation/v3/bucket-format.js +77 -0
  106. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  107. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  108. package/dist/storage/implementation/v3/chunking.js +30 -0
  109. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  110. package/dist/storage/implementation/v3/models.d.ts +101 -21
  111. package/dist/storage/implementation/v3/models.js +11 -28
  112. package/dist/storage/implementation/v3/models.js.map +1 -1
  113. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
  114. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
  115. package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
  116. package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
  117. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
  118. package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
  119. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
  120. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
  121. package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
  122. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
  123. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
  124. package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
  125. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  126. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  127. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  128. package/dist/storage/storage-index.d.ts +4 -4
  129. package/dist/storage/storage-index.js +4 -4
  130. package/dist/storage/storage-index.js.map +1 -1
  131. package/dist/types/types.d.ts +49 -3
  132. package/dist/types/types.js +48 -1
  133. package/dist/types/types.js.map +1 -1
  134. package/dist/utils/test-utils.d.ts +1 -2
  135. package/dist/utils/test-utils.js +7 -1
  136. package/dist/utils/test-utils.js.map +1 -1
  137. package/dist/utils/util.d.ts +9 -2
  138. package/dist/utils/util.js +128 -4
  139. package/dist/utils/util.js.map +1 -1
  140. package/package.json +12 -19
  141. package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
  142. package/src/storage/MongoBucketStorage.ts +347 -88
  143. package/src/storage/implementation/MongoBucketBatch.ts +147 -82
  144. package/src/storage/implementation/MongoChecksums.ts +93 -153
  145. package/src/storage/implementation/MongoCompactor.ts +102 -316
  146. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  147. package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
  148. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  149. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  150. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  151. package/src/storage/implementation/MongoStorageProvider.ts +46 -4
  152. package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
  153. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  154. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
  155. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
  156. package/src/storage/implementation/common/PersistedBatch.ts +19 -3
  157. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  158. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  159. package/src/storage/implementation/db.ts +19 -20
  160. package/src/storage/implementation/models.ts +20 -14
  161. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
  162. package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
  163. package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
  164. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
  165. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
  166. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  167. package/src/storage/implementation/v1/models.ts +11 -1
  168. package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
  169. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
  170. package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
  171. package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
  172. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
  173. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
  174. package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
  175. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  176. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
  177. package/src/storage/implementation/v3/bucket-format.ts +97 -0
  178. package/src/storage/implementation/v3/chunking.ts +40 -0
  179. package/src/storage/implementation/v3/models.ts +132 -55
  180. package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
  181. package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
  182. package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
  183. package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
  184. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  185. package/src/storage/storage-index.ts +12 -4
  186. package/src/types/types.ts +58 -1
  187. package/src/utils/test-utils.ts +8 -3
  188. package/src/utils/util.ts +185 -6
  189. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  190. package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
  191. package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
  192. package/test/src/helpers/MemoryObjectStorage.ts +53 -0
  193. package/test/src/helpers/s3TestFactory.ts +69 -0
  194. package/test/src/object_storage_lifecycle.test.ts +59 -0
  195. package/test/src/setup.ts +6 -1
  196. package/test/src/storage.test.ts +227 -13
  197. package/test/src/storage_compacting.test.ts +1951 -12
  198. package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
  199. package/test/src/storage_s3_checksums.test.ts +191 -0
  200. package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
  201. package/test/src/storage_s3_reading.test.ts +526 -0
  202. package/test/src/storage_s3_writing.test.ts +135 -0
  203. package/test/src/storage_sync.test.ts +1180 -73
  204. package/test/src/storeCurrentData.test.ts +227 -0
  205. package/test/src/util.ts +2 -1
  206. package/tsconfig.json +1 -4
  207. package/tsconfig.tsbuildinfo +1 -1
  208. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  209. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  210. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  211. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  212. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  213. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  214. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  215. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  216. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  217. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  218. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  219. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  220. package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
  221. package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
  222. package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
  223. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  224. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  225. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  226. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  227. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  228. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  229. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
  230. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
  231. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
  232. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  233. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  234. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  235. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  236. package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
  237. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  238. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
  239. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
@@ -1,15 +1,25 @@
1
- import { InternalOpId, SerializedSyncPlan, SyncRuleState } from '@powersync/service-core';
2
- import { BucketDefinitionId, ParameterIndexId } from '@powersync/service-sync-rules';
1
+ import {
2
+ InternalOpId,
3
+ PersistedDefinitionMapping,
4
+ SerializedSyncPlan,
5
+ SyncRuleState,
6
+ deserializeParameterLookup as deserializeParameterLookupCore
7
+ } from '@powersync/service-core';
8
+ import {
9
+ BucketDefinitionId,
10
+ ParameterIndexId,
11
+ ScopedParameterLookup,
12
+ SqliteJsonValue
13
+ } from '@powersync/service-sync-rules';
3
14
  import * as bson from 'bson';
4
- import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
5
15
  import {
6
- BucketDataDocumentBase,
7
16
  BucketDataKey,
8
17
  BucketParameterDocumentBase,
9
18
  BucketStateDocumentBase,
10
19
  CurrentBucket,
20
+ OpType,
11
21
  ReplicaId,
12
- SourceTableDocument,
22
+ SourceTableDocumentSnapshotStatus,
13
23
  SourceTableKey,
14
24
  SyncRuleCheckpointFields,
15
25
  SyncRuleDocumentBase,
@@ -19,7 +29,7 @@ import {
19
29
  /**
20
30
  * Embedded in sync_rules.sync_configs.
21
31
  */
22
- export interface SyncRuleConfigStateV3 extends SyncRuleCheckpointFields<bigint | null> {
32
+ export interface SyncRuleConfigStateV3 extends SyncRuleCheckpointFields {
23
33
  _id: bson.ObjectId;
24
34
 
25
35
  /**
@@ -47,6 +57,31 @@ export interface ReplicationStreamDocumentV3 extends SyncRuleDocumentBase {
47
57
  * but the model allows multiple configs in any state.
48
58
  */
49
59
  sync_configs: SyncRuleConfigStateV3[];
60
+
61
+ /**
62
+ * The monotonic head of the stream's op sequence: the highest op id persisted to bucket data,
63
+ * whether or not yet covered by a checkpoint.
64
+ *
65
+ * This is shared across all sync configs of the stream (they share the global op sequence).
66
+ * It is never cleared, only `$max`-advanced. A newly-appended config that replicates nothing
67
+ * adopts this value as its checkpoint rather than starting at 0.
68
+ *
69
+ * Stored as a mongo Long, nullable.
70
+ */
71
+ last_persisted_op?: bigint | null;
72
+
73
+ /**
74
+ * The stream's replication position: all source changes up to this LSN have been processed,
75
+ * and the resulting ops persisted. Replication resumes from here.
76
+ *
77
+ * Like {@link last_persisted_op}, this is shared across all sync configs of the stream -
78
+ * per-config last_checkpoint_lsn values are consistency markers, not replication positions.
79
+ *
80
+ * Set via setResumeLsn() (snapshot start, and per-batch progress during streaming), and
81
+ * advanced on every commit/keepalive - including checkpoint-blocked ones, since commit
82
+ * flushes first and blocking only delays consistency markers, not data persistence.
83
+ */
84
+ resume_lsn?: string | null;
50
85
  }
51
86
 
52
87
  /**
@@ -70,16 +105,7 @@ export interface SyncConfigDefinition {
70
105
  content: string;
71
106
  serialized_plan?: SerializedSyncPlan | null;
72
107
 
73
- rule_mapping: {
74
- /**
75
- * Map of uniqueName -> id, unique per replication stream.
76
- */
77
- definitions: Record<string, string>;
78
- /**
79
- * Map of (lookupName, queryId) -> id, unique per replication stream.
80
- */
81
- parameter_indexes: Record<string, string>;
82
- };
108
+ rule_mapping: PersistedDefinitionMapping;
83
109
  }
84
110
 
85
111
  export interface CurrentBucketV3 extends CurrentBucket {
@@ -108,50 +134,26 @@ export interface BucketParameterDocumentV3 extends BucketParameterDocumentBase<S
108
134
 
109
135
  export type BucketDataKeyV3 = BucketDataKey;
110
136
 
111
- export interface BucketDataDocumentV3 extends BucketDataDocumentBase {
112
- _id: BucketDataKeyV3;
113
- }
114
-
115
- export function serializeBucketDataV3(document: BucketDataDoc): BucketDataDocumentV3 {
116
- const { bucketKey, o } = document;
117
- return {
118
- _id: {
119
- b: bucketKey.bucket,
120
- o: o
121
- },
122
- // List fields directly, so that we don't accidentally persist any unknown fields
123
- op: document.op,
124
- source_table: document.source_table,
125
- source_key: document.source_key,
126
- table: document.table,
127
- row_id: document.row_id,
128
- checksum: document.checksum,
129
- data: document.data,
130
- target_op: document.target_op
131
- };
132
- }
133
-
134
- export function loadBucketDataDocumentV3(
135
- context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>,
136
- doc: BucketDataDocumentV3
137
- ): BucketDataDoc {
138
- const { _id, ...rest } = doc;
139
- return {
140
- bucketKey: {
141
- ...context,
142
- bucket: _id.b
143
- },
144
- o: _id.o,
145
- ...rest
146
- };
147
- }
148
-
149
137
  export function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3 {
150
138
  const { index: _index, ...rest } = document;
151
139
  return rest as BucketParameterDocumentV3;
152
140
  }
153
141
 
154
- export interface SourceTableDocumentV3 extends SourceTableDocument {
142
+ export interface ReplicaIdColumn {
143
+ name: string;
144
+ type_oid?: number;
145
+ type?: string;
146
+ }
147
+
148
+ export interface SourceTableDocumentV3 {
149
+ _id: bson.ObjectId;
150
+ connection_id: number;
151
+ relation_id: number | string | undefined;
152
+ schema_name: string;
153
+ table_name: string;
154
+ replica_id_columns: ReplicaIdColumn[];
155
+ snapshot_done: boolean;
156
+ snapshot_status: SourceTableDocumentSnapshotStatus | undefined;
155
157
  bucket_data_source_ids: BucketDefinitionId[];
156
158
  parameter_lookup_source_ids: ParameterIndexId[];
157
159
  latest_pending_delete?: InternalOpId | undefined;
@@ -162,3 +164,78 @@ export interface BucketStateDocumentV3 extends BucketStateDocumentBase {
162
164
  d: BucketDefinitionId;
163
165
  };
164
166
  }
167
+
168
+ export interface BucketOperation {
169
+ o: bigint;
170
+ op: OpType;
171
+ source_table?: bson.ObjectId;
172
+ source_key?: ReplicaId;
173
+ table?: string;
174
+ row_id?: string;
175
+ checksum: bigint;
176
+ data: string | null;
177
+ }
178
+
179
+ export interface StorageRef {
180
+ path: string;
181
+ file_size: number;
182
+ }
183
+
184
+ /** An S3 object that may be deleted once its grace period has elapsed. */
185
+ export interface ObjectStorageDeletionMarker {
186
+ _id: bson.ObjectId;
187
+ path: string;
188
+ delete_after: Date;
189
+ }
190
+
191
+ /**
192
+ * A non-empty chunk of a bucket's ordered operation stream.
193
+ *
194
+ * All writers maintain these invariants:
195
+ *
196
+ * - `ops` is non-empty and strictly ordered by `o`.
197
+ * - `min_op` is `ops[0].o` and `_id.o` is `ops[ops.length - 1].o`.
198
+ * - Documents for the same bucket have ordered, disjoint operation ranges,
199
+ * although gaps between ranges are allowed.
200
+ * - `checksum` is the sum of every operation checksum and `count` is
201
+ * `ops.length`.
202
+ *
203
+ * Metadata-only checksum and data queries rely on these invariants to identify
204
+ * boundary-straddling documents without reading `ops`.
205
+ */
206
+ export interface BucketDataDocumentV3 {
207
+ _id: BucketDataKey;
208
+ min_op: bigint;
209
+ checksum: bigint;
210
+ count: number;
211
+ size: number;
212
+ /**
213
+ * The greatest operation boundary that influenced this document's contents.
214
+ * Bucket-data reads propagate it so serving an earlier checkpoint can be
215
+ * invalidated when compaction changed the operations being returned.
216
+ */
217
+ target_op?: bigint | null;
218
+ ops?: BucketOperation[];
219
+ storage_ref?: StorageRef;
220
+ /**
221
+ * Present (and always true) when this document contains a CLEAR operation.
222
+ * In that case, this is the first document in the bucket: all preceding
223
+ * documents have been removed.
224
+ */
225
+ has_clear_op?: true;
226
+ }
227
+
228
+ export function serializeParameterLookup(lookup: ScopedParameterLookup): bson.Binary {
229
+ return new bson.Binary(bson.serialize({ l: lookup.values.slice(2) }));
230
+ }
231
+
232
+ export function deserializeParameterLookup(lookup: bson.Binary, indexId: ParameterIndexId): SqliteJsonValue[] {
233
+ return [indexId, '', ...deserializeParameterLookupCore(lookup)];
234
+ }
235
+
236
+ export function taggedBucketParameterDocumentToTagged(
237
+ document: TaggedBucketParameterDocument
238
+ ): BucketParameterDocumentV3 {
239
+ const { index: _index, ...rest } = document;
240
+ return rest as BucketParameterDocumentV3;
241
+ }
@@ -0,0 +1,54 @@
1
+ import { storage } from '@powersync/service-core';
2
+ import * as bson from 'bson';
3
+ import { BucketDataDocumentV3, BucketOperation } from '../models.js';
4
+ import { ObjectStorage } from './ObjectStorage.js';
5
+
6
+ export class BucketDataObjectStorage {
7
+ constructor(private readonly storage: ObjectStorage) {}
8
+
9
+ async store(path: string, ops: BucketOperation[]): Promise<{ fileSize: number }> {
10
+ const bsonBuffer = bson.serialize({ ops });
11
+ await this.storage.put(path, bsonBuffer, { contentType: 'application/bson', contentEncoding: null });
12
+ return { fileSize: bsonBuffer.byteLength };
13
+ }
14
+
15
+ async retrieve(path: string, options: { signal?: AbortSignal }): Promise<BucketOperation[]> {
16
+ const { data, metadata } = await this.storage.get(path, { signal: options.signal });
17
+ if (metadata.contentEncoding != null) {
18
+ throw new Error(`Unexpected content encoding: ${metadata.contentEncoding}`);
19
+ }
20
+ if (metadata.contentType !== 'application/bson') {
21
+ throw new Error(`Unexpected content type: ${metadata.contentType}`);
22
+ }
23
+ const wrapper = bson.deserialize(data, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS);
24
+ return wrapper.ops;
25
+ }
26
+
27
+ async delete(paths: string[]): Promise<void> {
28
+ return this.storage.delete(paths);
29
+ }
30
+ }
31
+
32
+ /**
33
+ * Load offloaded operations and patch them onto their MongoDB metadata documents.
34
+ * The object-storage implementation is responsible for limiting request
35
+ * concurrency across all callers.
36
+ */
37
+ export async function hydrateBucketDataDocuments(
38
+ documents: BucketDataDocumentV3[],
39
+ objectStorage: ObjectStorage | undefined,
40
+ options: { signal?: AbortSignal }
41
+ ): Promise<void> {
42
+ if (!objectStorage) {
43
+ return;
44
+ }
45
+
46
+ options.signal?.throwIfAborted();
47
+ const store = new BucketDataObjectStorage(objectStorage);
48
+ const storedDocuments = documents.filter((document) => document.storage_ref);
49
+ await Promise.all(
50
+ storedDocuments.map(async (document) => {
51
+ document.ops = await store.retrieve(document.storage_ref!.path, { signal: options.signal });
52
+ })
53
+ );
54
+ }
@@ -0,0 +1,29 @@
1
+ export interface ObjectStoragePutMetadata {
2
+ contentType: string;
3
+ contentEncoding: string | null;
4
+ }
5
+
6
+ export class ObjectStorageError extends Error {
7
+ readonly retryable: boolean;
8
+
9
+ constructor(message: string, options: { cause: unknown; retryable: boolean }) {
10
+ super(message, { cause: options.cause });
11
+ this.name = 'ObjectStorageError';
12
+ this.retryable = options.retryable;
13
+ }
14
+ }
15
+
16
+ export function isRetryableObjectStorageError(error: unknown): error is ObjectStorageError {
17
+ return error instanceof ObjectStorageError && error.retryable;
18
+ }
19
+
20
+ export interface ObjectStorage {
21
+ put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata): Promise<void>;
22
+ get(
23
+ path: string,
24
+ options?: { signal?: AbortSignal }
25
+ ): Promise<{ data: Uint8Array; metadata: ObjectStoragePutMetadata }>;
26
+ list(prefix: string, options?: { signal?: AbortSignal }): AsyncIterable<string>;
27
+ delete(paths: string[]): Promise<void>;
28
+ deletePrefix(prefix: string, options?: { signal?: AbortSignal }): Promise<{ objectCount: number }>;
29
+ }
@@ -0,0 +1,171 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+ import { Logger } from '@powersync/lib-services-framework';
3
+ import * as bson from 'bson';
4
+ import { createHash, randomUUID } from 'node:crypto';
5
+ import { ObjectStorageDeletionMarker } from '../models.js';
6
+ import { VersionedPowerSyncMongoV3 } from '../VersionedPowerSyncMongoV3.js';
7
+ import { BucketDataObjectStorage } from './BucketDataObjectStorage.js';
8
+ import { ObjectStorage } from './ObjectStorage.js';
9
+
10
+ export const OBJECT_STORAGE_UPLOAD_LEASE_MS = 60 * 60 * 1000;
11
+ export const OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS = 2 * 60 * 1000;
12
+ export const OBJECT_STORAGE_REFERENCE_GRACE_MS = 15 * 60 * 1000;
13
+ /**
14
+ * Protect against arbitrary-long bucket names and parameters exceeding the S3 key length limits.
15
+ *
16
+ * Any bucket name over this length is hashed.
17
+ */
18
+ const MAX_READABLE_BUCKET_SEGMENT_BYTES = 128;
19
+
20
+ export interface PreparedObjectStorageUpload {
21
+ markerId: bson.ObjectId;
22
+ path: string;
23
+ deleteAfter: Date;
24
+ }
25
+
26
+ /**
27
+ * Coordinates the MongoDB outbox used to make object-storage changes recoverable.
28
+ *
29
+ * A MongoDB collection is used to track files that will be uploaded, or should be deleted.
30
+ * By using a MongoDB collection, we can use the same transaction that creates or removes
31
+ * references to these files, making the operations atomic.
32
+ *
33
+ * These markers indicate which files should be deleted - either by being orphaned during create,
34
+ * or after references to them are removed.
35
+ */
36
+ export class ObjectStorageLifecycle {
37
+ readonly bucketData: BucketDataObjectStorage;
38
+
39
+ constructor(
40
+ private readonly db: VersionedPowerSyncMongoV3,
41
+ private readonly replicationStreamId: number,
42
+ private readonly objectStorage: ObjectStorage
43
+ ) {
44
+ this.bucketData = new BucketDataObjectStorage(objectStorage);
45
+ }
46
+
47
+ streamPrefix(): string {
48
+ return `bucket-data/${this.replicationStreamId}/`;
49
+ }
50
+
51
+ definitionPrefix(definitionId: string): string {
52
+ return `${this.streamPrefix()}${definitionId}/`;
53
+ }
54
+
55
+ allocatePath(definitionId: string, bucket: string, minOp: bigint, maxOp: bigint): string {
56
+ // Preserve readable paths for normal buckets, but bound user-controlled
57
+ // parameter values so they cannot consume the S3 key-length budget.
58
+ const escapedBucket = encodeURIComponent(bucket);
59
+ const bucketSegment =
60
+ Buffer.byteLength(escapedBucket, 'utf8') <= MAX_READABLE_BUCKET_SEGMENT_BYTES
61
+ ? escapedBucket
62
+ : `sha256-${createHash('sha256').update(bucket).digest('base64url')}`;
63
+ return `${this.definitionPrefix(definitionId)}${bucketSegment}/${minOp}-${maxOp}-${randomUUID()}.bson`;
64
+ }
65
+
66
+ /**
67
+ * Indicate that files will be uploaded to S3.
68
+ *
69
+ * These markers are expected to be removed using publishUploads(). If that does not happen within
70
+ * the grace period, the files will be deleted.
71
+ */
72
+ async prepareUploads(paths: string[], now = new Date()): Promise<PreparedObjectStorageUpload[]> {
73
+ const deleteAfter = new Date(now.getTime() + OBJECT_STORAGE_UPLOAD_LEASE_MS);
74
+ const uploads = paths.map((path) => ({ markerId: new bson.ObjectId(), path, deleteAfter }));
75
+ if (uploads.length) {
76
+ await this.db.pendingObjectStorageDeletes(this.replicationStreamId).insertMany(
77
+ uploads.map((upload) => ({
78
+ _id: upload.markerId,
79
+ path: upload.path,
80
+ delete_after: upload.deleteAfter
81
+ }))
82
+ );
83
+ }
84
+ return uploads;
85
+ }
86
+
87
+ canPublish(upload: PreparedObjectStorageUpload, now = new Date()): boolean {
88
+ return now.getTime() < upload.deleteAfter.getTime() - OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS;
89
+ }
90
+
91
+ /**
92
+ * Remove markers for files that have finished uploading, and are now referenced from other documents.
93
+ *
94
+ * Call this in the same transaction as the one creating the references.
95
+ */
96
+ async publishUploads(uploads: PreparedObjectStorageUpload[], session: mongo.ClientSession): Promise<void> {
97
+ for (const upload of uploads) {
98
+ if (!this.canPublish(upload)) {
99
+ throw new Error(`Object storage publication lease expired for ${upload.path}`);
100
+ }
101
+ }
102
+ if (uploads.length === 0) {
103
+ return;
104
+ }
105
+
106
+ const result = await this.db
107
+ .pendingObjectStorageDeletes(this.replicationStreamId)
108
+ .deleteMany({ _id: { $in: uploads.map((upload) => upload.markerId) } }, { session });
109
+ if (result.deletedCount !== uploads.length) {
110
+ throw new Error(`Missing object storage publication markers`);
111
+ }
112
+ }
113
+
114
+ /**
115
+ * Mark old paths for deletion.
116
+ *
117
+ * By marking for deletion instead of immediately deleting:
118
+ * 1. This avoids inconsistencies/orphaned files between MongoDB and S3, by allowing the markers to
119
+ * be persisted transactionally, while the deletes happen asynchronously and can be retried.
120
+ * 2. The deletes are delayed, giving clients a grace period to continue using the files for
121
+ * in-progress requests.
122
+ *
123
+ * Call this in the same transaction as the one removing the references to these files.
124
+ */
125
+ async retire(paths: Iterable<string>, session: mongo.ClientSession, now = new Date()): Promise<void> {
126
+ const markers: ObjectStorageDeletionMarker[] = Array.from(paths, (path) => ({
127
+ _id: new bson.ObjectId(),
128
+ path,
129
+ delete_after: new Date(now.getTime() + OBJECT_STORAGE_REFERENCE_GRACE_MS)
130
+ }));
131
+ if (markers.length) {
132
+ await this.db.pendingObjectStorageDeletes(this.replicationStreamId).insertMany(markers, { session });
133
+ }
134
+ }
135
+
136
+ /**
137
+ * Discover and delete objects by their storage prefix
138
+ * without touching the database.
139
+ */
140
+ async deletePrefix(prefix: string, options?: { signal?: AbortSignal }): Promise<{ objectCount: number }> {
141
+ return this.objectStorage.deletePrefix(prefix, { signal: options?.signal });
142
+ }
143
+
144
+ async cleanup(logger: Logger): Promise<void> {
145
+ const markers = this.db.pendingObjectStorageDeletes(this.replicationStreamId);
146
+ const cursor = markers.find({ delete_after: { $lte: new Date() } });
147
+ let batch: ObjectStorageDeletionMarker[] = [];
148
+
149
+ const deleteBatch = async () => {
150
+ if (batch.length === 0) {
151
+ return;
152
+ }
153
+ const deleting = batch;
154
+ batch = [];
155
+ try {
156
+ await this.bucketData.delete(deleting.map((marker) => marker.path));
157
+ await markers.deleteMany({ _id: { $in: deleting.map((marker) => marker._id) } });
158
+ } catch (error) {
159
+ logger.warn('Failed to clean up object storage deletion markers; will retry during the next compaction', error);
160
+ }
161
+ };
162
+
163
+ for await (const marker of cursor) {
164
+ batch.push(marker);
165
+ if (batch.length === 500) {
166
+ await deleteBatch();
167
+ }
168
+ }
169
+ await deleteBatch();
170
+ }
171
+ }