@powersync/service-module-mongodb-storage 0.15.3 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. package/CHANGELOG.md +54 -0
  2. package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js +1 -1
  3. package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js.map +1 -1
  4. package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js +3 -3
  5. package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map +1 -1
  6. package/dist/migrations/db/migrations/1770213298299-storage-version.js.map +1 -1
  7. package/dist/storage/MongoBucketStorage.d.ts +5 -3
  8. package/dist/storage/MongoBucketStorage.js +50 -36
  9. package/dist/storage/MongoBucketStorage.js.map +1 -1
  10. package/dist/storage/MongoReportStorage.js.map +1 -1
  11. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +17 -0
  12. package/dist/storage/implementation/BucketDefinitionMapping.js +58 -0
  13. package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
  14. package/dist/storage/implementation/MongoBucketBatch.d.ts +16 -14
  15. package/dist/storage/implementation/MongoBucketBatch.js +80 -115
  16. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  17. package/dist/storage/implementation/MongoBucketBatchShared.d.ts +5 -0
  18. package/dist/storage/implementation/MongoBucketBatchShared.js +8 -0
  19. package/dist/storage/implementation/MongoBucketBatchShared.js.map +1 -0
  20. package/dist/storage/implementation/MongoChecksums.d.ts +28 -17
  21. package/dist/storage/implementation/MongoChecksums.js +13 -72
  22. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  23. package/dist/storage/implementation/MongoCompactor.d.ts +98 -58
  24. package/dist/storage/implementation/MongoCompactor.js +229 -296
  25. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  26. package/dist/storage/implementation/MongoParameterCompactor.d.ts +11 -6
  27. package/dist/storage/implementation/MongoParameterCompactor.js +11 -8
  28. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  29. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +14 -0
  30. package/dist/storage/implementation/MongoPersistedSyncRules.js +64 -0
  31. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +1 -0
  32. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +3 -0
  33. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +9 -0
  34. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +1 -1
  35. package/dist/storage/implementation/MongoStorageProvider.js +1 -1
  36. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  37. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +49 -30
  38. package/dist/storage/implementation/MongoSyncBucketStorage.js +96 -388
  39. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  40. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +5 -3
  41. package/dist/storage/implementation/MongoSyncRulesLock.js +12 -10
  42. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  43. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +1 -1
  44. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  45. package/dist/storage/implementation/OperationBatch.js +1 -1
  46. package/dist/storage/implementation/common/BucketDataDoc.d.ts +35 -0
  47. package/dist/storage/implementation/common/BucketDataDoc.js +2 -0
  48. package/dist/storage/implementation/common/BucketDataDoc.js.map +1 -0
  49. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +13 -0
  50. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +2 -0
  51. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +1 -0
  52. package/dist/storage/implementation/common/PersistedBatch.d.ts +108 -0
  53. package/dist/storage/implementation/common/PersistedBatch.js +237 -0
  54. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -0
  55. package/dist/storage/implementation/common/SingleBucketStore.d.ts +54 -0
  56. package/dist/storage/implementation/common/SingleBucketStore.js +3 -0
  57. package/dist/storage/implementation/common/SingleBucketStore.js.map +1 -0
  58. package/dist/storage/implementation/common/SourceRecordStore.d.ts +36 -0
  59. package/dist/storage/implementation/common/SourceRecordStore.js +2 -0
  60. package/dist/storage/implementation/common/SourceRecordStore.js.map +1 -0
  61. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +27 -0
  62. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +57 -0
  63. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -0
  64. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +7 -0
  65. package/dist/storage/implementation/createMongoSyncBucketStorage.js +9 -0
  66. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -0
  67. package/dist/storage/implementation/db.d.ts +34 -34
  68. package/dist/storage/implementation/db.js +78 -98
  69. package/dist/storage/implementation/db.js.map +1 -1
  70. package/dist/storage/implementation/models.d.ts +63 -34
  71. package/dist/storage/implementation/models.js +21 -2
  72. package/dist/storage/implementation/models.js.map +1 -1
  73. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +13 -0
  74. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +22 -0
  75. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -0
  76. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +12 -0
  77. package/dist/storage/implementation/v1/MongoChecksumsV1.js +56 -0
  78. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -0
  79. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +23 -0
  80. package/dist/storage/implementation/v1/MongoCompactorV1.js +52 -0
  81. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -0
  82. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +9 -0
  83. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -0
  84. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -0
  85. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +41 -0
  86. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +283 -0
  87. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -0
  88. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +26 -0
  89. package/dist/storage/implementation/v1/PersistedBatchV1.js +183 -0
  90. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -0
  91. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +18 -0
  92. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +57 -0
  93. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -0
  94. package/dist/storage/implementation/v1/SourceRecordStoreV1.d.ts +19 -0
  95. package/dist/storage/implementation/v1/SourceRecordStoreV1.js +105 -0
  96. package/dist/storage/implementation/v1/SourceRecordStoreV1.js.map +1 -0
  97. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +12 -0
  98. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +20 -0
  99. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -0
  100. package/dist/storage/implementation/v1/models.d.ts +34 -0
  101. package/dist/storage/implementation/v1/models.js +37 -0
  102. package/dist/storage/implementation/v1/models.js.map +1 -0
  103. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +13 -0
  104. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +34 -0
  105. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -0
  106. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +15 -0
  107. package/dist/storage/implementation/v3/MongoChecksumsV3.js +84 -0
  108. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -0
  109. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +23 -0
  110. package/dist/storage/implementation/v3/MongoCompactorV3.js +68 -0
  111. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -0
  112. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +9 -0
  113. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +18 -0
  114. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
  115. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +5 -0
  116. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +9 -0
  117. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +1 -0
  118. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +41 -0
  119. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +407 -0
  120. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -0
  121. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +29 -0
  122. package/dist/storage/implementation/v3/PersistedBatchV3.js +259 -0
  123. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -0
  124. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +18 -0
  125. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +48 -0
  126. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +1 -0
  127. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +22 -0
  128. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +164 -0
  129. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -0
  130. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +21 -0
  131. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +71 -0
  132. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -0
  133. package/dist/storage/implementation/v3/models.d.ts +43 -0
  134. package/dist/storage/implementation/v3/models.js +34 -0
  135. package/dist/storage/implementation/v3/models.js.map +1 -0
  136. package/dist/storage/storage-index.d.ts +8 -5
  137. package/dist/storage/storage-index.js +8 -5
  138. package/dist/storage/storage-index.js.map +1 -1
  139. package/dist/utils/util.d.ts +11 -4
  140. package/dist/utils/util.js +25 -4
  141. package/dist/utils/util.js.map +1 -1
  142. package/package.json +9 -9
  143. package/src/migrations/db/migrations/1688556755264-initial-sync-rules.ts +1 -1
  144. package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +7 -7
  145. package/src/migrations/db/migrations/1770213298299-storage-version.ts +1 -1
  146. package/src/storage/MongoBucketStorage.ts +97 -62
  147. package/src/storage/MongoReportStorage.ts +2 -2
  148. package/src/storage/implementation/BucketDefinitionMapping.ts +72 -0
  149. package/src/storage/implementation/MongoBucketBatch.ts +110 -144
  150. package/src/storage/implementation/MongoBucketBatchShared.ts +11 -0
  151. package/src/storage/implementation/MongoChecksums.ts +53 -76
  152. package/src/storage/implementation/MongoCompactor.ts +374 -404
  153. package/src/storage/implementation/MongoParameterCompactor.ts +37 -24
  154. package/src/storage/implementation/MongoPersistedSyncRules.ts +76 -0
  155. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +18 -1
  156. package/src/storage/implementation/MongoStorageProvider.ts +1 -1
  157. package/src/storage/implementation/MongoSyncBucketStorage.ts +190 -457
  158. package/src/storage/implementation/MongoSyncRulesLock.ts +12 -14
  159. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +4 -2
  160. package/src/storage/implementation/OperationBatch.ts +1 -1
  161. package/src/storage/implementation/common/BucketDataDoc.ts +37 -0
  162. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +15 -0
  163. package/src/storage/implementation/common/PersistedBatch.ts +364 -0
  164. package/src/storage/implementation/common/SingleBucketStore.ts +63 -0
  165. package/src/storage/implementation/common/SourceRecordStore.ts +49 -0
  166. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +80 -0
  167. package/src/storage/implementation/createMongoSyncBucketStorage.ts +25 -0
  168. package/src/storage/implementation/db.ts +107 -128
  169. package/src/storage/implementation/models.ts +84 -38
  170. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +32 -0
  171. package/src/storage/implementation/v1/MongoChecksumsV1.ts +75 -0
  172. package/src/storage/implementation/v1/MongoCompactorV1.ts +93 -0
  173. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +26 -0
  174. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +448 -0
  175. package/src/storage/implementation/v1/PersistedBatchV1.ts +230 -0
  176. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +74 -0
  177. package/src/storage/implementation/v1/SourceRecordStoreV1.ts +156 -0
  178. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +28 -0
  179. package/src/storage/implementation/v1/models.ts +84 -0
  180. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +44 -0
  181. package/src/storage/implementation/v3/MongoChecksumsV3.ts +120 -0
  182. package/src/storage/implementation/v3/MongoCompactorV3.ts +107 -0
  183. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +24 -0
  184. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +12 -0
  185. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +550 -0
  186. package/src/storage/implementation/v3/PersistedBatchV3.ts +318 -0
  187. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +68 -0
  188. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +226 -0
  189. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +112 -0
  190. package/src/storage/implementation/v3/models.ts +96 -0
  191. package/src/storage/storage-index.ts +8 -5
  192. package/src/utils/util.ts +36 -7
  193. package/test/src/__snapshots__/storage_sync.test.ts.snap +282 -0
  194. package/test/src/connection-report-storage.test.ts +3 -3
  195. package/test/src/setup.ts +1 -1
  196. package/test/src/storage.test.ts +2 -2
  197. package/test/src/storage_compacting.test.ts +57 -29
  198. package/test/src/storage_sync.test.ts +351 -5
  199. package/test/tsconfig.json +0 -1
  200. package/tsconfig.tsbuildinfo +1 -1
  201. package/dist/storage/implementation/PersistedBatch.d.ts +0 -71
  202. package/dist/storage/implementation/PersistedBatch.js +0 -354
  203. package/dist/storage/implementation/PersistedBatch.js.map +0 -1
  204. package/src/storage/implementation/PersistedBatch.ts +0 -432
@@ -0,0 +1,43 @@
1
+ import { InternalOpId } from '@powersync/service-core';
2
+ import * as bson from 'bson';
3
+ import { BucketDefinitionId, ParameterIndexId } from '../BucketDefinitionMapping.js';
4
+ import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
5
+ import { BucketDataDocumentBase, BucketDataKey, BucketParameterDocumentBase, BucketStateDocumentBase, CurrentBucket, ReplicaId, SourceTableDocument, SourceTableKey, TaggedBucketParameterDocument } from '../models.js';
6
+ export interface CurrentBucketV3 extends CurrentBucket {
7
+ def: BucketDefinitionId;
8
+ }
9
+ export interface RecordedLookupV3 {
10
+ i: ParameterIndexId;
11
+ l: bson.Binary;
12
+ }
13
+ export interface CurrentDataDocumentV3 {
14
+ _id: ReplicaId;
15
+ data: bson.Binary | null;
16
+ buckets: CurrentBucketV3[];
17
+ lookups: RecordedLookupV3[];
18
+ /**
19
+ * If set, this can be deleted, once there is a consistent checkpoint >= pending_delete.
20
+ *
21
+ * This must only be set if buckets = [], lookups = [].
22
+ */
23
+ pending_delete?: bigint;
24
+ }
25
+ export interface BucketParameterDocumentV3 extends BucketParameterDocumentBase<SourceTableKey> {
26
+ }
27
+ export type BucketDataKeyV3 = BucketDataKey;
28
+ export interface BucketDataDocumentV3 extends BucketDataDocumentBase {
29
+ _id: BucketDataKeyV3;
30
+ }
31
+ export declare function serializeBucketDataV3(document: BucketDataDoc): BucketDataDocumentV3;
32
+ export declare function loadBucketDataDocumentV3(context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>, doc: BucketDataDocumentV3): BucketDataDoc;
33
+ export declare function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3;
34
+ export interface SourceTableDocumentV3 extends SourceTableDocument {
35
+ bucket_data_source_ids: BucketDefinitionId[];
36
+ parameter_lookup_source_ids: ParameterIndexId[];
37
+ latest_pending_delete?: InternalOpId | undefined;
38
+ }
39
+ export interface BucketStateDocumentV3 extends BucketStateDocumentBase {
40
+ _id: BucketStateDocumentBase['_id'] & {
41
+ d: BucketDefinitionId;
42
+ };
43
+ }
@@ -0,0 +1,34 @@
1
+ export function serializeBucketDataV3(document) {
2
+ const { bucketKey, o } = document;
3
+ return {
4
+ _id: {
5
+ b: bucketKey.bucket,
6
+ o: o
7
+ },
8
+ // List fields directly, so that we don't accidentally persist any unknown fields
9
+ op: document.op,
10
+ source_table: document.source_table,
11
+ source_key: document.source_key,
12
+ table: document.table,
13
+ row_id: document.row_id,
14
+ checksum: document.checksum,
15
+ data: document.data,
16
+ target_op: document.target_op
17
+ };
18
+ }
19
+ export function loadBucketDataDocumentV3(context, doc) {
20
+ const { _id, ...rest } = doc;
21
+ return {
22
+ bucketKey: {
23
+ ...context,
24
+ bucket: _id.b
25
+ },
26
+ o: _id.o,
27
+ ...rest
28
+ };
29
+ }
30
+ export function taggedBucketParameterDocumentToV3(document) {
31
+ const { index: _index, ...rest } = document;
32
+ return rest;
33
+ }
34
+ //# sourceMappingURL=models.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"models.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v3/models.ts"],"names":[],"mappings":"AA8CA,MAAM,UAAU,qBAAqB,CAAC,QAAuB;IAC3D,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,QAAQ,CAAC;IAClC,OAAO;QACL,GAAG,EAAE;YACH,CAAC,EAAE,SAAS,CAAC,MAAM;YACnB,CAAC,EAAE,CAAC;SACL;QACD,iFAAiF;QACjF,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,OAAgE,EAChE,GAAyB;IAEzB,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC;IAC7B,OAAO;QACL,SAAS,EAAE;YACT,GAAG,OAAO;YACV,MAAM,EAAE,GAAG,CAAC,CAAC;SACd;QACD,CAAC,EAAE,GAAG,CAAC,CAAC;QACR,GAAG,IAAI;KACR,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAAC,QAAuC;IACvF,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;IAC5C,OAAO,IAAiC,CAAC;AAC3C,CAAC"}
@@ -1,14 +1,17 @@
1
+ export * as test_utils from '../utils/test-utils.js';
2
+ export * from '../utils/util.js';
3
+ export * from './implementation/BucketDefinitionMapping.js';
4
+ export * from './implementation/common/PersistedBatch.js';
5
+ export * from './implementation/createMongoSyncBucketStorage.js';
1
6
  export * from './implementation/db.js';
2
7
  export * from './implementation/models.js';
3
- export * from './implementation/MongoBucketBatch.js';
4
8
  export * from './implementation/MongoIdSequence.js';
9
+ export * from './implementation/MongoPersistedSyncRules.js';
5
10
  export * from './implementation/MongoPersistedSyncRulesContent.js';
6
11
  export * from './implementation/MongoStorageProvider.js';
7
- export * from './implementation/MongoSyncBucketStorage.js';
8
12
  export * from './implementation/MongoSyncRulesLock.js';
9
13
  export * from './implementation/OperationBatch.js';
10
- export * from './implementation/PersistedBatch.js';
11
- export * from '../utils/util.js';
14
+ export * from './implementation/v1/models.js';
15
+ export * from './implementation/v3/models.js';
12
16
  export * from './MongoBucketStorage.js';
13
17
  export * from './MongoReportStorage.js';
14
- export * as test_utils from '../utils/test-utils.js';
@@ -1,15 +1,18 @@
1
+ export * as test_utils from '../utils/test-utils.js';
2
+ export * from '../utils/util.js';
3
+ export * from './implementation/BucketDefinitionMapping.js';
4
+ export * from './implementation/common/PersistedBatch.js';
5
+ export * from './implementation/createMongoSyncBucketStorage.js';
1
6
  export * from './implementation/db.js';
2
7
  export * from './implementation/models.js';
3
- export * from './implementation/MongoBucketBatch.js';
4
8
  export * from './implementation/MongoIdSequence.js';
9
+ export * from './implementation/MongoPersistedSyncRules.js';
5
10
  export * from './implementation/MongoPersistedSyncRulesContent.js';
6
11
  export * from './implementation/MongoStorageProvider.js';
7
- export * from './implementation/MongoSyncBucketStorage.js';
8
12
  export * from './implementation/MongoSyncRulesLock.js';
9
13
  export * from './implementation/OperationBatch.js';
10
- export * from './implementation/PersistedBatch.js';
11
- export * from '../utils/util.js';
14
+ export * from './implementation/v1/models.js';
15
+ export * from './implementation/v3/models.js';
12
16
  export * from './MongoBucketStorage.js';
13
17
  export * from './MongoReportStorage.js';
14
- export * as test_utils from '../utils/test-utils.js';
15
18
  //# sourceMappingURL=storage-index.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,sCAAsC,CAAC;AACrD,cAAc,qCAAqC,CAAC;AACpD,cAAc,oDAAoD,CAAC;AACnE,cAAc,0CAA0C,CAAC;AACzD,cAAc,4CAA4C,CAAC;AAC3D,cAAc,wCAAwC,CAAC;AACvD,cAAc,oCAAoC,CAAC;AACnD,cAAc,oCAAoC,CAAC;AACnD,cAAc,kBAAkB,CAAC;AACjC,cAAc,yBAAyB,CAAC;AACxC,cAAc,yBAAyB,CAAC;AACxC,OAAO,KAAK,UAAU,MAAM,wBAAwB,CAAC"}
1
+ {"version":3,"file":"storage-index.js","sourceRoot":"","sources":["../../src/storage/storage-index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,UAAU,MAAM,wBAAwB,CAAC;AACrD,cAAc,kBAAkB,CAAC;AACjC,cAAc,6CAA6C,CAAC;AAC5D,cAAc,2CAA2C,CAAC;AAC1D,cAAc,kDAAkD,CAAC;AACjE,cAAc,wBAAwB,CAAC;AACvC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,qCAAqC,CAAC;AACpD,cAAc,6CAA6C,CAAC;AAC5D,cAAc,oDAAoD,CAAC;AACnE,cAAc,0CAA0C,CAAC;AACzD,cAAc,wCAAwC,CAAC;AACvD,cAAc,oCAAoC,CAAC;AACnD,cAAc,+BAA+B,CAAC;AAC9C,cAAc,+BAA+B,CAAC;AAC9C,cAAc,yBAAyB,CAAC;AACxC,cAAc,yBAAyB,CAAC"}
@@ -1,7 +1,8 @@
1
- import * as bson from 'bson';
1
+ import * as lib_mongo from '@powersync/lib-service-mongodb';
2
2
  import { mongo } from '@powersync/lib-service-mongodb';
3
3
  import { storage, utils } from '@powersync/service-core';
4
- import { BucketDataDocument } from '../storage/implementation/models.js';
4
+ import * as bson from 'bson';
5
+ import { BucketDataDoc } from '../storage/implementation/common/BucketDataDoc.js';
5
6
  export declare function idPrefixFilter<T>(prefix: Partial<T>, rest: (keyof T)[]): mongo.Condition<T>;
6
7
  export declare function generateSlotName(prefix: string, sync_rules_id: number): string;
7
8
  /**
@@ -19,12 +20,18 @@ export declare function readSingleBatch<T>(cursor: mongo.AbstractCursor<T>): Pro
19
20
  data: T[];
20
21
  hasMore: boolean;
21
22
  }>;
22
- export declare function mapOpEntry(row: BucketDataDocument): utils.OplogEntry;
23
+ export declare function mapOpEntry(row: BucketDataDoc): utils.OplogEntry;
23
24
  export declare function replicaIdToSubkey(table: storage.SourceTableId, id: storage.ReplicaId): string;
24
25
  export declare function mongoTableId(table: storage.SourceTableId): bson.ObjectId;
25
26
  export declare function setSessionSnapshotTime(session: mongo.ClientSession, time: bson.Timestamp): void;
27
+ export declare function retryOnMongoMaxTimeMSExpired<T>(operation: () => Promise<T>, options: {
28
+ signal?: AbortSignal;
29
+ abortMessage?: string;
30
+ retryDelayMs: number;
31
+ onRetry?: (retryCount: number) => void;
32
+ }): Promise<T>;
26
33
  export declare const createPaginatedConnectionQuery: <T extends mongo.Document>(query: mongo.Filter<T>, collection: mongo.Collection<T>, limit: number, cursor?: string) => Promise<{
27
- items: mongo.WithId<T>[];
34
+ items: lib_mongo.mongo.WithId<T>[];
28
35
  count: number;
29
36
  /** Setting the cursor to the connected at date of the last item in the list */
30
37
  cursor: any;
@@ -1,8 +1,10 @@
1
+ import * as lib_mongo from '@powersync/lib-service-mongodb';
2
+ import { ReplicationAbortedError, ServiceAssertionError } from '@powersync/lib-services-framework';
3
+ import { storage, utils } from '@powersync/service-core';
1
4
  import * as bson from 'bson';
2
5
  import * as crypto from 'crypto';
6
+ import * as timers from 'node:timers/promises';
3
7
  import * as uuid from 'uuid';
4
- import { storage, utils } from '@powersync/service-core';
5
- import { ServiceAssertionError } from '@powersync/lib-services-framework';
6
8
  export function idPrefixFilter(prefix, rest) {
7
9
  let filter = {
8
10
  $gte: {
@@ -65,7 +67,7 @@ export async function readSingleBatch(cursor) {
65
67
  export function mapOpEntry(row) {
66
68
  if (row.op == 'PUT' || row.op == 'REMOVE') {
67
69
  return {
68
- op_id: utils.internalToExternalOpId(row._id.o),
70
+ op_id: utils.internalToExternalOpId(row.o),
69
71
  op: row.op,
70
72
  object_type: row.table,
71
73
  object_id: row.row_id,
@@ -77,7 +79,7 @@ export function mapOpEntry(row) {
77
79
  else {
78
80
  // MOVE, CLEAR
79
81
  return {
80
- op_id: utils.internalToExternalOpId(row._id.o),
82
+ op_id: utils.internalToExternalOpId(row.o),
81
83
  op: row.op,
82
84
  checksum: Number(row.checksum)
83
85
  };
@@ -120,6 +122,25 @@ export function setSessionSnapshotTime(session, time) {
120
122
  throw new ServiceAssertionError(`Session snapshotTime is already set`);
121
123
  }
122
124
  }
125
+ export async function retryOnMongoMaxTimeMSExpired(operation, options) {
126
+ let retryCount = 0;
127
+ while (true) {
128
+ if (options.signal?.aborted) {
129
+ throw new ReplicationAbortedError(options.abortMessage ?? 'Aborted MongoDB operation', options.signal.reason);
130
+ }
131
+ try {
132
+ return await operation();
133
+ }
134
+ catch (e) {
135
+ if (!lib_mongo.isMongoServerError(e) || e.codeName !== 'MaxTimeMSExpired') {
136
+ throw e;
137
+ }
138
+ retryCount += 1;
139
+ options.onRetry?.(retryCount);
140
+ await timers.setTimeout(options.retryDelayMs);
141
+ }
142
+ }
143
+ }
123
144
  export const createPaginatedConnectionQuery = async (query, collection, limit, cursor) => {
124
145
  const createQuery = (cursor) => {
125
146
  if (!cursor) {
@@ -1 +1 @@
1
- {"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/utils/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAE7B,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAG1E,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA+B;IACtE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAuB;IAChD,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YAC9C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAA4B,EAAE,EAAqB;IACnF,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAA4B;IACvD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,wCAAwC,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAA4B;IACjD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;SAAM,CAAC;QACN,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,OAA4B,EAAE,IAAoB;IACvF,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;IACxE,CAAC;IACD,IAAK,OAAe,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QACzC,OAAe,CAAC,YAAY,GAAG,IAAI,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,8BAA8B,GAAG,KAAK,EACjD,KAAsB,EACtB,UAA+B,EAC/B,KAAa,EACb,MAAe,EACf,EAAE;IACF,MAAM,WAAW,GAAG,CAAC,MAAe,EAAE,EAAE;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY;YACrC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE;YAC1D,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO;YACL,GAAG,KAAK;YACR,YAAY;SACM,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE;QACtD,IAAI,EAAE;YACJ,6FAA6F;YAC7F,YAAY,EAAE,CAAC,CAAC;SACjB;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B;;SAEK;IACL,OAAO;QACL,KAAK;QACL,KAAK;QACL,+EAA+E;QAC/E,MAAM,EAAE,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;QACxF,IAAI,EAAE,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC;KACzB,CAAC;AACJ,CAAC,CAAC"}
1
+ {"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/utils/util.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,uBAAuB,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AACnG,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,KAAK,MAAM,MAAM,QAAQ,CAAC;AACjC,OAAO,KAAK,MAAM,MAAM,sBAAsB,CAAC;AAC/C,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAG7B,MAAM,UAAU,cAAc,CAAI,MAAkB,EAAE,IAAiB;IACrE,IAAI,MAAM,GAAG;QACX,IAAI,EAAE;YACJ,GAAG,MAAM;SACH;QACR,GAAG,EAAE;YACH,GAAG,MAAM;SACH;KACT,CAAC;IAEF,KAAK,IAAI,GAAG,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,MAAc,EAAE,aAAqB;IACpE,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC1D,OAAO,GAAG,MAAM,GAAG,aAAa,IAAI,WAAW,EAAE,CAAC;AACpD,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,MAA+B;IACtE,IAAI,CAAC;QACH,IAAI,IAAS,CAAC;QACd,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,2CAA2C;QAC3C,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACtC,yCAAyC;QACzC,IAAI,GAAG,MAAM,CAAC,qBAAqB,EAAE,CAAC;QACtC,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC;YACnC,0CAA0C;YAC1C,wEAAwE;YACxE,uEAAuE;YACvE,oCAAoC;YACpC,EAAE;YACF,4EAA4E;YAC5E,2DAA2D;YAC3D,gCAAgC;YAChC,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;IAC3B,CAAC;YAAS,CAAC;QACT,iDAAiD;QACjD,uIAAuI;QACvI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,GAAkB;IAC3C,IAAI,GAAG,CAAC,EAAE,IAAI,KAAK,IAAI,GAAG,CAAC,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1C,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,WAAW,EAAE,GAAG,CAAC,KAAK;YACtB,SAAS,EAAE,GAAG,CAAC,MAAM;YACrB,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC9B,MAAM,EAAE,iBAAiB,CAAC,GAAG,CAAC,YAAa,EAAE,GAAG,CAAC,UAAW,CAAC;YAC7D,IAAI,EAAE,GAAG,CAAC,IAAI;SACf,CAAC;IACJ,CAAC;SAAM,CAAC;QACN,cAAc;QAEd,OAAO;YACL,KAAK,EAAE,KAAK,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1C,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/B,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAA4B,EAAE,EAAqB;IACnF,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC;QACvB,mDAAmD;QACnD,OAAO,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,oCAAoC;QACpC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;QAC3C,OAAO,IAAI,CAAC,EAAE,CAAC,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;IAC3C,CAAC;AACH,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,KAA4B;IACvD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,wCAAwC,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,KAA4B;IACjD,IAAI,OAAO,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,OAAO,KAAK,CAAC;IACf,CAAC;SAAM,CAAC;QACN,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC7B,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,OAA4B,EAAE,IAAoB;IACvF,gGAAgG;IAChG,IAAI,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC;QAC7B,MAAM,IAAI,qBAAqB,CAAC,oCAAoC,CAAC,CAAC;IACxE,CAAC;IACD,IAAK,OAAe,CAAC,YAAY,IAAI,IAAI,EAAE,CAAC;QACzC,OAAe,CAAC,YAAY,GAAG,IAAI,CAAC;IACvC,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,qBAAqB,CAAC,qCAAqC,CAAC,CAAC;IACzE,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,4BAA4B,CAChD,SAA2B,EAC3B,OAKC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,OAAO,IAAI,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,uBAAuB,CAAC,OAAO,CAAC,YAAY,IAAI,2BAA2B,EAAE,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChH,CAAC;QACD,IAAI,CAAC;YACH,OAAO,MAAM,SAAS,EAAE,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,QAAQ,KAAK,kBAAkB,EAAE,CAAC;gBAC1E,MAAM,CAAC,CAAC;YACV,CAAC;YACD,UAAU,IAAI,CAAC,CAAC;YAChB,OAAO,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,CAAC;YAC9B,MAAM,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,MAAM,8BAA8B,GAAG,KAAK,EACjD,KAAsB,EACtB,UAA+B,EAC/B,KAAa,EACb,MAAe,EACf,EAAE;IACF,MAAM,WAAW,GAAG,CAAC,MAAe,EAAE,EAAE;QACtC,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,OAAO,KAAK,CAAC;QACf,CAAC;QACD,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY;YACrC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE;YAC1D,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;QAC9B,OAAO;YACL,GAAG,KAAK;YACR,YAAY;SACM,CAAC;IACvB,CAAC,CAAC;IAEF,MAAM,UAAU,GAAG,UAAU,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE;QACtD,IAAI,EAAE;YACJ,6FAA6F;YAC7F,YAAY,EAAE,CAAC,CAAC;SACjB;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC;IACtD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAC3B;;SAEK;IACL,OAAO;QACL,KAAK;QACL,KAAK;QACL,+EAA+E;QAC/E,MAAM,EAAE,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,SAAS;QACxF,IAAI,EAAE,CAAC,CAAC,KAAK,KAAK,KAAK,CAAC;KACzB,CAAC;AACJ,CAAC,CAAC"}
package/package.json CHANGED
@@ -2,7 +2,7 @@
2
2
  "name": "@powersync/service-module-mongodb-storage",
3
3
  "repository": "https://github.com/powersync-ja/powersync-service",
4
4
  "types": "dist/index.d.ts",
5
- "version": "0.15.3",
5
+ "version": "0.16.0",
6
6
  "main": "dist/index.js",
7
7
  "license": "FSL-1.1-ALv2",
8
8
  "type": "module",
@@ -26,16 +26,16 @@
26
26
  "ix": "^5.0.0",
27
27
  "lru-cache": "^10.2.2",
28
28
  "ts-codec": "^1.3.0",
29
- "uuid": "^11.1.0",
30
- "@powersync/lib-service-mongodb": "0.6.23",
31
- "@powersync/lib-services-framework": "0.9.2",
32
- "@powersync/service-core": "1.20.4",
33
- "@powersync/service-jsonbig": "0.17.12",
34
- "@powersync/service-sync-rules": "0.34.1",
35
- "@powersync/service-types": "0.15.0"
29
+ "uuid": "^14.0.0",
30
+ "@powersync/lib-service-mongodb": "0.6.25",
31
+ "@powersync/lib-services-framework": "0.9.4",
32
+ "@powersync/service-core": "1.21.0",
33
+ "@powersync/service-jsonbig": "0.17.13",
34
+ "@powersync/service-sync-rules": "0.36.0",
35
+ "@powersync/service-types": "0.15.2"
36
36
  },
37
37
  "devDependencies": {
38
- "@powersync/service-core-tests": "0.15.3"
38
+ "@powersync/service-core-tests": "0.15.5"
39
39
  },
40
40
  "scripts": {
41
41
  "build": "tsc -b",
@@ -1,5 +1,5 @@
1
1
  export const up = async () => {
2
- // No-op - we don't auto-create sync rules anymore
2
+ // No-op - we don't auto-create sync config anymore
3
3
  };
4
4
 
5
5
  export const down = async () => {};
@@ -1,26 +1,26 @@
1
1
  import * as lib_mongo from '@powersync/lib-service-mongodb';
2
+ import { ServiceAssertionError } from '@powersync/lib-services-framework';
2
3
  import { storage as core_storage, migrations } from '@powersync/service-core';
3
4
  import * as storage from '../../../storage/storage-index.js';
4
5
  import { MongoStorageConfig } from '../../../types/types.js';
5
- import { ServiceAssertionError } from '@powersync/lib-services-framework';
6
6
 
7
7
  interface LegacySyncRulesDocument extends storage.SyncRuleDocument {
8
8
  /**
9
- * True if this is the active sync rules.
9
+ * True if this is the active replication stream.
10
10
  * requires `snapshot_done == true` and `replicating == true`.
11
11
  */
12
12
  active?: boolean;
13
13
 
14
14
  /**
15
- * True if this sync rules should be used for replication.
15
+ * True if this replication stream should be used for replication.
16
16
  *
17
- * During reprocessing, there is one sync rules with `replicating = true, active = true`,
17
+ * During reprocessing, there is one replication stream with `replicating = true, active = true`,
18
18
  * and one with `replicating = true, active = false, auto_activate = true`.
19
19
  */
20
20
  replicating?: boolean;
21
21
 
22
22
  /**
23
- * True if the sync rules should set `active = true` when `snapshot_done` = true.
23
+ * True if the replication stream should set `active = true` when `snapshot_done` = true.
24
24
  */
25
25
  auto_activate?: boolean;
26
26
  }
@@ -35,7 +35,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => {
35
35
  try {
36
36
  // We keep the old flags for existing deployments still shutting down.
37
37
 
38
- // 1. New sync rules: `active = false, snapshot_done = false, replicating = true, auto_activate = true`
38
+ // 1. New replication stream: `active = false, snapshot_done = false, replicating = true, auto_activate = true`
39
39
  await db.sync_rules.updateMany(
40
40
  {
41
41
  active: { $ne: true },
@@ -66,7 +66,7 @@ export const up: migrations.PowerSyncMigrationFunction = async (context) => {
66
66
  const remaining = await db.sync_rules.find({ state: null as any }).toArray();
67
67
  if (remaining.length > 0) {
68
68
  const slots = remaining.map((doc) => doc.slot_name).join(', ');
69
- throw new ServiceAssertionError(`Invalid state for sync rules: ${slots}`);
69
+ throw new ServiceAssertionError(`Invalid state for replication stream: ${slots}`);
70
70
  }
71
71
  } finally {
72
72
  await db.client.close();
@@ -1,4 +1,4 @@
1
- import { migrations, storage as core_storage } from '@powersync/service-core';
1
+ import { storage as core_storage, migrations } from '@powersync/service-core';
2
2
  import * as mongo_storage from '../../../storage/storage-index.js';
3
3
  import { MongoStorageConfig } from '../../../types/types.js';
4
4
 
@@ -1,23 +1,27 @@
1
1
  import { GetIntanceOptions, storage } from '@powersync/service-core';
2
2
 
3
- import { ErrorCode, ServiceError } from '@powersync/lib-services-framework';
3
+ import { DO_NOT_LOG, ErrorCode, ServiceError } from '@powersync/lib-services-framework';
4
4
  import { v4 as uuid } from 'uuid';
5
5
 
6
6
  import * as lib_mongo from '@powersync/lib-service-mongodb';
7
7
  import { mongo } from '@powersync/lib-service-mongodb';
8
8
 
9
+ import { generateSlotName } from '../utils/util.js';
10
+ import { BucketDefinitionMapping } from './implementation/BucketDefinitionMapping.js';
11
+ import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
12
+ import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
9
13
  import { PowerSyncMongo } from './implementation/db.js';
10
14
  import { getMongoStorageConfig, SyncRuleDocument } from './implementation/models.js';
11
- import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
12
- import { MongoSyncBucketStorage } from './implementation/MongoSyncBucketStorage.js';
13
- import { generateSlotName } from '../utils/util.js';
14
15
  import { MongoChecksumOptions } from './implementation/MongoChecksums.js';
16
+ import { MongoPersistedSyncRulesContent } from './implementation/MongoPersistedSyncRulesContent.js';
15
17
 
16
18
  export interface MongoBucketStorageOptions {
17
- checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
19
+ checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig' | 'mapping'>;
18
20
  }
19
21
 
20
22
  export class MongoBucketStorage extends storage.BucketStorageFactory {
23
+ [DO_NOT_LOG] = true;
24
+
21
25
  private readonly client: mongo.MongoClient;
22
26
  private readonly session: mongo.ClientSession;
23
27
  // TODO: This is still Postgres specific and needs to be reworked
@@ -51,7 +55,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
51
55
  id = Number(id);
52
56
  }
53
57
  const storageConfig = (syncRules as MongoPersistedSyncRulesContent).getStorageConfig();
54
- const storage = new MongoSyncBucketStorage(
58
+ const storage = createMongoSyncBucketStorage(
55
59
  this,
56
60
  id,
57
61
  syncRules as MongoPersistedSyncRulesContent,
@@ -98,7 +102,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
98
102
  const active = await this.getActiveSyncRulesContent();
99
103
 
100
104
  if (next != null && next.id == sync_rules_group_id) {
101
- // We need to redo the "next" sync rules
105
+ // We need to redo the "next" replication stream
102
106
  await this.updateSyncRules(next.asUpdateOptions());
103
107
  // Pro-actively stop replicating
104
108
  await this.db.sync_rules.updateOne(
@@ -114,7 +118,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
114
118
  );
115
119
  await this.db.notifyCheckpoint();
116
120
  } else if (next == null && active?.id == sync_rules_group_id) {
117
- // Slot removed for "active" sync rules, while there is no "next" one.
121
+ // Slot removed for "active" replication stream, while there is no "next" one.
118
122
  await this.updateSyncRules(active.asUpdateOptions());
119
123
 
120
124
  // In this case we keep the old one as active for clients, so that that existing clients
@@ -134,7 +138,7 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
134
138
  );
135
139
  await this.db.notifyCheckpoint();
136
140
  } else if (next != null && active?.id == sync_rules_group_id) {
137
- // Already have next sync rules, but need to stop replicating the active one.
141
+ // Already have next replication stream, but need to stop replicating the active one.
138
142
 
139
143
  await this.db.sync_rules.updateOne(
140
144
  {
@@ -152,14 +156,14 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
152
156
  }
153
157
 
154
158
  async updateSyncRules(options: storage.UpdateSyncRulesOptions): Promise<MongoPersistedSyncRulesContent> {
155
- const storageVersion = options.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
159
+ const storageVersion =
160
+ options.storageVersion ?? options.config.parsed.config.storageVersion ?? storage.CURRENT_STORAGE_VERSION;
156
161
  const storageConfig = getMongoStorageConfig(storageVersion);
157
- await this.db.initializeStorageVersion(storageConfig);
158
162
 
159
163
  let rules: MongoPersistedSyncRulesContent | undefined = undefined;
160
164
 
161
165
  await this.session.withTransaction(async () => {
162
- // Only have a single set of sync rules with PROCESSING.
166
+ // Only have a single replication stream with PROCESSING.
163
167
  await this.db.sync_rules.updateMany(
164
168
  {
165
169
  state: storage.SyncRuleState.PROCESSING
@@ -203,6 +207,10 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
203
207
  last_fatal_error_ts: null,
204
208
  last_keepalive_ts: null
205
209
  };
210
+ if (storageConfig.incrementalReprocessing) {
211
+ const parsed = options.config.parsed;
212
+ doc.rule_mapping = BucketDefinitionMapping.fromParsedSyncRules(parsed).serialize();
213
+ }
206
214
  await this.db.sync_rules.insertOne(doc);
207
215
  await this.db.notifyCheckpoint();
208
216
  rules = new MongoPersistedSyncRulesContent(this.db, doc);
@@ -294,63 +302,90 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
294
302
  }
295
303
  };
296
304
 
297
- const active_sync_rules = await this.getActiveSyncRules({ defaultSchema: 'public' });
298
- if (active_sync_rules == null) {
299
- return {
300
- operations_size_bytes: 0,
301
- parameters_size_bytes: 0,
302
- replication_size_bytes: 0
303
- };
304
- }
305
- const operations_aggregate = await this.db.bucket_data
305
+ // For now, we get storage metrics over all v1 and v3 collections.
306
+ // In the future, we may split these metrics to report separately for active replication streams versus processing streams.
306
307
 
307
- .aggregate([
308
- {
309
- $collStats: {
310
- storageStats: {}
311
- }
312
- }
313
- ])
314
- .toArray()
315
- .catch(ignoreNotExisting);
308
+ const aggregateStaticCollection = async <T extends mongo.Document>(collection: mongo.Collection<T>) => {
309
+ // We check whether the collection exists before getting the statistics. This avoids repeated
310
+ // errors in the MongoDB logs if the collection hasn't been created yet.
311
+ const exists =
312
+ (await this.db.db.listCollections({ name: collection.collectionName }, { nameOnly: true }).toArray()).length >
313
+ 0;
314
+ if (!exists) {
315
+ return [{ storageStats: { size: 0 } }];
316
+ }
316
317
 
317
- const parameters_aggregate = await this.db.bucket_parameters
318
- .aggregate([
319
- {
320
- $collStats: {
321
- storageStats: {}
318
+ return collection
319
+ .aggregate([
320
+ {
321
+ $collStats: {
322
+ storageStats: {}
323
+ }
322
324
  }
323
- }
324
- ])
325
- .toArray()
326
- .catch(ignoreNotExisting);
325
+ ])
326
+ .toArray()
327
+ .catch(ignoreNotExisting);
328
+ };
327
329
 
328
- const v1_replication_aggregate = await this.db.current_data
329
- .aggregate([
330
- {
331
- $collStats: {
332
- storageStats: {}
333
- }
334
- }
335
- ])
336
- .toArray()
337
- .catch(ignoreNotExisting);
330
+ const operations_aggregate = await aggregateStaticCollection(this.db.bucket_data);
331
+ const v3_operation_aggregates = await Promise.all(
332
+ (await this.db.listBucketDataCollectionsV3()).map((collection) =>
333
+ collection
334
+ .aggregate([
335
+ {
336
+ $collStats: {
337
+ storageStats: {}
338
+ }
339
+ }
340
+ ])
341
+ .toArray()
342
+ .catch(ignoreNotExisting)
343
+ )
344
+ );
338
345
 
339
- const v3_replication_aggregate = await this.db.v3_current_data
340
- .aggregate([
341
- {
342
- $collStats: {
343
- storageStats: {}
344
- }
345
- }
346
- ])
347
- .toArray()
348
- .catch(ignoreNotExisting);
346
+ const parameters_aggregate = await aggregateStaticCollection(this.db.bucket_parameters);
347
+
348
+ const v3_parameter_aggregates = await Promise.all(
349
+ (await this.db.listAllParameterIndexCollectionsV3()).map((collection) =>
350
+ collection
351
+ .aggregate([
352
+ {
353
+ $collStats: {
354
+ storageStats: {}
355
+ }
356
+ }
357
+ ])
358
+ .toArray()
359
+ .catch(ignoreNotExisting)
360
+ )
361
+ );
362
+
363
+ const v1_source_record_aggregate = await aggregateStaticCollection(this.db.current_data);
364
+
365
+ const source_record_aggregates = await Promise.all(
366
+ (await this.db.listAllSourceRecordCollectionsV3()).map((collection) =>
367
+ collection
368
+ .aggregate([
369
+ {
370
+ $collStats: {
371
+ storageStats: {}
372
+ }
373
+ }
374
+ ])
375
+ .toArray()
376
+ .catch(ignoreNotExisting)
377
+ )
378
+ );
349
379
  return {
350
- operations_size_bytes: Number(operations_aggregate[0].storageStats.size),
351
- parameters_size_bytes: Number(parameters_aggregate[0].storageStats.size),
380
+ operations_size_bytes:
381
+ Number(operations_aggregate[0].storageStats.size) +
382
+ v3_operation_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0),
383
+ parameters_size_bytes:
384
+ Number(parameters_aggregate[0].storageStats.size) +
385
+ v3_parameter_aggregates.reduce((total, aggregate) => total + Number(aggregate[0].storageStats.size), 0),
352
386
  replication_size_bytes:
353
- Number(v1_replication_aggregate[0].storageStats.size) + Number(v3_replication_aggregate[0].storageStats.size)
387
+ Number(v1_source_record_aggregate[0]?.storageStats?.size ?? 0) +
388
+ source_record_aggregates.reduce((total, aggregate) => total + Number(aggregate[0]?.storageStats?.size ?? 0), 0)
354
389
  };
355
390
  }
356
391
 
@@ -1,8 +1,8 @@
1
+ import { logger } from '@powersync/lib-services-framework';
1
2
  import { storage } from '@powersync/service-core';
2
3
  import { event_types } from '@powersync/service-types';
3
- import { PowerSyncMongo } from './implementation/db.js';
4
- import { logger } from '@powersync/lib-services-framework';
5
4
  import { createPaginatedConnectionQuery } from '../utils/util.js';
5
+ import { PowerSyncMongo } from './implementation/db.js';
6
6
 
7
7
  export class MongoReportStorage implements storage.ReportStorage {
8
8
  public readonly db: PowerSyncMongo;