@powersync/service-module-mongodb-storage 0.15.4 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. package/CHANGELOG.md +69 -0
  2. package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js +1 -1
  3. package/dist/migrations/db/migrations/1688556755264-initial-sync-rules.js.map +1 -1
  4. package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js +2 -2
  5. package/dist/migrations/db/migrations/1702295701188-sync-rule-state.js.map +1 -1
  6. package/dist/storage/MongoBucketStorage.d.ts +8 -6
  7. package/dist/storage/MongoBucketStorage.js +153 -66
  8. package/dist/storage/MongoBucketStorage.js.map +1 -1
  9. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +15 -0
  10. package/dist/storage/implementation/BucketDefinitionMapping.js +58 -0
  11. package/dist/storage/implementation/BucketDefinitionMapping.js.map +1 -0
  12. package/dist/storage/implementation/CheckpointState.d.ts +20 -0
  13. package/dist/storage/implementation/CheckpointState.js +31 -0
  14. package/dist/storage/implementation/CheckpointState.js.map +1 -0
  15. package/dist/storage/implementation/MongoBucketBatch.d.ts +48 -35
  16. package/dist/storage/implementation/MongoBucketBatch.js +118 -379
  17. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  18. package/dist/storage/implementation/MongoBucketBatchShared.d.ts +5 -0
  19. package/dist/storage/implementation/MongoBucketBatchShared.js +8 -0
  20. package/dist/storage/implementation/MongoBucketBatchShared.js.map +1 -0
  21. package/dist/storage/implementation/MongoChecksums.d.ts +29 -17
  22. package/dist/storage/implementation/MongoChecksums.js +13 -72
  23. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  24. package/dist/storage/implementation/MongoCompactor.d.ts +98 -58
  25. package/dist/storage/implementation/MongoCompactor.js +229 -296
  26. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  27. package/dist/storage/implementation/MongoParameterCompactor.d.ts +11 -6
  28. package/dist/storage/implementation/MongoParameterCompactor.js +11 -8
  29. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  30. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +14 -0
  31. package/dist/storage/implementation/MongoPersistedSyncRules.js +67 -0
  32. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +1 -0
  33. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +22 -5
  34. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +56 -13
  35. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +1 -1
  36. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +61 -32
  37. package/dist/storage/implementation/MongoSyncBucketStorage.js +85 -523
  38. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  39. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +10 -4
  40. package/dist/storage/implementation/MongoSyncRulesLock.js +19 -13
  41. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  42. package/dist/storage/implementation/MongoWriteCheckpointAPI.js +1 -1
  43. package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
  44. package/dist/storage/implementation/OperationBatch.js +1 -1
  45. package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +14 -0
  46. package/dist/storage/implementation/SyncRuleStateUpdate.js +36 -0
  47. package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -0
  48. package/dist/storage/implementation/common/BucketDataDoc.d.ts +35 -0
  49. package/dist/storage/implementation/common/BucketDataDoc.js +2 -0
  50. package/dist/storage/implementation/common/BucketDataDoc.js.map +1 -0
  51. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +13 -0
  52. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +2 -0
  53. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +1 -0
  54. package/dist/storage/implementation/common/PersistedBatch.d.ts +108 -0
  55. package/dist/storage/implementation/common/PersistedBatch.js +237 -0
  56. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -0
  57. package/dist/storage/implementation/common/SingleBucketStore.d.ts +54 -0
  58. package/dist/storage/implementation/common/SingleBucketStore.js +3 -0
  59. package/dist/storage/implementation/common/SingleBucketStore.js.map +1 -0
  60. package/dist/storage/implementation/common/SourceRecordStore.d.ts +35 -0
  61. package/dist/storage/implementation/common/SourceRecordStore.js +2 -0
  62. package/dist/storage/implementation/common/SourceRecordStore.js.map +1 -0
  63. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +27 -0
  64. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +57 -0
  65. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -0
  66. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +7 -0
  67. package/dist/storage/implementation/createMongoSyncBucketStorage.js +9 -0
  68. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -0
  69. package/dist/storage/implementation/db.d.ts +41 -36
  70. package/dist/storage/implementation/db.js +77 -99
  71. package/dist/storage/implementation/db.js.map +1 -1
  72. package/dist/storage/implementation/models.d.ts +79 -66
  73. package/dist/storage/implementation/models.js +20 -1
  74. package/dist/storage/implementation/models.js.map +1 -1
  75. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +27 -0
  76. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +407 -0
  77. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -0
  78. package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +12 -0
  79. package/dist/storage/implementation/v1/MongoChecksumsV1.js +56 -0
  80. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -0
  81. package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +23 -0
  82. package/dist/storage/implementation/v1/MongoCompactorV1.js +52 -0
  83. package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -0
  84. package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +9 -0
  85. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -0
  86. package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -0
  87. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +50 -0
  88. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +354 -0
  89. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -0
  90. package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +25 -0
  91. package/dist/storage/implementation/v1/PersistedBatchV1.js +183 -0
  92. package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -0
  93. package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +18 -0
  94. package/dist/storage/implementation/v1/SingleBucketStoreV1.js +57 -0
  95. package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -0
  96. package/dist/storage/implementation/v1/SourceRecordStoreV1.d.ts +19 -0
  97. package/dist/storage/implementation/v1/SourceRecordStoreV1.js +105 -0
  98. package/dist/storage/implementation/v1/SourceRecordStoreV1.js.map +1 -0
  99. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +12 -0
  100. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +20 -0
  101. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -0
  102. package/dist/storage/implementation/v1/models.d.ts +45 -0
  103. package/dist/storage/implementation/v1/models.js +37 -0
  104. package/dist/storage/implementation/v1/models.js.map +1 -0
  105. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +30 -0
  106. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +463 -0
  107. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -0
  108. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +15 -0
  109. package/dist/storage/implementation/v3/MongoChecksumsV3.js +84 -0
  110. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -0
  111. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +23 -0
  112. package/dist/storage/implementation/v3/MongoCompactorV3.js +68 -0
  113. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -0
  114. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +9 -0
  115. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +18 -0
  116. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
  117. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +4 -0
  118. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +9 -0
  119. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +1 -0
  120. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +63 -0
  121. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +508 -0
  122. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -0
  123. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +28 -0
  124. package/dist/storage/implementation/v3/PersistedBatchV3.js +259 -0
  125. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -0
  126. package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +18 -0
  127. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +48 -0
  128. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +1 -0
  129. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +22 -0
  130. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +164 -0
  131. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -0
  132. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +22 -0
  133. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +74 -0
  134. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -0
  135. package/dist/storage/implementation/v3/models.d.ts +101 -0
  136. package/dist/storage/implementation/v3/models.js +34 -0
  137. package/dist/storage/implementation/v3/models.js.map +1 -0
  138. package/dist/storage/storage-index.d.ts +6 -3
  139. package/dist/storage/storage-index.js +6 -3
  140. package/dist/storage/storage-index.js.map +1 -1
  141. package/dist/utils/util.d.ts +10 -3
  142. package/dist/utils/util.js +24 -3
  143. package/dist/utils/util.js.map +1 -1
  144. package/package.json +9 -9
  145. package/src/migrations/db/migrations/1688556755264-initial-sync-rules.ts +1 -1
  146. package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +7 -7
  147. package/src/storage/MongoBucketStorage.ts +254 -99
  148. package/src/storage/implementation/BucketDefinitionMapping.ts +75 -0
  149. package/src/storage/implementation/CheckpointState.ts +59 -0
  150. package/src/storage/implementation/MongoBucketBatch.ts +182 -490
  151. package/src/storage/implementation/MongoBucketBatchShared.ts +11 -0
  152. package/src/storage/implementation/MongoChecksums.ts +53 -75
  153. package/src/storage/implementation/MongoCompactor.ts +374 -404
  154. package/src/storage/implementation/MongoParameterCompactor.ts +37 -24
  155. package/src/storage/implementation/MongoPersistedSyncRules.ts +82 -0
  156. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +78 -16
  157. package/src/storage/implementation/MongoSyncBucketStorage.ts +179 -628
  158. package/src/storage/implementation/MongoSyncRulesLock.ts +20 -16
  159. package/src/storage/implementation/MongoWriteCheckpointAPI.ts +3 -1
  160. package/src/storage/implementation/OperationBatch.ts +1 -1
  161. package/src/storage/implementation/SyncRuleStateUpdate.ts +38 -0
  162. package/src/storage/implementation/common/BucketDataDoc.ts +37 -0
  163. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +15 -0
  164. package/src/storage/implementation/common/PersistedBatch.ts +364 -0
  165. package/src/storage/implementation/common/SingleBucketStore.ts +63 -0
  166. package/src/storage/implementation/common/SourceRecordStore.ts +48 -0
  167. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +80 -0
  168. package/src/storage/implementation/createMongoSyncBucketStorage.ts +25 -0
  169. package/src/storage/implementation/db.ts +110 -131
  170. package/src/storage/implementation/models.ts +102 -79
  171. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +509 -0
  172. package/src/storage/implementation/v1/MongoChecksumsV1.ts +75 -0
  173. package/src/storage/implementation/v1/MongoCompactorV1.ts +93 -0
  174. package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +26 -0
  175. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +543 -0
  176. package/src/storage/implementation/v1/PersistedBatchV1.ts +229 -0
  177. package/src/storage/implementation/v1/SingleBucketStoreV1.ts +74 -0
  178. package/src/storage/implementation/v1/SourceRecordStoreV1.ts +156 -0
  179. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +28 -0
  180. package/src/storage/implementation/v1/models.ts +99 -0
  181. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +607 -0
  182. package/src/storage/implementation/v3/MongoChecksumsV3.ts +120 -0
  183. package/src/storage/implementation/v3/MongoCompactorV3.ts +107 -0
  184. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +24 -0
  185. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +11 -0
  186. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +678 -0
  187. package/src/storage/implementation/v3/PersistedBatchV3.ts +317 -0
  188. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +68 -0
  189. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +226 -0
  190. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +117 -0
  191. package/src/storage/implementation/v3/models.ts +164 -0
  192. package/src/storage/storage-index.ts +6 -3
  193. package/src/utils/util.ts +34 -5
  194. package/test/src/storage_compacting.test.ts +57 -29
  195. package/test/src/storage_sync.test.ts +767 -5
  196. package/test/src/storeCurrentData.test.ts +211 -0
  197. package/test/tsconfig.json +0 -1
  198. package/tsconfig.tsbuildinfo +1 -1
  199. package/dist/storage/implementation/PersistedBatch.d.ts +0 -71
  200. package/dist/storage/implementation/PersistedBatch.js +0 -354
  201. package/dist/storage/implementation/PersistedBatch.js.map +0 -1
  202. package/src/storage/implementation/PersistedBatch.ts +0 -432
@@ -0,0 +1,57 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+ import { serializeBucketDataV1 } from './models.js';
3
+ export class SingleBucketStoreV1 {
4
+ db;
5
+ key;
6
+ collection;
7
+ constructor(db, key) {
8
+ this.db = db;
9
+ this.key = key;
10
+ this.collection = db.bucketDataV1;
11
+ }
12
+ docId(o) {
13
+ // `satisfies BucketDataKeyV1` checks that we use the correct type for V1 storage
14
+ // `as anyt` is to allow casting to the interface virtual type
15
+ return {
16
+ g: this.key.replicationStreamId,
17
+ b: this.key.bucket,
18
+ o
19
+ };
20
+ }
21
+ get minId() {
22
+ return {
23
+ g: this.key.replicationStreamId,
24
+ b: this.key.bucket,
25
+ o: new mongo.MinKey()
26
+ };
27
+ }
28
+ get maxId() {
29
+ return {
30
+ g: this.key.replicationStreamId,
31
+ b: this.key.bucket,
32
+ o: new mongo.MaxKey()
33
+ };
34
+ }
35
+ toPersistedDocument(source) {
36
+ return serializeBucketDataV1({ bucketKey: this.key, ...source });
37
+ }
38
+ fromPersistedDocument(doc) {
39
+ const document = doc;
40
+ const { _id, ...rest } = document;
41
+ return {
42
+ bucketKey: this.key,
43
+ o: _id.o,
44
+ ...rest
45
+ };
46
+ }
47
+ fromPartialPersistedDocument(doc) {
48
+ const document = doc;
49
+ const { _id, ...rest } = document;
50
+ return {
51
+ bucketKey: this.key,
52
+ o: _id.o,
53
+ ...rest
54
+ };
55
+ }
56
+ }
57
+ //# sourceMappingURL=SingleBucketStoreV1.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"SingleBucketStoreV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/SingleBucketStoreV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAUvD,OAAO,EAAyC,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAE3F,MAAM,OAAO,mBAAmB;IAIpB;IACQ;IAJF,UAAU,CAA8C;IAExE,YACU,EAA6B,EACrB,GAAc;QADtB,OAAE,GAAF,EAAE,CAA2B;QACrB,QAAG,GAAH,GAAG,CAAW;QAE9B,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,YAAsE,CAAC;IAC9F,CAAC;IAED,KAAK,CAAC,CAAe;QACnB,iFAAiF;QACjF,8DAA8D;QAC9D,OAAO;YACL,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,mBAAmB;YAC/B,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM;YAClB,CAAC;SAC+B,CAAC;IACrC,CAAC;IAED,IAAI,KAAK;QACP,OAAO;YACL,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,mBAAmB;YAC/B,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM;YAClB,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE;SACf,CAAC;IACX,CAAC;IAED,IAAI,KAAK;QACP,OAAO;YACL,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,mBAAmB;YAC/B,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM;YAClB,CAAC,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE;SACf,CAAC;IACX,CAAC;IAED,mBAAmB,CAAC,MAAwC;QAC1D,OAAO,qBAAqB,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,EAAE,CAAyC,CAAC;IAC3G,CAAC;IAED,qBAAqB,CAAC,GAA8B;QAClD,MAAM,QAAQ,GAAG,GAAsC,CAAC;QACxD,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;QAClC,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,GAAG;YACnB,CAAC,EAAE,GAAG,CAAC,CAAC;YACR,GAAG,IAAI;SACR,CAAC;IACJ,CAAC;IAED,4BAA4B,CAC1B,GAA+C;QAE/C,MAAM,QAAQ,GAAG,GAA4C,CAAC;QAC9D,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;QAClC,OAAO;YACL,SAAS,EAAE,IAAI,CAAC,GAAG;YACnB,CAAC,EAAE,GAAG,CAAC,CAAC;YACR,GAAG,IAAI;SACsC,CAAC;IAClD,CAAC;CACF"}
@@ -0,0 +1,19 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+ import { Logger } from '@powersync/lib-services-framework';
3
+ import { EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
4
+ import * as bson from 'bson';
5
+ import { LoadedSourceRecord, SourceRecordLookupEntry, SourceRecordLookupState, SourceRecordStore } from '../common/SourceRecordStore.js';
6
+ import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
7
+ export declare class SourceRecordStoreV1 implements SourceRecordStore {
8
+ private readonly db;
9
+ private readonly groupId;
10
+ constructor(db: VersionedPowerSyncMongoV1, groupId: number);
11
+ mapEvaluatedBuckets(evaluated: EvaluatedRow[]): LoadedSourceRecord['buckets'];
12
+ mapParameterLookups(paramEvaluated: EvaluatedParameters[]): SourceRecordLookupState[];
13
+ private createId;
14
+ private createLoadedDocument;
15
+ loadSizes(session: mongo.ClientSession, entries: SourceRecordLookupEntry[]): Promise<Map<string, number>>;
16
+ loadDocuments(session: mongo.ClientSession, entries: SourceRecordLookupEntry[], idsOnly: boolean): Promise<Map<string, LoadedSourceRecord>>;
17
+ loadTruncateBatch(session: mongo.ClientSession, sourceTableId: bson.ObjectId, limit: number): Promise<LoadedSourceRecord[]>;
18
+ postCommitCleanup(_lastCheckpoint: bigint, _logger: Logger): Promise<void>;
19
+ }
@@ -0,0 +1,105 @@
1
+ import { storage } from '@powersync/service-core';
2
+ import { idPrefixFilter } from '../../../utils/util.js';
3
+ import { cacheKey } from '../OperationBatch.js';
4
+ export class SourceRecordStoreV1 {
5
+ db;
6
+ groupId;
7
+ constructor(db, groupId) {
8
+ this.db = db;
9
+ this.groupId = groupId;
10
+ }
11
+ mapEvaluatedBuckets(evaluated) {
12
+ return evaluated.map((entry) => ({
13
+ definitionId: null,
14
+ bucket: entry.bucket,
15
+ table: entry.table,
16
+ id: entry.id
17
+ }));
18
+ }
19
+ mapParameterLookups(paramEvaluated) {
20
+ return paramEvaluated.map((entry) => ({
21
+ indexId: null,
22
+ lookup: storage.serializeLookup(entry.lookup)
23
+ }));
24
+ }
25
+ createId(sourceTableId, replicaId) {
26
+ return {
27
+ g: this.groupId,
28
+ t: sourceTableId,
29
+ k: replicaId
30
+ };
31
+ }
32
+ createLoadedDocument(sourceTableId, id, data, buckets, lookups) {
33
+ return {
34
+ sourceTableId,
35
+ replicaId: id.k,
36
+ data,
37
+ buckets: buckets.map((bucket) => ({
38
+ definitionId: null,
39
+ bucket: bucket.bucket,
40
+ table: bucket.table,
41
+ id: bucket.id
42
+ })),
43
+ lookups: lookups.map((lookup) => ({
44
+ indexId: null,
45
+ lookup
46
+ })),
47
+ cacheKey: cacheKey(sourceTableId, id.k)
48
+ };
49
+ }
50
+ async loadSizes(session, entries) {
51
+ const sizes = new Map();
52
+ const sizeCursor = this.db.sourceRecordsV1.aggregate([
53
+ {
54
+ $match: {
55
+ _id: {
56
+ $in: entries.map((entry) => this.createId(entry.sourceTableId, entry.replicaId))
57
+ }
58
+ }
59
+ },
60
+ {
61
+ $project: {
62
+ _id: 1,
63
+ size: { $bsonSize: '$$ROOT' }
64
+ }
65
+ }
66
+ ], { session });
67
+ for await (const doc of sizeCursor.stream()) {
68
+ sizes.set(cacheKey(doc._id.t, doc._id.k), doc.size);
69
+ }
70
+ return sizes;
71
+ }
72
+ async loadDocuments(session, entries, idsOnly) {
73
+ const documents = new Map();
74
+ const projection = idsOnly ? { _id: 1 } : undefined;
75
+ const cursor = this.db.sourceRecordsV1.find({
76
+ _id: {
77
+ $in: entries.map((entry) => this.createId(entry.sourceTableId, entry.replicaId))
78
+ }
79
+ }, { session, projection });
80
+ for await (const doc of cursor.stream()) {
81
+ const loaded = this.createLoadedDocument(doc._id.t, doc._id, idsOnly ? null : doc.data, idsOnly ? [] : doc.buckets, idsOnly ? [] : doc.lookups);
82
+ documents.set(loaded.cacheKey, loaded);
83
+ }
84
+ return documents;
85
+ }
86
+ async loadTruncateBatch(session, sourceTableId, limit) {
87
+ const cursor = this.db.sourceRecordsV1.find({
88
+ _id: idPrefixFilter({ g: this.groupId, t: sourceTableId }, ['k']),
89
+ pending_delete: { $exists: false }
90
+ }, {
91
+ projection: {
92
+ _id: 1,
93
+ buckets: 1,
94
+ lookups: 1
95
+ },
96
+ limit,
97
+ session
98
+ });
99
+ return (await cursor.toArray()).map((doc) => this.createLoadedDocument(sourceTableId, doc._id, null, doc.buckets, doc.lookups));
100
+ }
101
+ async postCommitCleanup(_lastCheckpoint, _logger) {
102
+ // No-op for V1.
103
+ }
104
+ }
105
+ //# sourceMappingURL=SourceRecordStoreV1.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"SourceRecordStoreV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/SourceRecordStoreV1.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAGlD,OAAO,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AACxD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAWhD,MAAM,OAAO,mBAAmB;IAEX;IACA;IAFnB,YACmB,EAA6B,EAC7B,OAAe;QADf,OAAE,GAAF,EAAE,CAA2B;QAC7B,YAAO,GAAP,OAAO,CAAQ;IAC/B,CAAC;IAEJ,mBAAmB,CAAC,SAAyB;QAC3C,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC/B,YAAY,EAAE,IAAI;YAClB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,EAAE,EAAE,KAAK,CAAC,EAAE;SACb,CAAC,CAAC,CAAC;IACN,CAAC;IAED,mBAAmB,CAAC,cAAqC;QACvD,OAAO,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACpC,OAAO,EAAE,IAAI;YACb,MAAM,EAAE,OAAO,CAAC,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC;SAC9C,CAAC,CAAC,CAAC;IACN,CAAC;IAEO,QAAQ,CAAC,aAA4B,EAAE,SAA4B;QACzE,OAAO;YACL,CAAC,EAAE,IAAI,CAAC,OAAO;YACf,CAAC,EAAE,aAAa;YAChB,CAAC,EAAE,SAAS;SACO,CAAC;IACxB,CAAC;IAEO,oBAAoB,CAC1B,aAA4B,EAC5B,EAAa,EACb,IAAwB,EACxB,OAAuC,EACvC,OAAuC;QAEvC,OAAO;YACL,aAAa;YACb,SAAS,EAAE,EAAE,CAAC,CAAC;YACf,IAAI;YACJ,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBAChC,YAAY,EAAE,IAAI;gBAClB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,EAAE,EAAE,MAAM,CAAC,EAAE;aACd,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBAChC,OAAO,EAAE,IAAI;gBACb,MAAM;aACP,CAAC,CAAC;YACH,QAAQ,EAAE,QAAQ,CAAC,aAAa,EAAE,EAAE,CAAC,CAAC,CAAC;SACxC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,OAA4B,EAAE,OAAkC;QAC9E,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;QACxC,MAAM,UAAU,GACd,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,SAAS,CAC/B;YACE;gBACE,MAAM,EAAE;oBACN,GAAG,EAAE;wBACH,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,SAAS,CAAc,CAAC;qBAC9F;iBACF;aACF;YACD;gBACE,QAAQ,EAAE;oBACR,GAAG,EAAE,CAAC;oBACN,IAAI,EAAE,EAAE,SAAS,EAAE,QAAQ,EAAE;iBAC9B;aACF;SACF,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,UAAU,CAAC,MAAM,EAAE,EAAE,CAAC;YAC5C,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,OAA4B,EAC5B,OAAkC,EAClC,OAAgB;QAEhB,MAAM,SAAS,GAAG,IAAI,GAAG,EAA8B,CAAC;QACxD,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;QACpD,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CACzC;YACE,GAAG,EAAE;gBACH,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,aAAa,EAAE,KAAK,CAAC,SAAS,CAAc,CAAC;aAC9F;SACF,EACD,EAAE,OAAO,EAAE,UAAU,EAAE,CACxB,CAAC;QACF,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,MAAM,MAAM,GAAG,IAAI,CAAC,oBAAoB,CACtC,GAAG,CAAC,GAAG,CAAC,CAAC,EACT,GAAG,CAAC,GAAG,EACP,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,EACzB,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAC1B,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAC3B,CAAC;YACF,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACzC,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,iBAAiB,CACrB,OAA4B,EAC5B,aAA4B,EAC5B,KAAa;QAEb,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,IAAI,CACzC;YACE,GAAG,EAAE,cAAc,CAAY,EAAE,CAAC,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,EAAE,aAAa,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC;YAC5E,cAAc,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE;SACnC,EACD;YACE,UAAU,EAAE;gBACV,GAAG,EAAE,CAAC;gBACN,OAAO,EAAE,CAAC;gBACV,OAAO,EAAE,CAAC;aACX;YACD,KAAK;YACL,OAAO;SACR,CACF,CAAC;QACF,OAAO,CAAC,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAC1C,IAAI,CAAC,oBAAoB,CAAC,aAAa,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,CAClF,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,eAAuB,EAAE,OAAe;QAC9D,gBAAgB;IAClB,CAAC;CACF"}
@@ -0,0 +1,12 @@
1
+ import { mongo } from '@powersync/lib-service-mongodb';
2
+ import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
3
+ import { CommonSourceTableDocument } from '../models.js';
4
+ import { BucketDataDocumentV1, BucketParameterDocument, BucketStateDocumentV1, CurrentDataDocument } from './models.js';
5
+ export declare class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
6
+ get sourceRecordsV1(): mongo.Collection<CurrentDataDocument>;
7
+ get bucketStateV1(): mongo.Collection<BucketStateDocumentV1>;
8
+ commonSourceTables(_replicationStreamId: number): mongo.Collection<CommonSourceTableDocument>;
9
+ initializeStreamStorage(_replicationStreamId: number): Promise<void>;
10
+ get bucketDataV1(): mongo.Collection<BucketDataDocumentV1>;
11
+ get parameterIndexV1(): mongo.Collection<BucketParameterDocument>;
12
+ }
@@ -0,0 +1,20 @@
1
+ import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
2
+ export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
3
+ get sourceRecordsV1() {
4
+ return this.upstream.current_data;
5
+ }
6
+ get bucketStateV1() {
7
+ return this.upstream.bucket_state;
8
+ }
9
+ commonSourceTables(_replicationStreamId) {
10
+ return this.upstream.source_tables;
11
+ }
12
+ async initializeStreamStorage(_replicationStreamId) { }
13
+ get bucketDataV1() {
14
+ return this.upstream.bucket_data;
15
+ }
16
+ get parameterIndexV1() {
17
+ return this.upstream.bucket_parameters;
18
+ }
19
+ }
20
+ //# sourceMappingURL=VersionedPowerSyncMongoV1.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"VersionedPowerSyncMongoV1.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,2BAA2B,EAAE,MAAM,0CAA0C,CAAC;AAIvF,MAAM,OAAO,yBAA0B,SAAQ,2BAA2B;IACxE,IAAI,eAAe;QACjB,OAAO,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;IACpC,CAAC;IAED,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC;IACpC,CAAC;IAED,kBAAkB,CAAC,oBAA4B;QAC7C,OAAO,IAAI,CAAC,QAAQ,CAAC,aAAmE,CAAC;IAC3F,CAAC;IAED,KAAK,CAAC,uBAAuB,CAAC,oBAA4B,IAAkB,CAAC;IAE7E,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,gBAAgB;QAClB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACzC,CAAC;CACF"}
@@ -0,0 +1,45 @@
1
+ import { SerializedSyncPlan } from '@powersync/service-core';
2
+ import * as bson from 'bson';
3
+ import { BucketDataDoc } from '../common/BucketDataDoc.js';
4
+ import { BucketDataDocumentBase, BucketParameterDocumentBase, BucketStateDocumentBase, CurrentBucket, SourceKey, SourceTableDocument, SyncRuleCheckpointFields, SyncRuleDocumentBase, TaggedBucketParameterDocument } from '../models.js';
5
+ export interface BucketDataKeyV1 {
6
+ /** group_id */
7
+ g: number;
8
+ /** bucket name */
9
+ b: string;
10
+ /** op_id */
11
+ o: bigint;
12
+ }
13
+ export interface CurrentDataDocument {
14
+ _id: SourceKey;
15
+ data: bson.Binary;
16
+ buckets: CurrentBucket[];
17
+ lookups: bson.Binary[];
18
+ }
19
+ export interface BucketParameterDocument extends BucketParameterDocumentBase<SourceKey> {
20
+ }
21
+ export interface BucketDataDocumentV1 extends BucketDataDocumentBase {
22
+ _id: BucketDataKeyV1;
23
+ }
24
+ export declare function serializeBucketDataV1(document: BucketDataDoc): BucketDataDocumentV1;
25
+ export declare function loadBucketDataDocumentV1(doc: BucketDataDocumentV1): BucketDataDoc;
26
+ export declare function taggedBucketParameterDocumentToV1(document: TaggedBucketParameterDocument): BucketParameterDocument;
27
+ export interface SourceTableDocumentV1 extends SourceTableDocument {
28
+ group_id: number;
29
+ }
30
+ export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckpointFields<string | null> {
31
+ content: string;
32
+ serialized_plan?: SerializedSyncPlan | null;
33
+ /**
34
+ * True if initial snapshot has been replicated.
35
+ *
36
+ * Can only be false if state == PROCESSING.
37
+ */
38
+ snapshot_done: boolean;
39
+ }
40
+ export interface BucketStateDocumentV1 extends BucketStateDocumentBase {
41
+ _id: BucketStateDocumentBase['_id'] & {
42
+ g: number;
43
+ };
44
+ }
45
+ export type BucketStateDocument = BucketStateDocumentV1;
@@ -0,0 +1,37 @@
1
+ import { LEGACY_BUCKET_DATA_DEFINITION_ID } from '../models.js';
2
+ export function serializeBucketDataV1(document) {
3
+ const { bucketKey, o } = document;
4
+ return {
5
+ _id: {
6
+ g: bucketKey.replicationStreamId,
7
+ b: bucketKey.bucket,
8
+ o: o
9
+ },
10
+ // List fields directly, so that we don't accidentally persist any unknown fields
11
+ op: document.op,
12
+ source_table: document.source_table,
13
+ source_key: document.source_key,
14
+ table: document.table,
15
+ row_id: document.row_id,
16
+ checksum: document.checksum,
17
+ data: document.data,
18
+ target_op: document.target_op
19
+ };
20
+ }
21
+ export function loadBucketDataDocumentV1(doc) {
22
+ const { _id, ...rest } = doc;
23
+ return {
24
+ bucketKey: {
25
+ replicationStreamId: _id.g,
26
+ definitionId: LEGACY_BUCKET_DATA_DEFINITION_ID,
27
+ bucket: _id.b
28
+ },
29
+ o: _id.o,
30
+ ...rest
31
+ };
32
+ }
33
+ export function taggedBucketParameterDocumentToV1(document) {
34
+ const { index: _index, ...rest } = document;
35
+ return rest;
36
+ }
37
+ //# sourceMappingURL=models.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"models.js","sourceRoot":"","sources":["../../../../src/storage/implementation/v1/models.ts"],"names":[],"mappings":"AAGA,OAAO,EAKL,gCAAgC,EAMjC,MAAM,cAAc,CAAC;AAwBtB,MAAM,UAAU,qBAAqB,CAAC,QAAuB;IAC3D,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,GAAG,QAAQ,CAAC;IAClC,OAAO;QACL,GAAG,EAAE;YACH,CAAC,EAAE,SAAS,CAAC,mBAAmB;YAChC,CAAC,EAAE,SAAS,CAAC,MAAM;YACnB,CAAC,EAAE,CAAC;SACL;QACD,iFAAiF;QACjF,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,UAAU,EAAE,QAAQ,CAAC,UAAU;QAC/B,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;KAC9B,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,GAAyB;IAChE,MAAM,EAAE,GAAG,EAAE,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC;IAC7B,OAAO;QACL,SAAS,EAAE;YACT,mBAAmB,EAAE,GAAG,CAAC,CAAC;YAC1B,YAAY,EAAE,gCAAgC;YAC9C,MAAM,EAAE,GAAG,CAAC,CAAC;SACd;QACD,CAAC,EAAE,GAAG,CAAC,CAAC;QACR,GAAG,IAAI;KACR,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iCAAiC,CAAC,QAAuC;IACvF,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,GAAG,QAAQ,CAAC;IAC5C,OAAO,IAA+B,CAAC;AACzC,CAAC"}
@@ -0,0 +1,30 @@
1
+ import { storage } from '@powersync/service-core';
2
+ import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
3
+ import { PersistedBatch } from '../common/PersistedBatch.js';
4
+ import { SourceRecordStore } from '../common/SourceRecordStore.js';
5
+ import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
6
+ export declare class MongoBucketBatchV3 extends MongoBucketBatch {
7
+ readonly db: VersionedPowerSyncMongoV3;
8
+ private readonly store;
9
+ private readonly syncConfigId;
10
+ private needsActivationV3;
11
+ private lastWaitingLogThrottledV3;
12
+ constructor(options: MongoBucketBatchOptions);
13
+ protected createPersistedBatch(writtenSize: number): PersistedBatch;
14
+ protected get sourceRecordStore(): SourceRecordStore;
15
+ protected cleanupDroppedSourceTables(sourceTables: storage.SourceTable[]): Promise<void>;
16
+ resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult>;
17
+ private sourceTableFromDocument;
18
+ private sourceTableMembershipsFromDocument;
19
+ getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null>;
20
+ commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult>;
21
+ keepalive(lsn: string): Promise<storage.CheckpointResult>;
22
+ setResumeLsn(lsn: string): Promise<void>;
23
+ private autoActivateV3;
24
+ markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
25
+ markSnapshotDone(no_checkpoint_before_lsn: string, options?: {
26
+ throwOnConflict?: boolean;
27
+ }): Promise<void>;
28
+ markTableSnapshotRequired(_table: storage.SourceTable): Promise<void>;
29
+ markTableSnapshotDone(tables: storage.SourceTable[], no_checkpoint_before_lsn?: string): Promise<storage.SourceTable[]>;
30
+ }