@powersync/service-module-mongodb-storage 0.17.0 → 0.18.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (185) hide show
  1. package/CHANGELOG.md +27 -0
  2. package/dist/storage/MongoBucketStorage.d.ts +34 -16
  3. package/dist/storage/MongoBucketStorage.js +219 -70
  4. package/dist/storage/MongoBucketStorage.js.map +1 -1
  5. package/dist/storage/implementation/MongoBucketBatch.d.ts +42 -23
  6. package/dist/storage/implementation/MongoBucketBatch.js +61 -44
  7. package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
  8. package/dist/storage/implementation/MongoChecksums.d.ts +10 -9
  9. package/dist/storage/implementation/MongoChecksums.js +3 -111
  10. package/dist/storage/implementation/MongoChecksums.js.map +1 -1
  11. package/dist/storage/implementation/MongoCompactor.d.ts +2 -0
  12. package/dist/storage/implementation/MongoCompactor.js +18 -9
  13. package/dist/storage/implementation/MongoCompactor.js.map +1 -1
  14. package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
  15. package/dist/storage/implementation/MongoHydrationState.js +49 -0
  16. package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
  17. package/dist/storage/implementation/MongoParameterCompactor.d.ts +6 -5
  18. package/dist/storage/implementation/MongoParameterCompactor.js +24 -1
  19. package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
  20. package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
  21. package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
  22. package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
  23. package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
  24. package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
  25. package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
  26. package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
  27. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
  28. package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
  29. package/dist/storage/implementation/MongoStorageProvider.js +5 -2
  30. package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
  31. package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +41 -23
  32. package/dist/storage/implementation/MongoSyncBucketStorage.js +51 -43
  33. package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
  34. package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
  35. package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
  36. package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
  37. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +6 -0
  38. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
  39. package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
  40. package/dist/storage/implementation/common/PersistedBatch.d.ts +1 -2
  41. package/dist/storage/implementation/common/PersistedBatch.js +5 -0
  42. package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
  43. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
  44. package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
  45. package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
  46. package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
  47. package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
  48. package/dist/storage/implementation/db.js +9 -14
  49. package/dist/storage/implementation/db.js.map +1 -1
  50. package/dist/storage/implementation/models.d.ts +4 -13
  51. package/dist/storage/implementation/models.js.map +1 -1
  52. package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
  53. package/dist/storage/implementation/v1/MongoBucketBatchV1.js +42 -26
  54. package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
  55. package/dist/storage/implementation/v1/MongoChecksumsV1.js +114 -3
  56. package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
  57. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +15 -17
  58. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +47 -45
  59. package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
  60. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
  61. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
  62. package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
  63. package/dist/storage/implementation/v1/models.d.ts +10 -1
  64. package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
  65. package/dist/storage/implementation/v3/MongoBucketBatchV3.js +286 -259
  66. package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
  67. package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +27 -7
  68. package/dist/storage/implementation/v3/MongoChecksumsV3.js +202 -18
  69. package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
  70. package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +19 -2
  71. package/dist/storage/implementation/v3/MongoCompactorV3.js +511 -22
  72. package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
  73. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +62 -0
  74. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +350 -0
  75. package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
  76. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +40 -28
  77. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +232 -116
  78. package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
  79. package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +1 -1
  80. package/dist/storage/implementation/v3/PersistedBatchV3.js +43 -20
  81. package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
  82. package/dist/storage/implementation/v3/SingleBucketStoreV3.js +52 -16
  83. package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +1 -1
  84. package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
  85. package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
  86. package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
  87. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +13 -12
  88. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +29 -27
  89. package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
  90. package/dist/storage/implementation/v3/bucket-format.d.ts +4 -0
  91. package/dist/storage/implementation/v3/bucket-format.js +58 -0
  92. package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
  93. package/dist/storage/implementation/v3/chunking.d.ts +10 -0
  94. package/dist/storage/implementation/v3/chunking.js +28 -0
  95. package/dist/storage/implementation/v3/chunking.js.map +1 -0
  96. package/dist/storage/implementation/v3/models.d.ts +64 -21
  97. package/dist/storage/implementation/v3/models.js +11 -28
  98. package/dist/storage/implementation/v3/models.js.map +1 -1
  99. package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
  100. package/dist/storage/implementation/v3/source-table-utils.js +214 -0
  101. package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
  102. package/dist/storage/storage-index.d.ts +4 -4
  103. package/dist/storage/storage-index.js +4 -4
  104. package/dist/storage/storage-index.js.map +1 -1
  105. package/dist/types/types.d.ts +2 -2
  106. package/dist/utils/test-utils.d.ts +1 -2
  107. package/dist/utils/test-utils.js +5 -1
  108. package/dist/utils/test-utils.js.map +1 -1
  109. package/dist/utils/util.d.ts +1 -1
  110. package/dist/utils/util.js +3 -2
  111. package/dist/utils/util.js.map +1 -1
  112. package/package.json +9 -19
  113. package/src/storage/MongoBucketStorage.ts +334 -87
  114. package/src/storage/implementation/MongoBucketBatch.ts +86 -58
  115. package/src/storage/implementation/MongoChecksums.ts +6 -138
  116. package/src/storage/implementation/MongoCompactor.ts +19 -9
  117. package/src/storage/implementation/MongoHydrationState.ts +61 -0
  118. package/src/storage/implementation/MongoParameterCompactor.ts +24 -5
  119. package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
  120. package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
  121. package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
  122. package/src/storage/implementation/MongoStorageProvider.ts +5 -2
  123. package/src/storage/implementation/MongoSyncBucketStorage.ts +70 -56
  124. package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
  125. package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +7 -0
  126. package/src/storage/implementation/common/PersistedBatch.ts +7 -3
  127. package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
  128. package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
  129. package/src/storage/implementation/db.ts +11 -19
  130. package/src/storage/implementation/models.ts +4 -14
  131. package/src/storage/implementation/v1/MongoBucketBatchV1.ts +55 -34
  132. package/src/storage/implementation/v1/MongoChecksumsV1.ts +137 -4
  133. package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +70 -64
  134. package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
  135. package/src/storage/implementation/v1/models.ts +11 -1
  136. package/src/storage/implementation/v3/MongoBucketBatchV3.ts +358 -314
  137. package/src/storage/implementation/v3/MongoChecksumsV3.ts +262 -26
  138. package/src/storage/implementation/v3/MongoCompactorV3.ts +646 -29
  139. package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +480 -0
  140. package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +315 -157
  141. package/src/storage/implementation/v3/PersistedBatchV3.ts +62 -36
  142. package/src/storage/implementation/v3/SingleBucketStoreV3.ts +59 -19
  143. package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
  144. package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +41 -36
  145. package/src/storage/implementation/v3/bucket-format.ts +70 -0
  146. package/src/storage/implementation/v3/chunking.ts +38 -0
  147. package/src/storage/implementation/v3/models.ts +93 -55
  148. package/src/storage/implementation/v3/source-table-utils.ts +392 -0
  149. package/src/storage/storage-index.ts +12 -4
  150. package/src/utils/test-utils.ts +6 -3
  151. package/src/utils/util.ts +3 -2
  152. package/test/src/__snapshots__/storage.test.ts.snap +0 -75
  153. package/test/src/__snapshots__/storage_sync.test.ts.snap +48 -48
  154. package/test/src/cleanup-stopped-sync-configs.test.ts +648 -0
  155. package/test/src/storage.test.ts +11 -11
  156. package/test/src/storage_compacting.test.ts +1956 -6
  157. package/test/src/storage_sync.test.ts +1077 -68
  158. package/test/src/storeCurrentData.test.ts +17 -7
  159. package/test/src/util.ts +2 -1
  160. package/tsconfig.json +1 -4
  161. package/tsconfig.tsbuildinfo +1 -1
  162. package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
  163. package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
  164. package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
  165. package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
  166. package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
  167. package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
  168. package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
  169. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
  170. package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
  171. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
  172. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
  173. package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
  174. package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
  175. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
  176. package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
  177. package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
  178. package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
  179. package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
  180. package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
  181. package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
  182. package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
  183. package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
  184. package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
  185. package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
@@ -66,7 +66,7 @@ async function storedRowPayload(
66
66
  data = doc?.data;
67
67
  } else {
68
68
  const db = (bucketStorage as any).db;
69
- const doc = await db.sourceRecordsV3(syncRulesId, table.id).findOne({});
69
+ const doc = await db.sourceRecords(syncRulesId, table.id).findOne({});
70
70
  data = doc?.data;
71
71
  }
72
72
  if (data == null) {
@@ -81,6 +81,7 @@ function registerStoreCurrentDataTests(storageVersion: number) {
81
81
  await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
82
82
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
83
83
  const bucketStorage = factory.getInstance(syncRules);
84
+ const syncRulesContent = syncRules.syncConfigContent[0];
84
85
 
85
86
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
86
87
 
@@ -114,6 +115,7 @@ function registerStoreCurrentDataTests(storageVersion: number) {
114
115
  await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
115
116
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
116
117
  const bucketStorage = factory.getInstance(syncRules);
118
+ const syncRulesContent = syncRules.syncConfigContent[0];
117
119
 
118
120
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
119
121
  const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
@@ -129,18 +131,21 @@ function registerStoreCurrentDataTests(storageVersion: number) {
129
131
  const checkpoint = flushResult!.flushed_op;
130
132
 
131
133
  const batch = await test_utils.fromAsync(
132
- bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)])
134
+ bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)])
133
135
  );
134
136
  expect(test_utils.getBatchData(batch)).toMatchObject([{ op: 'PUT', object_id: 'test1' }]);
135
137
 
136
138
  // No row payload retained in current_data.
137
- expect(await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.id, table)).toBeNull();
139
+ expect(
140
+ await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.replicationStreamId, table)
141
+ ).toBeNull();
138
142
  });
139
143
 
140
144
  test('storeCurrentData=true retains the row payload in current_data', async () => {
141
145
  await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
142
146
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
143
147
  const bucketStorage = factory.getInstance(syncRules);
148
+ const syncRulesContent = syncRules.syncConfigContent[0];
144
149
 
145
150
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
146
151
  const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
@@ -156,11 +161,13 @@ function registerStoreCurrentDataTests(storageVersion: number) {
156
161
  const checkpoint = flushResult!.flushed_op;
157
162
 
158
163
  const batch = await test_utils.fromAsync(
159
- bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)])
164
+ bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)])
160
165
  );
161
166
  expect(test_utils.getBatchData(batch)).toMatchObject([{ op: 'PUT', object_id: 'test1' }]);
162
167
 
163
- expect(await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.id, table)).toMatchObject({
168
+ expect(
169
+ await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.replicationStreamId, table)
170
+ ).toMatchObject({
164
171
  id: 'test1'
165
172
  });
166
173
  });
@@ -169,6 +176,7 @@ function registerStoreCurrentDataTests(storageVersion: number) {
169
176
  await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
170
177
  const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
171
178
  const bucketStorage = factory.getInstance(syncRules);
179
+ const syncRulesContent = syncRules.syncConfigContent[0];
172
180
 
173
181
  await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
174
182
  const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
@@ -192,13 +200,15 @@ function registerStoreCurrentDataTests(storageVersion: number) {
192
200
  const checkpoint = flushResult!.flushed_op;
193
201
 
194
202
  const batch = await test_utils.fromAsync(
195
- bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRules, 'global[]', 0n)])
203
+ bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)])
196
204
  );
197
205
  const data = test_utils.getBatchData(batch);
198
206
  expect(data.length).toBeGreaterThan(0);
199
207
  expect(data.at(-1)).toMatchObject({ op: 'PUT', object_id: 'test1' });
200
208
 
201
- expect(await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.id, table)).toBeNull();
209
+ expect(
210
+ await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.replicationStreamId, table)
211
+ ).toBeNull();
202
212
  });
203
213
  }
204
214
 
package/test/src/util.ts CHANGED
@@ -4,7 +4,8 @@ import { env } from './env.js';
4
4
 
5
5
  export const INITIALIZED_MONGO_STORAGE_FACTORY = mongoTestStorageFactoryGenerator({
6
6
  url: env.MONGO_TEST_URL,
7
- isCI: env.CI
7
+ isCI: env.CI,
8
+ supportsMultipleSyncConfigs: true
8
9
  });
9
10
 
10
11
  export const INITIALIZED_MONGO_REPORT_STORAGE_FACTORY = mongoTestReportStorageFactoryGenerator({
package/tsconfig.json CHANGED
@@ -2,10 +2,7 @@
2
2
  "extends": "../../tsconfig.base.json",
3
3
  "compilerOptions": {
4
4
  "rootDir": "src",
5
- "outDir": "dist",
6
- "esModuleInterop": true,
7
- "skipLibCheck": true,
8
- "sourceMap": true
5
+ "outDir": "dist"
9
6
  },
10
7
  "include": ["src"],
11
8
  "references": [