@powersync/service-module-mongodb-storage 0.17.0 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/dist/storage/MongoBucketStorage.d.ts +34 -16
- package/dist/storage/MongoBucketStorage.js +219 -70
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +42 -23
- package/dist/storage/implementation/MongoBucketBatch.js +61 -44
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +10 -9
- package/dist/storage/implementation/MongoChecksums.js +3 -111
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +2 -0
- package/dist/storage/implementation/MongoCompactor.js +18 -9
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +6 -5
- package/dist/storage/implementation/MongoParameterCompactor.js +24 -1
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +5 -2
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +41 -23
- package/dist/storage/implementation/MongoSyncBucketStorage.js +51 -43
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +6 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +1 -2
- package/dist/storage/implementation/common/PersistedBatch.js +5 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +9 -14
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +4 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +42 -26
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +114 -3
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +15 -17
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +47 -45
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +286 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +27 -7
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +202 -18
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +19 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +511 -22
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +62 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +350 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +40 -28
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +232 -116
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.js +43 -20
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +52 -16
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +13 -12
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +29 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +4 -0
- package/dist/storage/implementation/v3/bucket-format.js +58 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +28 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +64 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +2 -2
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +5 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +1 -1
- package/dist/utils/util.js +3 -2
- package/dist/utils/util.js.map +1 -1
- package/package.json +9 -19
- package/src/storage/MongoBucketStorage.ts +334 -87
- package/src/storage/implementation/MongoBucketBatch.ts +86 -58
- package/src/storage/implementation/MongoChecksums.ts +6 -138
- package/src/storage/implementation/MongoCompactor.ts +19 -9
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +24 -5
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +5 -2
- package/src/storage/implementation/MongoSyncBucketStorage.ts +70 -56
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +7 -0
- package/src/storage/implementation/common/PersistedBatch.ts +7 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +11 -19
- package/src/storage/implementation/models.ts +4 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +55 -34
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +137 -4
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +70 -64
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +358 -314
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +262 -26
- package/src/storage/implementation/v3/MongoCompactorV3.ts +646 -29
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +480 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +315 -157
- package/src/storage/implementation/v3/PersistedBatchV3.ts +62 -36
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +59 -19
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +41 -36
- package/src/storage/implementation/v3/bucket-format.ts +70 -0
- package/src/storage/implementation/v3/chunking.ts +38 -0
- package/src/storage/implementation/v3/models.ts +93 -55
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/utils/test-utils.ts +6 -3
- package/src/utils/util.ts +3 -2
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +48 -48
- package/test/src/cleanup-stopped-sync-configs.test.ts +648 -0
- package/test/src/storage.test.ts +11 -11
- package/test/src/storage_compacting.test.ts +1956 -6
- package/test/src/storage_sync.test.ts +1077 -68
- package/test/src/storeCurrentData.test.ts +17 -7
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
|
@@ -66,7 +66,7 @@ async function storedRowPayload(
|
|
|
66
66
|
data = doc?.data;
|
|
67
67
|
} else {
|
|
68
68
|
const db = (bucketStorage as any).db;
|
|
69
|
-
const doc = await db.
|
|
69
|
+
const doc = await db.sourceRecords(syncRulesId, table.id).findOne({});
|
|
70
70
|
data = doc?.data;
|
|
71
71
|
}
|
|
72
72
|
if (data == null) {
|
|
@@ -81,6 +81,7 @@ function registerStoreCurrentDataTests(storageVersion: number) {
|
|
|
81
81
|
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
82
82
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
|
|
83
83
|
const bucketStorage = factory.getInstance(syncRules);
|
|
84
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
84
85
|
|
|
85
86
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
86
87
|
|
|
@@ -114,6 +115,7 @@ function registerStoreCurrentDataTests(storageVersion: number) {
|
|
|
114
115
|
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
115
116
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
|
|
116
117
|
const bucketStorage = factory.getInstance(syncRules);
|
|
118
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
117
119
|
|
|
118
120
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
119
121
|
const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
@@ -129,18 +131,21 @@ function registerStoreCurrentDataTests(storageVersion: number) {
|
|
|
129
131
|
const checkpoint = flushResult!.flushed_op;
|
|
130
132
|
|
|
131
133
|
const batch = await test_utils.fromAsync(
|
|
132
|
-
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(
|
|
134
|
+
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)])
|
|
133
135
|
);
|
|
134
136
|
expect(test_utils.getBatchData(batch)).toMatchObject([{ op: 'PUT', object_id: 'test1' }]);
|
|
135
137
|
|
|
136
138
|
// No row payload retained in current_data.
|
|
137
|
-
expect(
|
|
139
|
+
expect(
|
|
140
|
+
await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.replicationStreamId, table)
|
|
141
|
+
).toBeNull();
|
|
138
142
|
});
|
|
139
143
|
|
|
140
144
|
test('storeCurrentData=true retains the row payload in current_data', async () => {
|
|
141
145
|
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
142
146
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
|
|
143
147
|
const bucketStorage = factory.getInstance(syncRules);
|
|
148
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
144
149
|
|
|
145
150
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
146
151
|
const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
@@ -156,11 +161,13 @@ function registerStoreCurrentDataTests(storageVersion: number) {
|
|
|
156
161
|
const checkpoint = flushResult!.flushed_op;
|
|
157
162
|
|
|
158
163
|
const batch = await test_utils.fromAsync(
|
|
159
|
-
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(
|
|
164
|
+
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)])
|
|
160
165
|
);
|
|
161
166
|
expect(test_utils.getBatchData(batch)).toMatchObject([{ op: 'PUT', object_id: 'test1' }]);
|
|
162
167
|
|
|
163
|
-
expect(
|
|
168
|
+
expect(
|
|
169
|
+
await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.replicationStreamId, table)
|
|
170
|
+
).toMatchObject({
|
|
164
171
|
id: 'test1'
|
|
165
172
|
});
|
|
166
173
|
});
|
|
@@ -169,6 +176,7 @@ function registerStoreCurrentDataTests(storageVersion: number) {
|
|
|
169
176
|
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
170
177
|
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion }));
|
|
171
178
|
const bucketStorage = factory.getInstance(syncRules);
|
|
179
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
172
180
|
|
|
173
181
|
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
174
182
|
const table = await test_utils.resolveTestTable(writer, 'test_data', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
@@ -192,13 +200,15 @@ function registerStoreCurrentDataTests(storageVersion: number) {
|
|
|
192
200
|
const checkpoint = flushResult!.flushed_op;
|
|
193
201
|
|
|
194
202
|
const batch = await test_utils.fromAsync(
|
|
195
|
-
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(
|
|
203
|
+
bucketStorage.getBucketDataBatch(checkpoint, [bucketRequest(syncRulesContent, 'global[]', 0n)])
|
|
196
204
|
);
|
|
197
205
|
const data = test_utils.getBatchData(batch);
|
|
198
206
|
expect(data.length).toBeGreaterThan(0);
|
|
199
207
|
expect(data.at(-1)).toMatchObject({ op: 'PUT', object_id: 'test1' });
|
|
200
208
|
|
|
201
|
-
expect(
|
|
209
|
+
expect(
|
|
210
|
+
await storedRowPayload(storageVersion, factory, bucketStorage, syncRules.replicationStreamId, table)
|
|
211
|
+
).toBeNull();
|
|
202
212
|
});
|
|
203
213
|
}
|
|
204
214
|
|
package/test/src/util.ts
CHANGED
|
@@ -4,7 +4,8 @@ import { env } from './env.js';
|
|
|
4
4
|
|
|
5
5
|
export const INITIALIZED_MONGO_STORAGE_FACTORY = mongoTestStorageFactoryGenerator({
|
|
6
6
|
url: env.MONGO_TEST_URL,
|
|
7
|
-
isCI: env.CI
|
|
7
|
+
isCI: env.CI,
|
|
8
|
+
supportsMultipleSyncConfigs: true
|
|
8
9
|
});
|
|
9
10
|
|
|
10
11
|
export const INITIALIZED_MONGO_REPORT_STORAGE_FACTORY = mongoTestReportStorageFactoryGenerator({
|
package/tsconfig.json
CHANGED