@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +177 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +71 -18
- package/dist/storage/MongoBucketStorage.js +516 -202
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
- package/dist/storage/implementation/MongoBucketBatch.js +140 -66
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
- package/dist/storage/implementation/MongoCompactor.js +87 -481
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
- package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +41 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
- package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
- package/dist/storage/implementation/common/PersistedBatch.js +63 -10
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +4 -1
- package/dist/storage/implementation/db.js +31 -16
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +55 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
- package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
- package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +79 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
- package/dist/storage/implementation/v3/compact-utils.js +216 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +187 -23
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
- package/dist/storage/implementation/v3/source-table-utils.js +206 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +5 -4
- package/dist/storage/storage-index.js +5 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +71 -3
- package/dist/types/types.js +58 -2
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +131 -5
- package/dist/utils/util.js.map +1 -1
- package/package.json +13 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +728 -242
- package/src/storage/implementation/MongoBucketBatch.ts +196 -91
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +106 -631
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
- package/src/storage/implementation/MongoStorageProvider.ts +48 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
- package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
- package/src/storage/implementation/common/PersistedBatch.ts +83 -15
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +40 -22
- package/src/storage/implementation/models.ts +58 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +12 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
- package/src/storage/implementation/v3/bucket-format.ts +99 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/compact-utils.ts +320 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +228 -58
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
- package/src/storage/implementation/v3/source-table-utils.ts +398 -0
- package/src/storage/storage-index.ts +14 -4
- package/src/types/types.ts +76 -2
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +188 -7
- package/test/src/__snapshots__/storage.test.ts.snap +4 -54
- package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
- package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
- package/test/src/compact-utils.test.ts +226 -0
- package/test/src/helpers/MemoryObjectStorage.ts +61 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/object_storage_usage.test.ts +255 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +478 -13
- package/test/src/storage_compacting.test.ts +2516 -107
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +190 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
- package/test/src/storage_s3_reading.test.ts +768 -0
- package/test/src/storage_s3_writing.test.ts +136 -0
- package/test/src/storage_sync.test.ts +1507 -76
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +3 -2
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js';
|
|
2
|
+
import { storage, updateSyncRulesFromYaml } from '@powersync/service-core';
|
|
3
|
+
import { bucketRequest, test_utils } from '@powersync/service-core-tests';
|
|
4
|
+
import { describe, expect, test } from 'vitest';
|
|
5
|
+
import { DEFAULT_INLINE_THRESHOLD_BYTES } from '../../src/storage/implementation/common/PersistedBatch.js';
|
|
6
|
+
import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
|
|
7
|
+
import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
|
|
8
|
+
import { env } from './env.js';
|
|
9
|
+
import { MemoryObjectStorage } from './helpers/MemoryObjectStorage.js';
|
|
10
|
+
|
|
11
|
+
const SYNC_RULES_YAML = `
|
|
12
|
+
bucket_definitions:
|
|
13
|
+
global:
|
|
14
|
+
data:
|
|
15
|
+
- SELECT id, description FROM items
|
|
16
|
+
`;
|
|
17
|
+
|
|
18
|
+
function s3Factory(threshold?: number) {
|
|
19
|
+
const memoryStorage = new MemoryObjectStorage();
|
|
20
|
+
const factoryGen = mongoTestStorageFactoryGenerator({
|
|
21
|
+
url: env.MONGO_TEST_URL,
|
|
22
|
+
isCI: env.CI,
|
|
23
|
+
objectStorage: memoryStorage,
|
|
24
|
+
inlineThresholdBytes: threshold
|
|
25
|
+
});
|
|
26
|
+
return { memoryStorage, factoryGen };
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
describe('Object storage inline threshold', () => {
|
|
30
|
+
test('stores small documents inline using the default threshold', async () => {
|
|
31
|
+
const { memoryStorage, factoryGen } = s3Factory();
|
|
32
|
+
await using factory = await factoryGen.factory();
|
|
33
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
|
|
34
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
35
|
+
expect(bucketStorage.inlineThresholdBytes).toBe(DEFAULT_INLINE_THRESHOLD_BYTES);
|
|
36
|
+
|
|
37
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
38
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
|
|
39
|
+
await writer.markAllSnapshotDone('1/1');
|
|
40
|
+
|
|
41
|
+
await writer.save({
|
|
42
|
+
sourceTable,
|
|
43
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
44
|
+
after: { id: 'small1', description: 'tiny' },
|
|
45
|
+
afterReplicaId: test_utils.rid('small1')
|
|
46
|
+
});
|
|
47
|
+
await writer.save({
|
|
48
|
+
sourceTable,
|
|
49
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
50
|
+
after: { id: 'small2', description: 'tiny' },
|
|
51
|
+
afterReplicaId: test_utils.rid('small2')
|
|
52
|
+
});
|
|
53
|
+
await writer.commit('1/1');
|
|
54
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
55
|
+
const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
|
|
56
|
+
|
|
57
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
58
|
+
const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
|
|
59
|
+
const documents = await db
|
|
60
|
+
.bucketData(bucketStorage.replicationStreamId, definitionId)
|
|
61
|
+
.find({ '_id.b': request.bucket })
|
|
62
|
+
.toArray();
|
|
63
|
+
expect(documents).not.toHaveLength(0);
|
|
64
|
+
for (const document of documents) {
|
|
65
|
+
expect(document.size).toBeLessThanOrEqual(DEFAULT_INLINE_THRESHOLD_BYTES);
|
|
66
|
+
expect(document.ops).toBeDefined();
|
|
67
|
+
expect(document.storage_ref).toBeUndefined();
|
|
68
|
+
}
|
|
69
|
+
expect(memoryStorage.store.size).toBe(0);
|
|
70
|
+
|
|
71
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
72
|
+
const data = test_utils.getBatchData(batch);
|
|
73
|
+
expect(data.map((op) => op.object_id)).toEqual(['small1', 'small2']);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
test('stores documents above a configured threshold in object storage', async () => {
|
|
77
|
+
const { memoryStorage, factoryGen } = s3Factory(256);
|
|
78
|
+
await using factory = await factoryGen.factory();
|
|
79
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
|
|
80
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
81
|
+
expect(bucketStorage.inlineThresholdBytes).toBe(256);
|
|
82
|
+
|
|
83
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
84
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
|
|
85
|
+
await writer.markAllSnapshotDone('1/1');
|
|
86
|
+
|
|
87
|
+
await writer.save({
|
|
88
|
+
sourceTable,
|
|
89
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
90
|
+
after: { id: 'large', description: 'value'.repeat(500) },
|
|
91
|
+
afterReplicaId: test_utils.rid('large')
|
|
92
|
+
});
|
|
93
|
+
await writer.commit('1/1');
|
|
94
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
95
|
+
const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
|
|
96
|
+
|
|
97
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
98
|
+
const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
|
|
99
|
+
const documents = await db
|
|
100
|
+
.bucketData(bucketStorage.replicationStreamId, definitionId)
|
|
101
|
+
.find({ '_id.b': request.bucket })
|
|
102
|
+
.toArray();
|
|
103
|
+
expect(documents).not.toHaveLength(0);
|
|
104
|
+
for (const document of documents) {
|
|
105
|
+
expect(document.size).toBeGreaterThan(256);
|
|
106
|
+
expect(document.ops).toBeUndefined();
|
|
107
|
+
expect(document.storage_ref).toBeDefined();
|
|
108
|
+
}
|
|
109
|
+
expect(new Set(memoryStorage.store.keys())).toEqual(
|
|
110
|
+
new Set(documents.map((document) => document.storage_ref!.path))
|
|
111
|
+
);
|
|
112
|
+
|
|
113
|
+
const batch = await test_utils.fromAsync(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
114
|
+
expect(test_utils.getBatchData(batch).map((op) => op.object_id)).toEqual(['large']);
|
|
115
|
+
});
|
|
116
|
+
});
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import {
|
|
2
|
+
addChecksums,
|
|
3
|
+
CheckpointChecksumInvalidatedError,
|
|
4
|
+
storage,
|
|
5
|
+
updateSyncRulesFromYaml
|
|
6
|
+
} from '@powersync/service-core';
|
|
7
|
+
import { bucketRequest, compactActive, test_utils } from '@powersync/service-core-tests';
|
|
8
|
+
import { describe, expect, test } from 'vitest';
|
|
9
|
+
import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js';
|
|
10
|
+
import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
|
|
11
|
+
import { env } from './env.js';
|
|
12
|
+
import { createS3TestStorageSuite } from './helpers/s3TestFactory.js';
|
|
13
|
+
|
|
14
|
+
const SYNC_RULES_YAML = `
|
|
15
|
+
bucket_definitions:
|
|
16
|
+
global:
|
|
17
|
+
data:
|
|
18
|
+
- SELECT id, description FROM items
|
|
19
|
+
`;
|
|
20
|
+
|
|
21
|
+
function s3Factory() {
|
|
22
|
+
const { objectStorage, factoryGen } = createS3TestStorageSuite({ url: env.MONGO_TEST_URL, isCI: env.CI });
|
|
23
|
+
return { memoryStorage: objectStorage, factoryGen };
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
describe('V3 checksums with S3 object storage', () => {
|
|
27
|
+
test('partial checksum with start straddling S3-backed document', async () => {
|
|
28
|
+
const { factoryGen } = s3Factory();
|
|
29
|
+
await using factory = await factoryGen.factory();
|
|
30
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
|
|
31
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
32
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
33
|
+
|
|
34
|
+
const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
|
|
35
|
+
const bucket = request.bucket;
|
|
36
|
+
const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
|
|
37
|
+
|
|
38
|
+
// Write 6 ops through S3 writer — all land in one S3-backed doc
|
|
39
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
40
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
|
|
41
|
+
await writer.markAllSnapshotDone('1/1');
|
|
42
|
+
for (const id of ['A', 'B', 'C', 'D', 'E', 'F']) {
|
|
43
|
+
await writer.save({
|
|
44
|
+
sourceTable,
|
|
45
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
46
|
+
after: { id, description: id },
|
|
47
|
+
afterReplicaId: test_utils.rid(id)
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
await writer.commit('1/1');
|
|
51
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
52
|
+
|
|
53
|
+
// Baseline: full checksum with no compacted_state
|
|
54
|
+
const full = (await bucketStorage.getChecksums(checkpoint, [request])).get(bucket)!;
|
|
55
|
+
|
|
56
|
+
// Set compacted_state.op_id = 3 to create a partial range starting after op 3.
|
|
57
|
+
// The doc has min_op=1, _id.o=6 so is_fully_included=false for the range (3, 6].
|
|
58
|
+
await db.bucketState(bucketStorage.replicationStreamId).updateOne(
|
|
59
|
+
{ _id: { d: definitionId, b: bucket } },
|
|
60
|
+
{
|
|
61
|
+
$set: {
|
|
62
|
+
last_op: 3n,
|
|
63
|
+
compacted_state: { op_id: 3n, count: 0, checksum: 0n, bytes: 0n, chunks: 0, at: new Date() }
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
{ upsert: true }
|
|
67
|
+
);
|
|
68
|
+
|
|
69
|
+
const partial = (await bucketStorage.getChecksums(checkpoint, [request])).get(bucket)!;
|
|
70
|
+
expect(partial.checksum).toBe(full.checksum);
|
|
71
|
+
expect(partial.count).toBe(full.count);
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test('partial checksum with end straddling S3-backed document invalidates the checkpoint', async () => {
|
|
75
|
+
const { factoryGen } = s3Factory();
|
|
76
|
+
await using factory = await factoryGen.factory();
|
|
77
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
|
|
78
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
79
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
80
|
+
|
|
81
|
+
const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
|
|
82
|
+
const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
|
|
83
|
+
|
|
84
|
+
// Write several operations into one S3-backed document that straddles the
|
|
85
|
+
// requested checkpoint.
|
|
86
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
87
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
|
|
88
|
+
await writer.markAllSnapshotDone('1/1');
|
|
89
|
+
for (let i = 1; i <= 12; i++) {
|
|
90
|
+
const id = `row${i}`;
|
|
91
|
+
await writer.save({
|
|
92
|
+
sourceTable,
|
|
93
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
94
|
+
after: { id, description: `value${i}`.repeat(50) },
|
|
95
|
+
afterReplicaId: test_utils.rid(id)
|
|
96
|
+
});
|
|
97
|
+
}
|
|
98
|
+
await writer.commit('1/1');
|
|
99
|
+
const documents = await db
|
|
100
|
+
.bucketData(bucketStorage.replicationStreamId, definitionId)
|
|
101
|
+
.find({ '_id.b': request.bucket })
|
|
102
|
+
.toArray();
|
|
103
|
+
expect(documents).toHaveLength(1);
|
|
104
|
+
expect(documents[0].storage_ref).toBeDefined();
|
|
105
|
+
expect(documents[0].ops).toBeUndefined();
|
|
106
|
+
|
|
107
|
+
// The document starts before op 7 and ends after it.
|
|
108
|
+
await expect(bucketStorage.getChecksums(test_utils.testCheckpoint(7n), [request])).rejects.toBeInstanceOf(
|
|
109
|
+
CheckpointChecksumInvalidatedError
|
|
110
|
+
);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test('checksum preserved after CLEAR-producing S3 compaction', async () => {
|
|
114
|
+
const { factoryGen } = s3Factory();
|
|
115
|
+
await using factory = await factoryGen.factory();
|
|
116
|
+
const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 }));
|
|
117
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
118
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
119
|
+
|
|
120
|
+
const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n);
|
|
121
|
+
const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0];
|
|
122
|
+
|
|
123
|
+
await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS);
|
|
124
|
+
const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1);
|
|
125
|
+
await writer.markAllSnapshotDone('1/1');
|
|
126
|
+
|
|
127
|
+
// A@1, A@2 get superseded by A@4 (same replica_id for dedup). B@3 independent.
|
|
128
|
+
await writer.save({
|
|
129
|
+
sourceTable,
|
|
130
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
131
|
+
after: { id: 'A', description: 'v1' },
|
|
132
|
+
afterReplicaId: test_utils.rid('A')
|
|
133
|
+
});
|
|
134
|
+
await writer.save({
|
|
135
|
+
sourceTable,
|
|
136
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
137
|
+
after: { id: 'A', description: 'v2' },
|
|
138
|
+
afterReplicaId: test_utils.rid('A')
|
|
139
|
+
});
|
|
140
|
+
await writer.save({
|
|
141
|
+
sourceTable,
|
|
142
|
+
tag: storage.SaveOperationTag.INSERT,
|
|
143
|
+
after: { id: 'B', description: 'beta' },
|
|
144
|
+
afterReplicaId: test_utils.rid('B')
|
|
145
|
+
});
|
|
146
|
+
await writer.save({
|
|
147
|
+
sourceTable,
|
|
148
|
+
tag: storage.SaveOperationTag.UPDATE,
|
|
149
|
+
after: { id: 'A', description: 'v4' },
|
|
150
|
+
afterReplicaId: test_utils.rid('A')
|
|
151
|
+
});
|
|
152
|
+
await writer.commit('1/1');
|
|
153
|
+
const checkpoint = await bucketStorage.getCheckpoint();
|
|
154
|
+
|
|
155
|
+
// Compact — produces CLEAR doc from collapsed MOVEs
|
|
156
|
+
await compactActive(factory, {
|
|
157
|
+
maxOpId: checkpoint.checkpoint,
|
|
158
|
+
compactBuckets: [request.bucket],
|
|
159
|
+
clearBatchLimit: 200,
|
|
160
|
+
moveBatchLimit: 10,
|
|
161
|
+
moveBatchQueryLimit: 10,
|
|
162
|
+
moveBatchByteLimit: 1024,
|
|
163
|
+
minBucketChanges: 1,
|
|
164
|
+
minChangeRatio: 0
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
// Shift compacted_state.op_id back to before the CLEAR doc so getChecksums
|
|
168
|
+
// queries the pipeline (not just reads compacted_state directly).
|
|
169
|
+
await db
|
|
170
|
+
.bucketState(bucketStorage.replicationStreamId)
|
|
171
|
+
.updateOne({ _id: { d: definitionId, b: request.bucket } }, { $set: { 'compacted_state.op_id': 0n } });
|
|
172
|
+
|
|
173
|
+
// Ground truth: sum of doc-level checksum fields
|
|
174
|
+
const docs = await db
|
|
175
|
+
.bucketData(bucketStorage.replicationStreamId, definitionId)
|
|
176
|
+
.find({ '_id.b': request.bucket })
|
|
177
|
+
.toArray();
|
|
178
|
+
const groundTruth = docs.reduce((sum: number, d: any) => addChecksums(sum, Number(d.checksum)), 0);
|
|
179
|
+
expect(docs.some((doc) => doc.has_clear_op)).toBe(true);
|
|
180
|
+
expect(docs.every((doc) => doc.storage_ref != null && doc.ops == null)).toBe(true);
|
|
181
|
+
|
|
182
|
+
const checksum = (await bucketStorage.getChecksums(checkpoint, [request])).get(request.bucket)!;
|
|
183
|
+
expect(checksum.checksum).toBe(groundTruth);
|
|
184
|
+
|
|
185
|
+
// The two superseded A operations collapse into CLEAR.
|
|
186
|
+
const batchAfter = await test_utils.getBatchArray(bucketStorage.getBucketDataBatch(checkpoint, [request]));
|
|
187
|
+
const dataAfter = batchAfter.flatMap((chunk) => chunk.chunkData.data);
|
|
188
|
+
expect(dataAfter).toMatchObject([{ op: 'CLEAR' }, { object_id: 'B', op: 'PUT' }, { object_id: 'A', op: 'PUT' }]);
|
|
189
|
+
});
|
|
190
|
+
});
|