@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -1,6 +1,18 @@
|
|
|
1
|
+
import { BucketDataDoc } from '@module/storage/implementation/common/BucketDataDoc.js';
|
|
2
|
+
import { MongoSyncBucketStorage } from '@module/storage/implementation/createMongoSyncBucketStorage.js';
|
|
3
|
+
import { loadBucketDataDocument, serializeBucketData } from '@module/storage/implementation/v3/bucket-format.js';
|
|
4
|
+
import { chunkBucketData, DEFAULT_MAX_DOC_SIZE_BYTES } from '@module/storage/implementation/v3/chunking.js';
|
|
5
|
+
import { BucketDataDocumentV3 } from '@module/storage/implementation/v3/models.js';
|
|
1
6
|
import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/VersionedPowerSyncMongoV3.js';
|
|
2
|
-
import {
|
|
7
|
+
import {
|
|
8
|
+
addChecksums,
|
|
9
|
+
CheckpointChecksumInvalidatedError,
|
|
10
|
+
storage,
|
|
11
|
+
SyncRulesBucketStorage,
|
|
12
|
+
updateSyncRulesFromYaml
|
|
13
|
+
} from '@powersync/service-core';
|
|
3
14
|
import { bucketRequest, register, test_utils } from '@powersync/service-core-tests';
|
|
15
|
+
import * as bson from 'bson';
|
|
4
16
|
import { describe, expect, test } from 'vitest';
|
|
5
17
|
import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js';
|
|
6
18
|
|
|
@@ -57,9 +69,10 @@ bucket_definitions:
|
|
|
57
69
|
`)
|
|
58
70
|
);
|
|
59
71
|
const bucketStorage = factory.getInstance(syncRules);
|
|
72
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
60
73
|
const { checkpoint } = await populate(bucketStorage, 1);
|
|
61
74
|
|
|
62
|
-
return { bucketStorage, checkpoint, factory, syncRules };
|
|
75
|
+
return { bucketStorage, checkpoint, factory, syncRules: syncRulesContent };
|
|
63
76
|
};
|
|
64
77
|
|
|
65
78
|
test('full compact', async () => {
|
|
@@ -70,7 +83,7 @@ bucket_definitions:
|
|
|
70
83
|
if (storageDb.storageConfig.incrementalReprocessing) {
|
|
71
84
|
// This should actually never happen on V3, but we test this anyway.
|
|
72
85
|
// Can remove this if it causes issues in the future.
|
|
73
|
-
await (storageDb as VersionedPowerSyncMongoV3).
|
|
86
|
+
await (storageDb as VersionedPowerSyncMongoV3).bucketState(bucketStorage.replicationStreamId).deleteMany({});
|
|
74
87
|
} else {
|
|
75
88
|
await factory.db.bucket_state.deleteMany({});
|
|
76
89
|
}
|
|
@@ -81,14 +94,13 @@ bucket_definitions:
|
|
|
81
94
|
moveBatchQueryLimit: 10,
|
|
82
95
|
minBucketChanges: 1,
|
|
83
96
|
minChangeRatio: 0,
|
|
84
|
-
maxOpId: checkpoint
|
|
85
|
-
signal: null as any
|
|
97
|
+
maxOpId: checkpoint
|
|
86
98
|
});
|
|
87
99
|
|
|
88
100
|
const users = ['u1', 'u2'];
|
|
89
101
|
const userRequests = users.map((user) => bucketRequest(syncRules, `by_user["${user}"]`));
|
|
90
102
|
const [u1Request, u2Request] = userRequests;
|
|
91
|
-
const checksumAfter = await bucketStorage.getChecksums(checkpoint, userRequests);
|
|
103
|
+
const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), userRequests);
|
|
92
104
|
expect(checksumAfter.get(u1Request.bucket)).toEqual({
|
|
93
105
|
bucket: u1Request.bucket,
|
|
94
106
|
checksum: -659469718,
|
|
@@ -115,7 +127,7 @@ bucket_definitions:
|
|
|
115
127
|
`)
|
|
116
128
|
);
|
|
117
129
|
const bucketStorage = factory.getInstance(syncRules);
|
|
118
|
-
const
|
|
130
|
+
const syncRulesContent = syncRules.syncConfigContent[0];
|
|
119
131
|
|
|
120
132
|
await populate(bucketStorage, 2);
|
|
121
133
|
const { checkpoint } = await bucketStorage.getCheckpoint();
|
|
@@ -141,9 +153,9 @@ bucket_definitions:
|
|
|
141
153
|
expect(result2.buckets).toEqual(0);
|
|
142
154
|
|
|
143
155
|
const users = ['u1', 'u2'];
|
|
144
|
-
const userRequests = users.map((user) => bucketRequest(
|
|
156
|
+
const userRequests = users.map((user) => bucketRequest(syncRulesContent, `by_user2["${user}"]`));
|
|
145
157
|
const [u1Request, u2Request] = userRequests;
|
|
146
|
-
const checksumAfter = await bucketStorage.getChecksums(checkpoint, userRequests);
|
|
158
|
+
const checksumAfter = await bucketStorage.getChecksums(test_utils.testCheckpoint(checkpoint), userRequests);
|
|
147
159
|
expect(checksumAfter.get(u1Request.bucket)).toEqual({
|
|
148
160
|
bucket: u1Request.bucket,
|
|
149
161
|
checksum: -659469718,
|
|
@@ -172,7 +184,9 @@ bucket_definitions:
|
|
|
172
184
|
// This typically happens when buckets get very large (> 2GiB). We don't want to create that much
|
|
173
185
|
// data in the tests, so we directly insert the bucket_state here.
|
|
174
186
|
if (storageDb.storageConfig.incrementalReprocessing) {
|
|
175
|
-
const bucketStateCollection = (storageDb as VersionedPowerSyncMongoV3).
|
|
187
|
+
const bucketStateCollection = (storageDb as VersionedPowerSyncMongoV3).bucketState(
|
|
188
|
+
bucketStorage.replicationStreamId
|
|
189
|
+
);
|
|
176
190
|
await bucketStateCollection.insertOne({
|
|
177
191
|
_id: {
|
|
178
192
|
d: '1',
|
|
@@ -193,7 +207,7 @@ bucket_definitions:
|
|
|
193
207
|
} else {
|
|
194
208
|
await factory.db.bucket_state.insertOne({
|
|
195
209
|
_id: {
|
|
196
|
-
g: bucketStorage.
|
|
210
|
+
g: bucketStorage.replicationStreamId,
|
|
197
211
|
b: 'global[]'
|
|
198
212
|
},
|
|
199
213
|
last_op: 5n,
|
|
@@ -226,7 +240,7 @@ bucket_definitions:
|
|
|
226
240
|
expect(typeof firstBatch.value[0].estimatedCount).toBe('number');
|
|
227
241
|
expect(firstBatch.value[0].dirtyRatio).toBeCloseTo(5 / 12);
|
|
228
242
|
|
|
229
|
-
const checksumBuckets = await
|
|
243
|
+
const checksumBuckets = await compactor.dirtyBucketBatchForChecksums({
|
|
230
244
|
minBucketChanges: 1
|
|
231
245
|
});
|
|
232
246
|
expect(checksumBuckets).toEqual([
|
|
@@ -243,3 +257,1928 @@ bucket_definitions:
|
|
|
243
257
|
describe('Mongo Sync Parameter Storage Compact', () => {
|
|
244
258
|
register.registerParameterCompactTests(INITIALIZED_MONGO_STORAGE_FACTORY);
|
|
245
259
|
});
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* V3 Invariant Verification
|
|
263
|
+
*
|
|
264
|
+
* Tests in this block exercise two levels:
|
|
265
|
+
*
|
|
266
|
+
* Unit tests (no MongoDB): call serializeBucketData(), chunkBucketData(), or
|
|
267
|
+
* loadBucketDataDocument() directly. These use makeBucketDataDoc() with fake
|
|
268
|
+
* bucket keys and source table IDs.
|
|
269
|
+
*
|
|
270
|
+
* Integration tests (full MongoDB): provision V3 storage, insert pre-serialized
|
|
271
|
+
* documents directly into collections, trigger compaction via
|
|
272
|
+
* bucketStorage.compact(), then read back and verify. Surfaces exercised:
|
|
273
|
+
*
|
|
274
|
+
* - collection.insertMany (direct bucket_data writes)
|
|
275
|
+
* - bucketStateCollection.insertOne (so compactor discovers buckets)
|
|
276
|
+
* - bucketStorage.compact() (full pipeline: dirtyBucketBatches →
|
|
277
|
+
* compactSingleBucket → delete-all + rechunk in transaction →
|
|
278
|
+
* writeBucketStateUpdates)
|
|
279
|
+
* - collection.find with _id.b filters (verify post-compaction state)
|
|
280
|
+
*
|
|
281
|
+
* The initial write path (MongoBucketBatchV3 → flushBucketDataShared →
|
|
282
|
+
* chunkBucketData → serializeBucketData → bulk write) is NOT exercised here —
|
|
283
|
+
* that path is covered by the existing shared tests in register-compacting-tests.ts.
|
|
284
|
+
*/
|
|
285
|
+
describe('V3 invariant verification', () => {
|
|
286
|
+
const BUCKET = 'global[]';
|
|
287
|
+
const TABLE = 'items';
|
|
288
|
+
|
|
289
|
+
function makeBucketDataDoc(overrides: Partial<BucketDataDoc> & { o: bigint }): BucketDataDoc {
|
|
290
|
+
return {
|
|
291
|
+
bucketKey: { replicationStreamId: 1, definitionId: '1', bucket: 'test[]' },
|
|
292
|
+
op: 'PUT',
|
|
293
|
+
source_table: new bson.ObjectId(),
|
|
294
|
+
source_key: 'key',
|
|
295
|
+
table: 'test',
|
|
296
|
+
row_id: 'row1',
|
|
297
|
+
checksum: 1n,
|
|
298
|
+
data: '{"id":"row1"}',
|
|
299
|
+
...overrides
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
async function setupV3Storage() {
|
|
304
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
305
|
+
const syncRules = await factory.updateSyncRules(
|
|
306
|
+
updateSyncRulesFromYaml(
|
|
307
|
+
`
|
|
308
|
+
bucket_definitions:
|
|
309
|
+
global:
|
|
310
|
+
data: [SELECT id as id, description FROM items]
|
|
311
|
+
`,
|
|
312
|
+
{ storageVersion: 3 }
|
|
313
|
+
)
|
|
314
|
+
);
|
|
315
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
316
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
317
|
+
const mapping = syncRules.syncConfigContent[0].mapping;
|
|
318
|
+
const definitionId = mapping.allBucketDefinitionIds()[0];
|
|
319
|
+
const collection = db.bucketData(bucketStorage.replicationStreamId, definitionId);
|
|
320
|
+
const bucketStateCollection = db.bucketState(bucketStorage.replicationStreamId);
|
|
321
|
+
const sourceTableId = new bson.ObjectId();
|
|
322
|
+
|
|
323
|
+
const ctx = {
|
|
324
|
+
replicationStreamId: bucketStorage.replicationStreamId,
|
|
325
|
+
definitionId,
|
|
326
|
+
bucket: BUCKET
|
|
327
|
+
};
|
|
328
|
+
|
|
329
|
+
return { bucketStorage, syncRules, db, collection, bucketStateCollection, sourceTableId, ctx, definitionId };
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
function makeOp(
|
|
333
|
+
opId: number,
|
|
334
|
+
rowId: string,
|
|
335
|
+
data: string,
|
|
336
|
+
ctx: { replicationStreamId: number; definitionId: string; bucket: string },
|
|
337
|
+
sourceTableId: bson.ObjectId,
|
|
338
|
+
overrides?: { op?: 'PUT' | 'REMOVE' }
|
|
339
|
+
): BucketDataDoc {
|
|
340
|
+
return {
|
|
341
|
+
bucketKey: {
|
|
342
|
+
replicationStreamId: ctx.replicationStreamId,
|
|
343
|
+
definitionId: ctx.definitionId,
|
|
344
|
+
bucket: ctx.bucket
|
|
345
|
+
},
|
|
346
|
+
o: BigInt(opId),
|
|
347
|
+
op: overrides?.op ?? 'PUT',
|
|
348
|
+
source_table: sourceTableId,
|
|
349
|
+
source_key: test_utils.rid(rowId),
|
|
350
|
+
table: TABLE,
|
|
351
|
+
row_id: rowId,
|
|
352
|
+
checksum: BigInt(opId * 7),
|
|
353
|
+
data: overrides?.op === 'REMOVE' ? null : JSON.stringify({ id: rowId, description: data })
|
|
354
|
+
};
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
async function insertDocs(collection: any, docs: BucketDataDocumentV3[]) {
|
|
358
|
+
await collection.insertMany(docs);
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
async function insertBucketState(bucketStateCollection: any, definitionId: string, lastOp: bigint) {
|
|
362
|
+
await bucketStateCollection.insertOne({
|
|
363
|
+
_id: { d: definitionId, b: BUCKET },
|
|
364
|
+
last_op: lastOp,
|
|
365
|
+
estimate_since_compact: { count: 10, bytes: 100 }
|
|
366
|
+
});
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
async function compact(bucketStorage: MongoSyncBucketStorage, maxOpId: bigint) {
|
|
370
|
+
await bucketStorage.compact({
|
|
371
|
+
clearBatchLimit: 200,
|
|
372
|
+
moveBatchLimit: 10,
|
|
373
|
+
moveBatchQueryLimit: 10,
|
|
374
|
+
minBucketChanges: 1,
|
|
375
|
+
minChangeRatio: 0,
|
|
376
|
+
maxOpId
|
|
377
|
+
});
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
test('1. ops[] ordering - preserves caller ordering (no implicit sort)', () => {
|
|
381
|
+
const ops = [
|
|
382
|
+
makeBucketDataDoc({ o: 5n, data: '{"id":"c"}' }),
|
|
383
|
+
makeBucketDataDoc({ o: 3n, data: '{"id":"a"}' }),
|
|
384
|
+
makeBucketDataDoc({ o: 7n, data: '{"id":"b"}' })
|
|
385
|
+
];
|
|
386
|
+
|
|
387
|
+
const doc = serializeBucketData('test[]', ops);
|
|
388
|
+
|
|
389
|
+
expect(doc.ops!.map((op) => op.o)).toEqual([5n, 3n, 7n]);
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
test('1. ops[] ordering - preserved after compaction', async () => {
|
|
393
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage();
|
|
394
|
+
const ops = [
|
|
395
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
396
|
+
makeOp(20, 'A', 'a2', ctx, sourceTableId),
|
|
397
|
+
makeOp(30, 'A', 'a3', ctx, sourceTableId)
|
|
398
|
+
];
|
|
399
|
+
const doc1 = serializeBucketData(BUCKET, ops);
|
|
400
|
+
await insertDocs(collection, [doc1]);
|
|
401
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 30n);
|
|
402
|
+
|
|
403
|
+
await compact(bucketStorage, 30n);
|
|
404
|
+
|
|
405
|
+
const docs = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
406
|
+
for (const doc of docs) {
|
|
407
|
+
for (let i = 1; i < doc.ops!.length; i++) {
|
|
408
|
+
expect(doc.ops![i].o).toBeGreaterThanOrEqual(doc.ops![i - 1].o);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
});
|
|
412
|
+
|
|
413
|
+
test('2. range metadata consistency - serializeBucketData fields', () => {
|
|
414
|
+
const ops = [
|
|
415
|
+
makeBucketDataDoc({ o: 3n, checksum: 10n, data: 'aaaa' }),
|
|
416
|
+
makeBucketDataDoc({ o: 5n, checksum: 20n, data: 'bbbbb' }),
|
|
417
|
+
makeBucketDataDoc({ o: 8n, checksum: 30n, data: 'cccccccc' })
|
|
418
|
+
];
|
|
419
|
+
|
|
420
|
+
const doc = serializeBucketData('test[]', ops);
|
|
421
|
+
|
|
422
|
+
expect(doc._id.o).toBe(8n);
|
|
423
|
+
expect(doc.min_op).toBe(3n);
|
|
424
|
+
expect(doc.count).toBe(3);
|
|
425
|
+
expect(doc.checksum).toBe(60n);
|
|
426
|
+
expect(doc.size).toBe(bson.calculateObjectSize(doc.ops!));
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
test('2. has_clear_op is only stored when true', () => {
|
|
430
|
+
const putDoc = serializeBucketData('test[]', [makeBucketDataDoc({ o: 1n })]);
|
|
431
|
+
const clearDoc = serializeBucketData('test[]', [makeBucketDataDoc({ o: 2n, op: 'CLEAR', data: null })]);
|
|
432
|
+
|
|
433
|
+
expect(putDoc).not.toHaveProperty('has_clear_op');
|
|
434
|
+
expect(clearDoc.has_clear_op).toBe(true);
|
|
435
|
+
});
|
|
436
|
+
|
|
437
|
+
test('2. range metadata consistency - after compaction', async () => {
|
|
438
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage();
|
|
439
|
+
const ops = [makeOp(10, 'A', 'a1', ctx, sourceTableId), makeOp(20, 'B', 'b1', ctx, sourceTableId)];
|
|
440
|
+
const doc1 = serializeBucketData(BUCKET, ops);
|
|
441
|
+
await insertDocs(collection, [doc1]);
|
|
442
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 20n);
|
|
443
|
+
|
|
444
|
+
await compact(bucketStorage, 20n);
|
|
445
|
+
|
|
446
|
+
const docs = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
447
|
+
for (const doc of docs) {
|
|
448
|
+
expect(doc._id.o).toBe(doc.ops!.reduce((max, op) => (op.o > max ? op.o : max), 0n));
|
|
449
|
+
expect(doc.min_op).toBe(doc.ops!.reduce((min, op) => (op.o < min ? op.o : min), doc.ops![0].o));
|
|
450
|
+
expect(doc.count).toBe(doc.ops!.length);
|
|
451
|
+
expect(doc.checksum).toBe(doc.ops!.reduce((sum, op) => sum + op.checksum, 0n));
|
|
452
|
+
expect(doc.size).toBe(bson.calculateObjectSize(doc.ops!));
|
|
453
|
+
}
|
|
454
|
+
});
|
|
455
|
+
|
|
456
|
+
test('4. no overlapping ranges - multiple documents', () => {
|
|
457
|
+
const opsA = [makeBucketDataDoc({ o: 1n }), makeBucketDataDoc({ o: 3n })];
|
|
458
|
+
const opsB = [makeBucketDataDoc({ o: 5n }), makeBucketDataDoc({ o: 8n })];
|
|
459
|
+
const opsC = [makeBucketDataDoc({ o: 10n }), makeBucketDataDoc({ o: 15n })];
|
|
460
|
+
|
|
461
|
+
const docA = serializeBucketData('test[]', opsA);
|
|
462
|
+
const docB = serializeBucketData('test[]', opsB);
|
|
463
|
+
const docC = serializeBucketData('test[]', opsC);
|
|
464
|
+
|
|
465
|
+
const docs = [docA, docB, docC];
|
|
466
|
+
|
|
467
|
+
for (let i = 0; i < docs.length; i++) {
|
|
468
|
+
for (let j = i + 1; j < docs.length; j++) {
|
|
469
|
+
const a = docs[i];
|
|
470
|
+
const b = docs[j];
|
|
471
|
+
const noOverlap = a.min_op > b._id.o || b.min_op > a._id.o;
|
|
472
|
+
expect(noOverlap).toBe(true);
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
});
|
|
476
|
+
|
|
477
|
+
test('4. no overlapping ranges - after compaction', async () => {
|
|
478
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage();
|
|
479
|
+
const ops1 = [makeOp(10, 'A', 'a1', ctx, sourceTableId), makeOp(20, 'B', 'b1', ctx, sourceTableId)];
|
|
480
|
+
const ops2 = [makeOp(30, 'C', 'c1', ctx, sourceTableId), makeOp(40, 'A', 'a2', ctx, sourceTableId)];
|
|
481
|
+
const ops3 = [makeOp(50, 'D', 'd1', ctx, sourceTableId), makeOp(60, 'E', 'e1', ctx, sourceTableId)];
|
|
482
|
+
const doc1 = serializeBucketData(BUCKET, ops1);
|
|
483
|
+
const doc2 = serializeBucketData(BUCKET, ops2);
|
|
484
|
+
const doc3 = serializeBucketData(BUCKET, ops3);
|
|
485
|
+
await insertDocs(collection, [doc1, doc2, doc3]);
|
|
486
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 60n);
|
|
487
|
+
|
|
488
|
+
await compact(bucketStorage, 60n);
|
|
489
|
+
|
|
490
|
+
const docs = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
491
|
+
for (let i = 0; i < docs.length; i++) {
|
|
492
|
+
for (let j = i + 1; j < docs.length; j++) {
|
|
493
|
+
const a = docs[i];
|
|
494
|
+
const b = docs[j];
|
|
495
|
+
const noOverlap = a.min_op > b._id.o || b.min_op > a._id.o;
|
|
496
|
+
expect(noOverlap).toBe(true);
|
|
497
|
+
}
|
|
498
|
+
}
|
|
499
|
+
});
|
|
500
|
+
|
|
501
|
+
test('6. compaction survivor integrity - superseded ops become MOVE tombstones', async () => {
|
|
502
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage();
|
|
503
|
+
const ops = [
|
|
504
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
505
|
+
makeOp(20, 'B', 'b1', ctx, sourceTableId),
|
|
506
|
+
makeOp(30, 'A', 'a2', ctx, sourceTableId)
|
|
507
|
+
];
|
|
508
|
+
const doc1 = serializeBucketData(BUCKET, ops);
|
|
509
|
+
await insertDocs(collection, [doc1]);
|
|
510
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 30n);
|
|
511
|
+
|
|
512
|
+
await compact(bucketStorage, 30n);
|
|
513
|
+
|
|
514
|
+
const docs = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
515
|
+
const allOps = docs.flatMap((d) => d.ops!);
|
|
516
|
+
const moveOps = allOps.filter((op) => op.op === 'MOVE');
|
|
517
|
+
expect(moveOps.length).toBe(1);
|
|
518
|
+
expect(moveOps[0].o).toBe(10n);
|
|
519
|
+
expect(moveOps[0].checksum).toBe(ops[0].checksum);
|
|
520
|
+
expect(moveOps[0].data).toBeNull();
|
|
521
|
+
const putOps = allOps.filter((op) => op.op === 'PUT');
|
|
522
|
+
expect(putOps.length).toBe(2);
|
|
523
|
+
expect(docs[0].target_op).toBe(30n);
|
|
524
|
+
});
|
|
525
|
+
|
|
526
|
+
test('6. compaction survivor integrity - MOVE ops preserved', async () => {
|
|
527
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage();
|
|
528
|
+
const ops = [
|
|
529
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
530
|
+
{ ...makeOp(20, 'A', 'a2', ctx, sourceTableId), op: 'MOVE' as const, data: null },
|
|
531
|
+
makeOp(30, 'A', 'a3', ctx, sourceTableId)
|
|
532
|
+
];
|
|
533
|
+
const doc1 = serializeBucketData(BUCKET, ops);
|
|
534
|
+
await insertDocs(collection, [doc1]);
|
|
535
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 30n);
|
|
536
|
+
|
|
537
|
+
await compact(bucketStorage, 30n);
|
|
538
|
+
|
|
539
|
+
const docs = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
540
|
+
const allOps = docs.flatMap((d) => d.ops!);
|
|
541
|
+
const moveOps = allOps.filter((op) => op.op === 'MOVE');
|
|
542
|
+
// Pre-existing MOVE@20 + new MOVE@10 are collapsed into CLEAR
|
|
543
|
+
expect(moveOps.length).toBe(0);
|
|
544
|
+
const clearOps = allOps.filter((op) => op.op === 'CLEAR');
|
|
545
|
+
expect(clearOps.length).toBe(1);
|
|
546
|
+
expect(clearOps[0].o).toBe(20n);
|
|
547
|
+
const putOps = allOps.filter((op) => op.op === 'PUT');
|
|
548
|
+
expect(putOps.length).toBe(1);
|
|
549
|
+
expect(putOps[0].o).toBe(30n);
|
|
550
|
+
});
|
|
551
|
+
|
|
552
|
+
test('6. compaction survivor integrity - CLEAR ops preserved', async () => {
|
|
553
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage();
|
|
554
|
+
const ops = [
|
|
555
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
556
|
+
makeOp(20, 'B', 'b1', ctx, sourceTableId),
|
|
557
|
+
{ ...makeOp(25, 'A', '', ctx, sourceTableId), op: 'CLEAR' as const, data: null, row_id: undefined }
|
|
558
|
+
];
|
|
559
|
+
const doc1 = serializeBucketData(BUCKET, ops);
|
|
560
|
+
await insertDocs(collection, [doc1]);
|
|
561
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 25n);
|
|
562
|
+
|
|
563
|
+
await compact(bucketStorage, 25n);
|
|
564
|
+
|
|
565
|
+
const docs = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
566
|
+
const allOps = docs.flatMap((d) => d.ops!);
|
|
567
|
+
const clearOps = allOps.filter((op) => op.op === 'CLEAR');
|
|
568
|
+
expect(clearOps.length).toBe(1);
|
|
569
|
+
});
|
|
570
|
+
|
|
571
|
+
test('7. superseded PUT becomes MOVE tombstone, REMOVE survives', async () => {
|
|
572
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage();
|
|
573
|
+
const ops = [
|
|
574
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
575
|
+
makeOp(20, 'A', 'a2', ctx, sourceTableId, { op: 'REMOVE' })
|
|
576
|
+
];
|
|
577
|
+
const doc1 = serializeBucketData(BUCKET, ops);
|
|
578
|
+
await insertDocs(collection, [doc1]);
|
|
579
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 20n);
|
|
580
|
+
|
|
581
|
+
await compact(bucketStorage, 20n);
|
|
582
|
+
|
|
583
|
+
const docs = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
584
|
+
const allOps = docs.flatMap((d) => d.ops!);
|
|
585
|
+
// MOVE@10 + REMOVE@20 collapsed into CLEAR@20
|
|
586
|
+
const clearOps = allOps.filter((op) => op.op === 'CLEAR');
|
|
587
|
+
expect(clearOps.length).toBe(1);
|
|
588
|
+
expect(clearOps[0].o).toBe(20n);
|
|
589
|
+
expect(allOps.filter((op) => op.op === 'REMOVE').length).toBe(0);
|
|
590
|
+
expect(allOps.filter((op) => op.op === 'MOVE').length).toBe(0);
|
|
591
|
+
});
|
|
592
|
+
|
|
593
|
+
test('7. empty bucket compact is a no-op', async () => {
|
|
594
|
+
const { bucketStorage, collection } = await setupV3Storage();
|
|
595
|
+
|
|
596
|
+
await bucketStorage.compact({
|
|
597
|
+
clearBatchLimit: 200,
|
|
598
|
+
moveBatchLimit: 10,
|
|
599
|
+
moveBatchQueryLimit: 10,
|
|
600
|
+
minBucketChanges: 1,
|
|
601
|
+
minChangeRatio: 0,
|
|
602
|
+
maxOpId: 1n
|
|
603
|
+
});
|
|
604
|
+
|
|
605
|
+
const docs = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
606
|
+
expect(docs.length).toBe(0);
|
|
607
|
+
});
|
|
608
|
+
|
|
609
|
+
test('8. BSON limit safety - two large ops get split', () => {
|
|
610
|
+
const halfMB = 600_000;
|
|
611
|
+
const ops = [
|
|
612
|
+
makeBucketDataDoc({ o: 1n, data: 'a'.repeat(halfMB) }),
|
|
613
|
+
makeBucketDataDoc({ o: 2n, data: 'b'.repeat(halfMB) })
|
|
614
|
+
];
|
|
615
|
+
|
|
616
|
+
const chunks = chunkBucketData(ops, DEFAULT_MAX_DOC_SIZE_BYTES);
|
|
617
|
+
expect(chunks.length).toBe(2);
|
|
618
|
+
});
|
|
619
|
+
|
|
620
|
+
test('8. BSON limit safety - single oversized op gets own chunk', () => {
|
|
621
|
+
const oversized = DEFAULT_MAX_DOC_SIZE_BYTES + 100_000;
|
|
622
|
+
const ops = [makeBucketDataDoc({ o: 1n, data: 'x'.repeat(oversized) })];
|
|
623
|
+
|
|
624
|
+
const chunks = chunkBucketData(ops, DEFAULT_MAX_DOC_SIZE_BYTES);
|
|
625
|
+
expect(chunks.length).toBe(1);
|
|
626
|
+
expect(chunks[0]).toHaveLength(1);
|
|
627
|
+
});
|
|
628
|
+
|
|
629
|
+
test('9. serialization fidelity - null data preserved', () => {
|
|
630
|
+
const ops = [makeBucketDataDoc({ o: 1n, data: null })];
|
|
631
|
+
const doc = serializeBucketData('test[]', ops);
|
|
632
|
+
expect(doc.ops![0].data).toBeNull();
|
|
633
|
+
|
|
634
|
+
const context = { replicationStreamId: 1, definitionId: '1' };
|
|
635
|
+
const deserialized = [...loadBucketDataDocument(context, doc)];
|
|
636
|
+
expect(deserialized[0].data).toBeNull();
|
|
637
|
+
});
|
|
638
|
+
|
|
639
|
+
test('9. serialization fidelity - empty string data preserved', () => {
|
|
640
|
+
const ops = [makeBucketDataDoc({ o: 1n, data: '' })];
|
|
641
|
+
const doc = serializeBucketData('test[]', ops);
|
|
642
|
+
expect(doc.ops![0].data).toBe('');
|
|
643
|
+
|
|
644
|
+
const context = { replicationStreamId: 1, definitionId: '1' };
|
|
645
|
+
const deserialized = [...loadBucketDataDocument(context, doc)];
|
|
646
|
+
expect(deserialized[0].data).toBe('');
|
|
647
|
+
});
|
|
648
|
+
|
|
649
|
+
test('9. serialization fidelity - unicode characters preserved', () => {
|
|
650
|
+
const unicodeData = '{"name":"日本語テスト","emoji":"🎉"}';
|
|
651
|
+
const ops = [makeBucketDataDoc({ o: 1n, data: unicodeData })];
|
|
652
|
+
const doc = serializeBucketData('test[]', ops);
|
|
653
|
+
expect(doc.ops![0].data).toBe(unicodeData);
|
|
654
|
+
|
|
655
|
+
const context = { replicationStreamId: 1, definitionId: '1' };
|
|
656
|
+
const deserialized = [...loadBucketDataDocument(context, doc)];
|
|
657
|
+
expect(deserialized[0].data).toBe(unicodeData);
|
|
658
|
+
});
|
|
659
|
+
|
|
660
|
+
test('10. document _id.o invariant - equals last ops[*].o (caller must sort)', () => {
|
|
661
|
+
const ops = [makeBucketDataDoc({ o: 10n }), makeBucketDataDoc({ o: 25n }), makeBucketDataDoc({ o: 7n })];
|
|
662
|
+
|
|
663
|
+
const doc = serializeBucketData('test[]', ops);
|
|
664
|
+
expect(doc._id.o).toBe(7n);
|
|
665
|
+
});
|
|
666
|
+
|
|
667
|
+
test('10. document _id.o invariant - equals max when pre-sorted', () => {
|
|
668
|
+
const ops = [makeBucketDataDoc({ o: 3n }), makeBucketDataDoc({ o: 10n }), makeBucketDataDoc({ o: 25n })];
|
|
669
|
+
|
|
670
|
+
const doc = serializeBucketData('test[]', ops);
|
|
671
|
+
const maxO = ops.reduce((max, op) => (op.o > max ? op.o : max), 0n);
|
|
672
|
+
expect(doc._id.o).toBe(maxO);
|
|
673
|
+
});
|
|
674
|
+
|
|
675
|
+
test('10. document _id.o invariant - every document after compaction', async () => {
|
|
676
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage();
|
|
677
|
+
const ops1 = [makeOp(10, 'A', 'a1', ctx, sourceTableId), makeOp(20, 'B', 'b1', ctx, sourceTableId)];
|
|
678
|
+
const ops2 = [makeOp(30, 'C', 'c1', ctx, sourceTableId), makeOp(40, 'A', 'a2', ctx, sourceTableId)];
|
|
679
|
+
const doc1 = serializeBucketData(BUCKET, ops1);
|
|
680
|
+
const doc2 = serializeBucketData(BUCKET, ops2);
|
|
681
|
+
await insertDocs(collection, [doc1, doc2]);
|
|
682
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 40n);
|
|
683
|
+
|
|
684
|
+
await compact(bucketStorage, 40n);
|
|
685
|
+
|
|
686
|
+
const docs = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
687
|
+
for (const doc of docs) {
|
|
688
|
+
const maxO = doc.ops!.reduce((max, op) => (op.o > max ? op.o : max), 0n);
|
|
689
|
+
expect(doc._id.o).toBe(maxO);
|
|
690
|
+
}
|
|
691
|
+
});
|
|
692
|
+
|
|
693
|
+
test('compaction with maxOpId filtering excludes a straddling document', async () => {
|
|
694
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage();
|
|
695
|
+
const ops = [
|
|
696
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
697
|
+
makeOp(20, 'B', 'b1', ctx, sourceTableId),
|
|
698
|
+
makeOp(30, 'C', 'c1', ctx, sourceTableId)
|
|
699
|
+
];
|
|
700
|
+
const doc1 = serializeBucketData(BUCKET, ops);
|
|
701
|
+
await insertDocs(collection, [doc1]);
|
|
702
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 30n);
|
|
703
|
+
|
|
704
|
+
await compact(bucketStorage, 15n);
|
|
705
|
+
|
|
706
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
707
|
+
const allOpsAfter = docsAfter.flatMap((d) => d.ops!);
|
|
708
|
+
|
|
709
|
+
// The document ends above maxOpId, so the database query excludes it as a unit.
|
|
710
|
+
expect(allOpsAfter.length).toBe(3);
|
|
711
|
+
const opsBelow = allOpsAfter.filter((op) => op.o <= 15n);
|
|
712
|
+
expect(opsBelow.length).toBe(1);
|
|
713
|
+
expect(opsBelow[0].op).toBe('PUT');
|
|
714
|
+
const opsAbove = allOpsAfter.filter((op) => op.o > 15n);
|
|
715
|
+
expect(opsAbove.length).toBe(2);
|
|
716
|
+
expect(opsAbove.every((op) => op.op === 'PUT')).toBe(true);
|
|
717
|
+
|
|
718
|
+
// Checksum preserved
|
|
719
|
+
const checksumAfter = docsAfter.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
720
|
+
expect(checksumAfter).toBe(Number(doc1.checksum));
|
|
721
|
+
});
|
|
722
|
+
|
|
723
|
+
test('checksum consistency - aggregation pipeline matches JavaScript addChecksums', async () => {
|
|
724
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3Storage();
|
|
725
|
+
const ops = [makeOp(10, 'A', 'a1', ctx, sourceTableId), makeOp(20, 'B', 'b1', ctx, sourceTableId)];
|
|
726
|
+
const doc1 = serializeBucketData(BUCKET, ops);
|
|
727
|
+
await insertDocs(collection, [doc1]);
|
|
728
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 20n);
|
|
729
|
+
|
|
730
|
+
const allOps = ops;
|
|
731
|
+
let jsChecksum = 0;
|
|
732
|
+
for (const op of allOps) {
|
|
733
|
+
jsChecksum = addChecksums(jsChecksum, Number(op.checksum));
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
const docs = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
737
|
+
const storedOps = docs.flatMap((d) => d.ops!);
|
|
738
|
+
let storedChecksum = 0;
|
|
739
|
+
for (const op of storedOps) {
|
|
740
|
+
storedChecksum = addChecksums(storedChecksum, Number(op.checksum));
|
|
741
|
+
}
|
|
742
|
+
|
|
743
|
+
expect(storedChecksum).toBe(jsChecksum);
|
|
744
|
+
});
|
|
745
|
+
});
|
|
746
|
+
|
|
747
|
+
describe('V3 checksum pipeline straddling', () => {
|
|
748
|
+
const BUCKET = 'global[]';
|
|
749
|
+
const TABLE = 'items';
|
|
750
|
+
|
|
751
|
+
async function setup() {
|
|
752
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
753
|
+
const syncRules = await factory.updateSyncRules(
|
|
754
|
+
updateSyncRulesFromYaml(
|
|
755
|
+
`
|
|
756
|
+
bucket_definitions:
|
|
757
|
+
global:
|
|
758
|
+
data: [SELECT id as id, description FROM items]
|
|
759
|
+
`,
|
|
760
|
+
{ storageVersion: 3 }
|
|
761
|
+
)
|
|
762
|
+
);
|
|
763
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
764
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
765
|
+
const mapping = syncRules.syncConfigContent[0].mapping;
|
|
766
|
+
const definitionId = mapping.allBucketDefinitionIds()[0];
|
|
767
|
+
const collection = db.bucketData(bucketStorage.replicationStreamId, definitionId);
|
|
768
|
+
const bucketStateCollection = db.bucketState(bucketStorage.replicationStreamId);
|
|
769
|
+
const sourceTableId = new bson.ObjectId();
|
|
770
|
+
const ctx = {
|
|
771
|
+
replicationStreamId: bucketStorage.replicationStreamId,
|
|
772
|
+
definitionId,
|
|
773
|
+
bucket: BUCKET
|
|
774
|
+
};
|
|
775
|
+
|
|
776
|
+
return { bucketStorage, syncRules, db, collection, bucketStateCollection, definitionId, sourceTableId, ctx };
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
function makeOp(
|
|
780
|
+
opId: number,
|
|
781
|
+
rowId: string,
|
|
782
|
+
data: string,
|
|
783
|
+
ctx: { replicationStreamId: number; definitionId: string; bucket: string },
|
|
784
|
+
sourceTableId: bson.ObjectId
|
|
785
|
+
): BucketDataDoc {
|
|
786
|
+
return {
|
|
787
|
+
bucketKey: {
|
|
788
|
+
replicationStreamId: ctx.replicationStreamId,
|
|
789
|
+
definitionId: ctx.definitionId,
|
|
790
|
+
bucket: ctx.bucket
|
|
791
|
+
},
|
|
792
|
+
o: BigInt(opId),
|
|
793
|
+
op: 'PUT',
|
|
794
|
+
source_table: sourceTableId,
|
|
795
|
+
source_key: test_utils.rid(rowId),
|
|
796
|
+
table: TABLE,
|
|
797
|
+
row_id: rowId,
|
|
798
|
+
checksum: BigInt(opId * 7),
|
|
799
|
+
data: JSON.stringify({ id: rowId, description: data })
|
|
800
|
+
};
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
function checksumRequest(): storage.BucketChecksumRequest {
|
|
804
|
+
return {
|
|
805
|
+
bucket: BUCKET,
|
|
806
|
+
source: {
|
|
807
|
+
uniqueName: 'global',
|
|
808
|
+
bucketParameters: [],
|
|
809
|
+
getSourceTables: () => new Set(),
|
|
810
|
+
tableSyncsData: () => false,
|
|
811
|
+
evaluateRow: () => [],
|
|
812
|
+
inferSchema: () => ({ objects: {} }),
|
|
813
|
+
bucketQuery: () => ({ ast: {} as any, parameters: [] })
|
|
814
|
+
} as any
|
|
815
|
+
};
|
|
816
|
+
}
|
|
817
|
+
|
|
818
|
+
test('start straddle recalculates a cached checksum from the beginning', async () => {
|
|
819
|
+
const { bucketStorage, collection, bucketStateCollection, definitionId, sourceTableId, ctx } = await setup();
|
|
820
|
+
|
|
821
|
+
// Single document with ops 10-60, min_op=10, _id.o=60
|
|
822
|
+
const ops = [
|
|
823
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
824
|
+
makeOp(20, 'B', 'b1', ctx, sourceTableId),
|
|
825
|
+
makeOp(30, 'C', 'c1', ctx, sourceTableId),
|
|
826
|
+
makeOp(40, 'D', 'd1', ctx, sourceTableId),
|
|
827
|
+
makeOp(50, 'E', 'e1', ctx, sourceTableId),
|
|
828
|
+
makeOp(60, 'F', 'f1', ctx, sourceTableId)
|
|
829
|
+
];
|
|
830
|
+
// Cache the first half before compaction changes the document boundary.
|
|
831
|
+
const firstDoc = serializeBucketData(BUCKET, ops.slice(0, 3));
|
|
832
|
+
await collection.insertOne(firstDoc);
|
|
833
|
+
|
|
834
|
+
const fullChecksum = ops.reduce((sum, op) => addChecksums(sum, Number(op.checksum)), 0);
|
|
835
|
+
await bucketStateCollection.insertOne({
|
|
836
|
+
_id: { d: definitionId, b: BUCKET },
|
|
837
|
+
last_op: 30n,
|
|
838
|
+
estimate_since_compact: { count: 3, bytes: 100 }
|
|
839
|
+
});
|
|
840
|
+
|
|
841
|
+
const request = checksumRequest();
|
|
842
|
+
|
|
843
|
+
const cached = await bucketStorage.getChecksums(test_utils.testCheckpoint(30n), [request]);
|
|
844
|
+
expect(cached.get(BUCKET)).toMatchObject({ count: 3 });
|
|
845
|
+
|
|
846
|
+
// Compaction rechunks the bucket into one document spanning the cached
|
|
847
|
+
// checkpoint. The requested endpoint is the end of the new document.
|
|
848
|
+
await collection.deleteMany({});
|
|
849
|
+
await collection.insertOne(serializeBucketData(BUCKET, ops, { targetOp: 60n }));
|
|
850
|
+
|
|
851
|
+
const result = await bucketStorage.getChecksums(test_utils.testCheckpoint(60n), [request]);
|
|
852
|
+
const checksumResult = result.get(BUCKET)!;
|
|
853
|
+
|
|
854
|
+
// The cached checkpoint at 30 is inside the new document. The full result
|
|
855
|
+
// must replace it rather than add the document checksum to it.
|
|
856
|
+
expect(checksumResult.checksum).toBe(fullChecksum);
|
|
857
|
+
expect(checksumResult.count).toBe(6);
|
|
858
|
+
});
|
|
859
|
+
|
|
860
|
+
test('end straddle invalidates the old checkpoint and a later checkpoint succeeds', async () => {
|
|
861
|
+
const { bucketStorage, collection, bucketStateCollection, definitionId, sourceTableId, ctx } = await setup();
|
|
862
|
+
|
|
863
|
+
// Document with ops 40-60, _id.o=60, min_op=40
|
|
864
|
+
const ops = [
|
|
865
|
+
makeOp(40, 'D', 'd1', ctx, sourceTableId),
|
|
866
|
+
makeOp(50, 'E', 'e1', ctx, sourceTableId),
|
|
867
|
+
makeOp(60, 'F', 'f1', ctx, sourceTableId)
|
|
868
|
+
];
|
|
869
|
+
const doc = serializeBucketData(BUCKET, ops, { targetOp: 60n });
|
|
870
|
+
await collection.insertMany([doc]);
|
|
871
|
+
|
|
872
|
+
const checksumAllOps = ops.reduce((sum, op) => addChecksums(sum, Number(op.checksum)), 0);
|
|
873
|
+
|
|
874
|
+
await bucketStateCollection.insertOne({
|
|
875
|
+
_id: { d: definitionId, b: BUCKET },
|
|
876
|
+
last_op: 0n,
|
|
877
|
+
estimate_since_compact: { count: 0, bytes: 0 }
|
|
878
|
+
});
|
|
879
|
+
|
|
880
|
+
const request = checksumRequest();
|
|
881
|
+
|
|
882
|
+
await expect(bucketStorage.getChecksums(test_utils.testCheckpoint(45n), [request])).rejects.toBeInstanceOf(
|
|
883
|
+
CheckpointChecksumInvalidatedError
|
|
884
|
+
);
|
|
885
|
+
|
|
886
|
+
const result = await bucketStorage.getChecksums(test_utils.testCheckpoint(60n), [request]);
|
|
887
|
+
expect(result.get(BUCKET)).toEqual({ bucket: BUCKET, checksum: checksumAllOps, count: 3 });
|
|
888
|
+
});
|
|
889
|
+
|
|
890
|
+
test('end straddle without target_op also invalidates the old checkpoint', async () => {
|
|
891
|
+
const { bucketStorage, collection, bucketStateCollection, definitionId, sourceTableId, ctx } = await setup();
|
|
892
|
+
const ops = [makeOp(40, 'D', 'd1', ctx, sourceTableId), makeOp(60, 'F', 'f1', ctx, sourceTableId)];
|
|
893
|
+
await collection.insertOne(serializeBucketData(BUCKET, ops));
|
|
894
|
+
await bucketStateCollection.insertOne({
|
|
895
|
+
_id: { d: definitionId, b: BUCKET },
|
|
896
|
+
last_op: 0n,
|
|
897
|
+
estimate_since_compact: { count: 0, bytes: 0 }
|
|
898
|
+
});
|
|
899
|
+
|
|
900
|
+
const request = checksumRequest();
|
|
901
|
+
await expect(bucketStorage.getChecksums(test_utils.testCheckpoint(45n), [request])).rejects.toBeInstanceOf(
|
|
902
|
+
CheckpointChecksumInvalidatedError
|
|
903
|
+
);
|
|
904
|
+
});
|
|
905
|
+
|
|
906
|
+
test('has_clear_op replaces a cached checksum', async () => {
|
|
907
|
+
const { bucketStorage, collection, bucketStateCollection, definitionId, sourceTableId, ctx } = await setup();
|
|
908
|
+
const beforeClear = makeOp(10, 'A', 'a1', ctx, sourceTableId);
|
|
909
|
+
const clear = { ...makeOp(20, 'clear', '', ctx, sourceTableId), op: 'CLEAR' as const, checksum: 101n, data: null };
|
|
910
|
+
const afterClear = makeOp(30, 'B', 'b1', ctx, sourceTableId);
|
|
911
|
+
|
|
912
|
+
await collection.insertOne(serializeBucketData(BUCKET, [beforeClear]));
|
|
913
|
+
await bucketStateCollection.insertOne({
|
|
914
|
+
_id: { d: definitionId, b: BUCKET },
|
|
915
|
+
last_op: 10n,
|
|
916
|
+
estimate_since_compact: { count: 1, bytes: 100 }
|
|
917
|
+
});
|
|
918
|
+
|
|
919
|
+
const request = checksumRequest();
|
|
920
|
+
const before = await bucketStorage.getChecksums(test_utils.testCheckpoint(10n), [request]);
|
|
921
|
+
expect(before.get(BUCKET)).toEqual({ bucket: BUCKET, checksum: 70, count: 1 });
|
|
922
|
+
|
|
923
|
+
// Compaction replaces everything preceding CLEAR, while the checksum cache
|
|
924
|
+
// still contains the old checkpoint.
|
|
925
|
+
await collection.deleteMany({});
|
|
926
|
+
await collection.insertOne(serializeBucketData(BUCKET, [clear, afterClear]));
|
|
927
|
+
await bucketStateCollection.updateOne(
|
|
928
|
+
{ _id: { d: definitionId, b: BUCKET } },
|
|
929
|
+
{ $set: { last_op: 30n, 'estimate_since_compact.count': 2 } }
|
|
930
|
+
);
|
|
931
|
+
|
|
932
|
+
const after = await bucketStorage.getChecksums(test_utils.testCheckpoint(30n), [request]);
|
|
933
|
+
expect(after.get(BUCKET)).toEqual({
|
|
934
|
+
bucket: BUCKET,
|
|
935
|
+
checksum: addChecksums(Number(clear.checksum), Number(afterClear.checksum)),
|
|
936
|
+
count: 2
|
|
937
|
+
});
|
|
938
|
+
});
|
|
939
|
+
});
|
|
940
|
+
|
|
941
|
+
describe('V3 compaction boundaries', () => {
|
|
942
|
+
const BUCKET = 'global[]';
|
|
943
|
+
const TABLE = 'items';
|
|
944
|
+
|
|
945
|
+
async function setupV3() {
|
|
946
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
947
|
+
const syncRules = await factory.updateSyncRules(
|
|
948
|
+
updateSyncRulesFromYaml(
|
|
949
|
+
`
|
|
950
|
+
bucket_definitions:
|
|
951
|
+
global:
|
|
952
|
+
data: [SELECT id as id, description FROM items]
|
|
953
|
+
`,
|
|
954
|
+
{ storageVersion: 3 }
|
|
955
|
+
)
|
|
956
|
+
);
|
|
957
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
958
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
959
|
+
const mapping = syncRules.syncConfigContent[0].mapping;
|
|
960
|
+
const definitionId = mapping.allBucketDefinitionIds()[0];
|
|
961
|
+
const collection = db.bucketData(bucketStorage.replicationStreamId, definitionId);
|
|
962
|
+
const bucketStateCollection = db.bucketState(bucketStorage.replicationStreamId);
|
|
963
|
+
const sourceTableId = new bson.ObjectId();
|
|
964
|
+
|
|
965
|
+
const ctx = {
|
|
966
|
+
replicationStreamId: bucketStorage.replicationStreamId,
|
|
967
|
+
definitionId,
|
|
968
|
+
bucket: BUCKET
|
|
969
|
+
};
|
|
970
|
+
|
|
971
|
+
return { bucketStorage, syncRules, db, collection, bucketStateCollection, sourceTableId, ctx };
|
|
972
|
+
}
|
|
973
|
+
|
|
974
|
+
function makeOp(
|
|
975
|
+
opId: number,
|
|
976
|
+
rowId: string,
|
|
977
|
+
data: string,
|
|
978
|
+
ctx: { replicationStreamId: number; definitionId: string; bucket: string },
|
|
979
|
+
sourceTableId: bson.ObjectId,
|
|
980
|
+
overrides?: { op?: 'PUT' | 'REMOVE' }
|
|
981
|
+
): BucketDataDoc {
|
|
982
|
+
return {
|
|
983
|
+
bucketKey: {
|
|
984
|
+
replicationStreamId: ctx.replicationStreamId,
|
|
985
|
+
definitionId: ctx.definitionId,
|
|
986
|
+
bucket: ctx.bucket
|
|
987
|
+
},
|
|
988
|
+
o: BigInt(opId),
|
|
989
|
+
op: overrides?.op ?? 'PUT',
|
|
990
|
+
source_table: sourceTableId,
|
|
991
|
+
source_key: test_utils.rid(rowId),
|
|
992
|
+
table: TABLE,
|
|
993
|
+
row_id: rowId,
|
|
994
|
+
checksum: BigInt(opId * 7),
|
|
995
|
+
data: overrides?.op === 'REMOVE' ? null : JSON.stringify({ id: rowId, description: data })
|
|
996
|
+
};
|
|
997
|
+
}
|
|
998
|
+
|
|
999
|
+
async function insertDocs(collection: any, docs: BucketDataDocumentV3[]) {
|
|
1000
|
+
await collection.insertMany(docs);
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
async function insertBucketState(bucketStateCollection: any, definitionId: string, lastOp: bigint) {
|
|
1004
|
+
await bucketStateCollection.insertOne({
|
|
1005
|
+
_id: { d: definitionId, b: BUCKET },
|
|
1006
|
+
last_op: lastOp,
|
|
1007
|
+
estimate_since_compact: { count: 10, bytes: 100 }
|
|
1008
|
+
});
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
async function compact(bucketStorage: MongoSyncBucketStorage, maxOpId: bigint) {
|
|
1012
|
+
await bucketStorage.compact({
|
|
1013
|
+
clearBatchLimit: 200,
|
|
1014
|
+
moveBatchLimit: 10,
|
|
1015
|
+
moveBatchQueryLimit: 10,
|
|
1016
|
+
minBucketChanges: 1,
|
|
1017
|
+
minChangeRatio: 0,
|
|
1018
|
+
maxOpId
|
|
1019
|
+
});
|
|
1020
|
+
}
|
|
1021
|
+
|
|
1022
|
+
async function readAllOps(collection: any): Promise<{ row_id: string; o: bigint; op: string }[]> {
|
|
1023
|
+
const docs = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1024
|
+
return docs.flatMap((d: any) =>
|
|
1025
|
+
d.ops!.map((op: any) => ({ row_id: op.row_id!, o: op.o, op: op.op, target_op: op.target_op ?? undefined }))
|
|
1026
|
+
);
|
|
1027
|
+
}
|
|
1028
|
+
|
|
1029
|
+
async function readAllDocs(collection: any): Promise<BucketDataDocumentV3[]> {
|
|
1030
|
+
return collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1031
|
+
}
|
|
1032
|
+
|
|
1033
|
+
test('1. superseded ops become MOVE tombstones', async () => {
|
|
1034
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1035
|
+
// Doc1: [A@10, B@20, A@30] — A@10 superseded by A@30, becomes MOVE tombstone
|
|
1036
|
+
const ops = [
|
|
1037
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
1038
|
+
makeOp(20, 'B', 'b1', ctx, sourceTableId),
|
|
1039
|
+
makeOp(30, 'A', 'a2', ctx, sourceTableId)
|
|
1040
|
+
];
|
|
1041
|
+
const doc1 = serializeBucketData(BUCKET, ops);
|
|
1042
|
+
await insertDocs(collection, [doc1]);
|
|
1043
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 30n);
|
|
1044
|
+
|
|
1045
|
+
await compact(bucketStorage, 30n);
|
|
1046
|
+
|
|
1047
|
+
const surviving = await readAllOps(collection);
|
|
1048
|
+
expect(surviving).toHaveLength(3);
|
|
1049
|
+
expect(surviving[0]).toMatchObject({ op: 'MOVE', o: 10n });
|
|
1050
|
+
expect(surviving[1]).toMatchObject({ row_id: 'B', o: 20n, op: 'PUT' });
|
|
1051
|
+
expect(surviving[2]).toMatchObject({ row_id: 'A', o: 30n, op: 'PUT' });
|
|
1052
|
+
});
|
|
1053
|
+
|
|
1054
|
+
test('2. first op in document superseded becomes MOVE tombstone', async () => {
|
|
1055
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1056
|
+
// Doc1: [A@10, B@20, C@30], Doc2: [A@40] — A@10 superseded by A@40
|
|
1057
|
+
const ops1 = [
|
|
1058
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
1059
|
+
makeOp(20, 'B', 'b1', ctx, sourceTableId),
|
|
1060
|
+
makeOp(30, 'C', 'c1', ctx, sourceTableId)
|
|
1061
|
+
];
|
|
1062
|
+
const ops2 = [makeOp(40, 'A', 'a2', ctx, sourceTableId)];
|
|
1063
|
+
const doc1 = serializeBucketData(BUCKET, ops1);
|
|
1064
|
+
const doc2 = serializeBucketData(BUCKET, ops2);
|
|
1065
|
+
await insertDocs(collection, [doc1, doc2]);
|
|
1066
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 40n);
|
|
1067
|
+
|
|
1068
|
+
await compact(bucketStorage, 40n);
|
|
1069
|
+
|
|
1070
|
+
const surviving = await readAllOps(collection);
|
|
1071
|
+
expect(surviving).toHaveLength(4);
|
|
1072
|
+
expect(surviving[0]).toMatchObject({ op: 'MOVE', o: 10n });
|
|
1073
|
+
expect(surviving[1]).toMatchObject({ row_id: 'B', o: 20n, op: 'PUT' });
|
|
1074
|
+
expect(surviving[2]).toMatchObject({ row_id: 'C', o: 30n, op: 'PUT' });
|
|
1075
|
+
expect(surviving[3]).toMatchObject({ row_id: 'A', o: 40n, op: 'PUT' });
|
|
1076
|
+
});
|
|
1077
|
+
|
|
1078
|
+
test('3. last op in document superseded becomes MOVE tombstone', async () => {
|
|
1079
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1080
|
+
// Doc1: [A@10, B@20], Doc2: [C@30, A@40] — A@10 superseded by A@40
|
|
1081
|
+
const ops1 = [makeOp(10, 'A', 'a1', ctx, sourceTableId), makeOp(20, 'B', 'b1', ctx, sourceTableId)];
|
|
1082
|
+
const ops2 = [makeOp(30, 'C', 'c1', ctx, sourceTableId), makeOp(40, 'A', 'a2', ctx, sourceTableId)];
|
|
1083
|
+
const doc1 = serializeBucketData(BUCKET, ops1);
|
|
1084
|
+
const doc2 = serializeBucketData(BUCKET, ops2);
|
|
1085
|
+
await insertDocs(collection, [doc1, doc2]);
|
|
1086
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 40n);
|
|
1087
|
+
|
|
1088
|
+
await compact(bucketStorage, 40n);
|
|
1089
|
+
|
|
1090
|
+
const surviving = await readAllOps(collection);
|
|
1091
|
+
expect(surviving).toHaveLength(4);
|
|
1092
|
+
expect(surviving[0]).toMatchObject({ op: 'MOVE', o: 10n });
|
|
1093
|
+
expect(surviving[1]).toMatchObject({ row_id: 'B', o: 20n, op: 'PUT' });
|
|
1094
|
+
expect(surviving[2]).toMatchObject({ row_id: 'C', o: 30n, op: 'PUT' });
|
|
1095
|
+
expect(surviving[3]).toMatchObject({ row_id: 'A', o: 40n, op: 'PUT' });
|
|
1096
|
+
});
|
|
1097
|
+
|
|
1098
|
+
test('4. cascading superseded ops become MOVE tombstones', async () => {
|
|
1099
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1100
|
+
// Doc1: [A@10, A@20] — A@10 superseded by A@20; A@20 superseded by A@30 REMOVE
|
|
1101
|
+
const ops1 = [makeOp(10, 'A', 'a1', ctx, sourceTableId), makeOp(20, 'A', 'a2', ctx, sourceTableId)];
|
|
1102
|
+
const ops2 = [makeOp(30, 'A', 'a3', ctx, sourceTableId, { op: 'REMOVE' })];
|
|
1103
|
+
const doc1 = serializeBucketData(BUCKET, ops1);
|
|
1104
|
+
const doc2 = serializeBucketData(BUCKET, ops2);
|
|
1105
|
+
await insertDocs(collection, [doc1, doc2]);
|
|
1106
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 30n);
|
|
1107
|
+
|
|
1108
|
+
await compact(bucketStorage, 30n);
|
|
1109
|
+
|
|
1110
|
+
const surviving = await readAllOps(collection);
|
|
1111
|
+
expect(surviving).toHaveLength(1);
|
|
1112
|
+
expect(surviving[0]).toMatchObject({ op: 'CLEAR', o: 30n });
|
|
1113
|
+
});
|
|
1114
|
+
|
|
1115
|
+
test('5. one surviving PUT per document plus MOVE tombstones', async () => {
|
|
1116
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1117
|
+
// Doc1: [A@10, B@20], Doc2: [C@30, A@40] — A@10 superseded, becomes MOVE
|
|
1118
|
+
const ops1 = [makeOp(10, 'A', 'a1', ctx, sourceTableId), makeOp(20, 'B', 'b1', ctx, sourceTableId)];
|
|
1119
|
+
const ops2 = [makeOp(30, 'C', 'c1', ctx, sourceTableId), makeOp(40, 'A', 'a2', ctx, sourceTableId)];
|
|
1120
|
+
const doc1 = serializeBucketData(BUCKET, ops1);
|
|
1121
|
+
const doc2 = serializeBucketData(BUCKET, ops2);
|
|
1122
|
+
await insertDocs(collection, [doc1, doc2]);
|
|
1123
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 40n);
|
|
1124
|
+
|
|
1125
|
+
await compact(bucketStorage, 40n);
|
|
1126
|
+
|
|
1127
|
+
const surviving = await readAllOps(collection);
|
|
1128
|
+
expect(surviving).toHaveLength(4);
|
|
1129
|
+
expect(surviving[0]).toMatchObject({ op: 'MOVE', o: 10n });
|
|
1130
|
+
expect(surviving[1]).toMatchObject({ row_id: 'B', o: 20n, op: 'PUT' });
|
|
1131
|
+
expect(surviving[2]).toMatchObject({ row_id: 'C', o: 30n, op: 'PUT' });
|
|
1132
|
+
expect(surviving[3]).toMatchObject({ row_id: 'A', o: 40n, op: 'PUT' });
|
|
1133
|
+
});
|
|
1134
|
+
|
|
1135
|
+
test('6. multiple superseded ops from different documents become MOVE tombstones', async () => {
|
|
1136
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1137
|
+
// 3 small docs, each with 2 ops, A@10 superseded by A@40
|
|
1138
|
+
// Doc1: [A@10, B@20] — A@10 becomes MOVE
|
|
1139
|
+
// Doc2: [C@30, A@40] — C@30 survives (A@40 is latest A)
|
|
1140
|
+
// Doc3: [D@50, E@60] — D@50, E@60 survive
|
|
1141
|
+
const ops1 = [makeOp(10, 'A', 'a1', ctx, sourceTableId), makeOp(20, 'B', 'b1', ctx, sourceTableId)];
|
|
1142
|
+
const ops2 = [makeOp(30, 'C', 'c1', ctx, sourceTableId), makeOp(40, 'A', 'a2', ctx, sourceTableId)];
|
|
1143
|
+
const ops3 = [makeOp(50, 'D', 'd1', ctx, sourceTableId), makeOp(60, 'E', 'e1', ctx, sourceTableId)];
|
|
1144
|
+
const doc1 = serializeBucketData(BUCKET, ops1);
|
|
1145
|
+
const doc2 = serializeBucketData(BUCKET, ops2);
|
|
1146
|
+
const doc3 = serializeBucketData(BUCKET, ops3);
|
|
1147
|
+
await insertDocs(collection, [doc1, doc2, doc3]);
|
|
1148
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 60n);
|
|
1149
|
+
|
|
1150
|
+
await compact(bucketStorage, 60n);
|
|
1151
|
+
|
|
1152
|
+
const surviving = await readAllOps(collection);
|
|
1153
|
+
expect(surviving).toHaveLength(6);
|
|
1154
|
+
expect(surviving[0]).toMatchObject({ op: 'MOVE', o: 10n });
|
|
1155
|
+
expect(surviving[1]).toMatchObject({ row_id: 'B', o: 20n, op: 'PUT' });
|
|
1156
|
+
expect(surviving[2]).toMatchObject({ row_id: 'C', o: 30n, op: 'PUT' });
|
|
1157
|
+
expect(surviving[3]).toMatchObject({ row_id: 'A', o: 40n, op: 'PUT' });
|
|
1158
|
+
expect(surviving[4]).toMatchObject({ row_id: 'D', o: 50n, op: 'PUT' });
|
|
1159
|
+
expect(surviving[5]).toMatchObject({ row_id: 'E', o: 60n, op: 'PUT' });
|
|
1160
|
+
});
|
|
1161
|
+
|
|
1162
|
+
test('7. superseded op at document boundary becomes MOVE tombstone', async () => {
|
|
1163
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1164
|
+
// Doc1: [A@10, B@20, C@30], Doc2: [D@40, A@50] — A@10 superseded by A@50
|
|
1165
|
+
const ops1 = [
|
|
1166
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
1167
|
+
makeOp(20, 'B', 'b1', ctx, sourceTableId),
|
|
1168
|
+
makeOp(30, 'C', 'c1', ctx, sourceTableId)
|
|
1169
|
+
];
|
|
1170
|
+
const ops2 = [makeOp(40, 'D', 'd1', ctx, sourceTableId), makeOp(50, 'A', 'a2', ctx, sourceTableId)];
|
|
1171
|
+
const doc1 = serializeBucketData(BUCKET, ops1);
|
|
1172
|
+
const doc2 = serializeBucketData(BUCKET, ops2);
|
|
1173
|
+
await insertDocs(collection, [doc1, doc2]);
|
|
1174
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 50n);
|
|
1175
|
+
|
|
1176
|
+
await compact(bucketStorage, 50n);
|
|
1177
|
+
|
|
1178
|
+
const surviving = await readAllOps(collection);
|
|
1179
|
+
expect(surviving).toHaveLength(5);
|
|
1180
|
+
expect(surviving[0]).toMatchObject({ op: 'MOVE', o: 10n });
|
|
1181
|
+
expect(surviving[1]).toMatchObject({ row_id: 'B', o: 20n, op: 'PUT' });
|
|
1182
|
+
expect(surviving[2]).toMatchObject({ row_id: 'C', o: 30n, op: 'PUT' });
|
|
1183
|
+
expect(surviving[3]).toMatchObject({ row_id: 'D', o: 40n, op: 'PUT' });
|
|
1184
|
+
expect(surviving[4]).toMatchObject({ row_id: 'A', o: 50n, op: 'PUT' });
|
|
1185
|
+
});
|
|
1186
|
+
|
|
1187
|
+
test('8. same row_id ops spanning document boundary produces MOVE tombstone', async () => {
|
|
1188
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1189
|
+
// Doc1: [A@10, B@20], Doc2: [A@30, C@40] — A@10 superseded by A@30
|
|
1190
|
+
const ops1 = [makeOp(10, 'A', 'a1', ctx, sourceTableId), makeOp(20, 'B', 'b1', ctx, sourceTableId)];
|
|
1191
|
+
const ops2 = [makeOp(30, 'A', 'a2', ctx, sourceTableId), makeOp(40, 'C', 'c1', ctx, sourceTableId)];
|
|
1192
|
+
const doc1 = serializeBucketData(BUCKET, ops1);
|
|
1193
|
+
const doc2 = serializeBucketData(BUCKET, ops2);
|
|
1194
|
+
await insertDocs(collection, [doc1, doc2]);
|
|
1195
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 40n);
|
|
1196
|
+
|
|
1197
|
+
await compact(bucketStorage, 40n);
|
|
1198
|
+
|
|
1199
|
+
const surviving = await readAllOps(collection);
|
|
1200
|
+
expect(surviving).toHaveLength(4);
|
|
1201
|
+
expect(surviving[0]).toMatchObject({ op: 'MOVE', o: 10n });
|
|
1202
|
+
expect(surviving[1]).toMatchObject({ row_id: 'B', o: 20n, op: 'PUT' });
|
|
1203
|
+
expect(surviving[2]).toMatchObject({ row_id: 'A', o: 30n, op: 'PUT' });
|
|
1204
|
+
expect(surviving[3]).toMatchObject({ row_id: 'C', o: 40n, op: 'PUT' });
|
|
1205
|
+
});
|
|
1206
|
+
});
|
|
1207
|
+
|
|
1208
|
+
describe('V3 MOVE tombstone properties', () => {
|
|
1209
|
+
const BUCKET = 'global[]';
|
|
1210
|
+
const TABLE = 'items';
|
|
1211
|
+
|
|
1212
|
+
async function setupV3() {
|
|
1213
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
1214
|
+
const syncRules = await factory.updateSyncRules(
|
|
1215
|
+
updateSyncRulesFromYaml(
|
|
1216
|
+
`
|
|
1217
|
+
bucket_definitions:
|
|
1218
|
+
global:
|
|
1219
|
+
data: [SELECT id as id, description FROM items]
|
|
1220
|
+
`,
|
|
1221
|
+
{ storageVersion: 3 }
|
|
1222
|
+
)
|
|
1223
|
+
);
|
|
1224
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
1225
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
1226
|
+
const mapping = syncRules.syncConfigContent[0].mapping;
|
|
1227
|
+
const definitionId = mapping.allBucketDefinitionIds()[0];
|
|
1228
|
+
const collection = db.bucketData(bucketStorage.replicationStreamId, definitionId);
|
|
1229
|
+
const bucketStateCollection = db.bucketState(bucketStorage.replicationStreamId);
|
|
1230
|
+
const sourceTableId = new bson.ObjectId();
|
|
1231
|
+
|
|
1232
|
+
const ctx = {
|
|
1233
|
+
replicationStreamId: bucketStorage.replicationStreamId,
|
|
1234
|
+
definitionId,
|
|
1235
|
+
bucket: BUCKET
|
|
1236
|
+
};
|
|
1237
|
+
|
|
1238
|
+
return { bucketStorage, syncRules, db, collection, bucketStateCollection, sourceTableId, ctx };
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
function makeOp(
|
|
1242
|
+
opId: number,
|
|
1243
|
+
rowId: string,
|
|
1244
|
+
data: string,
|
|
1245
|
+
ctx: { replicationStreamId: number; definitionId: string; bucket: string },
|
|
1246
|
+
sourceTableId: bson.ObjectId,
|
|
1247
|
+
overrides?: { op?: 'PUT' | 'REMOVE' }
|
|
1248
|
+
): BucketDataDoc {
|
|
1249
|
+
return {
|
|
1250
|
+
bucketKey: {
|
|
1251
|
+
replicationStreamId: ctx.replicationStreamId,
|
|
1252
|
+
definitionId: ctx.definitionId,
|
|
1253
|
+
bucket: ctx.bucket
|
|
1254
|
+
},
|
|
1255
|
+
o: BigInt(opId),
|
|
1256
|
+
op: overrides?.op ?? 'PUT',
|
|
1257
|
+
source_table: sourceTableId,
|
|
1258
|
+
source_key: test_utils.rid(rowId),
|
|
1259
|
+
table: TABLE,
|
|
1260
|
+
row_id: rowId,
|
|
1261
|
+
checksum: BigInt(opId * 7),
|
|
1262
|
+
data: overrides?.op === 'REMOVE' ? null : JSON.stringify({ id: rowId, description: data })
|
|
1263
|
+
};
|
|
1264
|
+
}
|
|
1265
|
+
|
|
1266
|
+
async function insertDocs(collection: any, docs: BucketDataDocumentV3[]) {
|
|
1267
|
+
await collection.insertMany(docs);
|
|
1268
|
+
}
|
|
1269
|
+
|
|
1270
|
+
async function insertBucketState(bucketStateCollection: any, definitionId: string, lastOp: bigint) {
|
|
1271
|
+
await bucketStateCollection.insertOne({
|
|
1272
|
+
_id: { d: definitionId, b: BUCKET },
|
|
1273
|
+
last_op: lastOp,
|
|
1274
|
+
estimate_since_compact: { count: 10, bytes: 100 }
|
|
1275
|
+
});
|
|
1276
|
+
}
|
|
1277
|
+
|
|
1278
|
+
async function compact(bucketStorage: MongoSyncBucketStorage, maxOpId: bigint) {
|
|
1279
|
+
await bucketStorage.compact({
|
|
1280
|
+
clearBatchLimit: 200,
|
|
1281
|
+
moveBatchLimit: 10,
|
|
1282
|
+
moveBatchQueryLimit: 10,
|
|
1283
|
+
minBucketChanges: 1,
|
|
1284
|
+
minChangeRatio: 0,
|
|
1285
|
+
maxOpId
|
|
1286
|
+
});
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
test('checksum preserved across compaction with superseded ops', async () => {
|
|
1290
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1291
|
+
const ops = [
|
|
1292
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
1293
|
+
makeOp(20, 'B', 'b1', ctx, sourceTableId),
|
|
1294
|
+
makeOp(30, 'A', 'a2', ctx, sourceTableId),
|
|
1295
|
+
makeOp(40, 'C', 'c1', ctx, sourceTableId),
|
|
1296
|
+
makeOp(50, 'B', 'b2', ctx, sourceTableId)
|
|
1297
|
+
];
|
|
1298
|
+
const doc1 = serializeBucketData(BUCKET, ops);
|
|
1299
|
+
await insertDocs(collection, [doc1]);
|
|
1300
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 50n);
|
|
1301
|
+
|
|
1302
|
+
const docsBefore = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
1303
|
+
const checksumBefore = docsBefore.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1304
|
+
|
|
1305
|
+
await compact(bucketStorage, 50n);
|
|
1306
|
+
|
|
1307
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1308
|
+
const checksumAfter = docsAfter.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1309
|
+
|
|
1310
|
+
expect(checksumAfter).toBe(checksumBefore);
|
|
1311
|
+
|
|
1312
|
+
const allOpsAfter = docsAfter.flatMap((d) => d.ops!);
|
|
1313
|
+
// Two MOVEs collapsed into one CLEAR
|
|
1314
|
+
expect(allOpsAfter.length).toBe(4);
|
|
1315
|
+
const clearOps = allOpsAfter.filter((op) => op.op === 'CLEAR');
|
|
1316
|
+
expect(clearOps.length).toBe(1);
|
|
1317
|
+
});
|
|
1318
|
+
|
|
1319
|
+
test('checksum preserved across compaction with multiple documents', async () => {
|
|
1320
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1321
|
+
const ops1 = [makeOp(10, 'A', 'a1', ctx, sourceTableId), makeOp(20, 'B', 'b1', ctx, sourceTableId)];
|
|
1322
|
+
const ops2 = [makeOp(30, 'C', 'c1', ctx, sourceTableId), makeOp(40, 'A', 'a2', ctx, sourceTableId)];
|
|
1323
|
+
const ops3 = [makeOp(50, 'D', 'd1', ctx, sourceTableId), makeOp(60, 'B', 'b2', ctx, sourceTableId)];
|
|
1324
|
+
await insertDocs(collection, [
|
|
1325
|
+
serializeBucketData(BUCKET, ops1),
|
|
1326
|
+
serializeBucketData(BUCKET, ops2),
|
|
1327
|
+
serializeBucketData(BUCKET, ops3)
|
|
1328
|
+
]);
|
|
1329
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 60n);
|
|
1330
|
+
|
|
1331
|
+
const docsBefore = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
1332
|
+
const checksumBefore = docsBefore.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1333
|
+
|
|
1334
|
+
await compact(bucketStorage, 60n);
|
|
1335
|
+
|
|
1336
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1337
|
+
const checksumAfter = docsAfter.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1338
|
+
|
|
1339
|
+
expect(checksumAfter).toBe(checksumBefore);
|
|
1340
|
+
|
|
1341
|
+
const allOpsAfter = docsAfter.flatMap((d) => d.ops!);
|
|
1342
|
+
// Two MOVEs collapsed into one CLEAR
|
|
1343
|
+
expect(allOpsAfter.length).toBe(5);
|
|
1344
|
+
const clearOps = allOpsAfter.filter((op) => op.op === 'CLEAR');
|
|
1345
|
+
expect(clearOps.length).toBe(1);
|
|
1346
|
+
});
|
|
1347
|
+
|
|
1348
|
+
test('tombstones have null data and pack densely after rechunking', async () => {
|
|
1349
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1350
|
+
const ops = [
|
|
1351
|
+
makeOp(10, 'A', 'x'.repeat(500_000), ctx, sourceTableId),
|
|
1352
|
+
makeOp(20, 'B', 'y'.repeat(500_000), ctx, sourceTableId),
|
|
1353
|
+
makeOp(30, 'A', 'z'.repeat(500_000), ctx, sourceTableId)
|
|
1354
|
+
];
|
|
1355
|
+
const doc1 = serializeBucketData(BUCKET, ops);
|
|
1356
|
+
await insertDocs(collection, [doc1]);
|
|
1357
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 30n);
|
|
1358
|
+
|
|
1359
|
+
await compact(bucketStorage, 30n);
|
|
1360
|
+
|
|
1361
|
+
const docs = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1362
|
+
const allOps = docs.flatMap((d) => d.ops!);
|
|
1363
|
+
const moveOps = allOps.filter((op) => op.op === 'MOVE');
|
|
1364
|
+
expect(moveOps.length).toBe(1);
|
|
1365
|
+
expect(moveOps[0].o).toBe(10n);
|
|
1366
|
+
expect(moveOps[0].data).toBeNull();
|
|
1367
|
+
|
|
1368
|
+
const putOps = allOps.filter((op) => op.op === 'PUT');
|
|
1369
|
+
expect(putOps.length).toBe(2);
|
|
1370
|
+
expect(putOps.every((op) => op.data != null)).toBe(true);
|
|
1371
|
+
|
|
1372
|
+
expect(docs.length).toBe(1);
|
|
1373
|
+
// Size calculation is not exact - we just check for a range
|
|
1374
|
+
expect(docs[0].size).toBeGreaterThan(1_000_000);
|
|
1375
|
+
expect(docs[0].size).toBeLessThan(1_001_000);
|
|
1376
|
+
});
|
|
1377
|
+
|
|
1378
|
+
test('tombstones and survivors end up in same document after rechunking', async () => {
|
|
1379
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1380
|
+
const ops1 = [
|
|
1381
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
1382
|
+
makeOp(20, 'B', 'b1', ctx, sourceTableId),
|
|
1383
|
+
makeOp(30, 'C', 'c1', ctx, sourceTableId)
|
|
1384
|
+
];
|
|
1385
|
+
const ops2 = [makeOp(40, 'D', 'd1', ctx, sourceTableId), makeOp(50, 'A', 'a2', ctx, sourceTableId)];
|
|
1386
|
+
await insertDocs(collection, [serializeBucketData(BUCKET, ops1), serializeBucketData(BUCKET, ops2)]);
|
|
1387
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 50n);
|
|
1388
|
+
|
|
1389
|
+
const checksumBefore = (await collection.find({ '_id.b': BUCKET }).toArray()).reduce(
|
|
1390
|
+
(sum, d) => addChecksums(sum, Number(d.checksum)),
|
|
1391
|
+
0
|
|
1392
|
+
);
|
|
1393
|
+
|
|
1394
|
+
await compact(bucketStorage, 50n);
|
|
1395
|
+
|
|
1396
|
+
const docs = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1397
|
+
const checksumAfter = docs.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1398
|
+
expect(checksumAfter).toBe(checksumBefore);
|
|
1399
|
+
|
|
1400
|
+
const allOps = docs.flatMap((d) => d.ops!);
|
|
1401
|
+
expect(allOps.length).toBe(5);
|
|
1402
|
+
const moveOp = allOps.find((op) => op.op === 'MOVE');
|
|
1403
|
+
expect(moveOp).toMatchObject({ o: 10n, data: null });
|
|
1404
|
+
|
|
1405
|
+
expect(docs.length).toBe(1);
|
|
1406
|
+
|
|
1407
|
+
const moveOpsInDoc = docs[0].ops!.filter((op) => op.op === 'MOVE');
|
|
1408
|
+
const putOpsInDoc = docs[0].ops!.filter((op) => op.op === 'PUT');
|
|
1409
|
+
expect(moveOpsInDoc.length).toBe(1);
|
|
1410
|
+
expect(putOpsInDoc.length).toBe(4);
|
|
1411
|
+
});
|
|
1412
|
+
});
|
|
1413
|
+
|
|
1414
|
+
/**
|
|
1415
|
+
* Streaming compactor tests
|
|
1416
|
+
*
|
|
1417
|
+
* These tests exercise the streaming compactor's batched behavior:
|
|
1418
|
+
* reverse-order batched reads, cross-batch seen map dedup with memory
|
|
1419
|
+
* bounding, scoped deletes via $in, and byte-based batch cutting.
|
|
1420
|
+
*/
|
|
1421
|
+
describe('Streaming compactor', () => {
|
|
1422
|
+
const BUCKET = 'global[]';
|
|
1423
|
+
const TABLE = 'items';
|
|
1424
|
+
|
|
1425
|
+
async function setupV3() {
|
|
1426
|
+
await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory();
|
|
1427
|
+
const syncRules = await factory.updateSyncRules(
|
|
1428
|
+
updateSyncRulesFromYaml(
|
|
1429
|
+
`
|
|
1430
|
+
bucket_definitions:
|
|
1431
|
+
global:
|
|
1432
|
+
data: [SELECT id as id, description FROM items]
|
|
1433
|
+
`,
|
|
1434
|
+
{ storageVersion: 3 }
|
|
1435
|
+
)
|
|
1436
|
+
);
|
|
1437
|
+
const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage;
|
|
1438
|
+
const db = bucketStorage.db as VersionedPowerSyncMongoV3;
|
|
1439
|
+
const definitionId = bucketStorage.storageIds.bucketDefinitionIds[0];
|
|
1440
|
+
const collection = db.bucketData(bucketStorage.replicationStreamId, definitionId);
|
|
1441
|
+
const bucketStateCollection = db.bucketState(bucketStorage.replicationStreamId);
|
|
1442
|
+
const sourceTableId = new bson.ObjectId();
|
|
1443
|
+
|
|
1444
|
+
const ctx = {
|
|
1445
|
+
replicationStreamId: bucketStorage.replicationStreamId,
|
|
1446
|
+
definitionId,
|
|
1447
|
+
bucket: BUCKET
|
|
1448
|
+
};
|
|
1449
|
+
|
|
1450
|
+
return { bucketStorage, syncRules, db, collection, bucketStateCollection, sourceTableId, ctx };
|
|
1451
|
+
}
|
|
1452
|
+
|
|
1453
|
+
function makeOp(
|
|
1454
|
+
opId: number,
|
|
1455
|
+
rowId: string,
|
|
1456
|
+
data: string,
|
|
1457
|
+
ctx: { replicationStreamId: number; definitionId: string; bucket: string },
|
|
1458
|
+
sourceTableId: bson.ObjectId,
|
|
1459
|
+
overrides?: { op?: 'PUT' | 'REMOVE' }
|
|
1460
|
+
): BucketDataDoc {
|
|
1461
|
+
return {
|
|
1462
|
+
bucketKey: {
|
|
1463
|
+
replicationStreamId: ctx.replicationStreamId,
|
|
1464
|
+
definitionId: ctx.definitionId,
|
|
1465
|
+
bucket: ctx.bucket
|
|
1466
|
+
},
|
|
1467
|
+
o: BigInt(opId),
|
|
1468
|
+
op: overrides?.op ?? 'PUT',
|
|
1469
|
+
source_table: sourceTableId,
|
|
1470
|
+
source_key: test_utils.rid(rowId),
|
|
1471
|
+
table: TABLE,
|
|
1472
|
+
row_id: rowId,
|
|
1473
|
+
checksum: BigInt(opId * 7),
|
|
1474
|
+
data: overrides?.op === 'REMOVE' ? null : JSON.stringify({ id: rowId, description: data })
|
|
1475
|
+
};
|
|
1476
|
+
}
|
|
1477
|
+
|
|
1478
|
+
async function insertDocs(collection: any, docs: BucketDataDocumentV3[]) {
|
|
1479
|
+
await collection.insertMany(docs);
|
|
1480
|
+
}
|
|
1481
|
+
|
|
1482
|
+
async function insertBucketState(bucketStateCollection: any, definitionId: string, lastOp: bigint) {
|
|
1483
|
+
await bucketStateCollection.insertOne({
|
|
1484
|
+
_id: { d: definitionId, b: BUCKET },
|
|
1485
|
+
last_op: lastOp,
|
|
1486
|
+
estimate_since_compact: { count: 10, bytes: 100 }
|
|
1487
|
+
});
|
|
1488
|
+
}
|
|
1489
|
+
|
|
1490
|
+
async function readAllOps(
|
|
1491
|
+
collection: any
|
|
1492
|
+
): Promise<{ row_id: string | undefined; o: bigint; op: string; target_op: bigint | undefined }[]> {
|
|
1493
|
+
const docs = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1494
|
+
return docs.flatMap((d: any) =>
|
|
1495
|
+
d.ops!.map((op: any) => ({
|
|
1496
|
+
row_id: op.row_id ?? undefined,
|
|
1497
|
+
o: op.o,
|
|
1498
|
+
op: op.op,
|
|
1499
|
+
target_op: op.target_op ?? undefined
|
|
1500
|
+
}))
|
|
1501
|
+
);
|
|
1502
|
+
}
|
|
1503
|
+
|
|
1504
|
+
test('1. multi-batch compaction preserves checksum and creates MOVE tombstones', async () => {
|
|
1505
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1506
|
+
|
|
1507
|
+
// 6 documents, 3-4 ops each, 3 rows with 2 versions spread across documents.
|
|
1508
|
+
// Rows A, B, C each have old+new versions:
|
|
1509
|
+
// A: new@24 (Doc1), old@10 (Doc4)
|
|
1510
|
+
// B: new@20 (Doc2), old@6 (Doc5)
|
|
1511
|
+
// C: new@16 (Doc3), old@2 (Doc6)
|
|
1512
|
+
// With moveBatchQueryLimit=2, the streaming compactor must process
|
|
1513
|
+
// 2 documents per batch (3 batches total) and still produce correct output.
|
|
1514
|
+
const doc1 = serializeBucketData(BUCKET, [
|
|
1515
|
+
makeOp(21, 'D', 'd1', ctx, sourceTableId),
|
|
1516
|
+
makeOp(22, 'E', 'e1', ctx, sourceTableId),
|
|
1517
|
+
makeOp(23, 'F', 'f1', ctx, sourceTableId),
|
|
1518
|
+
makeOp(24, 'A', 'a_new', ctx, sourceTableId)
|
|
1519
|
+
]);
|
|
1520
|
+
const doc2 = serializeBucketData(BUCKET, [
|
|
1521
|
+
makeOp(18, 'G', 'g1', ctx, sourceTableId),
|
|
1522
|
+
makeOp(19, 'H', 'h1', ctx, sourceTableId),
|
|
1523
|
+
makeOp(20, 'B', 'b_new', ctx, sourceTableId)
|
|
1524
|
+
]);
|
|
1525
|
+
const doc3 = serializeBucketData(BUCKET, [
|
|
1526
|
+
makeOp(14, 'I', 'i1', ctx, sourceTableId),
|
|
1527
|
+
makeOp(15, 'J', 'j1', ctx, sourceTableId),
|
|
1528
|
+
makeOp(16, 'C', 'c_new', ctx, sourceTableId)
|
|
1529
|
+
]);
|
|
1530
|
+
const doc4 = serializeBucketData(BUCKET, [
|
|
1531
|
+
makeOp(10, 'A', 'a_old', ctx, sourceTableId),
|
|
1532
|
+
makeOp(11, 'K', 'k1', ctx, sourceTableId),
|
|
1533
|
+
makeOp(12, 'L', 'l1', ctx, sourceTableId)
|
|
1534
|
+
]);
|
|
1535
|
+
const doc5 = serializeBucketData(BUCKET, [
|
|
1536
|
+
makeOp(6, 'B', 'b_old', ctx, sourceTableId),
|
|
1537
|
+
makeOp(7, 'M', 'm1', ctx, sourceTableId),
|
|
1538
|
+
makeOp(8, 'N', 'n1', ctx, sourceTableId)
|
|
1539
|
+
]);
|
|
1540
|
+
const doc6 = serializeBucketData(BUCKET, [
|
|
1541
|
+
makeOp(2, 'C', 'c_old', ctx, sourceTableId),
|
|
1542
|
+
makeOp(3, 'O', 'o1', ctx, sourceTableId),
|
|
1543
|
+
makeOp(4, 'P', 'p1', ctx, sourceTableId)
|
|
1544
|
+
]);
|
|
1545
|
+
await insertDocs(collection, [doc1, doc2, doc3, doc4, doc5, doc6]);
|
|
1546
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 24n);
|
|
1547
|
+
|
|
1548
|
+
// Compute checksum before compaction
|
|
1549
|
+
const docsBefore = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
1550
|
+
const checksumBefore = docsBefore.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1551
|
+
|
|
1552
|
+
// Compact with moveBatchQueryLimit=2 to force 3 batches of 2 documents each
|
|
1553
|
+
await bucketStorage.compact({
|
|
1554
|
+
clearBatchLimit: 200,
|
|
1555
|
+
moveBatchLimit: 10,
|
|
1556
|
+
moveBatchQueryLimit: 2,
|
|
1557
|
+
minBucketChanges: 1,
|
|
1558
|
+
minChangeRatio: 0,
|
|
1559
|
+
maxOpId: 24n
|
|
1560
|
+
});
|
|
1561
|
+
|
|
1562
|
+
// Verify checksum preserved
|
|
1563
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1564
|
+
const checksumAfter = docsAfter.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1565
|
+
expect(checksumAfter).toBe(checksumBefore);
|
|
1566
|
+
|
|
1567
|
+
// Verify MOVE tombstones for superseded old versions (A@10, B@6, C@2)
|
|
1568
|
+
const allOps = await readAllOps(collection);
|
|
1569
|
+
const moveOps = allOps.filter((op) => op.op === 'MOVE');
|
|
1570
|
+
expect(moveOps.length).toBe(3);
|
|
1571
|
+
expect(moveOps.map((op) => op.o).sort((a, b) => (a < b ? -1 : a > b ? 1 : 0))).toEqual([2n, 6n, 10n]);
|
|
1572
|
+
|
|
1573
|
+
// Verify the newer version of each duplicated row survives as PUT
|
|
1574
|
+
const putOps = allOps.filter((op) => op.op === 'PUT');
|
|
1575
|
+
expect(putOps.length).toBe(16);
|
|
1576
|
+
const putRowIds = putOps.map((op) => op.row_id).sort();
|
|
1577
|
+
expect(putRowIds).toContain('A');
|
|
1578
|
+
expect(putRowIds).toContain('B');
|
|
1579
|
+
expect(putRowIds).toContain('C');
|
|
1580
|
+
});
|
|
1581
|
+
|
|
1582
|
+
test('2. scoped delete isolation - ops above maxOpId preserved', async () => {
|
|
1583
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1584
|
+
|
|
1585
|
+
// Insert ops in _id.o range 100-500 via direct document insertion
|
|
1586
|
+
const doc1 = serializeBucketData(BUCKET, [
|
|
1587
|
+
makeOp(100, 'A', 'a1', ctx, sourceTableId),
|
|
1588
|
+
makeOp(200, 'B', 'b1', ctx, sourceTableId),
|
|
1589
|
+
makeOp(300, 'C', 'c1', ctx, sourceTableId)
|
|
1590
|
+
]);
|
|
1591
|
+
const doc2 = serializeBucketData(BUCKET, [
|
|
1592
|
+
makeOp(400, 'D', 'd1', ctx, sourceTableId),
|
|
1593
|
+
makeOp(500, 'E', 'e1', ctx, sourceTableId)
|
|
1594
|
+
]);
|
|
1595
|
+
// One extra document with _id.o=600 — clearly outside the batch range
|
|
1596
|
+
const docOutside = serializeBucketData(BUCKET, [makeOp(600, 'F', 'f_outside', ctx, sourceTableId)]);
|
|
1597
|
+
|
|
1598
|
+
await insertDocs(collection, [doc1, doc2, docOutside]);
|
|
1599
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 600n);
|
|
1600
|
+
|
|
1601
|
+
// Run compaction with maxOpId=500n
|
|
1602
|
+
await bucketStorage.compact({
|
|
1603
|
+
clearBatchLimit: 200,
|
|
1604
|
+
moveBatchLimit: 10,
|
|
1605
|
+
moveBatchQueryLimit: 2,
|
|
1606
|
+
minBucketChanges: 1,
|
|
1607
|
+
minChangeRatio: 0,
|
|
1608
|
+
maxOpId: 500n
|
|
1609
|
+
});
|
|
1610
|
+
|
|
1611
|
+
// The document at _id.o=600 must still be present and unmodified after compaction.
|
|
1612
|
+
// The streaming compactor should only touch documents containing ops <= maxOpId.
|
|
1613
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1614
|
+
const allOpsAfter = docsAfter.flatMap((d) => d.ops!);
|
|
1615
|
+
const op600 = allOpsAfter.find((op) => op.o === 600n);
|
|
1616
|
+
expect(op600).toBeDefined();
|
|
1617
|
+
expect(op600!.op).toBe('PUT');
|
|
1618
|
+
expect(op600!.row_id).toBe('F');
|
|
1619
|
+
});
|
|
1620
|
+
|
|
1621
|
+
test('3. seen map overflow - some old ops pass through without tombstoning', async () => {
|
|
1622
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1623
|
+
|
|
1624
|
+
// 20 rows, each with 2 versions (old + new). 40 ops total.
|
|
1625
|
+
// memoryLimitMB=0.001 → idLimitBytes ≈ 1048 bytes.
|
|
1626
|
+
// The seen map adds ~140 bytes per entry (key.length + 140), so it can hold ~7 entries
|
|
1627
|
+
// before overflowing. The streaming compactor processes ops newest-first; after ~7 rows
|
|
1628
|
+
// are tracked, the seen map overflows and subsequent old-version ops are not tombstoned.
|
|
1629
|
+
const rows = Array.from({ length: 20 }, (_, i) => `row${i + 1}`);
|
|
1630
|
+
const allOps: BucketDataDoc[] = [];
|
|
1631
|
+
|
|
1632
|
+
// Old versions at op_ids 1-20 (processed last by descending-order compactor)
|
|
1633
|
+
for (let i = 0; i < 20; i++) {
|
|
1634
|
+
allOps.push(makeOp(i + 1, rows[i], `old_${rows[i]}`, ctx, sourceTableId));
|
|
1635
|
+
}
|
|
1636
|
+
// New versions at op_ids 21-40 (processed first by descending-order compactor)
|
|
1637
|
+
for (let i = 0; i < 20; i++) {
|
|
1638
|
+
allOps.push(makeOp(i + 21, rows[i], `new_${rows[i]}`, ctx, sourceTableId));
|
|
1639
|
+
}
|
|
1640
|
+
|
|
1641
|
+
// Single document containing all ops
|
|
1642
|
+
const doc = serializeBucketData(BUCKET, allOps);
|
|
1643
|
+
await insertDocs(collection, [doc]);
|
|
1644
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 40n);
|
|
1645
|
+
|
|
1646
|
+
// Compute checksum before compaction
|
|
1647
|
+
const docsBefore = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
1648
|
+
const checksumBefore = docsBefore.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1649
|
+
|
|
1650
|
+
await bucketStorage.compact({
|
|
1651
|
+
clearBatchLimit: 200,
|
|
1652
|
+
moveBatchLimit: 10,
|
|
1653
|
+
moveBatchQueryLimit: 100,
|
|
1654
|
+
minBucketChanges: 1,
|
|
1655
|
+
minChangeRatio: 0,
|
|
1656
|
+
maxOpId: 40n,
|
|
1657
|
+
memoryLimitMB: 0.001
|
|
1658
|
+
});
|
|
1659
|
+
|
|
1660
|
+
// Bucket checksum must be correct even with overflow — old ops that pass through
|
|
1661
|
+
// as PUT still contribute their original checksum.
|
|
1662
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1663
|
+
const checksumAfter = docsAfter.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1664
|
+
expect(checksumAfter).toBe(checksumBefore);
|
|
1665
|
+
|
|
1666
|
+
// With seen map overflow, fewer than 20 old-version ops become MOVE tombstones.
|
|
1667
|
+
// Some old-version ops pass through as PUT because the seen map was full when
|
|
1668
|
+
// the compactor encountered them. With ~1048 bytes and ~176 bytes per entry,
|
|
1669
|
+
// approximately 5-7 rows are tracked before overflow, producing ~5-7 MOVE
|
|
1670
|
+
// tombstones from the 20 duplicate pairs.
|
|
1671
|
+
const opsAfter = await readAllOps(collection);
|
|
1672
|
+
const moveOps = opsAfter.filter((op) => op.op === 'MOVE');
|
|
1673
|
+
expect(moveOps.length).toBeGreaterThan(0);
|
|
1674
|
+
expect(moveOps.length).toBeLessThanOrEqual(12);
|
|
1675
|
+
});
|
|
1676
|
+
|
|
1677
|
+
test('5. scoped delete does not remove sandwiched non-processable docs', async () => {
|
|
1678
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1679
|
+
|
|
1680
|
+
// Doc1: processable (has ops <= maxOpId=300), _id.o = 300
|
|
1681
|
+
const doc1 = serializeBucketData(BUCKET, [
|
|
1682
|
+
makeOp(200, 'A', 'a1', ctx, sourceTableId),
|
|
1683
|
+
makeOp(300, 'B', 'b1', ctx, sourceTableId)
|
|
1684
|
+
]);
|
|
1685
|
+
// Doc2: NON-processable (all ops > maxOpId=300), _id.o = 350
|
|
1686
|
+
// This doc is sandwiched between doc1 (_id.o=300) and doc3 (_id.o=500)
|
|
1687
|
+
// in _id.o sort order. A continuous range delete [200, 500] would
|
|
1688
|
+
// incorrectly catch this doc.
|
|
1689
|
+
const doc2 = serializeBucketData(BUCKET, [
|
|
1690
|
+
makeOp(340, 'X', 'x_sandwich', ctx, sourceTableId),
|
|
1691
|
+
makeOp(350, 'Y', 'y_sandwich', ctx, sourceTableId)
|
|
1692
|
+
]);
|
|
1693
|
+
// Doc3: processable (has ops <= maxOpId=300), _id.o = 400
|
|
1694
|
+
const doc3 = serializeBucketData(BUCKET, [
|
|
1695
|
+
makeOp(250, 'C', 'c1', ctx, sourceTableId),
|
|
1696
|
+
makeOp(400, 'D', 'd1', ctx, sourceTableId)
|
|
1697
|
+
]);
|
|
1698
|
+
|
|
1699
|
+
await insertDocs(collection, [doc1, doc2, doc3]);
|
|
1700
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 400n);
|
|
1701
|
+
|
|
1702
|
+
await bucketStorage.compact({
|
|
1703
|
+
clearBatchLimit: 200,
|
|
1704
|
+
moveBatchLimit: 10,
|
|
1705
|
+
moveBatchQueryLimit: 10,
|
|
1706
|
+
minBucketChanges: 1,
|
|
1707
|
+
minChangeRatio: 0,
|
|
1708
|
+
maxOpId: 300n
|
|
1709
|
+
});
|
|
1710
|
+
|
|
1711
|
+
// The sandwiched document (doc2, ops 340+350) must survive because ALL
|
|
1712
|
+
// its ops are > maxOpId — it should not be touched by compaction.
|
|
1713
|
+
const opsAfter = await readAllOps(collection);
|
|
1714
|
+
const sandwichOps = opsAfter.filter((op) => op.row_id === 'X' || op.row_id === 'Y');
|
|
1715
|
+
expect(sandwichOps.length).toBe(2);
|
|
1716
|
+
expect(sandwichOps.every((op) => op.op === 'PUT')).toBe(true);
|
|
1717
|
+
});
|
|
1718
|
+
|
|
1719
|
+
test('6. byte-based read batches do not constrain output merging', async () => {
|
|
1720
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1721
|
+
|
|
1722
|
+
// 6 documents, each with 2 PUT ops. With moveBatchByteLimit=400,
|
|
1723
|
+
// the byte limit should be hit after 1-2 documents, forcing multiple
|
|
1724
|
+
// batch iterations even though the document count limit (10) is never reached.
|
|
1725
|
+
const rows = Array.from({ length: 12 }, (_, i) => `row${i + 1}`);
|
|
1726
|
+
const docs: BucketDataDocumentV3[] = [];
|
|
1727
|
+
for (let i = 0; i < 6; i++) {
|
|
1728
|
+
docs.push(
|
|
1729
|
+
serializeBucketData(BUCKET, [
|
|
1730
|
+
makeOp(i * 2 + 1, rows[i * 2], `data_${rows[i * 2]}`, ctx, sourceTableId),
|
|
1731
|
+
makeOp(i * 2 + 2, rows[i * 2 + 1], `data_${rows[i * 2 + 1]}`, ctx, sourceTableId)
|
|
1732
|
+
])
|
|
1733
|
+
);
|
|
1734
|
+
}
|
|
1735
|
+
expect(docs[0].size + docs[1].size).toBeGreaterThan(400);
|
|
1736
|
+
await insertDocs(collection, docs);
|
|
1737
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 12n);
|
|
1738
|
+
|
|
1739
|
+
const docsBefore = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
1740
|
+
const checksumBefore = docsBefore.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1741
|
+
|
|
1742
|
+
await bucketStorage.compact({
|
|
1743
|
+
clearBatchLimit: 200,
|
|
1744
|
+
moveBatchLimit: 10,
|
|
1745
|
+
moveBatchQueryLimit: 10,
|
|
1746
|
+
moveBatchByteLimit: 400,
|
|
1747
|
+
minBucketChanges: 1,
|
|
1748
|
+
minChangeRatio: 0,
|
|
1749
|
+
maxOpId: 12n
|
|
1750
|
+
});
|
|
1751
|
+
|
|
1752
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1753
|
+
const checksumAfter = docsAfter.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1754
|
+
expect(checksumAfter).toBe(checksumBefore);
|
|
1755
|
+
|
|
1756
|
+
// All 12 ops should survive — no superseding, just byte-based batching
|
|
1757
|
+
const allOps = await readAllOps(collection);
|
|
1758
|
+
expect(allOps.length).toBe(12);
|
|
1759
|
+
expect(allOps.every((op) => op.op === 'PUT')).toBe(true);
|
|
1760
|
+
|
|
1761
|
+
// Read batches are only a memory bound. The pending output group crosses
|
|
1762
|
+
// those boundaries and merges all post-compaction results that fit.
|
|
1763
|
+
expect(docsAfter).toHaveLength(1);
|
|
1764
|
+
expect(docsAfter[0]).toMatchObject({
|
|
1765
|
+
min_op: docs[0].min_op,
|
|
1766
|
+
_id: docs[docs.length - 1]._id
|
|
1767
|
+
});
|
|
1768
|
+
});
|
|
1769
|
+
|
|
1770
|
+
test('7. cross-batch seen map overflow - dedup continues across batches', async () => {
|
|
1771
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1772
|
+
|
|
1773
|
+
// 10 rows (A-J), 2 versions each. 20 docs total, 1 PUT per doc.
|
|
1774
|
+
// New versions at opIds 20-11 (docs 1-10), old versions at opIds 10-1 (docs 11-20).
|
|
1775
|
+
// Processed newest-first: new versions fill the seen map; old versions that
|
|
1776
|
+
// were tracked get tombstoned; old versions beyond map capacity survive as PUT.
|
|
1777
|
+
const rows = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J'];
|
|
1778
|
+
const docs: BucketDataDocumentV3[] = [];
|
|
1779
|
+
|
|
1780
|
+
// New versions first (higher opIds — processed first in reverse order)
|
|
1781
|
+
for (let i = 0; i < 10; i++) {
|
|
1782
|
+
docs.push(
|
|
1783
|
+
serializeBucketData(BUCKET, [makeOp(20 - i, rows[i], `${rows[i].toLowerCase()}_new`, ctx, sourceTableId)])
|
|
1784
|
+
);
|
|
1785
|
+
}
|
|
1786
|
+
// Old versions (lower opIds — processed later)
|
|
1787
|
+
for (let i = 0; i < 10; i++) {
|
|
1788
|
+
docs.push(
|
|
1789
|
+
serializeBucketData(BUCKET, [makeOp(10 - i, rows[i], `${rows[i].toLowerCase()}_old`, ctx, sourceTableId)])
|
|
1790
|
+
);
|
|
1791
|
+
}
|
|
1792
|
+
await insertDocs(collection, docs);
|
|
1793
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 20n);
|
|
1794
|
+
|
|
1795
|
+
const docsBefore = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
1796
|
+
const checksumBefore = docsBefore.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1797
|
+
|
|
1798
|
+
// moveBatchQueryLimit=5 forces 4 cross-batch iterations. memoryLimitMB=0.001
|
|
1799
|
+
// limits the seen map to ~6-7 entries, so overflow occurs mid-processing.
|
|
1800
|
+
await bucketStorage.compact({
|
|
1801
|
+
clearBatchLimit: 200,
|
|
1802
|
+
moveBatchLimit: 10,
|
|
1803
|
+
moveBatchQueryLimit: 5,
|
|
1804
|
+
minBucketChanges: 1,
|
|
1805
|
+
minChangeRatio: 0,
|
|
1806
|
+
maxOpId: 20n,
|
|
1807
|
+
memoryLimitMB: 0.001
|
|
1808
|
+
});
|
|
1809
|
+
|
|
1810
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1811
|
+
const checksumAfter = docsAfter.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1812
|
+
expect(checksumAfter).toBe(checksumBefore);
|
|
1813
|
+
|
|
1814
|
+
const allOps = await readAllOps(collection);
|
|
1815
|
+
const moveOps = allOps.filter((op) => op.op === 'MOVE');
|
|
1816
|
+
|
|
1817
|
+
// At least some rows were tracked before overflow → MOVE tombstones exist
|
|
1818
|
+
expect(moveOps.length).toBeGreaterThan(0);
|
|
1819
|
+
|
|
1820
|
+
// Some old-version ops (o <= 10) survive as PUT — overflow prevented tracking
|
|
1821
|
+
const oldVersionPutOps = allOps.filter((op) => op.op === 'PUT' && op.o <= 10n);
|
|
1822
|
+
expect(oldVersionPutOps.length).toBeGreaterThan(0);
|
|
1823
|
+
|
|
1824
|
+
// Not all 10 old versions became MOVEs (overflow occurred)
|
|
1825
|
+
expect(moveOps.length).toBeLessThan(10);
|
|
1826
|
+
|
|
1827
|
+
// MOVEs have target_op stored at the document level (not per-op in V3).
|
|
1828
|
+
// Documents containing MOVEs should have non-null target_op.
|
|
1829
|
+
const docsWithMoves = docsAfter.filter((d) => d.ops!.some((op: any) => op.op === 'MOVE'));
|
|
1830
|
+
expect(docsWithMoves.length).toBeGreaterThan(0);
|
|
1831
|
+
for (const doc of docsWithMoves) {
|
|
1832
|
+
expect(doc.target_op).toBeDefined();
|
|
1833
|
+
expect(doc.target_op).not.toBeNull();
|
|
1834
|
+
}
|
|
1835
|
+
});
|
|
1836
|
+
|
|
1837
|
+
test('8. all ops above maxOpId - paginates without writing', async () => {
|
|
1838
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1839
|
+
|
|
1840
|
+
// 5 docs, each with 1 PUT op at high opIds (all > maxOpId=10).
|
|
1841
|
+
// The compactor should paginate through batches where processableDocs
|
|
1842
|
+
// is always empty, terminating cleanly without touching any data.
|
|
1843
|
+
const docs: BucketDataDocumentV3[] = [];
|
|
1844
|
+
for (let i = 0; i < 5; i++) {
|
|
1845
|
+
docs.push(serializeBucketData(BUCKET, [makeOp((i + 1) * 100, `row${i}`, `data${i}`, ctx, sourceTableId)]));
|
|
1846
|
+
}
|
|
1847
|
+
await insertDocs(collection, docs);
|
|
1848
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 500n);
|
|
1849
|
+
|
|
1850
|
+
const docsBefore = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1851
|
+
const checksumBefore = docsBefore.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1852
|
+
|
|
1853
|
+
await bucketStorage.compact({
|
|
1854
|
+
clearBatchLimit: 200,
|
|
1855
|
+
moveBatchLimit: 10,
|
|
1856
|
+
moveBatchQueryLimit: 2,
|
|
1857
|
+
minBucketChanges: 1,
|
|
1858
|
+
minChangeRatio: 0,
|
|
1859
|
+
maxOpId: 10n
|
|
1860
|
+
});
|
|
1861
|
+
|
|
1862
|
+
// All ops are > maxOpId, so processableDocs is always empty.
|
|
1863
|
+
// The compactor should paginate through all documents without
|
|
1864
|
+
// deleting or modifying anything.
|
|
1865
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1866
|
+
const checksumAfter = docsAfter.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1867
|
+
expect(checksumAfter).toBe(checksumBefore);
|
|
1868
|
+
|
|
1869
|
+
// All 5 docs should still exist with their PUT ops intact
|
|
1870
|
+
expect(docsAfter.length).toBe(5);
|
|
1871
|
+
const allOps = await readAllOps(collection);
|
|
1872
|
+
expect(allOps.length).toBe(5);
|
|
1873
|
+
expect(allOps.every((op) => op.op === 'PUT')).toBe(true);
|
|
1874
|
+
});
|
|
1875
|
+
|
|
1876
|
+
test('9. maxOpId filtering leaves a straddling document untouched', async () => {
|
|
1877
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1878
|
+
|
|
1879
|
+
// Single document with two ops: one <= maxOpId, one > maxOpId
|
|
1880
|
+
const doc = serializeBucketData(BUCKET, [
|
|
1881
|
+
makeOp(200, 'A', 'old_A', ctx, sourceTableId),
|
|
1882
|
+
makeOp(400, 'B', 'new_B', ctx, sourceTableId)
|
|
1883
|
+
]);
|
|
1884
|
+
await insertDocs(collection, [doc]);
|
|
1885
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 400n);
|
|
1886
|
+
|
|
1887
|
+
const docsBefore = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
1888
|
+
const checksumBefore = docsBefore.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1889
|
+
|
|
1890
|
+
await bucketStorage.compact({
|
|
1891
|
+
clearBatchLimit: 200,
|
|
1892
|
+
moveBatchLimit: 10,
|
|
1893
|
+
moveBatchQueryLimit: 10,
|
|
1894
|
+
minBucketChanges: 1,
|
|
1895
|
+
minChangeRatio: 0,
|
|
1896
|
+
maxOpId: 300n
|
|
1897
|
+
});
|
|
1898
|
+
|
|
1899
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1900
|
+
const checksumAfter = docsAfter.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1901
|
+
expect(checksumAfter).toBe(checksumBefore);
|
|
1902
|
+
|
|
1903
|
+
const allOps = await readAllOps(collection);
|
|
1904
|
+
expect(allOps.length).toBe(2);
|
|
1905
|
+
|
|
1906
|
+
const op400 = allOps.find((op) => op.o === 400n);
|
|
1907
|
+
expect(op400).toBeDefined();
|
|
1908
|
+
expect(op400!.op).toBe('PUT');
|
|
1909
|
+
expect(op400!.row_id).toBe('B');
|
|
1910
|
+
});
|
|
1911
|
+
|
|
1912
|
+
test('10. compacted documents have non-overlapping ranges', async () => {
|
|
1913
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1914
|
+
|
|
1915
|
+
// Four documents with truly disjoint [min_op, _id.o] ranges.
|
|
1916
|
+
// maxOpId=350: only doc1 is processable; doc2 straddles the horizon.
|
|
1917
|
+
// Compaction must preserve range disjointness after rechunking.
|
|
1918
|
+
const doc1 = serializeBucketData(BUCKET, [
|
|
1919
|
+
makeOp(100, 'A', 'a1', ctx, sourceTableId),
|
|
1920
|
+
makeOp(200, 'B', 'b1', ctx, sourceTableId)
|
|
1921
|
+
]); // range [100, 200]
|
|
1922
|
+
const doc2 = serializeBucketData(BUCKET, [
|
|
1923
|
+
makeOp(300, 'C', 'c1', ctx, sourceTableId),
|
|
1924
|
+
makeOp(400, 'D', 'd1', ctx, sourceTableId)
|
|
1925
|
+
]); // range [300, 400], mixed (300≤350, 400>350)
|
|
1926
|
+
const doc3 = serializeBucketData(BUCKET, [
|
|
1927
|
+
makeOp(500, 'E', 'e1', ctx, sourceTableId),
|
|
1928
|
+
makeOp(600, 'F', 'f1', ctx, sourceTableId)
|
|
1929
|
+
]); // range [500, 600], all >350
|
|
1930
|
+
const doc4 = serializeBucketData(BUCKET, [
|
|
1931
|
+
makeOp(700, 'G', 'g1', ctx, sourceTableId),
|
|
1932
|
+
makeOp(800, 'H', 'h1', ctx, sourceTableId)
|
|
1933
|
+
]); // range [700, 800], all >350
|
|
1934
|
+
|
|
1935
|
+
await insertDocs(collection, [doc1, doc2, doc3, doc4]);
|
|
1936
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 800n);
|
|
1937
|
+
|
|
1938
|
+
const docsBefore = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
1939
|
+
const checksumBefore = docsBefore.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1940
|
+
|
|
1941
|
+
await bucketStorage.compact({
|
|
1942
|
+
clearBatchLimit: 200,
|
|
1943
|
+
moveBatchLimit: 10,
|
|
1944
|
+
moveBatchQueryLimit: 10,
|
|
1945
|
+
minBucketChanges: 1,
|
|
1946
|
+
minChangeRatio: 0,
|
|
1947
|
+
maxOpId: 350n
|
|
1948
|
+
});
|
|
1949
|
+
|
|
1950
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).sort({ '_id.o': 1 }).toArray();
|
|
1951
|
+
|
|
1952
|
+
// Every pair of documents must have disjoint [min_op, _id.o] ranges.
|
|
1953
|
+
const overlaps = docsAfter.flatMap((a, i) =>
|
|
1954
|
+
docsAfter
|
|
1955
|
+
.slice(i + 1)
|
|
1956
|
+
.filter((b) => a._id.o >= b.min_op && b._id.o >= a.min_op)
|
|
1957
|
+
.map((b) => `[${a.min_op}, ${a._id.o}] vs [${b.min_op}, ${b._id.o}]`)
|
|
1958
|
+
);
|
|
1959
|
+
expect(overlaps).toEqual([]);
|
|
1960
|
+
|
|
1961
|
+
// Checksum must be preserved across compaction
|
|
1962
|
+
const checksumAfter = docsAfter.reduce((sum, d) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1963
|
+
expect(checksumAfter).toBe(checksumBefore);
|
|
1964
|
+
});
|
|
1965
|
+
|
|
1966
|
+
describe('V3 CLEAR pass', () => {
|
|
1967
|
+
async function doCompact(bucketStorage: MongoSyncBucketStorage, maxOpId: bigint) {
|
|
1968
|
+
await bucketStorage.compact({
|
|
1969
|
+
clearBatchLimit: 200,
|
|
1970
|
+
moveBatchLimit: 10,
|
|
1971
|
+
moveBatchQueryLimit: 10,
|
|
1972
|
+
minBucketChanges: 1,
|
|
1973
|
+
minChangeRatio: 0,
|
|
1974
|
+
maxOpId
|
|
1975
|
+
});
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1978
|
+
// Helper to find a CLEAR op in a bucket
|
|
1979
|
+
async function hasClearOp(collection: any): Promise<boolean> {
|
|
1980
|
+
const allOps = await readAllOps(collection);
|
|
1981
|
+
return allOps.some((op) => op.op === 'CLEAR');
|
|
1982
|
+
}
|
|
1983
|
+
|
|
1984
|
+
// Helper to assert checksum preservation before/after compact
|
|
1985
|
+
async function assertChecksumPreserved(collection: any, compactFn: () => Promise<void>) {
|
|
1986
|
+
const docsBefore = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
1987
|
+
const checksumBefore = docsBefore.reduce((sum: number, d: any) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1988
|
+
await compactFn();
|
|
1989
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
1990
|
+
const checksumAfter = docsAfter.reduce((sum: number, d: any) => addChecksums(sum, Number(d.checksum)), 0);
|
|
1991
|
+
expect(checksumAfter).toBe(checksumBefore);
|
|
1992
|
+
}
|
|
1993
|
+
|
|
1994
|
+
test('duplicate rows produce CLEAR op', async () => {
|
|
1995
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
1996
|
+
|
|
1997
|
+
const doc = serializeBucketData(BUCKET, [
|
|
1998
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
1999
|
+
makeOp(20, 'A', 'a2', ctx, sourceTableId),
|
|
2000
|
+
makeOp(30, 'A', 'a3', ctx, sourceTableId)
|
|
2001
|
+
]);
|
|
2002
|
+
await insertDocs(collection, [doc]);
|
|
2003
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 30n);
|
|
2004
|
+
|
|
2005
|
+
await doCompact(bucketStorage, 30n);
|
|
2006
|
+
|
|
2007
|
+
expect(await hasClearOp(collection)).toBe(true);
|
|
2008
|
+
});
|
|
2009
|
+
|
|
2010
|
+
test('unique rows produce no CLEAR op', async () => {
|
|
2011
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
2012
|
+
|
|
2013
|
+
const doc = serializeBucketData(BUCKET, [
|
|
2014
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
2015
|
+
makeOp(20, 'B', 'b1', ctx, sourceTableId)
|
|
2016
|
+
]);
|
|
2017
|
+
await insertDocs(collection, [doc]);
|
|
2018
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 20n);
|
|
2019
|
+
|
|
2020
|
+
await doCompact(bucketStorage, 20n);
|
|
2021
|
+
|
|
2022
|
+
expect(await hasClearOp(collection)).toBe(false);
|
|
2023
|
+
const allOps = await readAllOps(collection);
|
|
2024
|
+
expect(allOps.length).toBe(2);
|
|
2025
|
+
expect(allOps.every((op) => op.op === 'PUT')).toBe(true);
|
|
2026
|
+
});
|
|
2027
|
+
|
|
2028
|
+
test('end-to-end: duplicate + unique rows, checksum preserved', async () => {
|
|
2029
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
2030
|
+
|
|
2031
|
+
const doc = serializeBucketData(BUCKET, [
|
|
2032
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
2033
|
+
makeOp(20, 'A', 'a2', ctx, sourceTableId),
|
|
2034
|
+
makeOp(30, 'B', 'b1', ctx, sourceTableId),
|
|
2035
|
+
makeOp(40, 'A', 'a3', ctx, sourceTableId)
|
|
2036
|
+
]);
|
|
2037
|
+
await insertDocs(collection, [doc]);
|
|
2038
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 40n);
|
|
2039
|
+
|
|
2040
|
+
await assertChecksumPreserved(collection, () => doCompact(bucketStorage, 40n));
|
|
2041
|
+
|
|
2042
|
+
expect(await hasClearOp(collection)).toBe(true);
|
|
2043
|
+
const allOps = await readAllOps(collection);
|
|
2044
|
+
const putOps = allOps.filter((op) => op.op === 'PUT');
|
|
2045
|
+
expect(putOps.length).toBe(2);
|
|
2046
|
+
expect(putOps.some((op) => op.row_id === 'B' && op.o === 30n)).toBe(true);
|
|
2047
|
+
expect(putOps.some((op) => op.row_id === 'A' && op.o === 40n)).toBe(true);
|
|
2048
|
+
});
|
|
2049
|
+
|
|
2050
|
+
// 15 MOVE ops, each in its own document. moveBatchQueryLimit=1
|
|
2051
|
+
// forces the MOVE pass to produce 15 separate rechunked documents
|
|
2052
|
+
// (one per batch). clearBatchLimit=5 forces the CLEAR read loop
|
|
2053
|
+
// to paginate through 3 iterations.
|
|
2054
|
+
test('read pagination iterates across multiple read batches', async () => {
|
|
2055
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
2056
|
+
|
|
2057
|
+
const docs: BucketDataDocumentV3[] = [];
|
|
2058
|
+
for (let i = 1; i <= 15; i++) {
|
|
2059
|
+
const op = { ...makeOp(i, `row_${i}`, '', ctx, sourceTableId), op: 'MOVE' as const, data: null };
|
|
2060
|
+
docs.push(serializeBucketData(BUCKET, [op]));
|
|
2061
|
+
}
|
|
2062
|
+
await insertDocs(collection, docs);
|
|
2063
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 15n);
|
|
2064
|
+
|
|
2065
|
+
const docsBefore = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
2066
|
+
const checksumBefore = docsBefore.reduce((sum: number, d: any) => addChecksums(sum, Number(d.checksum)), 0);
|
|
2067
|
+
|
|
2068
|
+
await bucketStorage.compact({
|
|
2069
|
+
clearBatchLimit: 5,
|
|
2070
|
+
moveBatchLimit: 1,
|
|
2071
|
+
moveBatchQueryLimit: 1,
|
|
2072
|
+
minBucketChanges: 1,
|
|
2073
|
+
minChangeRatio: 0,
|
|
2074
|
+
maxOpId: 15n
|
|
2075
|
+
});
|
|
2076
|
+
|
|
2077
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
2078
|
+
const checksumAfter = docsAfter.reduce((sum: number, d: any) => addChecksums(sum, Number(d.checksum)), 0);
|
|
2079
|
+
expect(checksumAfter).toBe(checksumBefore);
|
|
2080
|
+
|
|
2081
|
+
expect(await hasClearOp(collection)).toBe(true);
|
|
2082
|
+
const allOps = await readAllOps(collection);
|
|
2083
|
+
expect(allOps.length).toBe(1);
|
|
2084
|
+
expect(allOps[0].op).toBe('CLEAR');
|
|
2085
|
+
});
|
|
2086
|
+
|
|
2087
|
+
test('CLEAR document carries target_op from collapsed MOVE tombstones', async () => {
|
|
2088
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
2089
|
+
|
|
2090
|
+
// Three versions of row A: ops 10, 20, 30.
|
|
2091
|
+
// MOVE pass: 30 survives, 20 and 10 become MOVE tombstones (target=30).
|
|
2092
|
+
// CLEAR pass: collapses both MOVEs. The CLEAR doc should carry target_op=30.
|
|
2093
|
+
const doc = serializeBucketData(BUCKET, [
|
|
2094
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
2095
|
+
makeOp(20, 'A', 'a2', ctx, sourceTableId),
|
|
2096
|
+
makeOp(30, 'A', 'a3', ctx, sourceTableId)
|
|
2097
|
+
]);
|
|
2098
|
+
await insertDocs(collection, [doc]);
|
|
2099
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 30n);
|
|
2100
|
+
|
|
2101
|
+
await doCompact(bucketStorage, 30n);
|
|
2102
|
+
|
|
2103
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
2104
|
+
const clearDoc = docsAfter.find((d: any) => d.ops!.some((op: any) => op.op === 'CLEAR'));
|
|
2105
|
+
expect(clearDoc).toBeDefined();
|
|
2106
|
+
expect(clearDoc!.target_op).toBe(30n);
|
|
2107
|
+
});
|
|
2108
|
+
|
|
2109
|
+
test('straddling document is excluded from compacted state', async () => {
|
|
2110
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
2111
|
+
|
|
2112
|
+
// Row A at 10, 30, 50 — ops 10+30 ≤ maxOpId=40, op 50 > 40 (pass-through)
|
|
2113
|
+
// Row B at 20 — ≤ 40, first occurrence
|
|
2114
|
+
// Row C at 60 — > 40 (pass-through, no earlier version)
|
|
2115
|
+
const doc = serializeBucketData(BUCKET, [
|
|
2116
|
+
makeOp(10, 'A', 'a1', ctx, sourceTableId),
|
|
2117
|
+
makeOp(20, 'B', 'b1', ctx, sourceTableId),
|
|
2118
|
+
makeOp(30, 'A', 'a2', ctx, sourceTableId),
|
|
2119
|
+
makeOp(50, 'A', 'a3', ctx, sourceTableId),
|
|
2120
|
+
makeOp(60, 'C', 'c1', ctx, sourceTableId)
|
|
2121
|
+
]);
|
|
2122
|
+
await insertDocs(collection, [doc]);
|
|
2123
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 60n);
|
|
2124
|
+
|
|
2125
|
+
await doCompact(bucketStorage, 40n);
|
|
2126
|
+
|
|
2127
|
+
const state = await bucketStateCollection.findOne({
|
|
2128
|
+
_id: { d: ctx.definitionId, b: BUCKET }
|
|
2129
|
+
});
|
|
2130
|
+
expect(state).toBeDefined();
|
|
2131
|
+
expect(state!.compacted_state).toBeUndefined();
|
|
2132
|
+
|
|
2133
|
+
// The document ends beyond maxOpId, so it is not read or modified.
|
|
2134
|
+
const allOps = await readAllOps(collection);
|
|
2135
|
+
expect(allOps.length).toBe(5);
|
|
2136
|
+
expect(allOps.every((op) => op.op == 'PUT')).toBe(true);
|
|
2137
|
+
});
|
|
2138
|
+
|
|
2139
|
+
test('upperBound pagination prevents re-reading replacement documents', async () => {
|
|
2140
|
+
const { bucketStorage, collection, bucketStateCollection, ctx, sourceTableId } = await setupV3();
|
|
2141
|
+
|
|
2142
|
+
// Two distinct-row PUTs at ~600KB each. Rechunking produces two
|
|
2143
|
+
// documents: C1(_id.o=10) and C2(_id.o=30). The fix sets upperBound
|
|
2144
|
+
// from the oldest replacement C1, so _id < { o: 10 } excludes both.
|
|
2145
|
+
// No re-read. Both PUTs survive with intact data and valid row_ids.
|
|
2146
|
+
const largeData = 'x'.repeat(600_000);
|
|
2147
|
+
const doc = serializeBucketData(BUCKET, [
|
|
2148
|
+
makeOp(10, 'A', largeData, ctx, sourceTableId),
|
|
2149
|
+
makeOp(30, 'B', largeData, ctx, sourceTableId)
|
|
2150
|
+
]);
|
|
2151
|
+
await insertDocs(collection, [doc]);
|
|
2152
|
+
await insertBucketState(bucketStateCollection, ctx.definitionId, 40n);
|
|
2153
|
+
|
|
2154
|
+
await bucketStorage.compact({
|
|
2155
|
+
clearBatchLimit: 200,
|
|
2156
|
+
moveBatchLimit: 10,
|
|
2157
|
+
moveBatchQueryLimit: 1,
|
|
2158
|
+
minBucketChanges: 1,
|
|
2159
|
+
minChangeRatio: 0,
|
|
2160
|
+
maxOpId: 40n
|
|
2161
|
+
});
|
|
2162
|
+
|
|
2163
|
+
const docsAfter = await collection.find({ '_id.b': BUCKET }).toArray();
|
|
2164
|
+
const rawOps = docsAfter.flatMap((d: any) => d.ops) as any[];
|
|
2165
|
+
|
|
2166
|
+
// Row A@10 must survive as PUT with data intact
|
|
2167
|
+
const rowA = rawOps.find((op: any) => op.row_id === 'A' && op.o === 10n);
|
|
2168
|
+
expect(rowA).toBeDefined();
|
|
2169
|
+
expect(rowA!.op).toBe('PUT');
|
|
2170
|
+
expect(rowA!.data).not.toBeNull();
|
|
2171
|
+
|
|
2172
|
+
// Row B@30 must survive as PUT with data intact
|
|
2173
|
+
const rowB = rawOps.find((op: any) => op.row_id === 'B' && op.o === 30n);
|
|
2174
|
+
expect(rowB).toBeDefined();
|
|
2175
|
+
expect(rowB!.op).toBe('PUT');
|
|
2176
|
+
expect(rowB!.data).not.toBeNull();
|
|
2177
|
+
|
|
2178
|
+
// Checksum preserved
|
|
2179
|
+
const expectedChecksum = [10, 30].reduce((sum, id) => addChecksums(sum, id * 7), 0);
|
|
2180
|
+
const checksumAfter = docsAfter.reduce((s: number, d: any) => addChecksums(s, Number(d.checksum)), 0);
|
|
2181
|
+
expect(checksumAfter).toBe(expectedChecksum);
|
|
2182
|
+
});
|
|
2183
|
+
});
|
|
2184
|
+
});
|