@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -1,19 +1,100 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
-
import { storage } from '@powersync/service-core';
|
|
2
|
+
import { logger, ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { addChecksums, InternalOpId, storage, utils } from '@powersync/service-core';
|
|
4
4
|
import { BucketDefinitionId } from '@powersync/service-sync-rules';
|
|
5
|
-
import {
|
|
6
|
-
import { BucketStateDocumentBase } from '../models.js';
|
|
7
|
-
import { DirtyBucket, MongoCompactor } from '../MongoCompactor.js';
|
|
8
|
-
import {
|
|
5
|
+
import { BucketDataDoc } from '../common/BucketDataDoc.js';
|
|
6
|
+
import { BucketDataKey, BucketStateDocumentBase } from '../models.js';
|
|
7
|
+
import { ConcurrentCompactionError, DirtyBucket, MongoCompactor } from '../MongoCompactor.js';
|
|
8
|
+
import { cacheKey } from '../OperationBatch.js';
|
|
9
|
+
import { loadBucketDataDocument, maxOpId, serializeBucketData } from './bucket-format.js';
|
|
10
|
+
import { BucketDataContextV3 } from './BucketDataContextV3.js';
|
|
11
|
+
import { DEFAULT_MAX_DOC_SIZE_BYTES } from './chunking.js';
|
|
12
|
+
import { BucketDataDocumentV3, BucketStateDocumentV3 } from './models.js';
|
|
13
|
+
import { DefinitionChecksumOperations, MongoChecksumsV3 } from './MongoChecksumsV3.js';
|
|
9
14
|
import type { MongoSyncBucketStorageV3 } from './MongoSyncBucketStorageV3.js';
|
|
10
|
-
import {
|
|
15
|
+
import { BucketDataObjectStorage, hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js';
|
|
16
|
+
import { ObjectStorageLifecycle, PreparedObjectStorageUpload } from './object-storage/ObjectStorageLifecycle.js';
|
|
11
17
|
import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
|
|
12
18
|
|
|
19
|
+
interface PendingCompactionGroup {
|
|
20
|
+
/**
|
|
21
|
+
* Input documents are ordered from oldest to newest, matching `ops`.
|
|
22
|
+
* Keeping the inputs intact lets unchanged singletons retain their object.
|
|
23
|
+
*/
|
|
24
|
+
inputs: BucketDataDocumentV3[];
|
|
25
|
+
ops: BucketDataDoc[];
|
|
26
|
+
changed: boolean;
|
|
27
|
+
targetOp: InternalOpId | null;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Read one bounded prefix from a descending compaction cursor.
|
|
32
|
+
*
|
|
33
|
+
* The document that would cross the byte limit is deliberately not returned:
|
|
34
|
+
* pagination resumes below the last returned `_id`, so that document remains
|
|
35
|
+
* eligible for the next query. The first document is always accepted to ensure
|
|
36
|
+
* progress when a single document exceeds the configured byte limit.
|
|
37
|
+
*
|
|
38
|
+
* `hasMore` is conservative when the document limit is reached. An extra empty
|
|
39
|
+
* query is preferable to exhausting the cursor just to determine whether the
|
|
40
|
+
* limited MongoDB query contained another document.
|
|
41
|
+
*/
|
|
42
|
+
async function readCompactionBatch(
|
|
43
|
+
cursor: mongo.AggregationCursor<BucketDataDocumentV3>,
|
|
44
|
+
options: { byteLimit: number; documentLimit: number }
|
|
45
|
+
): Promise<{ documents: BucketDataDocumentV3[]; hasMore: boolean }> {
|
|
46
|
+
const documents: BucketDataDocumentV3[] = [];
|
|
47
|
+
let cumulativeBytes = 0;
|
|
48
|
+
|
|
49
|
+
try {
|
|
50
|
+
for await (const document of cursor) {
|
|
51
|
+
if (documents.length > 0 && cumulativeBytes + document.size > options.byteLimit) {
|
|
52
|
+
return { documents, hasMore: true };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
documents.push(document);
|
|
56
|
+
cumulativeBytes += document.size;
|
|
57
|
+
|
|
58
|
+
if (documents.length >= options.documentLimit) {
|
|
59
|
+
return { documents, hasMore: true };
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
return { documents, hasMore: false };
|
|
63
|
+
} finally {
|
|
64
|
+
await cursor.close();
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
13
68
|
export class MongoCompactorV3 extends MongoCompactor {
|
|
14
69
|
declare protected readonly db: VersionedPowerSyncMongoV3;
|
|
15
70
|
declare protected readonly storage: MongoSyncBucketStorageV3;
|
|
16
71
|
|
|
72
|
+
override async compact(): Promise<void> {
|
|
73
|
+
if (this.storage.objectStorage) {
|
|
74
|
+
// Clean these before compacting - should be quick in most cases.
|
|
75
|
+
try {
|
|
76
|
+
await this.objectStorageLifecycle.cleanup(this.logger);
|
|
77
|
+
} catch (e) {
|
|
78
|
+
// In this case, still continue normal compact process
|
|
79
|
+
this.logger.error(`Failed to clean up object storage deletion markers before compaction`, e);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
await super.compact();
|
|
83
|
+
if (this.storage.objectStorage) {
|
|
84
|
+
// Cleanup for any produced during compacting.
|
|
85
|
+
// Note that markers only expire after a delay, so this may skip many produced during this compact
|
|
86
|
+
// run. However, during long compact runs, this may also have many ones it can clean up.
|
|
87
|
+
await this.objectStorageLifecycle.cleanup(this.logger);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
private get objectStorageLifecycle(): ObjectStorageLifecycle {
|
|
92
|
+
if (!this.storage.objectStorage) {
|
|
93
|
+
throw new Error('Object storage is not configured');
|
|
94
|
+
}
|
|
95
|
+
return new ObjectStorageLifecycle(this.db, this.group_id, this.storage.objectStorage);
|
|
96
|
+
}
|
|
97
|
+
|
|
17
98
|
public async *dirtyBucketBatches(options: {
|
|
18
99
|
minBucketChanges: number;
|
|
19
100
|
minChangeRatio: number;
|
|
@@ -21,11 +102,11 @@ export class MongoCompactorV3 extends MongoCompactor {
|
|
|
21
102
|
if (options.minBucketChanges <= 0) {
|
|
22
103
|
throw new ReplicationAssertionError('minBucketChanges must be >= 1');
|
|
23
104
|
}
|
|
24
|
-
|
|
105
|
+
const collection = this.db.bucketState(this.group_id) as unknown as mongo.Collection<BucketStateDocumentBase>;
|
|
25
106
|
yield* this.dirtyBucketBatchesForCollection(
|
|
26
|
-
|
|
27
|
-
{ d: new mongo.MinKey()
|
|
28
|
-
{ d: new mongo.MaxKey()
|
|
107
|
+
collection,
|
|
108
|
+
{ d: new mongo.MinKey(), b: new mongo.MinKey() } as unknown as BucketStateDocumentV3['_id'],
|
|
109
|
+
{ d: new mongo.MaxKey(), b: new mongo.MaxKey() } as unknown as BucketStateDocumentV3['_id'],
|
|
29
110
|
options,
|
|
30
111
|
(bucketState) => (bucketState as BucketStateDocumentV3)._id.d
|
|
31
112
|
);
|
|
@@ -36,27 +117,37 @@ export class MongoCompactorV3 extends MongoCompactor {
|
|
|
36
117
|
throw new ReplicationAssertionError('minBucketChanges must be >= 1');
|
|
37
118
|
}
|
|
38
119
|
return this.dirtyBucketBatchForChecksumsForCollection(
|
|
39
|
-
this.db.
|
|
120
|
+
this.db.bucketState(this.group_id) as unknown as mongo.Collection<BucketStateDocumentBase>,
|
|
40
121
|
{
|
|
41
122
|
'estimate_since_compact.count': { $gte: options.minBucketChanges }
|
|
42
|
-
}
|
|
123
|
+
} as unknown as mongo.Filter<BucketStateDocumentBase>,
|
|
43
124
|
(bucketState) => (bucketState as BucketStateDocumentV3)._id.d
|
|
44
125
|
);
|
|
45
126
|
}
|
|
46
127
|
|
|
47
128
|
protected async writeBucketStateUpdates(): Promise<void> {
|
|
48
129
|
await this.db
|
|
49
|
-
.
|
|
50
|
-
.bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation<BucketStateDocumentV3>[], {
|
|
130
|
+
.bucketState(this.group_id)
|
|
131
|
+
.bulkWrite(this.bucketStateUpdates as mongo.AnyBulkWriteOperation<BucketStateDocumentV3>[], {
|
|
132
|
+
ordered: false
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* The compactor operates on persisted definition ids only - never on parsed sources.
|
|
138
|
+
* This narrowed view makes the source-resolving checksum methods unreachable here.
|
|
139
|
+
*/
|
|
140
|
+
private get definitionChecksums(): DefinitionChecksumOperations {
|
|
141
|
+
return this.storage.checksums as MongoChecksumsV3;
|
|
51
142
|
}
|
|
52
143
|
|
|
53
144
|
protected async computeChecksumsForBuckets(
|
|
54
145
|
buckets: Pick<DirtyBucket, 'bucket' | 'definitionId'>[]
|
|
55
146
|
): Promise<storage.PartialChecksumMap> {
|
|
56
|
-
return this.
|
|
147
|
+
return this.definitionChecksums.computePartialChecksumsDirectByDefinition(
|
|
57
148
|
buckets.map(({ bucket, definitionId }) => {
|
|
58
149
|
if (definitionId == null) {
|
|
59
|
-
throw new ServiceAssertionError(`Missing definitionId for
|
|
150
|
+
throw new ServiceAssertionError(`Missing definitionId for bucket checksum update on bucket ${bucket}`);
|
|
60
151
|
}
|
|
61
152
|
return {
|
|
62
153
|
bucket,
|
|
@@ -82,26 +173,750 @@ export class MongoCompactorV3 extends MongoCompactor {
|
|
|
82
173
|
};
|
|
83
174
|
}
|
|
84
175
|
|
|
85
|
-
|
|
176
|
+
private async getBucketDataContext(
|
|
86
177
|
bucket: string,
|
|
87
178
|
definitionId: BucketDefinitionId | null
|
|
88
|
-
): Promise<
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
179
|
+
): Promise<BucketDataContextV3 | null> {
|
|
180
|
+
let resolvedDefinitionId = definitionId;
|
|
181
|
+
|
|
182
|
+
if (resolvedDefinitionId == null) {
|
|
183
|
+
const allDefinitionIds = this.storage.storageIds.bucketDefinitionIds;
|
|
184
|
+
if (allDefinitionIds.length > 0) {
|
|
185
|
+
const potentialIds = allDefinitionIds.map((id) => ({ d: id, b: bucket }));
|
|
186
|
+
const bucketState = await this.db.bucketState(this.group_id).findOne({
|
|
187
|
+
_id: { $in: potentialIds }
|
|
188
|
+
});
|
|
189
|
+
if (bucketState != null) {
|
|
190
|
+
resolvedDefinitionId = bucketState._id.d;
|
|
191
|
+
}
|
|
94
192
|
}
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
if (resolvedDefinitionId == null) {
|
|
196
|
+
return null;
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
return new BucketDataContextV3(this.db, {
|
|
200
|
+
bucket,
|
|
201
|
+
definitionId: resolvedDefinitionId,
|
|
202
|
+
replicationStreamId: this.group_id
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
protected override async compactSingleBucket(bucket: string, definitionId: BucketDefinitionId | null = null) {
|
|
207
|
+
const bucketContext = await this.getBucketDataContext(bucket, definitionId);
|
|
208
|
+
if (bucketContext == null) {
|
|
209
|
+
return;
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const resolvedDefinitionId = bucketContext.key.definitionId;
|
|
213
|
+
const collection = this.db.bucketData(this.group_id, resolvedDefinitionId);
|
|
214
|
+
const context = { replicationStreamId: this.group_id, definitionId: resolvedDefinitionId };
|
|
215
|
+
|
|
216
|
+
const lowerBound = bucketContext.minId;
|
|
217
|
+
let upperBound = bucketContext.docId(this.maxOpId + 1n);
|
|
218
|
+
|
|
219
|
+
let totalChecksum = 0;
|
|
220
|
+
let totalOpCount = 0;
|
|
221
|
+
let totalOpBytes = 0;
|
|
222
|
+
|
|
223
|
+
let lastNotPut: bigint | null = null;
|
|
224
|
+
let opsSincePut = 0;
|
|
225
|
+
let compactedOpId: bigint | null = null;
|
|
226
|
+
let clearBoundary: { opId: bigint; documentId: BucketDataKey } | null = null;
|
|
227
|
+
const seen = new Map<string, bigint>();
|
|
228
|
+
let trackingSize = 0;
|
|
229
|
+
let pendingGroup: PendingCompactionGroup | null = null;
|
|
230
|
+
|
|
231
|
+
// --- Read batch from MongoDB ---
|
|
232
|
+
while (true) {
|
|
233
|
+
this.signal?.throwIfAborted();
|
|
234
|
+
|
|
235
|
+
const pipeline: mongo.Document[] = [
|
|
236
|
+
{
|
|
237
|
+
$match: {
|
|
238
|
+
_id: {
|
|
239
|
+
$gte: lowerBound,
|
|
240
|
+
$lt: upperBound
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
},
|
|
244
|
+
{ $sort: { _id: -1 } },
|
|
245
|
+
{ $limit: this.moveBatchQueryLimit },
|
|
246
|
+
{
|
|
247
|
+
$project: {
|
|
248
|
+
_id: 1,
|
|
249
|
+
min_op: 1,
|
|
250
|
+
checksum: 1,
|
|
251
|
+
count: 1,
|
|
252
|
+
size: 1,
|
|
253
|
+
target_op: 1,
|
|
254
|
+
ops: 1,
|
|
255
|
+
storage_ref: 1
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
];
|
|
259
|
+
|
|
260
|
+
const batch = await readCompactionBatch(
|
|
261
|
+
collection.aggregate<BucketDataDocumentV3>(pipeline, {
|
|
262
|
+
batchSize: this.moveBatchQueryLimit + 1
|
|
263
|
+
}),
|
|
264
|
+
{
|
|
265
|
+
byteLimit: this.moveBatchByteLimit,
|
|
266
|
+
documentLimit: this.moveBatchQueryLimit
|
|
267
|
+
}
|
|
268
|
+
);
|
|
269
|
+
const batchDocs = batch.documents;
|
|
270
|
+
|
|
271
|
+
if (batchDocs.length == 0) {
|
|
272
|
+
// No more documents in this bucket — compaction complete.
|
|
273
|
+
break;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
await hydrateBucketDataDocuments(batchDocs, this.storage.objectStorage, { signal: this.signal });
|
|
277
|
+
|
|
278
|
+
// Compact each document independently, then greedily merge adjacent
|
|
279
|
+
// post-compaction results. This preserves existing boundaries unless
|
|
280
|
+
// merging is useful, and writes each final object at most once.
|
|
281
|
+
for (const doc of batchDocs) {
|
|
282
|
+
compactedOpId ??= doc._id.o;
|
|
283
|
+
const originalOps = Array.from(loadBucketDataDocument(context, doc));
|
|
284
|
+
|
|
285
|
+
let changed = false;
|
|
286
|
+
const compactedOps: BucketDataDoc[] = [];
|
|
287
|
+
let maxTargetOp: InternalOpId | null = doc.target_op ?? null;
|
|
288
|
+
for (let index = originalOps.length - 1; index >= 0; index--) {
|
|
289
|
+
const op = originalOps[index];
|
|
290
|
+
if (op.op == 'PUT' || op.op == 'REMOVE') {
|
|
291
|
+
const key = `${op.table}/${op.row_id}/${cacheKey(op.source_table!, op.source_key!)}`;
|
|
292
|
+
const targetOp = seen.get(key);
|
|
293
|
+
if (targetOp != null) {
|
|
294
|
+
maxTargetOp = maxOpId(maxTargetOp, targetOp);
|
|
295
|
+
compactedOps.push({
|
|
296
|
+
...op,
|
|
297
|
+
op: 'MOVE',
|
|
298
|
+
table: undefined,
|
|
299
|
+
row_id: undefined,
|
|
300
|
+
source_table: undefined,
|
|
301
|
+
source_key: undefined,
|
|
302
|
+
data: null
|
|
303
|
+
});
|
|
304
|
+
changed = true;
|
|
305
|
+
if (lastNotPut == null) {
|
|
306
|
+
lastNotPut = op.o;
|
|
307
|
+
}
|
|
308
|
+
opsSincePut += 1;
|
|
309
|
+
} else {
|
|
310
|
+
if (trackingSize < this.idLimitBytes) {
|
|
311
|
+
seen.set(utils.flatstr(key), op.o);
|
|
312
|
+
trackingSize += key.length + 140;
|
|
313
|
+
}
|
|
314
|
+
compactedOps.push(op);
|
|
315
|
+
if (op.op == 'PUT') {
|
|
316
|
+
lastNotPut = null;
|
|
317
|
+
opsSincePut = 0;
|
|
318
|
+
} else {
|
|
319
|
+
if (lastNotPut == null) {
|
|
320
|
+
lastNotPut = op.o;
|
|
321
|
+
}
|
|
322
|
+
opsSincePut += 1;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
} else {
|
|
326
|
+
compactedOps.push(op);
|
|
327
|
+
if (op.op != 'CLEAR') {
|
|
328
|
+
if (lastNotPut == null) {
|
|
329
|
+
lastNotPut = op.o;
|
|
330
|
+
}
|
|
331
|
+
opsSincePut += 1;
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
compactedOps.reverse();
|
|
336
|
+
|
|
337
|
+
for (const op of compactedOps) {
|
|
338
|
+
totalChecksum = addChecksums(totalChecksum, Number(op.checksum));
|
|
339
|
+
totalOpBytes += op.data?.length ?? 0;
|
|
340
|
+
}
|
|
341
|
+
totalOpCount += compactedOps.length;
|
|
342
|
+
|
|
343
|
+
const candidate: PendingCompactionGroup = {
|
|
344
|
+
inputs: [doc],
|
|
345
|
+
ops: compactedOps,
|
|
346
|
+
changed,
|
|
347
|
+
targetOp: maxTargetOp
|
|
348
|
+
};
|
|
349
|
+
|
|
350
|
+
if (pendingGroup == null) {
|
|
351
|
+
pendingGroup = candidate;
|
|
352
|
+
} else {
|
|
353
|
+
const mergedOps: BucketDataDoc[] = [...candidate.ops, ...pendingGroup.ops];
|
|
354
|
+
const mergedSize = serializeBucketData(bucket, mergedOps, { targetOp: maxTargetOp }).size;
|
|
355
|
+
if (mergedSize <= DEFAULT_MAX_DOC_SIZE_BYTES) {
|
|
356
|
+
pendingGroup = {
|
|
357
|
+
inputs: [...candidate.inputs, ...pendingGroup.inputs],
|
|
358
|
+
ops: mergedOps,
|
|
359
|
+
changed: candidate.changed || pendingGroup.changed,
|
|
360
|
+
targetOp: maxOpId(maxTargetOp, pendingGroup.targetOp)
|
|
361
|
+
};
|
|
362
|
+
} else {
|
|
363
|
+
const flushedGroup = pendingGroup;
|
|
364
|
+
const documentId = await this.flushCompactionGroup(bucket, flushedGroup, bucketContext, context);
|
|
365
|
+
if (
|
|
366
|
+
lastNotPut != null &&
|
|
367
|
+
flushedGroup.ops[0].o <= lastNotPut &&
|
|
368
|
+
flushedGroup.ops[flushedGroup.ops.length - 1].o >= lastNotPut
|
|
369
|
+
) {
|
|
370
|
+
clearBoundary = { opId: lastNotPut, documentId };
|
|
371
|
+
}
|
|
372
|
+
pendingGroup = candidate;
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
// --- Advance to next batch ---
|
|
378
|
+
upperBound = batchDocs[batchDocs.length - 1]._id as typeof upperBound;
|
|
379
|
+
|
|
380
|
+
if (!batch.hasMore) {
|
|
381
|
+
break;
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
this.logger.info(`Compacted batch of ${batchDocs.length} documents for bucket ${bucket}`);
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
if (pendingGroup != null) {
|
|
388
|
+
const documentId = await this.flushCompactionGroup(bucket, pendingGroup, bucketContext, context);
|
|
389
|
+
if (
|
|
390
|
+
lastNotPut != null &&
|
|
391
|
+
pendingGroup.ops[0].o <= lastNotPut &&
|
|
392
|
+
pendingGroup.ops[pendingGroup.ops.length - 1].o >= lastNotPut
|
|
393
|
+
) {
|
|
394
|
+
clearBoundary = { opId: lastNotPut, documentId };
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
if (compactedOpId == null) {
|
|
398
|
+
return;
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
// --- Clear: collapse leading MOVE/REMOVE/CLEAR sequence ---
|
|
402
|
+
if (lastNotPut != null && opsSincePut >= 2) {
|
|
403
|
+
if (clearBoundary == null || clearBoundary.opId != lastNotPut) {
|
|
404
|
+
throw new ReplicationAssertionError(`Missing CLEAR boundary document for bucket ${bucket}`);
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
totalOpCount += await this.clearBucketLeading(
|
|
408
|
+
lastNotPut,
|
|
409
|
+
clearBoundary.documentId,
|
|
410
|
+
bucketContext,
|
|
411
|
+
collection,
|
|
412
|
+
context
|
|
413
|
+
);
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
// --- Finalize: update bucket checksums and state ---
|
|
417
|
+
this.updateBucketChecksums(
|
|
418
|
+
{
|
|
419
|
+
bucket,
|
|
420
|
+
definitionId: resolvedDefinitionId,
|
|
421
|
+
seen: new Map(),
|
|
422
|
+
trackingSize: 0,
|
|
423
|
+
lastNotPut: lastNotPut,
|
|
424
|
+
opsSincePut: opsSincePut,
|
|
425
|
+
checksum: totalChecksum,
|
|
426
|
+
opCount: totalOpCount,
|
|
427
|
+
opBytes: totalOpBytes
|
|
428
|
+
},
|
|
429
|
+
compactedOpId
|
|
430
|
+
);
|
|
431
|
+
if (this.bucketStateUpdates.length > 0) {
|
|
432
|
+
await this.writeBucketStateUpdates();
|
|
433
|
+
this.bucketStateUpdates = [];
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
logger.info(`Compacted bucket ${bucket}: ${totalOpCount} surviving ops`);
|
|
437
|
+
}
|
|
438
|
+
|
|
439
|
+
/**
|
|
440
|
+
* Persist replacement objects before starting the transaction, then atomically
|
|
441
|
+
* publish their lifecycle markers alongside the MongoDB document replacement.
|
|
442
|
+
* If verification or the transaction fails, the prepared markers retain enough
|
|
443
|
+
* information for the uploaded objects to be cleaned up later.
|
|
444
|
+
*/
|
|
445
|
+
private async flushCompactionGroup(
|
|
446
|
+
bucket: string,
|
|
447
|
+
group: PendingCompactionGroup,
|
|
448
|
+
bucketContext: BucketDataContextV3,
|
|
449
|
+
context: { replicationStreamId: number; definitionId: string }
|
|
450
|
+
): Promise<BucketDataKey> {
|
|
451
|
+
if (group.inputs.length == 1 && !group.changed) {
|
|
452
|
+
return group.inputs[0]._id;
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
const inputs = group.inputs;
|
|
456
|
+
const idsToDelete = inputs.map((doc) => doc._id);
|
|
457
|
+
const expectedDocCount = inputs.length;
|
|
458
|
+
const expectedChecksum = inputs.reduce((sum, doc) => sum + doc.checksum, 0n);
|
|
459
|
+
const expectedOpCount = inputs.reduce((sum, doc) => sum + doc.count, 0);
|
|
460
|
+
const oldStoragePaths = inputs.flatMap((doc) => (doc.storage_ref ? [doc.storage_ref.path] : []));
|
|
461
|
+
const {
|
|
462
|
+
documents,
|
|
463
|
+
storagePaths: newStoragePaths,
|
|
464
|
+
uploads
|
|
465
|
+
} = await this.persistBucketData(bucket, [group.ops], context, undefined, { targetOp: group.targetOp });
|
|
466
|
+
const session = this.db.client.startSession();
|
|
467
|
+
try {
|
|
468
|
+
await session.withTransaction(
|
|
469
|
+
async () => {
|
|
470
|
+
const verification = await bucketContext.collection
|
|
471
|
+
.aggregate<{ docCount: number; checksumSum: bigint | null; opCountSum: number | null }>(
|
|
472
|
+
[
|
|
473
|
+
{ $match: { _id: { $in: idsToDelete } } },
|
|
474
|
+
{
|
|
475
|
+
$group: {
|
|
476
|
+
_id: null,
|
|
477
|
+
docCount: { $sum: 1 },
|
|
478
|
+
checksumSum: { $sum: '$checksum' },
|
|
479
|
+
opCountSum: { $sum: '$count' }
|
|
480
|
+
}
|
|
481
|
+
}
|
|
482
|
+
],
|
|
483
|
+
{ session }
|
|
484
|
+
)
|
|
485
|
+
.next();
|
|
486
|
+
|
|
487
|
+
if (
|
|
488
|
+
verification == null ||
|
|
489
|
+
verification.docCount !== expectedDocCount ||
|
|
490
|
+
verification.checksumSum !== expectedChecksum ||
|
|
491
|
+
verification.opCountSum !== expectedOpCount
|
|
492
|
+
) {
|
|
493
|
+
throw new ConcurrentCompactionError(
|
|
494
|
+
`Inputs changed while compacting bucket ${bucket}; restarting from the latest bucket state`
|
|
495
|
+
);
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
await bucketContext.collection.deleteMany({ _id: { $in: idsToDelete } }, { session });
|
|
499
|
+
await bucketContext.collection.insertMany(documents, { session });
|
|
500
|
+
await this.finishObjectStorageReplacement(oldStoragePaths, newStoragePaths, uploads, session);
|
|
501
|
+
},
|
|
502
|
+
{
|
|
503
|
+
writeConcern: { w: 'majority' },
|
|
504
|
+
readConcern: { level: 'snapshot' }
|
|
505
|
+
}
|
|
506
|
+
);
|
|
507
|
+
} finally {
|
|
508
|
+
await session.endSession();
|
|
509
|
+
}
|
|
510
|
+
return documents[0]._id;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
/**
|
|
514
|
+
* Collapse the leading sequence of MOVE/REMOVE/CLEAR ops at the start
|
|
515
|
+
* of the bucket into a single CLEAR op. Reads whole clearable documents
|
|
516
|
+
* before the known boundary document, then splits that boundary document
|
|
517
|
+
* if it contains ops on both sides of lastNotPut.
|
|
518
|
+
*
|
|
519
|
+
* Returns the op count diff after replacing cleared ops with CLEAR ops.
|
|
520
|
+
*/
|
|
521
|
+
private async clearBucketLeading(
|
|
522
|
+
lastNotPut: bigint,
|
|
523
|
+
boundaryDocId: BucketDataKey,
|
|
524
|
+
bucketContext: BucketDataContextV3,
|
|
525
|
+
collection: mongo.Collection<BucketDataDocumentV3 & { bsonSize?: number | bigint }>,
|
|
526
|
+
context: { replicationStreamId: number; definitionId: string }
|
|
527
|
+
): Promise<number> {
|
|
528
|
+
let opCountDiff = 0;
|
|
529
|
+
const session = this.db.client.startSession();
|
|
530
|
+
try {
|
|
531
|
+
let done = false;
|
|
532
|
+
// First step is to clear full chunks that contain only CLEAR/MOVE/REMOVE operations.
|
|
533
|
+
// There can be many of them, so we do one batch at a time.
|
|
534
|
+
while (!done) {
|
|
535
|
+
const batch = await this.clearLeadingFullDocuments(
|
|
536
|
+
session,
|
|
537
|
+
lastNotPut,
|
|
538
|
+
boundaryDocId,
|
|
539
|
+
bucketContext,
|
|
540
|
+
collection,
|
|
541
|
+
context
|
|
542
|
+
);
|
|
543
|
+
done = batch.done;
|
|
544
|
+
opCountDiff += batch.opCountDiff;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
// The final step is to process the "boundary" document: It may contain some CLEAR/MOVE/REMOVE operations,
|
|
548
|
+
// potentially followed by PUT operations. This is only a single document, so no need for batching.
|
|
549
|
+
opCountDiff += await this.clearBoundaryDocument(
|
|
550
|
+
session,
|
|
551
|
+
lastNotPut,
|
|
552
|
+
boundaryDocId,
|
|
553
|
+
bucketContext,
|
|
554
|
+
collection,
|
|
555
|
+
context
|
|
556
|
+
);
|
|
557
|
+
} finally {
|
|
558
|
+
await session.endSession();
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
return opCountDiff;
|
|
562
|
+
}
|
|
563
|
+
|
|
564
|
+
private async clearLeadingFullDocuments(
|
|
565
|
+
session: mongo.ClientSession,
|
|
566
|
+
lastNotPut: bigint,
|
|
567
|
+
boundaryDocId: BucketDataKey,
|
|
568
|
+
bucketContext: BucketDataContextV3,
|
|
569
|
+
collection: mongo.Collection<BucketDataDocumentV3 & { bsonSize?: number | bigint }>,
|
|
570
|
+
context: { replicationStreamId: number; definitionId: string }
|
|
571
|
+
): Promise<{ done: boolean; opCountDiff: number }> {
|
|
572
|
+
const bucket = bucketContext.key.bucket;
|
|
573
|
+
this.signal?.throwIfAborted();
|
|
574
|
+
const prepared = await this.prepareCompactionUploads(bucket, context, [lastNotPut]);
|
|
575
|
+
let done = false;
|
|
576
|
+
let opCountDiff = 0;
|
|
577
|
+
|
|
578
|
+
await session.withTransaction(
|
|
579
|
+
async () => {
|
|
580
|
+
done = false;
|
|
581
|
+
opCountDiff = 0;
|
|
582
|
+
const oldStoragePaths: string[] = [];
|
|
583
|
+
const query = collection.find(
|
|
584
|
+
{
|
|
585
|
+
_id: {
|
|
586
|
+
$gte: bucketContext.minId,
|
|
587
|
+
$lt: boundaryDocId
|
|
588
|
+
}
|
|
589
|
+
},
|
|
590
|
+
{
|
|
591
|
+
session,
|
|
592
|
+
sort: { _id: 1 },
|
|
593
|
+
projection: {
|
|
594
|
+
_id: 1,
|
|
595
|
+
min_op: 1,
|
|
596
|
+
checksum: 1,
|
|
597
|
+
count: 1,
|
|
598
|
+
target_op: 1,
|
|
599
|
+
has_clear_op: 1,
|
|
600
|
+
storage_ref: 1
|
|
601
|
+
},
|
|
602
|
+
limit: this.clearBatchLimit
|
|
603
|
+
}
|
|
604
|
+
);
|
|
605
|
+
|
|
606
|
+
let combinedChecksum = 0;
|
|
607
|
+
let clearedOpCount = 0;
|
|
608
|
+
let maxTargetOp: bigint | null = null;
|
|
609
|
+
let lastDocId: BucketDataKey | null = null;
|
|
610
|
+
let clearOpCount = 0;
|
|
611
|
+
let gotNonClearOp = false;
|
|
612
|
+
|
|
613
|
+
for await (const doc of query.stream()) {
|
|
614
|
+
if (doc.min_op > lastNotPut) {
|
|
615
|
+
throw new ReplicationAssertionError(
|
|
616
|
+
`Unexpected document before CLEAR boundary with min_op ${doc.min_op} > ${lastNotPut} in bucket ${bucket}`
|
|
617
|
+
);
|
|
618
|
+
}
|
|
619
|
+
|
|
620
|
+
lastDocId = doc._id;
|
|
621
|
+
if (doc.storage_ref) {
|
|
622
|
+
oldStoragePaths.push(doc.storage_ref.path);
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
// The compaction scan established that every operation before the
|
|
626
|
+
// boundary is MOVE/REMOVE/CLEAR. Root metadata is sufficient to fold
|
|
627
|
+
// whole documents into one CLEAR, so avoid downloading their payloads.
|
|
628
|
+
if (doc.has_clear_op) {
|
|
629
|
+
clearOpCount++;
|
|
630
|
+
if (clearOpCount > 1) {
|
|
631
|
+
throw new ReplicationAssertionError(`Unexpected multiple CLEAR operations in bucket ${bucket}`);
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
if (!doc.has_clear_op || doc.count > 1) {
|
|
635
|
+
gotNonClearOp = true;
|
|
636
|
+
}
|
|
637
|
+
combinedChecksum = addChecksums(combinedChecksum, Number(doc.checksum));
|
|
638
|
+
clearedOpCount += doc.count;
|
|
639
|
+
if (doc.target_op != null && (maxTargetOp == null || doc.target_op > maxTargetOp)) {
|
|
640
|
+
maxTargetOp = doc.target_op;
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
|
|
644
|
+
if (!gotNonClearOp) {
|
|
645
|
+
done = true;
|
|
646
|
+
return;
|
|
647
|
+
}
|
|
648
|
+
|
|
649
|
+
this.logger.info(`Flushing CLEAR for ${clearedOpCount} ops at ${lastDocId?.o}`);
|
|
650
|
+
await collection.deleteMany(
|
|
651
|
+
{
|
|
652
|
+
_id: {
|
|
653
|
+
$gte: bucketContext.minId,
|
|
654
|
+
$lte: lastDocId!
|
|
655
|
+
}
|
|
656
|
+
},
|
|
657
|
+
{ session }
|
|
658
|
+
);
|
|
659
|
+
|
|
660
|
+
const clearOp = {
|
|
661
|
+
bucketKey: { ...context, bucket },
|
|
662
|
+
o: lastDocId!.o,
|
|
663
|
+
op: 'CLEAR' as const,
|
|
664
|
+
checksum: BigInt(combinedChecksum),
|
|
665
|
+
data: null
|
|
666
|
+
} satisfies BucketDataDoc;
|
|
667
|
+
const persisted = await this.persistBucketData(bucket, [[clearOp]], context, prepared, {
|
|
668
|
+
targetOp: maxTargetOp
|
|
669
|
+
});
|
|
670
|
+
await collection.insertOne(persisted.documents[0], { session });
|
|
671
|
+
await this.finishObjectStorageReplacement(oldStoragePaths, persisted.storagePaths, persisted.uploads, session);
|
|
672
|
+
|
|
673
|
+
opCountDiff = -clearedOpCount + 1;
|
|
674
|
+
},
|
|
675
|
+
{
|
|
676
|
+
writeConcern: { w: 'majority' },
|
|
677
|
+
readConcern: { level: 'snapshot' }
|
|
678
|
+
}
|
|
679
|
+
);
|
|
680
|
+
|
|
681
|
+
return { done, opCountDiff };
|
|
682
|
+
}
|
|
683
|
+
|
|
684
|
+
private async clearBoundaryDocument(
|
|
685
|
+
session: mongo.ClientSession,
|
|
686
|
+
lastNotPut: bigint,
|
|
687
|
+
boundaryDocId: BucketDataKey,
|
|
688
|
+
bucketContext: BucketDataContextV3,
|
|
689
|
+
collection: mongo.Collection<BucketDataDocumentV3 & { bsonSize?: number | bigint }>,
|
|
690
|
+
context: { replicationStreamId: number; definitionId: string }
|
|
691
|
+
): Promise<number> {
|
|
692
|
+
const bucket = bucketContext.key.bucket;
|
|
693
|
+
this.signal?.throwIfAborted();
|
|
694
|
+
const prepared = await this.prepareCompactionUploads(bucket, context, [lastNotPut, boundaryDocId.o]);
|
|
695
|
+
let opCountDiff = 0;
|
|
696
|
+
|
|
697
|
+
await session.withTransaction(
|
|
698
|
+
async () => {
|
|
699
|
+
opCountDiff = 0;
|
|
700
|
+
const oldStoragePaths: string[] = [];
|
|
701
|
+
const query = collection.find(
|
|
702
|
+
{
|
|
703
|
+
// This is a range query, but should only ever return two documents:
|
|
704
|
+
// 1. The CLEAR op from the previous clearLeadingFullDocuments.
|
|
705
|
+
// 2. The boundary document.
|
|
706
|
+
_id: {
|
|
707
|
+
$gte: bucketContext.minId,
|
|
708
|
+
$lte: boundaryDocId
|
|
709
|
+
}
|
|
710
|
+
},
|
|
711
|
+
{
|
|
712
|
+
session,
|
|
713
|
+
sort: { _id: 1 },
|
|
714
|
+
projection: {
|
|
715
|
+
_id: 1,
|
|
716
|
+
min_op: 1,
|
|
717
|
+
checksum: 1,
|
|
718
|
+
count: 1,
|
|
719
|
+
target_op: 1,
|
|
720
|
+
ops: 1,
|
|
721
|
+
storage_ref: 1
|
|
722
|
+
},
|
|
723
|
+
limit: 3
|
|
724
|
+
}
|
|
725
|
+
);
|
|
726
|
+
|
|
727
|
+
let docsRead = 0;
|
|
728
|
+
let combinedChecksum = 0;
|
|
729
|
+
let clearedOpCount = 0;
|
|
730
|
+
let maxTargetOp: bigint | null = null;
|
|
731
|
+
const boundarySurvivors: BucketDataDoc[] = [];
|
|
732
|
+
|
|
733
|
+
for await (const doc of query.stream()) {
|
|
734
|
+
docsRead++;
|
|
735
|
+
if (docsRead > 2) {
|
|
736
|
+
throw new ReplicationAssertionError(`Unexpected extra document before CLEAR boundary in bucket ${bucket}`);
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
const isBoundaryDoc = doc._id.o == boundaryDocId.o;
|
|
740
|
+
if (doc.storage_ref) {
|
|
741
|
+
oldStoragePaths.push(doc.storage_ref.path);
|
|
742
|
+
}
|
|
743
|
+
await hydrateBucketDataDocuments([doc], this.storage.objectStorage, { signal: this.signal });
|
|
744
|
+
maxTargetOp = maxOpId(maxTargetOp, doc.target_op);
|
|
745
|
+
for (const op of loadBucketDataDocument(context, doc)) {
|
|
746
|
+
if (!isBoundaryDoc && op.op != 'CLEAR') {
|
|
747
|
+
throw new ReplicationAssertionError(
|
|
748
|
+
`Unexpected ${op.op} operation before CLEAR boundary in bucket ${bucket}`
|
|
749
|
+
);
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
if (op.o <= lastNotPut) {
|
|
753
|
+
if (op.op == 'PUT') {
|
|
754
|
+
throw new ReplicationAssertionError(
|
|
755
|
+
`Unexpected PUT at op ${op.o} in CLEAR region for bucket ${bucket}`
|
|
756
|
+
);
|
|
757
|
+
}
|
|
758
|
+
combinedChecksum = addChecksums(combinedChecksum, Number(op.checksum));
|
|
759
|
+
clearedOpCount++;
|
|
760
|
+
} else if (isBoundaryDoc) {
|
|
761
|
+
boundarySurvivors.push(op);
|
|
762
|
+
} else {
|
|
763
|
+
throw new ReplicationAssertionError(
|
|
764
|
+
`Unexpected op ${op.o} after CLEAR boundary ${lastNotPut} in bucket ${bucket}`
|
|
765
|
+
);
|
|
766
|
+
}
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
|
|
770
|
+
if (clearedOpCount == 0) {
|
|
771
|
+
throw new Error(`CLEAR boundary document not found for bucket ${bucket}`);
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
this.logger.info(`Flushing CLEAR for ${clearedOpCount} ops at ${lastNotPut}`);
|
|
775
|
+
await collection.deleteMany(
|
|
776
|
+
{
|
|
777
|
+
_id: {
|
|
778
|
+
$gte: bucketContext.minId,
|
|
779
|
+
$lte: boundaryDocId
|
|
780
|
+
}
|
|
781
|
+
},
|
|
782
|
+
{ session }
|
|
783
|
+
);
|
|
784
|
+
|
|
785
|
+
const clearOp = {
|
|
786
|
+
bucketKey: { ...context, bucket },
|
|
787
|
+
o: lastNotPut,
|
|
788
|
+
op: 'CLEAR' as const,
|
|
789
|
+
checksum: BigInt(combinedChecksum),
|
|
790
|
+
data: null
|
|
791
|
+
} satisfies BucketDataDoc;
|
|
792
|
+
const chunks: BucketDataDoc[][] = [[clearOp]];
|
|
793
|
+
if (boundarySurvivors.length > 0) {
|
|
794
|
+
// These operations are a subset of one existing document, so keeping
|
|
795
|
+
// them together cannot increase its stored ops payload.
|
|
796
|
+
chunks.push(boundarySurvivors);
|
|
797
|
+
}
|
|
798
|
+
const persisted = await this.persistBucketData(bucket, chunks, context, prepared, {
|
|
799
|
+
targetOp: maxTargetOp ?? undefined
|
|
800
|
+
});
|
|
801
|
+
await collection.insertMany(persisted.documents, { session });
|
|
802
|
+
await this.finishObjectStorageReplacement(oldStoragePaths, persisted.storagePaths, persisted.uploads, session);
|
|
803
|
+
|
|
804
|
+
opCountDiff = -clearedOpCount + 1;
|
|
805
|
+
},
|
|
806
|
+
{
|
|
807
|
+
writeConcern: { w: 'majority' },
|
|
808
|
+
readConcern: { level: 'snapshot' }
|
|
809
|
+
}
|
|
810
|
+
);
|
|
811
|
+
|
|
812
|
+
return opCountDiff;
|
|
813
|
+
}
|
|
814
|
+
|
|
815
|
+
/**
|
|
816
|
+
* Reserve stable object paths before starting a retryable MongoDB transaction.
|
|
817
|
+
* Each retry can safely overwrite the same paths, while the pre-existing
|
|
818
|
+
* deletion markers remain visible to the transaction that publishes them.
|
|
819
|
+
*
|
|
820
|
+
* CLEAR compaction reserves its maximum output count. Unused markers remain
|
|
821
|
+
* pending so they can clean up a path that an earlier transaction attempt may
|
|
822
|
+
* have uploaded before retrying with fewer output documents.
|
|
823
|
+
*/
|
|
824
|
+
private async prepareCompactionUploads(
|
|
825
|
+
bucket: string,
|
|
826
|
+
context: { replicationStreamId: number; definitionId: string },
|
|
827
|
+
opIdHints: bigint[]
|
|
828
|
+
): Promise<PreparedObjectStorageUpload[]> {
|
|
829
|
+
if (!this.storage.objectStorage) {
|
|
830
|
+
return [];
|
|
831
|
+
}
|
|
832
|
+
|
|
833
|
+
const lifecycle = this.objectStorageLifecycle;
|
|
834
|
+
const paths = opIdHints.map((opIdHint) => lifecycle.allocatePath(context.definitionId, bucket, opIdHint, opIdHint));
|
|
835
|
+
return lifecycle.prepareUploads(paths);
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
/** Publish replacement uploads and retire superseded objects in the same transaction. */
|
|
839
|
+
private async finishObjectStorageReplacement(
|
|
840
|
+
oldStoragePaths: Iterable<string>,
|
|
841
|
+
newStoragePaths: Set<string>,
|
|
842
|
+
uploads: PreparedObjectStorageUpload[],
|
|
843
|
+
session: mongo.ClientSession
|
|
844
|
+
): Promise<void> {
|
|
845
|
+
if (!this.storage.objectStorage) {
|
|
846
|
+
return;
|
|
847
|
+
}
|
|
848
|
+
await this.objectStorageLifecycle.publishUploads(uploads, session);
|
|
849
|
+
await this.objectStorageLifecycle.retire(
|
|
850
|
+
Array.from(oldStoragePaths).filter((path) => !newStoragePaths.has(path)),
|
|
851
|
+
session
|
|
852
|
+
);
|
|
853
|
+
}
|
|
854
|
+
|
|
855
|
+
private async persistBucketData(
|
|
856
|
+
bucket: string,
|
|
857
|
+
chunks: BucketDataDoc[][],
|
|
858
|
+
context: { replicationStreamId: number; definitionId: string },
|
|
859
|
+
preparedUploads?: PreparedObjectStorageUpload[],
|
|
860
|
+
options?: { targetOp?: InternalOpId | null }
|
|
861
|
+
): Promise<{ documents: BucketDataDocumentV3[]; storagePaths: Set<string>; uploads: PreparedObjectStorageUpload[] }> {
|
|
862
|
+
const serializedChunks = chunks.map((chunk) => serializeBucketData(bucket, chunk, options));
|
|
863
|
+
if (!this.storage.objectStorage) {
|
|
864
|
+
return {
|
|
865
|
+
documents: serializedChunks,
|
|
866
|
+
storagePaths: new Set(),
|
|
867
|
+
uploads: []
|
|
868
|
+
};
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
const store = new BucketDataObjectStorage(this.storage.objectStorage);
|
|
872
|
+
const storagePaths = new Set<string>();
|
|
873
|
+
const lifecycle = this.objectStorageLifecycle;
|
|
874
|
+
// Base placement on the final compacted size. Unchanged documents are not
|
|
875
|
+
// rewritten, while small MOVE/merge results and CLEAR ops stay inline.
|
|
876
|
+
const storedIndexes = serializedChunks.flatMap((document, index) =>
|
|
877
|
+
document.size > this.storage.inlineThresholdBytes ? [index] : []
|
|
878
|
+
);
|
|
879
|
+
const uploadsByIndex = new Map<number, PreparedObjectStorageUpload>();
|
|
880
|
+
|
|
881
|
+
if (preparedUploads) {
|
|
882
|
+
for (const index of storedIndexes) {
|
|
883
|
+
const upload = preparedUploads[index];
|
|
884
|
+
if (!upload) {
|
|
885
|
+
throw new ServiceAssertionError(
|
|
886
|
+
`Missing prepared object storage path for compacted document at index ${index}`
|
|
887
|
+
);
|
|
888
|
+
}
|
|
889
|
+
uploadsByIndex.set(index, upload);
|
|
101
890
|
}
|
|
102
|
-
|
|
891
|
+
} else {
|
|
892
|
+
const paths = storedIndexes.map((index) => {
|
|
893
|
+
const chunk = chunks[index];
|
|
894
|
+
return lifecycle.allocatePath(context.definitionId, bucket, chunk[0].o, chunk[chunk.length - 1].o);
|
|
895
|
+
});
|
|
896
|
+
const prepared = await lifecycle.prepareUploads(paths);
|
|
897
|
+
storedIndexes.forEach((index, preparedIndex) => uploadsByIndex.set(index, prepared[preparedIndex]));
|
|
103
898
|
}
|
|
104
899
|
|
|
105
|
-
|
|
900
|
+
// S3ObjectStorage applies one shared concurrency limit across all callers,
|
|
901
|
+
// so compaction can schedule its uploads together without creating a
|
|
902
|
+
// separate limiter here.
|
|
903
|
+
const documents = await Promise.all(
|
|
904
|
+
serializedChunks.map(async (serialized, index) => {
|
|
905
|
+
const upload = uploadsByIndex.get(index);
|
|
906
|
+
if (!upload) {
|
|
907
|
+
return serialized;
|
|
908
|
+
}
|
|
909
|
+
|
|
910
|
+
const { ops, ...metadata } = serialized;
|
|
911
|
+
const { fileSize } = await store.store(upload.path, ops!);
|
|
912
|
+
storagePaths.add(upload.path);
|
|
913
|
+
return {
|
|
914
|
+
...metadata,
|
|
915
|
+
storage_ref: { path: upload.path, file_size: fileSize }
|
|
916
|
+
};
|
|
917
|
+
})
|
|
918
|
+
);
|
|
919
|
+
|
|
920
|
+
return { documents, storagePaths, uploads: Array.from(uploadsByIndex.values()) };
|
|
106
921
|
}
|
|
107
922
|
}
|