@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -1,38 +1,75 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
-
import { internalToExternalOpId,
|
|
3
|
+
import { internalToExternalOpId, ParameterSetLimitExceededError, storage } from '@powersync/service-core';
|
|
4
4
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
5
5
|
import * as bson from 'bson';
|
|
6
6
|
import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
|
|
7
|
-
import {
|
|
7
|
+
import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
|
|
8
8
|
import { MongoSyncBucketStorage } from '../MongoSyncBucketStorage.js';
|
|
9
|
-
import {
|
|
9
|
+
import { loadBucketDataDocument, maxOpId } from './bucket-format.js';
|
|
10
|
+
import { deserializeParameterLookup, serializeParameterLookup } from './models.js';
|
|
10
11
|
import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js';
|
|
11
12
|
import { MongoChecksumsV3 } from './MongoChecksumsV3.js';
|
|
12
13
|
import { MongoCompactorV3 } from './MongoCompactorV3.js';
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
14
|
+
import { MongoStoppedSyncConfigCleanup } from './MongoStoppedSyncConfigCleanup.js';
|
|
15
|
+
import { hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js';
|
|
16
|
+
import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
|
|
17
|
+
const BUCKET_DATA_FETCH_BATCH_LIMIT_BYTES = 16 * 1024 * 1024;
|
|
18
|
+
/**
|
|
19
|
+
* Keep the documents hydrated for one sync response within a bounded payload or slightly higher.
|
|
20
|
+
*
|
|
21
|
+
* This deserializes on-demand, so no deserialization is performed for discarded data.
|
|
22
|
+
*/
|
|
23
|
+
function cutBucketDataBatch(rawDocuments) {
|
|
24
|
+
let cumulativeBytes = 0;
|
|
25
|
+
let documents = [];
|
|
26
|
+
for (const raw of rawDocuments) {
|
|
27
|
+
const doc = bson.deserialize(raw, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS);
|
|
28
|
+
documents.push(doc);
|
|
29
|
+
cumulativeBytes += doc.size;
|
|
30
|
+
if (cumulativeBytes > BUCKET_DATA_FETCH_BATCH_LIMIT_BYTES) {
|
|
31
|
+
return {
|
|
32
|
+
documents,
|
|
33
|
+
wasCut: documents.length < rawDocuments.length
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
return { documents, wasCut: false };
|
|
38
|
+
}
|
|
39
|
+
function extractRowsFromDocument(doc, context, bucketStart, endOpId) {
|
|
40
|
+
const rows = [];
|
|
41
|
+
for (const row of loadBucketDataDocument(context, doc)) {
|
|
42
|
+
// In theory a binary search could be faster than a linear scan to find the start.
|
|
43
|
+
// In practice, most cases should not filter out anything here.
|
|
44
|
+
if (row.o <= bucketStart) {
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
if (row.o > endOpId) {
|
|
48
|
+
break;
|
|
49
|
+
}
|
|
50
|
+
rows.push(row);
|
|
51
|
+
}
|
|
52
|
+
return rows;
|
|
53
|
+
}
|
|
15
54
|
export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
if (!(sync_rules instanceof MongoPersistedSyncRulesContentV3)) {
|
|
55
|
+
constructor(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options) {
|
|
56
|
+
super(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
|
|
57
|
+
if (replicationStream.syncConfigIds.length == 0) {
|
|
20
58
|
throw new ServiceAssertionError('Missing sync config id for storage v3');
|
|
21
59
|
}
|
|
22
|
-
this.syncRulesV3 = sync_rules;
|
|
23
60
|
}
|
|
24
|
-
get
|
|
25
|
-
return this.
|
|
61
|
+
get syncConfigIds() {
|
|
62
|
+
return this.replicationStream.syncConfigIds;
|
|
26
63
|
}
|
|
27
64
|
get syncRulesCollection() {
|
|
28
65
|
return this.db.sync_rules;
|
|
29
66
|
}
|
|
30
67
|
syncConfigMatch(extra = {}) {
|
|
31
68
|
return {
|
|
32
|
-
_id: this.
|
|
69
|
+
_id: this.replicationStreamId,
|
|
33
70
|
sync_configs: {
|
|
34
71
|
$elemMatch: {
|
|
35
|
-
_id: this.
|
|
72
|
+
_id: { $in: this.syncConfigIds },
|
|
36
73
|
...extra
|
|
37
74
|
}
|
|
38
75
|
}
|
|
@@ -41,39 +78,25 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
41
78
|
syncConfigProjection(extra = {}) {
|
|
42
79
|
return {
|
|
43
80
|
...extra,
|
|
44
|
-
sync_configs:
|
|
45
|
-
$elemMatch: {
|
|
46
|
-
_id: this.syncConfigId
|
|
47
|
-
}
|
|
48
|
-
}
|
|
81
|
+
sync_configs: 1
|
|
49
82
|
};
|
|
50
83
|
}
|
|
51
|
-
|
|
52
|
-
return
|
|
53
|
-
}
|
|
54
|
-
/**
|
|
55
|
-
* For now, we only support a single sync config per replication stream.
|
|
56
|
-
*
|
|
57
|
-
* In the future we'll add support for multiple.
|
|
58
|
-
*/
|
|
59
|
-
selectedSyncConfig(doc) {
|
|
60
|
-
return doc?.sync_configs?.[0] ?? null;
|
|
84
|
+
selectedSyncConfigs(doc) {
|
|
85
|
+
return doc?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ?? [];
|
|
61
86
|
}
|
|
62
87
|
async initializeVersionStorage() {
|
|
63
|
-
const
|
|
64
|
-
for (
|
|
65
|
-
const collection = this.db.
|
|
66
|
-
await this.db.db
|
|
67
|
-
.createCollection(collection, { clusteredIndex: { name: '_id', unique: true, key: { _id: 1 } } })
|
|
68
|
-
.catch((error) => {
|
|
88
|
+
const storageIds = this.storageIds;
|
|
89
|
+
for (const source of storageIds.bucketDefinitionIds) {
|
|
90
|
+
const collection = this.db.bucketData(this.replicationStreamId, source).collectionName;
|
|
91
|
+
await this.db.db.createCollection(collection, {}).catch((error) => {
|
|
69
92
|
if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceExists') {
|
|
70
93
|
return;
|
|
71
94
|
}
|
|
72
95
|
throw error;
|
|
73
96
|
});
|
|
74
97
|
}
|
|
75
|
-
for (
|
|
76
|
-
await this.db.
|
|
98
|
+
for (const indexId of storageIds.parameterIndexIds) {
|
|
99
|
+
await this.db.parameterIndex(this.replicationStreamId, indexId).createIndex({
|
|
77
100
|
lookup: 1,
|
|
78
101
|
key: 1,
|
|
79
102
|
_id: -1
|
|
@@ -83,20 +106,34 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
83
106
|
}
|
|
84
107
|
}
|
|
85
108
|
createMongoChecksums(options) {
|
|
86
|
-
return new MongoChecksumsV3(this.db, this.
|
|
109
|
+
return new MongoChecksumsV3(this.db, this.replicationStreamId, {
|
|
87
110
|
...options.checksumOptions,
|
|
111
|
+
checksumCacheTtlMs: options.checksumCacheTtlMs,
|
|
88
112
|
storageConfig: options?.storageConfig,
|
|
89
|
-
|
|
113
|
+
syncConfigMapping: () => this.singleSyncConfigMapping()
|
|
90
114
|
});
|
|
91
115
|
}
|
|
92
116
|
createMongoCompactor(options) {
|
|
93
117
|
return new MongoCompactorV3(this, this.db, options);
|
|
94
118
|
}
|
|
95
119
|
createMongoParameterCompactor(checkpoint, options) {
|
|
96
|
-
return new
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
120
|
+
return new MongoParameterCompactor(this.db, this.replicationStreamId, checkpoint, options, () => this.db
|
|
121
|
+
.listParameterIndexCollections(this.replicationStreamId)
|
|
122
|
+
.then((collections) => collections.map((c) => c.collection)));
|
|
123
|
+
}
|
|
124
|
+
async fetchPersistedOpHead() {
|
|
125
|
+
const doc = await this.syncRulesCollection.findOne({ _id: this.replicationStreamId }, { projection: { last_persisted_op: 1 } });
|
|
126
|
+
return doc?.last_persisted_op == null ? null : BigInt(doc.last_persisted_op);
|
|
127
|
+
}
|
|
128
|
+
async createWriterImpl(options) {
|
|
129
|
+
const doc = await this.syncRulesCollection.findOne({ _id: this.replicationStreamId }, { projection: { resume_lsn: 1 } });
|
|
130
|
+
return new MongoBucketBatchV3({
|
|
131
|
+
...this.writerBatchOptions(options),
|
|
132
|
+
// The stream-level replication position - per-config checkpoint LSNs are consistency
|
|
133
|
+
// markers and do not affect where replication resumes.
|
|
134
|
+
resumeFromLsn: doc?.resume_lsn ?? null,
|
|
135
|
+
syncConfigIds: this.syncConfigIds
|
|
136
|
+
});
|
|
100
137
|
}
|
|
101
138
|
async fetchCheckpointState(session) {
|
|
102
139
|
const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch({
|
|
@@ -105,8 +142,16 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
105
142
|
session,
|
|
106
143
|
projection: this.syncConfigProjection()
|
|
107
144
|
});
|
|
108
|
-
|
|
109
|
-
|
|
145
|
+
// Checkpoints are served from the single active config. A PROCESSING config in the same
|
|
146
|
+
// stream (incremental reprocessing) does not affect checkpoints until it is activated.
|
|
147
|
+
const syncConfigs = this.selectedSyncConfigs(doc).filter((config) => [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED].includes(config.state));
|
|
148
|
+
if (syncConfigs.length > 1) {
|
|
149
|
+
// Activation atomically replaces the previous active config, so this cannot happen unless
|
|
150
|
+
// the stored state is corrupt.
|
|
151
|
+
throw new ServiceAssertionError(`Expected a single active sync config, got ${syncConfigs.map((config) => config._id.toHexString()).join(', ')}`);
|
|
152
|
+
}
|
|
153
|
+
const syncConfig = syncConfigs[0];
|
|
154
|
+
if (syncConfig == null || !syncConfig.snapshot_done) {
|
|
110
155
|
return null;
|
|
111
156
|
}
|
|
112
157
|
return {
|
|
@@ -114,22 +159,9 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
114
159
|
lsn: syncConfig.last_checkpoint_lsn ?? null
|
|
115
160
|
};
|
|
116
161
|
}
|
|
117
|
-
async getWriterSyncState() {
|
|
118
|
-
const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), {
|
|
119
|
-
projection: this.syncConfigProjection({ snapshot_lsn: 1 })
|
|
120
|
-
});
|
|
121
|
-
const syncConfig = this.selectedSyncConfig(doc);
|
|
122
|
-
const checkpointLsn = syncConfig?.last_checkpoint_lsn ?? null;
|
|
123
|
-
return {
|
|
124
|
-
lastCheckpointLsn: checkpointLsn,
|
|
125
|
-
resumeFromLsn: maxLsn(checkpointLsn, doc?.snapshot_lsn),
|
|
126
|
-
keepaliveOp: syncConfig?.keepalive_op ?? null,
|
|
127
|
-
syncConfigId: this.syncConfigId
|
|
128
|
-
};
|
|
129
|
-
}
|
|
130
162
|
async terminateSyncRuleState() {
|
|
131
163
|
await this.db.sync_rules.updateOne({
|
|
132
|
-
_id: this.
|
|
164
|
+
_id: this.replicationStreamId
|
|
133
165
|
}, {
|
|
134
166
|
$set: {
|
|
135
167
|
state: storage.SyncRuleState.TERMINATED,
|
|
@@ -140,43 +172,69 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
140
172
|
}
|
|
141
173
|
async getStatusImpl() {
|
|
142
174
|
const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), {
|
|
143
|
-
projection: this.syncConfigProjection({
|
|
175
|
+
projection: this.syncConfigProjection({ resume_lsn: 1 })
|
|
144
176
|
});
|
|
145
|
-
const
|
|
146
|
-
if (doc == null ||
|
|
177
|
+
const syncConfigs = this.selectedSyncConfigs(doc);
|
|
178
|
+
if (doc == null || syncConfigs.length == 0) {
|
|
147
179
|
throw new ServiceAssertionError('Cannot find replication stream status');
|
|
148
180
|
}
|
|
149
181
|
return {
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
checkpoint_lsn: syncConfig.last_checkpoint_lsn ?? null,
|
|
154
|
-
keepalive_op: syncConfig.keepalive_op ?? null
|
|
182
|
+
snapshotDone: syncConfigs.every((config) => config.snapshot_done ?? false) &&
|
|
183
|
+
syncConfigs.every((config) => config.last_checkpoint_lsn != null),
|
|
184
|
+
resumeLsn: doc.resume_lsn ?? null
|
|
155
185
|
};
|
|
156
186
|
}
|
|
157
187
|
async clearSyncRuleState() {
|
|
158
|
-
|
|
188
|
+
// Clearing resets the entire replication stream (bucket data and the op sequence), so reset
|
|
189
|
+
// the checkpoint state for _all_ embedded sync configs, not only the ones selected for this
|
|
190
|
+
// storage instance. This maintains the invariant that no config has a last_checkpoint past
|
|
191
|
+
// the stream-level last_persisted_op.
|
|
192
|
+
await this.syncRulesCollection.updateOne({ _id: this.replicationStreamId }, {
|
|
159
193
|
$set: {
|
|
160
194
|
persisted_lsn: null,
|
|
161
|
-
'sync_configs.$[
|
|
162
|
-
'sync_configs.$[
|
|
163
|
-
'sync_configs.$[
|
|
164
|
-
'sync_configs.$[
|
|
165
|
-
'sync_configs.$[config].keepalive_op': null
|
|
195
|
+
'sync_configs.$[].snapshot_done': false,
|
|
196
|
+
'sync_configs.$[].last_checkpoint_lsn': null,
|
|
197
|
+
'sync_configs.$[].last_checkpoint': null,
|
|
198
|
+
'sync_configs.$[].no_checkpoint_before': null
|
|
166
199
|
},
|
|
167
200
|
$unset: {
|
|
168
|
-
|
|
201
|
+
resume_lsn: 1,
|
|
202
|
+
last_persisted_op: 1
|
|
169
203
|
}
|
|
170
204
|
}, {
|
|
171
|
-
maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
|
|
172
|
-
arrayFilters: this.syncConfigArrayFilters()
|
|
205
|
+
maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
|
|
173
206
|
});
|
|
174
207
|
}
|
|
208
|
+
/**
|
|
209
|
+
* The persisted mapping of the single sync config that read operations are served from.
|
|
210
|
+
*
|
|
211
|
+
* Reads always operate on a single sync config: read-path storage instances are
|
|
212
|
+
* constructed for the active sync config only (see MongoBucketStorage.getActiveSyncConfig),
|
|
213
|
+
* and checkpoints are served from the single active config (see fetchCheckpointState).
|
|
214
|
+
* Within a single sync config, unique names are the persistence key of its rule mapping,
|
|
215
|
+
* so its name-keyed {@link SingleSyncConfigBucketDefinitionMapping} resolves sources from
|
|
216
|
+
* any parse of that config unambiguously - no parsed-set identity is required.
|
|
217
|
+
*
|
|
218
|
+
* Throws on storage instances with multiple sync configs (replication-side instances),
|
|
219
|
+
* which must not serve reads.
|
|
220
|
+
*/
|
|
221
|
+
singleSyncConfigMapping() {
|
|
222
|
+
const content = this.replicationStream.syncConfigContent;
|
|
223
|
+
if (content.length != 1) {
|
|
224
|
+
throw new ServiceAssertionError(`Read operations require a storage instance with a single sync config, got ${content.length}`);
|
|
225
|
+
}
|
|
226
|
+
return content[0].mapping;
|
|
227
|
+
}
|
|
175
228
|
get versionContext() {
|
|
229
|
+
const self = this;
|
|
176
230
|
return {
|
|
177
231
|
db: this.db,
|
|
178
|
-
|
|
179
|
-
|
|
232
|
+
objectStorage: this.objectStorage,
|
|
233
|
+
replicationStreamId: this.replicationStreamId,
|
|
234
|
+
readPreference: this.readPreference,
|
|
235
|
+
get mapping() {
|
|
236
|
+
return self.singleSyncConfigMapping();
|
|
237
|
+
}
|
|
180
238
|
};
|
|
181
239
|
}
|
|
182
240
|
getParameterSetsImpl(checkpoint, lookups, limit) {
|
|
@@ -185,24 +243,37 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
185
243
|
getBucketDataBatchImpl(checkpoint, dataBuckets, options) {
|
|
186
244
|
return getBucketDataBatchV3(this.versionContext, checkpoint, dataBuckets, options);
|
|
187
245
|
}
|
|
188
|
-
async clearBucketData(
|
|
189
|
-
for (const collection of await this.db.
|
|
246
|
+
async clearBucketData(signal) {
|
|
247
|
+
for (const collection of await this.db.listBucketDataCollections(this.replicationStreamId)) {
|
|
190
248
|
await collection.drop();
|
|
191
249
|
}
|
|
250
|
+
if (this.objectStorage) {
|
|
251
|
+
const lifecycle = new ObjectStorageLifecycle(this.db, this.replicationStreamId, this.objectStorage);
|
|
252
|
+
await lifecycle.deletePrefix(lifecycle.streamPrefix(), { signal });
|
|
253
|
+
}
|
|
254
|
+
await this.db
|
|
255
|
+
.pendingObjectStorageDeletes(this.replicationStreamId)
|
|
256
|
+
.drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
|
|
257
|
+
.catch((error) => {
|
|
258
|
+
if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
throw error;
|
|
262
|
+
});
|
|
192
263
|
}
|
|
193
264
|
async clearParameterIndexes(_signal) {
|
|
194
|
-
for (const collection of await this.db.
|
|
265
|
+
for (const collection of await this.db.listParameterIndexCollections(this.replicationStreamId)) {
|
|
195
266
|
await collection.collection.drop();
|
|
196
267
|
}
|
|
197
268
|
}
|
|
198
269
|
async clearSourceRecords(_signal) {
|
|
199
|
-
for (const collection of await this.db.
|
|
270
|
+
for (const collection of await this.db.listSourceRecordCollections(this.replicationStreamId)) {
|
|
200
271
|
await collection.drop();
|
|
201
272
|
}
|
|
202
273
|
}
|
|
203
274
|
async clearBucketState(_signal) {
|
|
204
275
|
await this.db
|
|
205
|
-
.
|
|
276
|
+
.bucketState(this.replicationStreamId)
|
|
206
277
|
.drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
|
|
207
278
|
.catch((error) => {
|
|
208
279
|
if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
|
|
@@ -213,7 +284,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
213
284
|
}
|
|
214
285
|
async clearSourceTables(_signal) {
|
|
215
286
|
await this.db
|
|
216
|
-
.
|
|
287
|
+
.sourceTables(this.replicationStreamId)
|
|
217
288
|
.drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
|
|
218
289
|
.catch((error) => {
|
|
219
290
|
if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
|
|
@@ -222,6 +293,18 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
222
293
|
throw error;
|
|
223
294
|
});
|
|
224
295
|
}
|
|
296
|
+
async cleanupStoppedSyncConfigs(options) {
|
|
297
|
+
return new MongoStoppedSyncConfigCleanup({
|
|
298
|
+
db: this.db,
|
|
299
|
+
replicationStreamId: this.replicationStreamId,
|
|
300
|
+
signal: options.signal,
|
|
301
|
+
logger: options.logger ?? this.logger,
|
|
302
|
+
defaultSchema: options.defaultSchema,
|
|
303
|
+
sourceConnectionTag: options.sourceConnectionTag,
|
|
304
|
+
objectStorage: this.objectStorage,
|
|
305
|
+
clearBatchThrottleRate: this.clearBatchThrottleRate
|
|
306
|
+
}).run();
|
|
307
|
+
}
|
|
225
308
|
getDataBucketChangesImpl(options) {
|
|
226
309
|
return getDataBucketChangesV3(this.versionContext, options);
|
|
227
310
|
}
|
|
@@ -234,8 +317,8 @@ export async function getParameterSetsV3(ctx, checkpoint, lookups, limit) {
|
|
|
234
317
|
setSessionSnapshotTime(session, checkpoint.snapshotTime);
|
|
235
318
|
const buildLookupPipeline = (lookup, index) => {
|
|
236
319
|
const indexId = lookup.indexId;
|
|
237
|
-
const collection = ctx.db.
|
|
238
|
-
const lookupFilter =
|
|
320
|
+
const collection = ctx.db.parameterIndex(ctx.replicationStreamId, indexId);
|
|
321
|
+
const lookupFilter = serializeParameterLookup(lookup);
|
|
239
322
|
return {
|
|
240
323
|
collection,
|
|
241
324
|
pipeline: [
|
|
@@ -313,13 +396,24 @@ export async function* getBucketDataBatchV3(ctx, checkpoint, dataBuckets, option
|
|
|
313
396
|
if (dataBuckets.length == 0) {
|
|
314
397
|
return;
|
|
315
398
|
}
|
|
316
|
-
if (checkpoint == null) {
|
|
399
|
+
if (checkpoint.checkpoint == null) {
|
|
317
400
|
throw new Error('checkpoint is null');
|
|
318
401
|
}
|
|
402
|
+
const readPreference = options?.requestHint == 'bulk' ? ctx.readPreference : undefined;
|
|
403
|
+
const readConcern = ctx.readPreference == null ? undefined : 'majority';
|
|
404
|
+
const session = readPreference == null || checkpoint.snapshotTime == null
|
|
405
|
+
? undefined
|
|
406
|
+
: ctx.db.client.startSession({ causalConsistency: true });
|
|
407
|
+
await using _ = { [Symbol.asyncDispose]: async () => session?.endSession() };
|
|
408
|
+
if (session != null) {
|
|
409
|
+
session.advanceOperationTime(checkpoint.snapshotTime);
|
|
410
|
+
session.advanceClusterTime(checkpoint.clusterTime);
|
|
411
|
+
}
|
|
319
412
|
const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
|
|
320
413
|
const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
|
|
321
|
-
const end = checkpoint;
|
|
322
|
-
|
|
414
|
+
const end = checkpoint.checkpoint;
|
|
415
|
+
// Group requests by definition, so that we can query each definition's bucket data in a single query.
|
|
416
|
+
// We only return the results of a single query per batch.
|
|
323
417
|
const requestsByDefinition = new Map();
|
|
324
418
|
for (const request of dataBuckets) {
|
|
325
419
|
const definitionId = ctx.mapping.bucketSourceId(request.source);
|
|
@@ -328,50 +422,68 @@ export async function* getBucketDataBatchV3(ctx, checkpoint, dataBuckets, option
|
|
|
328
422
|
requestsByDefinition.set(definitionId, requests);
|
|
329
423
|
}
|
|
330
424
|
const definitionGroups = Array.from(requestsByDefinition.entries());
|
|
331
|
-
for (
|
|
332
|
-
const [definitionId, requests] = definitionGroups[groupIndex];
|
|
425
|
+
for (const [groupIndex, [definitionId, requests]] of definitionGroups.entries()) {
|
|
333
426
|
const hasLaterDefinitionGroups = groupIndex < definitionGroups.length - 1;
|
|
334
427
|
const bucketMap = new Map(requests.map((request) => [request.bucket, request.start]));
|
|
335
428
|
const filters = Array.from(bucketMap.entries()).map(([bucket, start]) => ({
|
|
336
429
|
_id: {
|
|
337
|
-
$gt: {
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
b: bucket,
|
|
343
|
-
o: end
|
|
344
|
-
}
|
|
345
|
-
}
|
|
430
|
+
$gt: { b: bucket, o: start },
|
|
431
|
+
$lte: { b: bucket, o: new bson.MaxKey() }
|
|
432
|
+
},
|
|
433
|
+
min_op: { $lte: end }
|
|
434
|
+
// MongoDB Filter<T> doesn't accept compound _id ranges or dotted field paths in its type.
|
|
346
435
|
}));
|
|
347
|
-
const
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
436
|
+
const collection = ctx.db.bucketData(ctx.replicationStreamId, definitionId);
|
|
437
|
+
// MongoDB Filter<T> doesn't accept the $or operator in its type.
|
|
438
|
+
const filter = { $or: filters };
|
|
439
|
+
const context = { replicationStreamId: ctx.replicationStreamId, definitionId };
|
|
440
|
+
const cursorOptions = { limit: batchLimit, batchSize: batchLimit + 1 };
|
|
441
|
+
// raw: true returns Buffers, but the driver typing doesn't reflect that
|
|
442
|
+
// without an explicit cast to FindCursor<Buffer>.
|
|
443
|
+
const cursor = collection.find(filter, {
|
|
444
|
+
session,
|
|
445
|
+
readPreference,
|
|
446
|
+
readConcern,
|
|
351
447
|
sort: { _id: 1 },
|
|
352
|
-
limit: remainingLimit,
|
|
353
|
-
batchSize: remainingLimit + 1,
|
|
354
448
|
raw: true,
|
|
355
|
-
maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS
|
|
449
|
+
maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS,
|
|
450
|
+
...cursorOptions
|
|
356
451
|
});
|
|
357
|
-
let { data, hasMore
|
|
452
|
+
let { data: rawData, hasMore } = await readSingleBatch(cursor).catch((e) => {
|
|
358
453
|
throw lib_mongo.mapQueryError(e, 'while reading bucket data');
|
|
359
454
|
});
|
|
360
|
-
if (
|
|
361
|
-
|
|
455
|
+
if (rawData.length >= cursorOptions.limit) {
|
|
456
|
+
hasMore = true;
|
|
362
457
|
}
|
|
363
|
-
|
|
364
|
-
|
|
458
|
+
// Deserialize the raw documents and cut the batch to a bounded size. Any data not
|
|
459
|
+
// making the cut will be read in the next round, using a fresh query.
|
|
460
|
+
const cutBatch = cutBucketDataBatch(rawData);
|
|
461
|
+
const docs = cutBatch.documents;
|
|
462
|
+
if (cutBatch.wasCut) {
|
|
463
|
+
hasMore = true;
|
|
365
464
|
}
|
|
366
|
-
|
|
367
|
-
|
|
465
|
+
// Hydrate any operations from object storage.
|
|
466
|
+
// In the future we can do this in a more pipelined fashion, but for now we hydrate
|
|
467
|
+
// the entire batch at once.
|
|
468
|
+
await hydrateBucketDataDocuments(docs, ctx.objectStorage, { signal: options?.signal });
|
|
469
|
+
let currentChunkSizeBytes = 0;
|
|
368
470
|
let currentChunk = null;
|
|
369
471
|
let targetOp = null;
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
472
|
+
let seenBuckets = new Set();
|
|
473
|
+
const batchHasMore = hasMore;
|
|
474
|
+
for (const doc of docs) {
|
|
475
|
+
const bucket = doc._id.b;
|
|
476
|
+
seenBuckets.add(bucket);
|
|
477
|
+
const bucketStart = bucketMap.get(bucket);
|
|
478
|
+
if (bucketStart == null) {
|
|
479
|
+
throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`);
|
|
480
|
+
}
|
|
481
|
+
let start = undefined;
|
|
482
|
+
// Reached a new bucket or size limit: yield the current chunk and start a new one.
|
|
483
|
+
if (currentChunk == null ||
|
|
484
|
+
currentChunk.bucket != bucket ||
|
|
485
|
+
currentChunkSizeBytes >= chunkSizeLimitBytes ||
|
|
486
|
+
currentChunk.data.length >= batchLimit) {
|
|
375
487
|
if (currentChunk != null) {
|
|
376
488
|
if (currentChunk.bucket == bucket) {
|
|
377
489
|
currentChunk.has_more = true;
|
|
@@ -379,16 +491,12 @@ export async function* getBucketDataBatchV3(ctx, checkpoint, dataBuckets, option
|
|
|
379
491
|
}
|
|
380
492
|
const yieldChunk = currentChunk;
|
|
381
493
|
currentChunk = null;
|
|
382
|
-
|
|
383
|
-
yield { chunkData: yieldChunk, targetOp
|
|
494
|
+
currentChunkSizeBytes = 0;
|
|
495
|
+
yield { chunkData: yieldChunk, targetOp };
|
|
384
496
|
targetOp = null;
|
|
385
497
|
}
|
|
386
498
|
if (start == null) {
|
|
387
|
-
|
|
388
|
-
if (startOpId == null) {
|
|
389
|
-
throw new Error(`data for unexpected bucket: ${bucket}`);
|
|
390
|
-
}
|
|
391
|
-
start = internalToExternalOpId(startOpId);
|
|
499
|
+
start = internalToExternalOpId(bucketStart);
|
|
392
500
|
}
|
|
393
501
|
currentChunk = {
|
|
394
502
|
bucket,
|
|
@@ -398,28 +506,52 @@ export async function* getBucketDataBatchV3(ctx, checkpoint, dataBuckets, option
|
|
|
398
506
|
next_after: start
|
|
399
507
|
};
|
|
400
508
|
}
|
|
401
|
-
const
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
currentChunk.next_after = entry.op_id;
|
|
407
|
-
chunkSizeBytes += rawData.byteLength;
|
|
509
|
+
const rows = extractRowsFromDocument(doc, context, bucketStart, end);
|
|
510
|
+
currentChunk.data.push(...rows.map(mapOpEntry));
|
|
511
|
+
currentChunk.next_after = currentChunk.data.at(-1)?.op_id ?? internalToExternalOpId(end);
|
|
512
|
+
targetOp = maxOpId(targetOp, doc.target_op);
|
|
513
|
+
currentChunkSizeBytes += doc.size;
|
|
408
514
|
}
|
|
409
515
|
if (currentChunk != null) {
|
|
410
516
|
const yieldChunk = currentChunk;
|
|
411
|
-
|
|
412
|
-
|
|
517
|
+
// The last chunk may contain more data that was cut in this batch.
|
|
518
|
+
yieldChunk.has_more = batchHasMore;
|
|
519
|
+
yield { chunkData: yieldChunk, targetOp };
|
|
520
|
+
}
|
|
521
|
+
if (!batchHasMore) {
|
|
522
|
+
for (const bucket of bucketMap.keys()) {
|
|
523
|
+
if (!seenBuckets.has(bucket)) {
|
|
524
|
+
// We processed everything for this definition group, but this bucket had no data in the batch.
|
|
525
|
+
// Yield an empty chunk to indicate that it is complete.
|
|
526
|
+
// This prevents re-querying the same bucket in the next batch.
|
|
527
|
+
yield {
|
|
528
|
+
chunkData: {
|
|
529
|
+
bucket,
|
|
530
|
+
after: internalToExternalOpId(bucketMap.get(bucket)),
|
|
531
|
+
has_more: false,
|
|
532
|
+
data: [],
|
|
533
|
+
next_after: internalToExternalOpId(end)
|
|
534
|
+
},
|
|
535
|
+
targetOp
|
|
536
|
+
};
|
|
537
|
+
}
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
if (currentChunk != null) {
|
|
541
|
+
// We yielded data in this group (aside from empty buckets).
|
|
542
|
+
// Return to the caller to allow them to process it before continuing to the next group.
|
|
543
|
+
yield { hasMore: batchHasMore || hasLaterDefinitionGroups };
|
|
544
|
+
break;
|
|
413
545
|
}
|
|
414
|
-
|
|
415
|
-
|
|
546
|
+
else {
|
|
547
|
+
// No data in this definition group - continue in the next group.
|
|
416
548
|
}
|
|
417
549
|
}
|
|
418
550
|
}
|
|
419
551
|
export async function getDataBucketChangesV3(ctx, options) {
|
|
420
552
|
const limit = 1000;
|
|
421
553
|
const bucketStateUpdates = await ctx.db
|
|
422
|
-
.
|
|
554
|
+
.bucketState(ctx.replicationStreamId)
|
|
423
555
|
.aggregate([
|
|
424
556
|
{
|
|
425
557
|
$match: {
|
|
@@ -454,7 +586,7 @@ export async function getParameterBucketChangesV3(ctx, options) {
|
|
|
454
586
|
const indexIds = ctx.mapping.allParameterIndexIds();
|
|
455
587
|
const collections = indexIds.map((indexId) => ({
|
|
456
588
|
indexId,
|
|
457
|
-
collection: ctx.db.
|
|
589
|
+
collection: ctx.db.parameterIndex(ctx.replicationStreamId, indexId)
|
|
458
590
|
}));
|
|
459
591
|
if (collections.length == 0) {
|
|
460
592
|
return {
|
|
@@ -502,7 +634,7 @@ export async function getParameterBucketChangesV3(ctx, options) {
|
|
|
502
634
|
invalidateParameterBuckets: invalidateParameterUpdates,
|
|
503
635
|
updatedParameterLookups: invalidateParameterUpdates
|
|
504
636
|
? new Set()
|
|
505
|
-
: new Set(parameterUpdates.map((p) => JSONBig.stringify(
|
|
637
|
+
: new Set(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookup(p.lookup, p.indexId))))
|
|
506
638
|
};
|
|
507
639
|
}
|
|
508
640
|
//# sourceMappingURL=MongoSyncBucketStorageV3.js.map
|