@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -6,9 +6,9 @@ import {
|
|
|
6
6
|
GetCheckpointChangesOptions,
|
|
7
7
|
InternalOpId,
|
|
8
8
|
internalToExternalOpId,
|
|
9
|
-
maxLsn,
|
|
10
9
|
ParameterSetLimitExceededError,
|
|
11
10
|
ProtocolOpId,
|
|
11
|
+
SingleSyncConfigBucketDefinitionMapping,
|
|
12
12
|
storage,
|
|
13
13
|
utils
|
|
14
14
|
} from '@powersync/service-core';
|
|
@@ -17,57 +17,114 @@ import { ParameterLookupRows, ScopedParameterLookup, SqliteJsonRow } from '@powe
|
|
|
17
17
|
import * as bson from 'bson';
|
|
18
18
|
import { mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
|
|
19
19
|
import { MongoBucketStorage } from '../../MongoBucketStorage.js';
|
|
20
|
-
import {
|
|
21
|
-
|
|
22
|
-
MongoSyncBucketStorageContext
|
|
23
|
-
} from '../common/MongoSyncBucketStorageContext.js';
|
|
24
|
-
import { MongoBucketBatchOptions } from '../MongoBucketBatch.js';
|
|
20
|
+
import { BucketDataDoc } from '../common/BucketDataDoc.js';
|
|
21
|
+
import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js';
|
|
25
22
|
import { MongoChecksums } from '../MongoChecksums.js';
|
|
26
23
|
import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
|
|
27
24
|
import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
|
|
28
|
-
import {
|
|
29
|
-
MongoPersistedSyncRulesContentV1,
|
|
30
|
-
MongoPersistedSyncRulesContentV3
|
|
31
|
-
} from '../MongoPersistedSyncRulesContent.js';
|
|
25
|
+
import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js';
|
|
32
26
|
import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
|
|
27
|
+
import { loadBucketDataDocument, maxOpId } from './bucket-format.js';
|
|
33
28
|
import {
|
|
34
29
|
BucketDataDocumentV3,
|
|
35
30
|
BucketParameterDocumentV3,
|
|
36
|
-
|
|
31
|
+
deserializeParameterLookup,
|
|
37
32
|
ReplicationStreamDocumentV3,
|
|
33
|
+
serializeParameterLookup,
|
|
38
34
|
SyncRuleConfigStateV3
|
|
39
35
|
} from './models.js';
|
|
40
36
|
import { MongoBucketBatchV3 } from './MongoBucketBatchV3.js';
|
|
41
37
|
import { MongoChecksumsV3 } from './MongoChecksumsV3.js';
|
|
42
38
|
import { MongoCompactorV3 } from './MongoCompactorV3.js';
|
|
43
|
-
import {
|
|
44
|
-
import {
|
|
39
|
+
import { MongoStoppedSyncConfigCleanup } from './MongoStoppedSyncConfigCleanup.js';
|
|
40
|
+
import { hydrateBucketDataDocuments } from './object-storage/BucketDataObjectStorage.js';
|
|
41
|
+
import { ObjectStorage } from './object-storage/ObjectStorage.js';
|
|
42
|
+
import { ObjectStorageLifecycle } from './object-storage/ObjectStorageLifecycle.js';
|
|
45
43
|
import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
|
|
46
44
|
|
|
45
|
+
export interface MongoSyncBucketStorageContextV3 {
|
|
46
|
+
db: VersionedPowerSyncMongoV3;
|
|
47
|
+
replicationStreamId: number;
|
|
48
|
+
readPreference?: mongo.ReadPreference;
|
|
49
|
+
objectStorage: ObjectStorage | undefined;
|
|
50
|
+
/**
|
|
51
|
+
* Persisted mapping of the single sync config that read operations are served from.
|
|
52
|
+
*
|
|
53
|
+
* Implemented as a lazy getter: accessing it on a storage instance with multiple sync
|
|
54
|
+
* configs throws, but operations that don't use it remain unaffected.
|
|
55
|
+
*/
|
|
56
|
+
readonly mapping: SingleSyncConfigBucketDefinitionMapping;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const BUCKET_DATA_FETCH_BATCH_LIMIT_BYTES = 16 * 1024 * 1024;
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Keep the documents hydrated for one sync response within a bounded payload or slightly higher.
|
|
63
|
+
*
|
|
64
|
+
* This deserializes on-demand, so no deserialization is performed for discarded data.
|
|
65
|
+
*/
|
|
66
|
+
function cutBucketDataBatch(rawDocuments: Buffer[]): {
|
|
67
|
+
documents: BucketDataDocumentV3[];
|
|
68
|
+
wasCut: boolean;
|
|
69
|
+
} {
|
|
70
|
+
let cumulativeBytes = 0;
|
|
71
|
+
let documents: BucketDataDocumentV3[] = [];
|
|
72
|
+
for (const raw of rawDocuments) {
|
|
73
|
+
const doc = bson.deserialize(raw, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS) as BucketDataDocumentV3;
|
|
74
|
+
documents.push(doc);
|
|
75
|
+
cumulativeBytes += doc.size;
|
|
76
|
+
if (cumulativeBytes > BUCKET_DATA_FETCH_BATCH_LIMIT_BYTES) {
|
|
77
|
+
return {
|
|
78
|
+
documents,
|
|
79
|
+
wasCut: documents.length < rawDocuments.length
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return { documents, wasCut: false };
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function extractRowsFromDocument(
|
|
87
|
+
doc: BucketDataDocumentV3,
|
|
88
|
+
context: { replicationStreamId: number; definitionId: string },
|
|
89
|
+
bucketStart: InternalOpId,
|
|
90
|
+
endOpId: InternalOpId
|
|
91
|
+
): BucketDataDoc[] {
|
|
92
|
+
const rows: BucketDataDoc[] = [];
|
|
93
|
+
for (const row of loadBucketDataDocument(context, doc)) {
|
|
94
|
+
// In theory a binary search could be faster than a linear scan to find the start.
|
|
95
|
+
// In practice, most cases should not filter out anything here.
|
|
96
|
+
if (row.o <= bucketStart) {
|
|
97
|
+
continue;
|
|
98
|
+
}
|
|
99
|
+
if (row.o > endOpId) {
|
|
100
|
+
break;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
rows.push(row);
|
|
104
|
+
}
|
|
105
|
+
return rows;
|
|
106
|
+
}
|
|
107
|
+
|
|
47
108
|
export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
48
|
-
// Declare types to be more specific
|
|
49
109
|
declare readonly db: VersionedPowerSyncMongoV3;
|
|
50
110
|
declare readonly checksums: MongoChecksumsV3;
|
|
51
111
|
|
|
52
|
-
private readonly syncRulesV3: MongoPersistedSyncRulesContentV3;
|
|
53
|
-
|
|
54
112
|
constructor(
|
|
55
113
|
factory: MongoBucketStorage,
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
114
|
+
replicationStreamId: number,
|
|
115
|
+
replicationStream: MongoPersistedReplicationStream,
|
|
116
|
+
replicationStreamName: string,
|
|
59
117
|
writeCheckpointMode: storage.WriteCheckpointMode | undefined,
|
|
60
118
|
options: MongoSyncBucketStorageOptions
|
|
61
119
|
) {
|
|
62
|
-
super(factory,
|
|
63
|
-
if (
|
|
120
|
+
super(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
|
|
121
|
+
if (replicationStream.syncConfigIds.length == 0) {
|
|
64
122
|
throw new ServiceAssertionError('Missing sync config id for storage v3');
|
|
65
123
|
}
|
|
66
|
-
this.syncRulesV3 = sync_rules;
|
|
67
124
|
}
|
|
68
125
|
|
|
69
|
-
private get
|
|
70
|
-
return this.
|
|
126
|
+
private get syncConfigIds(): bson.ObjectId[] {
|
|
127
|
+
return this.replicationStream.syncConfigIds;
|
|
71
128
|
}
|
|
72
129
|
|
|
73
130
|
private get syncRulesCollection(): mongo.Collection<ReplicationStreamDocumentV3> {
|
|
@@ -76,10 +133,10 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
76
133
|
|
|
77
134
|
private syncConfigMatch(extra: mongo.Document = {}): mongo.Filter<ReplicationStreamDocumentV3> {
|
|
78
135
|
return {
|
|
79
|
-
_id: this.
|
|
136
|
+
_id: this.replicationStreamId,
|
|
80
137
|
sync_configs: {
|
|
81
138
|
$elemMatch: {
|
|
82
|
-
_id: this.
|
|
139
|
+
_id: { $in: this.syncConfigIds },
|
|
83
140
|
...extra
|
|
84
141
|
}
|
|
85
142
|
}
|
|
@@ -89,44 +146,27 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
89
146
|
private syncConfigProjection(extra: mongo.Document = {}): mongo.Document {
|
|
90
147
|
return {
|
|
91
148
|
...extra,
|
|
92
|
-
sync_configs:
|
|
93
|
-
$elemMatch: {
|
|
94
|
-
_id: this.syncConfigId
|
|
95
|
-
}
|
|
96
|
-
}
|
|
149
|
+
sync_configs: 1
|
|
97
150
|
};
|
|
98
151
|
}
|
|
99
152
|
|
|
100
|
-
private
|
|
101
|
-
return
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
/**
|
|
105
|
-
* For now, we only support a single sync config per replication stream.
|
|
106
|
-
*
|
|
107
|
-
* In the future we'll add support for multiple.
|
|
108
|
-
*/
|
|
109
|
-
private selectedSyncConfig(
|
|
110
|
-
doc: Pick<ReplicationStreamDocumentV3, 'sync_configs'> | null
|
|
111
|
-
): SyncRuleConfigStateV3 | null {
|
|
112
|
-
return doc?.sync_configs?.[0] ?? null;
|
|
153
|
+
private selectedSyncConfigs(doc: Pick<ReplicationStreamDocumentV3, 'sync_configs'> | null): SyncRuleConfigStateV3[] {
|
|
154
|
+
return doc?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ?? [];
|
|
113
155
|
}
|
|
114
156
|
|
|
115
157
|
protected async initializeVersionStorage(): Promise<void> {
|
|
116
|
-
const
|
|
117
|
-
for (
|
|
118
|
-
const collection = this.db.
|
|
119
|
-
await this.db.db
|
|
120
|
-
.
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
throw error;
|
|
126
|
-
});
|
|
158
|
+
const storageIds = this.storageIds;
|
|
159
|
+
for (const source of storageIds.bucketDefinitionIds) {
|
|
160
|
+
const collection = this.db.bucketData(this.replicationStreamId, source).collectionName;
|
|
161
|
+
await this.db.db.createCollection(collection, {}).catch((error) => {
|
|
162
|
+
if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceExists') {
|
|
163
|
+
return;
|
|
164
|
+
}
|
|
165
|
+
throw error;
|
|
166
|
+
});
|
|
127
167
|
}
|
|
128
|
-
for (
|
|
129
|
-
await this.db.
|
|
168
|
+
for (const indexId of storageIds.parameterIndexIds) {
|
|
169
|
+
await this.db.parameterIndex(this.replicationStreamId, indexId).createIndex(
|
|
130
170
|
{
|
|
131
171
|
lookup: 1,
|
|
132
172
|
key: 1,
|
|
@@ -140,10 +180,11 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
140
180
|
}
|
|
141
181
|
|
|
142
182
|
protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums {
|
|
143
|
-
return new MongoChecksumsV3(this.db, this.
|
|
183
|
+
return new MongoChecksumsV3(this.db, this.replicationStreamId, {
|
|
144
184
|
...options.checksumOptions,
|
|
185
|
+
checksumCacheTtlMs: options.checksumCacheTtlMs,
|
|
145
186
|
storageConfig: options?.storageConfig,
|
|
146
|
-
|
|
187
|
+
syncConfigMapping: () => this.singleSyncConfigMapping()
|
|
147
188
|
});
|
|
148
189
|
}
|
|
149
190
|
|
|
@@ -155,11 +196,36 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
155
196
|
checkpoint: InternalOpId,
|
|
156
197
|
options: storage.CompactOptions
|
|
157
198
|
): MongoParameterCompactor {
|
|
158
|
-
return new
|
|
199
|
+
return new MongoParameterCompactor(this.db, this.replicationStreamId, checkpoint, options, () =>
|
|
200
|
+
this.db
|
|
201
|
+
.listParameterIndexCollections(this.replicationStreamId)
|
|
202
|
+
.then((collections) =>
|
|
203
|
+
collections.map((c) => c.collection as unknown as lib_mongo.mongo.Collection<lib_mongo.mongo.Document>)
|
|
204
|
+
)
|
|
205
|
+
);
|
|
159
206
|
}
|
|
160
207
|
|
|
161
|
-
protected
|
|
162
|
-
|
|
208
|
+
protected async fetchPersistedOpHead(): Promise<InternalOpId | null> {
|
|
209
|
+
const doc = await this.syncRulesCollection.findOne(
|
|
210
|
+
{ _id: this.replicationStreamId },
|
|
211
|
+
{ projection: { last_persisted_op: 1 } }
|
|
212
|
+
);
|
|
213
|
+
return doc?.last_persisted_op == null ? null : BigInt(doc.last_persisted_op);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
protected async createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
|
|
217
|
+
const doc = await this.syncRulesCollection.findOne(
|
|
218
|
+
{ _id: this.replicationStreamId },
|
|
219
|
+
{ projection: { resume_lsn: 1 } }
|
|
220
|
+
);
|
|
221
|
+
|
|
222
|
+
return new MongoBucketBatchV3({
|
|
223
|
+
...this.writerBatchOptions(options),
|
|
224
|
+
// The stream-level replication position - per-config checkpoint LSNs are consistency
|
|
225
|
+
// markers and do not affect where replication resumes.
|
|
226
|
+
resumeFromLsn: doc?.resume_lsn ?? null,
|
|
227
|
+
syncConfigIds: this.syncConfigIds
|
|
228
|
+
});
|
|
163
229
|
}
|
|
164
230
|
|
|
165
231
|
protected async fetchCheckpointState(
|
|
@@ -174,8 +240,20 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
174
240
|
projection: this.syncConfigProjection()
|
|
175
241
|
}
|
|
176
242
|
);
|
|
177
|
-
|
|
178
|
-
|
|
243
|
+
// Checkpoints are served from the single active config. A PROCESSING config in the same
|
|
244
|
+
// stream (incremental reprocessing) does not affect checkpoints until it is activated.
|
|
245
|
+
const syncConfigs = this.selectedSyncConfigs(doc).filter((config) =>
|
|
246
|
+
[storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED].includes(config.state)
|
|
247
|
+
);
|
|
248
|
+
if (syncConfigs.length > 1) {
|
|
249
|
+
// Activation atomically replaces the previous active config, so this cannot happen unless
|
|
250
|
+
// the stored state is corrupt.
|
|
251
|
+
throw new ServiceAssertionError(
|
|
252
|
+
`Expected a single active sync config, got ${syncConfigs.map((config) => config._id.toHexString()).join(', ')}`
|
|
253
|
+
);
|
|
254
|
+
}
|
|
255
|
+
const syncConfig = syncConfigs[0];
|
|
256
|
+
if (syncConfig == null || !syncConfig.snapshot_done) {
|
|
179
257
|
return null;
|
|
180
258
|
}
|
|
181
259
|
return {
|
|
@@ -184,24 +262,10 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
184
262
|
};
|
|
185
263
|
}
|
|
186
264
|
|
|
187
|
-
protected async getWriterSyncState() {
|
|
188
|
-
const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), {
|
|
189
|
-
projection: this.syncConfigProjection({ snapshot_lsn: 1 })
|
|
190
|
-
});
|
|
191
|
-
const syncConfig = this.selectedSyncConfig(doc);
|
|
192
|
-
const checkpointLsn = syncConfig?.last_checkpoint_lsn ?? null;
|
|
193
|
-
return {
|
|
194
|
-
lastCheckpointLsn: checkpointLsn,
|
|
195
|
-
resumeFromLsn: maxLsn(checkpointLsn, doc?.snapshot_lsn),
|
|
196
|
-
keepaliveOp: syncConfig?.keepalive_op ?? null,
|
|
197
|
-
syncConfigId: this.syncConfigId
|
|
198
|
-
};
|
|
199
|
-
}
|
|
200
|
-
|
|
201
265
|
protected async terminateSyncRuleState(): Promise<void> {
|
|
202
266
|
await this.db.sync_rules.updateOne(
|
|
203
267
|
{
|
|
204
|
-
_id: this.
|
|
268
|
+
_id: this.replicationStreamId
|
|
205
269
|
},
|
|
206
270
|
{
|
|
207
271
|
$set: {
|
|
@@ -213,52 +277,82 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
213
277
|
);
|
|
214
278
|
}
|
|
215
279
|
|
|
216
|
-
protected async getStatusImpl(): Promise<storage.
|
|
280
|
+
protected async getStatusImpl(): Promise<storage.ReplicationStreamStatus> {
|
|
217
281
|
const doc = await this.syncRulesCollection.findOne(this.syncConfigMatch(), {
|
|
218
|
-
projection: this.syncConfigProjection({
|
|
282
|
+
projection: this.syncConfigProjection({ resume_lsn: 1 })
|
|
219
283
|
});
|
|
220
|
-
const
|
|
221
|
-
if (doc == null ||
|
|
284
|
+
const syncConfigs = this.selectedSyncConfigs(doc);
|
|
285
|
+
if (doc == null || syncConfigs.length == 0) {
|
|
222
286
|
throw new ServiceAssertionError('Cannot find replication stream status');
|
|
223
287
|
}
|
|
224
288
|
|
|
225
289
|
return {
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
keepalive_op: syncConfig.keepalive_op ?? null
|
|
290
|
+
snapshotDone:
|
|
291
|
+
syncConfigs.every((config) => config.snapshot_done ?? false) &&
|
|
292
|
+
syncConfigs.every((config) => config.last_checkpoint_lsn != null),
|
|
293
|
+
resumeLsn: doc.resume_lsn ?? null
|
|
231
294
|
};
|
|
232
295
|
}
|
|
233
296
|
|
|
234
297
|
protected async clearSyncRuleState(): Promise<void> {
|
|
298
|
+
// Clearing resets the entire replication stream (bucket data and the op sequence), so reset
|
|
299
|
+
// the checkpoint state for _all_ embedded sync configs, not only the ones selected for this
|
|
300
|
+
// storage instance. This maintains the invariant that no config has a last_checkpoint past
|
|
301
|
+
// the stream-level last_persisted_op.
|
|
235
302
|
await this.syncRulesCollection.updateOne(
|
|
236
|
-
this.
|
|
303
|
+
{ _id: this.replicationStreamId },
|
|
237
304
|
{
|
|
238
305
|
$set: {
|
|
239
306
|
persisted_lsn: null,
|
|
240
|
-
'sync_configs.$[
|
|
241
|
-
'sync_configs.$[
|
|
242
|
-
'sync_configs.$[
|
|
243
|
-
'sync_configs.$[
|
|
244
|
-
'sync_configs.$[config].keepalive_op': null
|
|
307
|
+
'sync_configs.$[].snapshot_done': false,
|
|
308
|
+
'sync_configs.$[].last_checkpoint_lsn': null,
|
|
309
|
+
'sync_configs.$[].last_checkpoint': null,
|
|
310
|
+
'sync_configs.$[].no_checkpoint_before': null
|
|
245
311
|
},
|
|
246
312
|
$unset: {
|
|
247
|
-
|
|
313
|
+
resume_lsn: 1,
|
|
314
|
+
last_persisted_op: 1
|
|
248
315
|
}
|
|
249
316
|
},
|
|
250
317
|
{
|
|
251
|
-
maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
|
|
252
|
-
arrayFilters: this.syncConfigArrayFilters()
|
|
318
|
+
maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS
|
|
253
319
|
}
|
|
254
320
|
);
|
|
255
321
|
}
|
|
256
322
|
|
|
257
|
-
|
|
323
|
+
/**
|
|
324
|
+
* The persisted mapping of the single sync config that read operations are served from.
|
|
325
|
+
*
|
|
326
|
+
* Reads always operate on a single sync config: read-path storage instances are
|
|
327
|
+
* constructed for the active sync config only (see MongoBucketStorage.getActiveSyncConfig),
|
|
328
|
+
* and checkpoints are served from the single active config (see fetchCheckpointState).
|
|
329
|
+
* Within a single sync config, unique names are the persistence key of its rule mapping,
|
|
330
|
+
* so its name-keyed {@link SingleSyncConfigBucketDefinitionMapping} resolves sources from
|
|
331
|
+
* any parse of that config unambiguously - no parsed-set identity is required.
|
|
332
|
+
*
|
|
333
|
+
* Throws on storage instances with multiple sync configs (replication-side instances),
|
|
334
|
+
* which must not serve reads.
|
|
335
|
+
*/
|
|
336
|
+
private singleSyncConfigMapping(): SingleSyncConfigBucketDefinitionMapping {
|
|
337
|
+
const content = this.replicationStream.syncConfigContent;
|
|
338
|
+
if (content.length != 1) {
|
|
339
|
+
throw new ServiceAssertionError(
|
|
340
|
+
`Read operations require a storage instance with a single sync config, got ${content.length}`
|
|
341
|
+
);
|
|
342
|
+
}
|
|
343
|
+
return content[0].mapping;
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
protected get versionContext(): MongoSyncBucketStorageContextV3 {
|
|
347
|
+
const self = this;
|
|
258
348
|
return {
|
|
259
349
|
db: this.db,
|
|
260
|
-
|
|
261
|
-
|
|
350
|
+
objectStorage: this.objectStorage,
|
|
351
|
+
replicationStreamId: this.replicationStreamId,
|
|
352
|
+
readPreference: this.readPreference,
|
|
353
|
+
get mapping() {
|
|
354
|
+
return self.singleSyncConfigMapping();
|
|
355
|
+
}
|
|
262
356
|
};
|
|
263
357
|
}
|
|
264
358
|
|
|
@@ -271,34 +365,47 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
271
365
|
}
|
|
272
366
|
|
|
273
367
|
protected getBucketDataBatchImpl(
|
|
274
|
-
checkpoint:
|
|
368
|
+
checkpoint: MongoSyncBucketStorageCheckpoint,
|
|
275
369
|
dataBuckets: storage.BucketDataRequest[],
|
|
276
370
|
options?: storage.BucketDataBatchOptions
|
|
277
|
-
): AsyncIterable<storage.SyncBucketDataChunk> {
|
|
371
|
+
): AsyncIterable<storage.SyncBucketDataChunk | storage.SyncBucketDataBatchEnd> {
|
|
278
372
|
return getBucketDataBatchV3(this.versionContext, checkpoint, dataBuckets, options);
|
|
279
373
|
}
|
|
280
374
|
|
|
281
|
-
protected async clearBucketData(
|
|
282
|
-
for (const collection of await this.db.
|
|
375
|
+
protected async clearBucketData(signal?: AbortSignal): Promise<void> {
|
|
376
|
+
for (const collection of await this.db.listBucketDataCollections(this.replicationStreamId)) {
|
|
283
377
|
await collection.drop();
|
|
284
378
|
}
|
|
379
|
+
if (this.objectStorage) {
|
|
380
|
+
const lifecycle = new ObjectStorageLifecycle(this.db, this.replicationStreamId, this.objectStorage);
|
|
381
|
+
await lifecycle.deletePrefix(lifecycle.streamPrefix(), { signal });
|
|
382
|
+
}
|
|
383
|
+
await this.db
|
|
384
|
+
.pendingObjectStorageDeletes(this.replicationStreamId)
|
|
385
|
+
.drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
|
|
386
|
+
.catch((error) => {
|
|
387
|
+
if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
|
|
388
|
+
return;
|
|
389
|
+
}
|
|
390
|
+
throw error;
|
|
391
|
+
});
|
|
285
392
|
}
|
|
286
393
|
|
|
287
394
|
protected async clearParameterIndexes(_signal?: AbortSignal): Promise<void> {
|
|
288
|
-
for (const collection of await this.db.
|
|
395
|
+
for (const collection of await this.db.listParameterIndexCollections(this.replicationStreamId)) {
|
|
289
396
|
await collection.collection.drop();
|
|
290
397
|
}
|
|
291
398
|
}
|
|
292
399
|
|
|
293
400
|
protected async clearSourceRecords(_signal?: AbortSignal): Promise<void> {
|
|
294
|
-
for (const collection of await this.db.
|
|
401
|
+
for (const collection of await this.db.listSourceRecordCollections(this.replicationStreamId)) {
|
|
295
402
|
await collection.drop();
|
|
296
403
|
}
|
|
297
404
|
}
|
|
298
405
|
|
|
299
406
|
protected async clearBucketState(_signal?: AbortSignal): Promise<void> {
|
|
300
407
|
await this.db
|
|
301
|
-
.
|
|
408
|
+
.bucketState(this.replicationStreamId)
|
|
302
409
|
.drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
|
|
303
410
|
.catch((error) => {
|
|
304
411
|
if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
|
|
@@ -310,7 +417,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
310
417
|
|
|
311
418
|
protected async clearSourceTables(_signal?: AbortSignal): Promise<void> {
|
|
312
419
|
await this.db
|
|
313
|
-
.
|
|
420
|
+
.sourceTables(this.replicationStreamId)
|
|
314
421
|
.drop({ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS })
|
|
315
422
|
.catch((error) => {
|
|
316
423
|
if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
|
|
@@ -320,6 +427,21 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
320
427
|
});
|
|
321
428
|
}
|
|
322
429
|
|
|
430
|
+
async cleanupStoppedSyncConfigs(
|
|
431
|
+
options: storage.CleanupStoppedSyncConfigsOptions
|
|
432
|
+
): Promise<storage.CleanupStoppedSyncConfigsResult> {
|
|
433
|
+
return new MongoStoppedSyncConfigCleanup({
|
|
434
|
+
db: this.db,
|
|
435
|
+
replicationStreamId: this.replicationStreamId,
|
|
436
|
+
signal: options.signal,
|
|
437
|
+
logger: options.logger ?? this.logger,
|
|
438
|
+
defaultSchema: options.defaultSchema,
|
|
439
|
+
sourceConnectionTag: options.sourceConnectionTag,
|
|
440
|
+
objectStorage: this.objectStorage,
|
|
441
|
+
clearBatchThrottleRate: this.clearBatchThrottleRate
|
|
442
|
+
}).run();
|
|
443
|
+
}
|
|
444
|
+
|
|
323
445
|
protected getDataBucketChangesImpl(
|
|
324
446
|
options: GetCheckpointChangesOptions
|
|
325
447
|
): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>> {
|
|
@@ -334,7 +456,7 @@ export class MongoSyncBucketStorageV3 extends MongoSyncBucketStorage {
|
|
|
334
456
|
}
|
|
335
457
|
|
|
336
458
|
export async function getParameterSetsV3(
|
|
337
|
-
ctx:
|
|
459
|
+
ctx: MongoSyncBucketStorageContextV3,
|
|
338
460
|
checkpoint: MongoSyncBucketStorageCheckpoint,
|
|
339
461
|
lookups: ScopedParameterLookup[],
|
|
340
462
|
limit: number
|
|
@@ -350,8 +472,8 @@ export async function getParameterSetsV3(
|
|
|
350
472
|
pipeline: mongo.Document[];
|
|
351
473
|
} => {
|
|
352
474
|
const indexId = lookup.indexId;
|
|
353
|
-
const collection = ctx.db.
|
|
354
|
-
const lookupFilter =
|
|
475
|
+
const collection = ctx.db.parameterIndex(ctx.replicationStreamId, indexId);
|
|
476
|
+
const lookupFilter = serializeParameterLookup(lookup);
|
|
355
477
|
|
|
356
478
|
return {
|
|
357
479
|
collection,
|
|
@@ -434,24 +556,38 @@ export async function getParameterSetsV3(
|
|
|
434
556
|
}
|
|
435
557
|
|
|
436
558
|
export async function* getBucketDataBatchV3(
|
|
437
|
-
ctx:
|
|
438
|
-
checkpoint:
|
|
559
|
+
ctx: MongoSyncBucketStorageContextV3,
|
|
560
|
+
checkpoint: MongoSyncBucketStorageCheckpoint,
|
|
439
561
|
dataBuckets: storage.BucketDataRequest[],
|
|
440
562
|
options?: storage.BucketDataBatchOptions
|
|
441
|
-
): AsyncIterable<storage.SyncBucketDataChunk> {
|
|
563
|
+
): AsyncIterable<storage.SyncBucketDataChunk | storage.SyncBucketDataBatchEnd> {
|
|
442
564
|
if (dataBuckets.length == 0) {
|
|
443
565
|
return;
|
|
444
566
|
}
|
|
445
567
|
|
|
446
|
-
if (checkpoint == null) {
|
|
568
|
+
if (checkpoint.checkpoint == null) {
|
|
447
569
|
throw new Error('checkpoint is null');
|
|
448
570
|
}
|
|
449
571
|
|
|
572
|
+
const readPreference = options?.requestHint == 'bulk' ? ctx.readPreference : undefined;
|
|
573
|
+
const readConcern = ctx.readPreference == null ? undefined : 'majority';
|
|
574
|
+
const session =
|
|
575
|
+
readPreference == null || checkpoint.snapshotTime == null
|
|
576
|
+
? undefined
|
|
577
|
+
: ctx.db.client.startSession({ causalConsistency: true });
|
|
578
|
+
await using _ = { [Symbol.asyncDispose]: async () => session?.endSession() };
|
|
579
|
+
|
|
580
|
+
if (session != null) {
|
|
581
|
+
session.advanceOperationTime(checkpoint.snapshotTime);
|
|
582
|
+
session.advanceClusterTime(checkpoint.clusterTime);
|
|
583
|
+
}
|
|
584
|
+
|
|
450
585
|
const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
|
|
451
586
|
const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
|
|
452
|
-
const end = checkpoint;
|
|
453
|
-
let remainingLimit = batchLimit;
|
|
587
|
+
const end = checkpoint.checkpoint;
|
|
454
588
|
|
|
589
|
+
// Group requests by definition, so that we can query each definition's bucket data in a single query.
|
|
590
|
+
// We only return the results of a single query per batch.
|
|
455
591
|
const requestsByDefinition = new Map<string, storage.BucketDataRequest[]>();
|
|
456
592
|
for (const request of dataBuckets) {
|
|
457
593
|
const definitionId = ctx.mapping.bucketSourceId(request.source);
|
|
@@ -461,62 +597,80 @@ export async function* getBucketDataBatchV3(
|
|
|
461
597
|
}
|
|
462
598
|
|
|
463
599
|
const definitionGroups = Array.from(requestsByDefinition.entries());
|
|
464
|
-
for (
|
|
465
|
-
const [definitionId, requests] = definitionGroups[groupIndex];
|
|
600
|
+
for (const [groupIndex, [definitionId, requests]] of definitionGroups.entries()) {
|
|
466
601
|
const hasLaterDefinitionGroups = groupIndex < definitionGroups.length - 1;
|
|
467
602
|
const bucketMap = new Map(requests.map((request) => [request.bucket, request.start]));
|
|
468
|
-
const filters
|
|
603
|
+
const filters = Array.from(bucketMap.entries()).map(([bucket, start]) => ({
|
|
469
604
|
_id: {
|
|
470
|
-
$gt: {
|
|
471
|
-
|
|
472
|
-
o: start
|
|
473
|
-
},
|
|
474
|
-
$lte: {
|
|
475
|
-
b: bucket,
|
|
476
|
-
o: end as any
|
|
477
|
-
}
|
|
478
|
-
}
|
|
479
|
-
}));
|
|
480
|
-
|
|
481
|
-
const cursor = ctx.db.bucketDataV3(ctx.group_id, definitionId).find(
|
|
482
|
-
{
|
|
483
|
-
$or: filters
|
|
605
|
+
$gt: { b: bucket, o: start },
|
|
606
|
+
$lte: { b: bucket, o: new bson.MaxKey() }
|
|
484
607
|
},
|
|
485
|
-
{
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
608
|
+
min_op: { $lte: end }
|
|
609
|
+
// MongoDB Filter<T> doesn't accept compound _id ranges or dotted field paths in its type.
|
|
610
|
+
})) as unknown as mongo.Filter<BucketDataDocumentV3>[];
|
|
611
|
+
|
|
612
|
+
const collection = ctx.db.bucketData(ctx.replicationStreamId, definitionId);
|
|
613
|
+
// MongoDB Filter<T> doesn't accept the $or operator in its type.
|
|
614
|
+
const filter = { $or: filters } as unknown as mongo.Filter<BucketDataDocumentV3>;
|
|
615
|
+
const context = { replicationStreamId: ctx.replicationStreamId, definitionId };
|
|
616
|
+
|
|
617
|
+
const cursorOptions = { limit: batchLimit, batchSize: batchLimit + 1 };
|
|
618
|
+
|
|
619
|
+
// raw: true returns Buffers, but the driver typing doesn't reflect that
|
|
620
|
+
// without an explicit cast to FindCursor<Buffer>.
|
|
621
|
+
const cursor = collection.find(filter, {
|
|
622
|
+
session,
|
|
623
|
+
readPreference,
|
|
624
|
+
readConcern,
|
|
625
|
+
sort: { _id: 1 },
|
|
626
|
+
raw: true,
|
|
627
|
+
maxTimeMS: lib_mongo.db.MONGO_OPERATION_TIMEOUT_MS,
|
|
628
|
+
...cursorOptions
|
|
629
|
+
}) as unknown as mongo.FindCursor<Buffer>;
|
|
630
|
+
|
|
631
|
+
let { data: rawData, hasMore } = await readSingleBatch(cursor).catch((e: unknown) => {
|
|
496
632
|
throw lib_mongo.mapQueryError(e, 'while reading bucket data');
|
|
497
633
|
});
|
|
498
|
-
|
|
499
|
-
|
|
634
|
+
|
|
635
|
+
if (rawData.length >= cursorOptions.limit) {
|
|
636
|
+
hasMore = true;
|
|
500
637
|
}
|
|
501
|
-
|
|
502
|
-
|
|
638
|
+
|
|
639
|
+
// Deserialize the raw documents and cut the batch to a bounded size. Any data not
|
|
640
|
+
// making the cut will be read in the next round, using a fresh query.
|
|
641
|
+
const cutBatch = cutBucketDataBatch(rawData);
|
|
642
|
+
const docs = cutBatch.documents;
|
|
643
|
+
if (cutBatch.wasCut) {
|
|
644
|
+
hasMore = true;
|
|
503
645
|
}
|
|
504
646
|
|
|
505
|
-
|
|
647
|
+
// Hydrate any operations from object storage.
|
|
648
|
+
// In the future we can do this in a more pipelined fashion, but for now we hydrate
|
|
649
|
+
// the entire batch at once.
|
|
650
|
+
await hydrateBucketDataDocuments(docs, ctx.objectStorage, { signal: options?.signal });
|
|
506
651
|
|
|
507
|
-
let
|
|
652
|
+
let currentChunkSizeBytes = 0;
|
|
508
653
|
let currentChunk: utils.SyncBucketData | null = null;
|
|
509
654
|
let targetOp: InternalOpId | null = null;
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
);
|
|
516
|
-
const
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
655
|
+
let seenBuckets = new Set<string>();
|
|
656
|
+
const batchHasMore = hasMore;
|
|
657
|
+
|
|
658
|
+
for (const doc of docs) {
|
|
659
|
+
const bucket = doc._id.b;
|
|
660
|
+
seenBuckets.add(bucket);
|
|
661
|
+
const bucketStart = bucketMap.get(bucket);
|
|
662
|
+
if (bucketStart == null) {
|
|
663
|
+
throw new ServiceAssertionError(`data for unexpected bucket: ${bucket}`);
|
|
664
|
+
}
|
|
665
|
+
let start: ProtocolOpId | undefined = undefined;
|
|
666
|
+
|
|
667
|
+
// Reached a new bucket or size limit: yield the current chunk and start a new one.
|
|
668
|
+
if (
|
|
669
|
+
currentChunk == null ||
|
|
670
|
+
currentChunk.bucket != bucket ||
|
|
671
|
+
currentChunkSizeBytes >= chunkSizeLimitBytes ||
|
|
672
|
+
currentChunk.data.length >= batchLimit
|
|
673
|
+
) {
|
|
520
674
|
if (currentChunk != null) {
|
|
521
675
|
if (currentChunk.bucket == bucket) {
|
|
522
676
|
currentChunk.has_more = true;
|
|
@@ -525,18 +679,15 @@ export async function* getBucketDataBatchV3(
|
|
|
525
679
|
|
|
526
680
|
const yieldChunk = currentChunk;
|
|
527
681
|
currentChunk = null;
|
|
528
|
-
|
|
529
|
-
yield { chunkData: yieldChunk, targetOp
|
|
682
|
+
currentChunkSizeBytes = 0;
|
|
683
|
+
yield { chunkData: yieldChunk, targetOp };
|
|
530
684
|
targetOp = null;
|
|
531
685
|
}
|
|
532
686
|
|
|
533
687
|
if (start == null) {
|
|
534
|
-
|
|
535
|
-
if (startOpId == null) {
|
|
536
|
-
throw new Error(`data for unexpected bucket: ${bucket}`);
|
|
537
|
-
}
|
|
538
|
-
start = internalToExternalOpId(startOpId);
|
|
688
|
+
start = internalToExternalOpId(bucketStart);
|
|
539
689
|
}
|
|
690
|
+
|
|
540
691
|
currentChunk = {
|
|
541
692
|
bucket,
|
|
542
693
|
after: start,
|
|
@@ -546,35 +697,58 @@ export async function* getBucketDataBatchV3(
|
|
|
546
697
|
};
|
|
547
698
|
}
|
|
548
699
|
|
|
549
|
-
const
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
currentChunk.data.push(entry);
|
|
555
|
-
currentChunk.next_after = entry.op_id;
|
|
556
|
-
chunkSizeBytes += rawData.byteLength;
|
|
700
|
+
const rows = extractRowsFromDocument(doc, context, bucketStart, end);
|
|
701
|
+
currentChunk.data.push(...rows.map(mapOpEntry));
|
|
702
|
+
currentChunk.next_after = currentChunk.data.at(-1)?.op_id ?? internalToExternalOpId(end);
|
|
703
|
+
targetOp = maxOpId(targetOp, doc.target_op);
|
|
704
|
+
currentChunkSizeBytes += doc.size;
|
|
557
705
|
}
|
|
558
706
|
|
|
559
707
|
if (currentChunk != null) {
|
|
560
708
|
const yieldChunk = currentChunk;
|
|
561
|
-
|
|
562
|
-
|
|
709
|
+
// The last chunk may contain more data that was cut in this batch.
|
|
710
|
+
yieldChunk.has_more = batchHasMore;
|
|
711
|
+
yield { chunkData: yieldChunk, targetOp };
|
|
563
712
|
}
|
|
564
713
|
|
|
565
|
-
if (batchHasMore
|
|
566
|
-
|
|
714
|
+
if (!batchHasMore) {
|
|
715
|
+
for (const bucket of bucketMap.keys()) {
|
|
716
|
+
if (!seenBuckets.has(bucket)) {
|
|
717
|
+
// We processed everything for this definition group, but this bucket had no data in the batch.
|
|
718
|
+
// Yield an empty chunk to indicate that it is complete.
|
|
719
|
+
// This prevents re-querying the same bucket in the next batch.
|
|
720
|
+
yield {
|
|
721
|
+
chunkData: {
|
|
722
|
+
bucket,
|
|
723
|
+
after: internalToExternalOpId(bucketMap.get(bucket)!),
|
|
724
|
+
has_more: false,
|
|
725
|
+
data: [],
|
|
726
|
+
next_after: internalToExternalOpId(end)
|
|
727
|
+
},
|
|
728
|
+
targetOp
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
if (currentChunk != null) {
|
|
735
|
+
// We yielded data in this group (aside from empty buckets).
|
|
736
|
+
// Return to the caller to allow them to process it before continuing to the next group.
|
|
737
|
+
yield { hasMore: batchHasMore || hasLaterDefinitionGroups };
|
|
738
|
+
break;
|
|
739
|
+
} else {
|
|
740
|
+
// No data in this definition group - continue in the next group.
|
|
567
741
|
}
|
|
568
742
|
}
|
|
569
743
|
}
|
|
570
744
|
|
|
571
745
|
export async function getDataBucketChangesV3(
|
|
572
|
-
ctx:
|
|
746
|
+
ctx: MongoSyncBucketStorageContextV3,
|
|
573
747
|
options: GetCheckpointChangesOptions
|
|
574
748
|
): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>> {
|
|
575
749
|
const limit = 1000;
|
|
576
750
|
const bucketStateUpdates = await ctx.db
|
|
577
|
-
.
|
|
751
|
+
.bucketState(ctx.replicationStreamId)
|
|
578
752
|
.aggregate<{ _id: string; last_op: bigint }>(
|
|
579
753
|
[
|
|
580
754
|
{
|
|
@@ -611,14 +785,14 @@ export async function getDataBucketChangesV3(
|
|
|
611
785
|
}
|
|
612
786
|
|
|
613
787
|
export async function getParameterBucketChangesV3(
|
|
614
|
-
ctx:
|
|
788
|
+
ctx: MongoSyncBucketStorageContextV3,
|
|
615
789
|
options: GetCheckpointChangesOptions
|
|
616
790
|
): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
|
|
617
791
|
const limit = 1000;
|
|
618
792
|
const indexIds = ctx.mapping.allParameterIndexIds();
|
|
619
793
|
const collections = indexIds.map((indexId) => ({
|
|
620
794
|
indexId,
|
|
621
|
-
collection: ctx.db.
|
|
795
|
+
collection: ctx.db.parameterIndex(ctx.replicationStreamId, indexId)
|
|
622
796
|
}));
|
|
623
797
|
if (collections.length == 0) {
|
|
624
798
|
return {
|
|
@@ -671,8 +845,6 @@ export async function getParameterBucketChangesV3(
|
|
|
671
845
|
invalidateParameterBuckets: invalidateParameterUpdates,
|
|
672
846
|
updatedParameterLookups: invalidateParameterUpdates
|
|
673
847
|
? new Set<string>()
|
|
674
|
-
: new Set<string>(
|
|
675
|
-
parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookupV3(p.lookup, p.indexId)))
|
|
676
|
-
)
|
|
848
|
+
: new Set<string>(parameterUpdates.map((p) => JSONBig.stringify(deserializeParameterLookup(p.lookup, p.indexId))))
|
|
677
849
|
};
|
|
678
850
|
}
|