@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -3,39 +3,88 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
|
3
3
|
import { storage } from '@powersync/service-core';
|
|
4
4
|
import * as bson from 'bson';
|
|
5
5
|
import { mongoTableId } from '../../../utils/util.js';
|
|
6
|
-
import {
|
|
6
|
+
import { canCheckpointState } from '../CheckpointState.js';
|
|
7
7
|
import { MongoBucketBatch } from '../MongoBucketBatch.js';
|
|
8
8
|
import { syncRuleStateUpdatePipeline } from '../SyncRuleStateUpdate.js';
|
|
9
9
|
import { PersistedBatchV3 } from './PersistedBatchV3.js';
|
|
10
10
|
import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js';
|
|
11
|
-
|
|
12
|
-
return left.length == right.length && left.every((value, index) => value == right[index]);
|
|
13
|
-
}
|
|
11
|
+
import { createNewSourceTable, designateEventCarrier, overlappingSourceTableFilter, planSourceTableReconciliation, sourceTableDesiredResolution, sourceTableFromDocument } from './source-table-utils.js';
|
|
14
12
|
export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
15
13
|
store;
|
|
16
|
-
|
|
14
|
+
syncConfigIds;
|
|
17
15
|
needsActivationV3 = true;
|
|
18
16
|
lastWaitingLogThrottledV3 = 0;
|
|
19
17
|
constructor(options) {
|
|
20
18
|
super(options);
|
|
21
|
-
|
|
19
|
+
const syncConfigIds = options.syncConfigIds ?? [];
|
|
20
|
+
if (syncConfigIds.length == 0) {
|
|
22
21
|
throw new ReplicationAssertionError('Missing sync config id for v3 batch');
|
|
23
22
|
}
|
|
24
|
-
this.
|
|
25
|
-
this.store = new SourceRecordStoreV3(this.db, this.
|
|
23
|
+
this.syncConfigIds = syncConfigIds;
|
|
24
|
+
this.store = new SourceRecordStoreV3(this.db, this.replicationStreamId, this.mapping);
|
|
26
25
|
}
|
|
27
26
|
createPersistedBatch(writtenSize) {
|
|
28
|
-
return new PersistedBatchV3(this.db, this.
|
|
29
|
-
logger: this.logger
|
|
27
|
+
return new PersistedBatchV3(this.db, this.replicationStreamId, this.mapping, writtenSize, {
|
|
28
|
+
logger: this.logger,
|
|
29
|
+
objectStorage: this.options.objectStorage,
|
|
30
|
+
inlineThresholdBytes: this.options.inlineThresholdBytes
|
|
30
31
|
});
|
|
31
32
|
}
|
|
32
33
|
get sourceRecordStore() {
|
|
33
34
|
return this.store;
|
|
34
35
|
}
|
|
36
|
+
async onReplicationTransactionFlush(session, lastOp) {
|
|
37
|
+
// Durably advance the stream-level head of persisted ops within the flush transaction.
|
|
38
|
+
// This ensures a checkpoint created later (even by an empty commit, or by a freshly-appended
|
|
39
|
+
// config that replicates nothing) covers all ops persisted before a potential crash.
|
|
40
|
+
await this.db.sync_rules.updateOne({
|
|
41
|
+
_id: this.replicationStreamId
|
|
42
|
+
}, {
|
|
43
|
+
$max: {
|
|
44
|
+
last_persisted_op: lastOp
|
|
45
|
+
}
|
|
46
|
+
}, { session });
|
|
47
|
+
}
|
|
48
|
+
selectedSyncConfigObjectIds(syncConfigIds) {
|
|
49
|
+
const selectedIds = new Set(this.syncConfigIds.map((id) => id.toHexString()));
|
|
50
|
+
return syncConfigIds
|
|
51
|
+
.filter((id) => selectedIds.has(id) && bson.ObjectId.isValid(id))
|
|
52
|
+
.map((id) => new bson.ObjectId(id));
|
|
53
|
+
}
|
|
54
|
+
relevantSyncConfigIds(table) {
|
|
55
|
+
const bucketDataSourceIds = table.bucketDataSources.map((source) => this.mapping.bucketSourceId(source));
|
|
56
|
+
const parameterLookupSourceIds = table.parameterLookupSources.map((source) => this.mapping.parameterLookupId(source));
|
|
57
|
+
return this.selectedSyncConfigObjectIds(this.mapping.syncConfigIdsForSourceTable(this.syncConfigIds.map((id) => id.toHexString()), table.ref, bucketDataSourceIds, parameterLookupSourceIds));
|
|
58
|
+
}
|
|
59
|
+
relevantSyncConfigIdsForTables(tables) {
|
|
60
|
+
const ids = new Map();
|
|
61
|
+
for (const table of tables) {
|
|
62
|
+
for (const id of this.relevantSyncConfigIds(table)) {
|
|
63
|
+
ids.set(id.toHexString(), id);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return [...ids.values()];
|
|
67
|
+
}
|
|
68
|
+
snapshotBlockingSourceTablesFilter() {
|
|
69
|
+
const clauses = this.syncConfigIds.flatMap((syncConfigId) => {
|
|
70
|
+
const filter = this.mapping.snapshotBlockingSourceTablesFilter(syncConfigId.toHexString());
|
|
71
|
+
return filter.$or ?? [];
|
|
72
|
+
});
|
|
73
|
+
if (clauses.length == 0) {
|
|
74
|
+
return {
|
|
75
|
+
snapshot_done: false,
|
|
76
|
+
_id: { $exists: false }
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
return {
|
|
80
|
+
snapshot_done: false,
|
|
81
|
+
$or: clauses
|
|
82
|
+
};
|
|
83
|
+
}
|
|
35
84
|
async cleanupDroppedSourceTables(sourceTables) {
|
|
36
85
|
for (const table of sourceTables) {
|
|
37
86
|
await this.db
|
|
38
|
-
.
|
|
87
|
+
.sourceRecords(this.replicationStreamId, mongoTableId(table.id))
|
|
39
88
|
.drop()
|
|
40
89
|
.catch((error) => {
|
|
41
90
|
if (lib_mongo.isMongoServerError(error) && error.codeName === 'NamespaceNotFound') {
|
|
@@ -46,179 +95,91 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
46
95
|
}
|
|
47
96
|
}
|
|
48
97
|
async resolveTables(options) {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
const
|
|
98
|
+
// The test-only override is a whole parsed set, so the sync rules and the mapping
|
|
99
|
+
// used below always come from the same parse.
|
|
100
|
+
const parsedOverride = options.parsedSyncConfig;
|
|
101
|
+
const syncConfig = parsedOverride?.hydratedSyncConfig ?? this.sync_rules;
|
|
102
|
+
const mapping = parsedOverride?.mapping ?? this.mapping;
|
|
52
103
|
const { connection_id, source } = options;
|
|
53
|
-
const
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
104
|
+
const context = {
|
|
105
|
+
connectionId: connection_id,
|
|
106
|
+
connectionTag: source.connectionTag,
|
|
107
|
+
identity: {
|
|
108
|
+
schema: source.schema,
|
|
109
|
+
name: source.name,
|
|
110
|
+
objectId: source.objectId,
|
|
111
|
+
replicaIdColumns: source.replicaIdColumns.map((column) => ({
|
|
112
|
+
name: column.name,
|
|
113
|
+
type: column.type,
|
|
114
|
+
type_oid: column.typeId
|
|
115
|
+
}))
|
|
116
|
+
},
|
|
117
|
+
storeCurrentData: source.sendsCompleteRows !== true,
|
|
118
|
+
syncConfig,
|
|
119
|
+
mapping,
|
|
120
|
+
desired: sourceTableDesiredResolution(syncConfig, source, mapping)
|
|
121
|
+
};
|
|
59
122
|
let result = null;
|
|
60
|
-
const
|
|
61
|
-
await
|
|
62
|
-
|
|
63
|
-
const
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
const
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
const retainedDocIds = [];
|
|
82
|
-
const tables = [];
|
|
83
|
-
let retainedEventOnlyTable = false;
|
|
84
|
-
for (const doc of exactDocs) {
|
|
85
|
-
const bucketDataSourceIds = doc.bucket_data_source_ids.filter((id) => desiredBucketIds.has(id));
|
|
86
|
-
const parameterLookupSourceIds = doc.parameter_lookup_source_ids.filter((id) => desiredLookupIds.has(id));
|
|
87
|
-
const coversDesiredMembership = bucketDataSourceIds.length > 0 || parameterLookupSourceIds.length > 0;
|
|
88
|
-
const coversEventOnlyTable = !desiredHasMembership && triggersEvent && !retainedEventOnlyTable;
|
|
89
|
-
for (const id of bucketDataSourceIds) {
|
|
90
|
-
coveredBucketIds.add(id);
|
|
91
|
-
}
|
|
92
|
-
for (const id of parameterLookupSourceIds) {
|
|
93
|
-
coveredLookupIds.add(id);
|
|
94
|
-
}
|
|
95
|
-
if (!sameStringArray(doc.bucket_data_source_ids, bucketDataSourceIds) ||
|
|
96
|
-
!sameStringArray(doc.parameter_lookup_source_ids, parameterLookupSourceIds)) {
|
|
97
|
-
await col.updateOne({ _id: doc._id }, {
|
|
98
|
-
$set: {
|
|
99
|
-
bucket_data_source_ids: bucketDataSourceIds,
|
|
100
|
-
parameter_lookup_source_ids: parameterLookupSourceIds
|
|
101
|
-
}
|
|
102
|
-
}, { session });
|
|
103
|
-
}
|
|
104
|
-
if (coversDesiredMembership || coversEventOnlyTable) {
|
|
105
|
-
if (coversEventOnlyTable) {
|
|
106
|
-
retainedEventOnlyTable = true;
|
|
123
|
+
const session = this.db.client.startSession();
|
|
124
|
+
await using _ = { [Symbol.asyncDispose]: () => session.endSession() };
|
|
125
|
+
await session.withTransaction(async () => {
|
|
126
|
+
const col = this.db.sourceTables(this.replicationStreamId);
|
|
127
|
+
// Fetch every persisted source-table doc that can overlap this physical table.
|
|
128
|
+
// Exact-identity docs are candidates for reuse; non-exact overlaps are possible drops.
|
|
129
|
+
const candidateDocs = await col
|
|
130
|
+
.find(overlappingSourceTableFilter(connection_id, context.identity), { session })
|
|
131
|
+
.toArray();
|
|
132
|
+
// Pure planning: which docs to retain (and narrow), whether a new doc is needed for
|
|
133
|
+
// uncovered memberships, and which docs conflict with the current identity.
|
|
134
|
+
const plan = planSourceTableReconciliation(candidateDocs, context);
|
|
135
|
+
// Persist narrowing for incomplete snapshots only. Snapshot-complete docs keep stale
|
|
136
|
+
// coverage ids so compatible future configs can reuse already-snapshotted data.
|
|
137
|
+
// Narrowing occurs after removing a sync config, meaning we don't process those
|
|
138
|
+
// definitions anymore.
|
|
139
|
+
for (const update of plan.narrowingUpdates) {
|
|
140
|
+
await col.updateOne({ _id: update.id }, {
|
|
141
|
+
$set: {
|
|
142
|
+
bucket_data_source_ids: update.memberships.bucketDataSourceIds,
|
|
143
|
+
parameter_lookup_source_ids: update.memberships.parameterLookupSourceIds
|
|
107
144
|
}
|
|
108
|
-
|
|
109
|
-
tables.push(this.sourceTableFromDocument({
|
|
110
|
-
...doc,
|
|
111
|
-
bucket_data_source_ids: bucketDataSourceIds,
|
|
112
|
-
parameter_lookup_source_ids: parameterLookupSourceIds
|
|
113
|
-
}, connectionTag, syncRules, {
|
|
114
|
-
bucketDataSources: bucketDataSourceIds.map((id) => bucketSourceById.get(id)),
|
|
115
|
-
parameterLookupSources: parameterLookupSourceIds.map((id) => parameterLookupSourceById.get(id))
|
|
116
|
-
}));
|
|
117
|
-
}
|
|
145
|
+
}, { session });
|
|
118
146
|
}
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
if (
|
|
147
|
+
// Any desired membership not covered by an existing doc gets a new source table.
|
|
148
|
+
// That table snapshots only the uncovered memberships.
|
|
149
|
+
if (plan.newTableMemberships != null) {
|
|
122
150
|
const id = options.idGenerator ? options.idGenerator() : new bson.ObjectId();
|
|
123
|
-
const
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
replicaIdColumns,
|
|
128
|
-
snapshotComplete: false,
|
|
129
|
-
bucketDataSources: uncoveredBucketIds.map((id) => bucketSourceById.get(id)),
|
|
130
|
-
parameterLookupSources: uncoveredLookupIds.map((id) => parameterLookupSourceById.get(id))
|
|
131
|
-
});
|
|
132
|
-
sourceTable.syncData = uncoveredBucketIds.length > 0;
|
|
133
|
-
sourceTable.syncParameters = uncoveredLookupIds.length > 0;
|
|
134
|
-
sourceTable.syncEvent = triggersEvent;
|
|
135
|
-
const createDoc = {
|
|
136
|
-
_id: id,
|
|
137
|
-
connection_id,
|
|
138
|
-
relation_id: objectId,
|
|
139
|
-
schema_name: schema,
|
|
140
|
-
table_name: name,
|
|
141
|
-
replica_id_columns: null,
|
|
142
|
-
replica_id_columns2: normalizedReplicaIdColumns,
|
|
143
|
-
snapshot_done: false,
|
|
144
|
-
snapshot_status: undefined,
|
|
145
|
-
bucket_data_source_ids: uncoveredBucketIds,
|
|
146
|
-
parameter_lookup_source_ids: uncoveredLookupIds
|
|
147
|
-
};
|
|
148
|
-
await col.insertOne(createDoc, { session });
|
|
149
|
-
initializeSourceRecordsFor.push(createDoc._id);
|
|
150
|
-
retainedDocIds.push(createDoc._id);
|
|
151
|
-
tables.push(sourceTable);
|
|
152
|
-
}
|
|
153
|
-
const conflictFilter = [{ schema_name: schema, table_name: name }];
|
|
154
|
-
if (objectId != null) {
|
|
155
|
-
conflictFilter.push({ relation_id: objectId });
|
|
151
|
+
const { doc, table } = createNewSourceTable(id, plan.newTableMemberships, context);
|
|
152
|
+
await col.insertOne(doc, { session });
|
|
153
|
+
await this.db.initializeSourceRecordsCollection(this.replicationStreamId, doc._id, session);
|
|
154
|
+
plan.tables.push(table);
|
|
156
155
|
}
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
connection_id,
|
|
160
|
-
_id: { $nin: retainedDocIds },
|
|
161
|
-
$or: conflictFilter
|
|
162
|
-
}, { session })
|
|
163
|
-
.toArray();
|
|
156
|
+
// If memberships are split across multiple source tables, only one may fire events.
|
|
157
|
+
designateEventCarrier(plan.tables, context.desired.triggersEvent);
|
|
164
158
|
result = {
|
|
165
|
-
tables,
|
|
166
|
-
dropTables:
|
|
159
|
+
tables: plan.tables,
|
|
160
|
+
dropTables: plan.dropDocs.map((doc) => sourceTableFromDocument(doc, context.connectionTag, syncConfig, mapping))
|
|
167
161
|
};
|
|
168
162
|
});
|
|
169
|
-
for (const sourceTableId of initializeSourceRecordsFor) {
|
|
170
|
-
await this.db.initializeSourceRecordsCollection(this.group_id, sourceTableId);
|
|
171
|
-
}
|
|
172
163
|
return result;
|
|
173
164
|
}
|
|
174
|
-
sourceTableFromDocument(doc, connectionTag, syncRules, memberships) {
|
|
175
|
-
const resolvedMemberships = memberships ?? this.sourceTableMembershipsFromDocument(doc, syncRules);
|
|
176
|
-
const table = new storage.SourceTable({
|
|
177
|
-
id: doc._id,
|
|
178
|
-
ref: {
|
|
179
|
-
connectionTag,
|
|
180
|
-
schema: doc.schema_name,
|
|
181
|
-
name: doc.table_name
|
|
182
|
-
},
|
|
183
|
-
objectId: doc.relation_id,
|
|
184
|
-
replicaIdColumns: doc.replica_id_columns2.map((c) => ({ name: c.name, typeId: c.type_oid, type: c.type })),
|
|
185
|
-
snapshotComplete: doc.snapshot_done ?? true,
|
|
186
|
-
bucketDataSources: resolvedMemberships.bucketDataSources,
|
|
187
|
-
parameterLookupSources: resolvedMemberships.parameterLookupSources
|
|
188
|
-
});
|
|
189
|
-
table.syncData = table.bucketDataSources.length > 0;
|
|
190
|
-
table.syncParameters = table.parameterLookupSources.length > 0;
|
|
191
|
-
table.syncEvent = syncRules.tableTriggersEvent(table.ref);
|
|
192
|
-
table.snapshotStatus =
|
|
193
|
-
doc.snapshot_status == null
|
|
194
|
-
? undefined
|
|
195
|
-
: {
|
|
196
|
-
lastKey: doc.snapshot_status.last_key?.buffer ?? null,
|
|
197
|
-
totalEstimatedCount: doc.snapshot_status.total_estimated_count,
|
|
198
|
-
replicatedCount: doc.snapshot_status.replicated_count
|
|
199
|
-
};
|
|
200
|
-
return table;
|
|
201
|
-
}
|
|
202
|
-
sourceTableMembershipsFromDocument(doc, syncRules) {
|
|
203
|
-
const bucketDataSourceIds = new Set(doc.bucket_data_source_ids);
|
|
204
|
-
const parameterLookupSourceIds = new Set(doc.parameter_lookup_source_ids);
|
|
205
|
-
return {
|
|
206
|
-
bucketDataSources: syncRules.bucketDataSources.filter((source) => bucketDataSourceIds.has(this.mapping.bucketSourceId(source))),
|
|
207
|
-
parameterLookupSources: syncRules.bucketParameterLookupSources.filter((source) => parameterLookupSourceIds.has(this.mapping.parameterLookupId(source)))
|
|
208
|
-
};
|
|
209
|
-
}
|
|
210
165
|
async getSourceTableStatus(table) {
|
|
211
166
|
const doc = (await this.db
|
|
212
|
-
.
|
|
167
|
+
.sourceTables(this.replicationStreamId)
|
|
213
168
|
.findOne({ _id: mongoTableId(table.id) }, { session: this.session }));
|
|
214
169
|
if (doc == null) {
|
|
215
170
|
return null;
|
|
216
171
|
}
|
|
217
|
-
|
|
172
|
+
const refreshed = sourceTableFromDocument(doc, table.ref.connectionTag, this.sync_rules, this.mapping);
|
|
173
|
+
// The event-carrier designation is decided per resolveTables result and not persisted -
|
|
174
|
+
// preserve the caller's designation instead of recomputing it from the ref, so that
|
|
175
|
+
// refreshing a non-carrier table does not make it fire events.
|
|
176
|
+
refreshed.syncEvent = table.syncEvent;
|
|
177
|
+
return refreshed;
|
|
218
178
|
}
|
|
219
179
|
async commit(lsn, options) {
|
|
220
180
|
const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
|
|
221
181
|
await this.flush(options);
|
|
182
|
+
using _ = this.tracer.span('storage', 'commit');
|
|
222
183
|
const now = new Date();
|
|
223
184
|
await this.db.write_checkpoints.updateMany({
|
|
224
185
|
processed_at_lsn: null,
|
|
@@ -230,94 +191,145 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
230
191
|
}, {
|
|
231
192
|
session: this.session
|
|
232
193
|
});
|
|
233
|
-
const preUpdateDocument = await this.db.sync_rules.findOne({
|
|
234
|
-
_id: this.
|
|
235
|
-
'sync_configs._id': this.
|
|
194
|
+
const preUpdateDocument = (await this.db.sync_rules.findOne({
|
|
195
|
+
_id: this.replicationStreamId,
|
|
196
|
+
'sync_configs._id': { $in: this.syncConfigIds }
|
|
236
197
|
}, {
|
|
237
198
|
session: this.session,
|
|
238
199
|
projection: {
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
$elemMatch: {
|
|
242
|
-
_id: this.syncConfigId
|
|
243
|
-
}
|
|
244
|
-
}
|
|
200
|
+
sync_configs: 1,
|
|
201
|
+
last_persisted_op: 1
|
|
245
202
|
}
|
|
246
|
-
});
|
|
247
|
-
const
|
|
248
|
-
if (
|
|
249
|
-
throw new ReplicationAssertionError(`Failed to update checkpoint - no matching sync_config for _id: ${this.
|
|
203
|
+
}));
|
|
204
|
+
const states = preUpdateDocument?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ?? [];
|
|
205
|
+
if (states.length == 0) {
|
|
206
|
+
throw new ReplicationAssertionError(`Failed to update checkpoint - no matching sync_config for _id: ${this.replicationStreamId}/${this.syncConfigIds
|
|
207
|
+
.map((id) => id.toHexString())
|
|
208
|
+
.join(',')}`);
|
|
250
209
|
}
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
persistedOp: this.persisted_op,
|
|
259
|
-
createEmptyCheckpoints
|
|
260
|
-
});
|
|
261
|
-
const updateSet = {
|
|
262
|
-
last_keepalive_ts: now,
|
|
263
|
-
last_fatal_error: null,
|
|
264
|
-
last_fatal_error_ts: null,
|
|
265
|
-
'sync_configs.$[config].keepalive_op': checkpointState.newKeepaliveOp,
|
|
266
|
-
'sync_configs.$[config].last_checkpoint': checkpointState.newLastCheckpoint
|
|
267
|
-
};
|
|
268
|
-
if (checkpointState.checkpointCreated) {
|
|
269
|
-
updateSet['sync_configs.$[config].last_checkpoint_lsn'] = lsn;
|
|
270
|
-
updateSet['snapshot_lsn'] = null;
|
|
271
|
-
updateSet['last_checkpoint_ts'] = now;
|
|
210
|
+
// The replication job / MongoBucketBatch must be constructed with all replicating (PROCESSING / ACTIVE)
|
|
211
|
+
// sync configs in the stream, otherwise we'll get inconsistencies. Configs in other states (e.g. STOP)
|
|
212
|
+
// remain embedded in the document and are ignored here.
|
|
213
|
+
const missingSyncConfig = preUpdateDocument.sync_configs.find((config) => [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE].includes(config.state) &&
|
|
214
|
+
!this.syncConfigIds.some((id) => id.equals(config._id)));
|
|
215
|
+
if (missingSyncConfig != null) {
|
|
216
|
+
throw new ReplicationAssertionError(`Replication job not configured for sync config ${missingSyncConfig._id}`);
|
|
272
217
|
}
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
218
|
+
// Effective head of the stream's op sequence.
|
|
219
|
+
// last_persisted_op is $max-advanced durably in the same transaction as every flush, and this
|
|
220
|
+
// read uses the same session, so it covers all ops persisted by this batch.
|
|
221
|
+
const newCheckpoint = preUpdateDocument?.last_persisted_op == null ? 0n : BigInt(preUpdateDocument.last_persisted_op);
|
|
222
|
+
let checkpointBlocked = false;
|
|
223
|
+
let checkpointCreated = false;
|
|
224
|
+
let checkpointLogState = null;
|
|
225
|
+
const unblockedConfigIds = [];
|
|
226
|
+
for (const state of states) {
|
|
227
|
+
if (state.last_checkpoint != null && state.last_checkpoint > newCheckpoint) {
|
|
228
|
+
// last_persisted_op is $max-advanced durably in the same transaction as every flush, and
|
|
229
|
+
// checkpoints are only ever created at that head, so a checkpoint past the head means the
|
|
230
|
+
// op sequence or the stored state is corrupt.
|
|
231
|
+
throw new ReplicationAssertionError(`Invariant violation: sync config ${state._id} has last_checkpoint ${state.last_checkpoint} > stream head ${newCheckpoint}`);
|
|
232
|
+
}
|
|
233
|
+
const canCheckpoint = canCheckpointState(lsn, {
|
|
234
|
+
snapshotDone: state.snapshot_done === true,
|
|
235
|
+
lastCheckpointLsn: state.last_checkpoint_lsn,
|
|
236
|
+
noCheckpointBefore: state.no_checkpoint_before
|
|
237
|
+
});
|
|
238
|
+
if (!canCheckpoint) {
|
|
239
|
+
checkpointBlocked = true;
|
|
240
|
+
// Log the first blocked config's state.
|
|
241
|
+
checkpointLogState ??= {
|
|
285
242
|
snapshot_done: state.snapshot_done,
|
|
286
243
|
last_checkpoint_lsn: state.last_checkpoint_lsn,
|
|
287
244
|
no_checkpoint_before: state.no_checkpoint_before
|
|
288
|
-
}
|
|
245
|
+
};
|
|
246
|
+
continue;
|
|
247
|
+
}
|
|
248
|
+
checkpointCreated ||= createEmptyCheckpoints || state.last_checkpoint !== newCheckpoint;
|
|
249
|
+
unblockedConfigIds.push(state._id);
|
|
250
|
+
}
|
|
251
|
+
// Every commit advances the stream's resume position: commit() flushes first, so all
|
|
252
|
+
// source changes up to this lsn have been persisted, even when checkpoints are blocked.
|
|
253
|
+
// In the future we could also advance this on flush, when the connector provides the
|
|
254
|
+
// current position (see setResumeLsn, which connectors may already call after flushing).
|
|
255
|
+
const resumeLsnUpdate = { resume_lsn: lsn };
|
|
256
|
+
if (unblockedConfigIds.length > 0) {
|
|
257
|
+
// All unblocked configs get the SAME new value, so we apply it with a single updateOne
|
|
258
|
+
// (single-document atomicity).
|
|
259
|
+
const updateSet = {
|
|
260
|
+
last_keepalive_ts: now,
|
|
261
|
+
last_fatal_error: null,
|
|
262
|
+
last_fatal_error_ts: null
|
|
263
|
+
};
|
|
264
|
+
// Only advance checkpoint fields when an actual (non-empty) checkpoint is created, matching
|
|
265
|
+
// the previous per-config / v1 behaviour.
|
|
266
|
+
if (checkpointCreated) {
|
|
267
|
+
updateSet['sync_configs.$[config].last_checkpoint'] = newCheckpoint;
|
|
268
|
+
updateSet['sync_configs.$[config].last_checkpoint_lsn'] = lsn;
|
|
269
|
+
updateSet['last_checkpoint_ts'] = now;
|
|
270
|
+
}
|
|
271
|
+
await this.db.sync_rules.updateOne({
|
|
272
|
+
_id: this.replicationStreamId,
|
|
273
|
+
'sync_configs._id': { $in: unblockedConfigIds }
|
|
274
|
+
}, { $set: updateSet, $max: resumeLsnUpdate }, {
|
|
275
|
+
session: this.session,
|
|
276
|
+
arrayFilters: checkpointCreated ? [{ 'config._id': { $in: unblockedConfigIds } }] : undefined
|
|
277
|
+
});
|
|
278
|
+
}
|
|
279
|
+
else {
|
|
280
|
+
// All selected configs are blocked - only update keepalive/error tracking and the
|
|
281
|
+
// resume position.
|
|
282
|
+
await this.db.sync_rules.updateOne({
|
|
283
|
+
_id: this.replicationStreamId
|
|
284
|
+
}, {
|
|
285
|
+
$set: {
|
|
286
|
+
last_keepalive_ts: now,
|
|
287
|
+
last_fatal_error: null,
|
|
288
|
+
last_fatal_error_ts: null
|
|
289
|
+
},
|
|
290
|
+
$max: resumeLsnUpdate
|
|
291
|
+
}, { session: this.session });
|
|
292
|
+
}
|
|
293
|
+
if (checkpointBlocked) {
|
|
294
|
+
if (Date.now() - this.lastWaitingLogThrottledV3 > 5_000) {
|
|
295
|
+
this.logger.info(`Waiting before creating checkpoint, currently at ${lsn}. Current state: ${JSON.stringify(checkpointLogState)}`);
|
|
289
296
|
this.lastWaitingLogThrottledV3 = Date.now();
|
|
290
297
|
}
|
|
291
298
|
}
|
|
292
299
|
else {
|
|
293
|
-
if (
|
|
294
|
-
this.logger.debug(`Created checkpoint at ${lsn}
|
|
300
|
+
if (checkpointCreated) {
|
|
301
|
+
this.logger.debug(`Created checkpoint at ${lsn}`);
|
|
295
302
|
}
|
|
296
303
|
await this.autoActivateV3(lsn);
|
|
304
|
+
// All configs are now checkpointed at newCheckpoint (the stream head).
|
|
305
|
+
await this.sourceRecordStore.postCommitCleanup(newCheckpoint, this.logger);
|
|
306
|
+
}
|
|
307
|
+
if (checkpointCreated) {
|
|
297
308
|
await this.db.notifyCheckpoint();
|
|
298
|
-
this.persisted_op = null;
|
|
299
|
-
this.last_checkpoint_lsn = lsn;
|
|
300
|
-
if (checkpointState.newLastCheckpoint != null) {
|
|
301
|
-
await this.sourceRecordStore.postCommitCleanup(checkpointState.newLastCheckpoint, this.logger);
|
|
302
|
-
}
|
|
303
309
|
}
|
|
304
310
|
return {
|
|
305
|
-
checkpointBlocked
|
|
306
|
-
checkpointCreated
|
|
311
|
+
checkpointBlocked,
|
|
312
|
+
checkpointCreated
|
|
307
313
|
};
|
|
308
314
|
}
|
|
309
315
|
async keepalive(lsn) {
|
|
310
316
|
return await this.commit(lsn, { createEmptyCheckpoints: true });
|
|
311
317
|
}
|
|
312
318
|
async setResumeLsn(lsn) {
|
|
319
|
+
using _ = this.tracer.span('storage', 'set_resume_lsn');
|
|
313
320
|
await this.db.sync_rules.updateOne({
|
|
314
|
-
_id: this.
|
|
315
|
-
'sync_configs._id': this.syncConfigId
|
|
321
|
+
_id: this.replicationStreamId
|
|
316
322
|
}, {
|
|
317
323
|
$set: {
|
|
318
|
-
|
|
324
|
+
resume_lsn: lsn
|
|
319
325
|
}
|
|
320
|
-
}, {
|
|
326
|
+
}, {
|
|
327
|
+
session: this.session,
|
|
328
|
+
// Losing occasional resume LSN is fine. That may mean reprocessing
|
|
329
|
+
// some source changes in some edge cases, which is not an issue since
|
|
330
|
+
// changes are processed in an idempotent way.
|
|
331
|
+
writeConcern: { w: 1 }
|
|
332
|
+
});
|
|
321
333
|
}
|
|
322
334
|
async autoActivateV3(lsn) {
|
|
323
335
|
if (!this.needsActivationV3) {
|
|
@@ -325,30 +337,32 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
325
337
|
}
|
|
326
338
|
const session = this.session;
|
|
327
339
|
let activated = false;
|
|
340
|
+
let needsFutureActivationCheck = true;
|
|
328
341
|
await session.withTransaction(async () => {
|
|
342
|
+
// Reset on transaction retries.
|
|
343
|
+
needsFutureActivationCheck = true;
|
|
344
|
+
activated = false;
|
|
329
345
|
const doc = await this.db.sync_rules.findOne({
|
|
330
|
-
_id: this.
|
|
331
|
-
'sync_configs._id': this.
|
|
346
|
+
_id: this.replicationStreamId,
|
|
347
|
+
'sync_configs._id': { $in: this.syncConfigIds }
|
|
332
348
|
}, {
|
|
333
349
|
session,
|
|
334
350
|
projection: {
|
|
335
351
|
state: 1,
|
|
336
|
-
sync_configs:
|
|
337
|
-
$elemMatch: {
|
|
338
|
-
_id: this.syncConfigId
|
|
339
|
-
}
|
|
340
|
-
}
|
|
352
|
+
sync_configs: 1
|
|
341
353
|
}
|
|
342
354
|
});
|
|
343
|
-
const
|
|
344
|
-
if (doc
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
355
|
+
const states = doc?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ?? [];
|
|
356
|
+
if (doc == null || states.length == 0) {
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
const processingStates = states.filter((state) => state.state == storage.SyncRuleState.PROCESSING);
|
|
360
|
+
if (doc.state == storage.SyncRuleState.PROCESSING &&
|
|
361
|
+
processingStates.length == states.length &&
|
|
362
|
+
states.every((state) => state.snapshot_done && state.last_checkpoint != null)) {
|
|
349
363
|
await this.db.sync_rules.updateOne({
|
|
350
|
-
_id: this.
|
|
351
|
-
'sync_configs._id': this.
|
|
364
|
+
_id: this.replicationStreamId,
|
|
365
|
+
'sync_configs._id': { $in: this.syncConfigIds }
|
|
352
366
|
}, {
|
|
353
367
|
$set: {
|
|
354
368
|
state: storage.SyncRuleState.ACTIVE,
|
|
@@ -356,16 +370,39 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
356
370
|
}
|
|
357
371
|
}, {
|
|
358
372
|
session,
|
|
359
|
-
arrayFilters: [{ 'config._id': this.
|
|
373
|
+
arrayFilters: [{ 'config._id': { $in: this.syncConfigIds } }]
|
|
360
374
|
});
|
|
361
375
|
await this.db.sync_rules.updateMany({
|
|
362
|
-
_id: { $ne: this.
|
|
376
|
+
_id: { $ne: this.replicationStreamId },
|
|
363
377
|
state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
|
|
364
378
|
}, syncRuleStateUpdatePipeline(storage.SyncRuleState.STOP), { session });
|
|
365
379
|
activated = true;
|
|
366
380
|
}
|
|
367
|
-
else if (doc
|
|
368
|
-
|
|
381
|
+
else if (doc.state == storage.SyncRuleState.ACTIVE &&
|
|
382
|
+
processingStates.length > 0 &&
|
|
383
|
+
processingStates.every((state) => state.snapshot_done && state.last_checkpoint != null)) {
|
|
384
|
+
await this.db.sync_rules.updateOne({
|
|
385
|
+
_id: this.replicationStreamId,
|
|
386
|
+
'sync_configs._id': { $in: processingStates.map((state) => state._id) }
|
|
387
|
+
}, {
|
|
388
|
+
$set: {
|
|
389
|
+
'sync_configs.$[activeConfig].state': storage.SyncRuleState.STOP,
|
|
390
|
+
'sync_configs.$[processingConfig].state': storage.SyncRuleState.ACTIVE
|
|
391
|
+
}
|
|
392
|
+
}, {
|
|
393
|
+
session,
|
|
394
|
+
arrayFilters: [
|
|
395
|
+
{ 'activeConfig.state': storage.SyncRuleState.ACTIVE },
|
|
396
|
+
{ 'processingConfig._id': { $in: processingStates.map((state) => state._id) } }
|
|
397
|
+
]
|
|
398
|
+
});
|
|
399
|
+
activated = true;
|
|
400
|
+
}
|
|
401
|
+
else if (doc.state != storage.SyncRuleState.PROCESSING && doc.state != storage.SyncRuleState.ACTIVE) {
|
|
402
|
+
needsFutureActivationCheck = false;
|
|
403
|
+
}
|
|
404
|
+
else if (doc.state == storage.SyncRuleState.ACTIVE && processingStates.length == 0) {
|
|
405
|
+
needsFutureActivationCheck = false;
|
|
369
406
|
}
|
|
370
407
|
});
|
|
371
408
|
if (activated) {
|
|
@@ -373,11 +410,14 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
373
410
|
await this.db.notifyCheckpoint();
|
|
374
411
|
this.needsActivationV3 = false;
|
|
375
412
|
}
|
|
413
|
+
else if (!needsFutureActivationCheck) {
|
|
414
|
+
this.needsActivationV3 = false;
|
|
415
|
+
}
|
|
376
416
|
}
|
|
377
417
|
async markAllSnapshotDone(no_checkpoint_before_lsn) {
|
|
378
418
|
await this.db.sync_rules.updateOne({
|
|
379
|
-
_id: this.
|
|
380
|
-
'sync_configs._id': this.
|
|
419
|
+
_id: this.replicationStreamId,
|
|
420
|
+
'sync_configs._id': { $in: this.syncConfigIds }
|
|
381
421
|
}, {
|
|
382
422
|
$set: {
|
|
383
423
|
'sync_configs.$[config].snapshot_done': true,
|
|
@@ -388,18 +428,22 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
388
428
|
}
|
|
389
429
|
}, {
|
|
390
430
|
session: this.session,
|
|
391
|
-
arrayFilters: [{ 'config._id': this.
|
|
431
|
+
arrayFilters: [{ 'config._id': { $in: this.syncConfigIds } }]
|
|
392
432
|
});
|
|
393
433
|
}
|
|
394
434
|
async markSnapshotDone(no_checkpoint_before_lsn, options) {
|
|
395
435
|
await this.withTransaction(async () => {
|
|
396
436
|
// Protect against race conditions
|
|
397
|
-
const
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
437
|
+
const blockingTables = await this.db
|
|
438
|
+
.sourceTables(this.replicationStreamId)
|
|
439
|
+
.find(this.snapshotBlockingSourceTablesFilter(), {
|
|
440
|
+
session: this.session,
|
|
441
|
+
projection: { schema_name: 1, table_name: 1 }
|
|
442
|
+
})
|
|
443
|
+
.toArray();
|
|
444
|
+
if (blockingTables.length > 0) {
|
|
401
445
|
if (options?.throwOnConflict ?? true) {
|
|
402
|
-
throw new ReplicationAssertionError(`Cannot mark snapshot done while
|
|
446
|
+
throw new ReplicationAssertionError(`Cannot mark snapshot done while source tables still require snapshotting. Tables: ${blockingTables.map((t) => `${t.schema_name}.${t.table_name}`).join(', ')}`);
|
|
403
447
|
}
|
|
404
448
|
else {
|
|
405
449
|
return;
|
|
@@ -408,24 +452,29 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
408
452
|
await this.markAllSnapshotDone(no_checkpoint_before_lsn);
|
|
409
453
|
});
|
|
410
454
|
}
|
|
411
|
-
async markTableSnapshotRequired(
|
|
455
|
+
async markTableSnapshotRequired(table) {
|
|
456
|
+
const syncConfigIds = this.relevantSyncConfigIds(table);
|
|
457
|
+
if (syncConfigIds.length == 0) {
|
|
458
|
+
return;
|
|
459
|
+
}
|
|
412
460
|
await this.db.sync_rules.updateOne({
|
|
413
|
-
_id: this.
|
|
414
|
-
'sync_configs._id':
|
|
461
|
+
_id: this.replicationStreamId,
|
|
462
|
+
'sync_configs._id': { $in: syncConfigIds }
|
|
415
463
|
}, {
|
|
416
464
|
$set: {
|
|
417
465
|
'sync_configs.$[config].snapshot_done': false
|
|
418
466
|
}
|
|
419
467
|
}, {
|
|
420
468
|
session: this.session,
|
|
421
|
-
arrayFilters: [{ 'config._id':
|
|
469
|
+
arrayFilters: [{ 'config._id': { $in: syncConfigIds } }]
|
|
422
470
|
});
|
|
423
471
|
}
|
|
424
472
|
async markTableSnapshotDone(tables, no_checkpoint_before_lsn) {
|
|
425
473
|
const session = this.session;
|
|
426
474
|
const ids = tables.map((table) => mongoTableId(table.id));
|
|
475
|
+
const syncConfigIds = this.relevantSyncConfigIdsForTables(tables);
|
|
427
476
|
await this.withTransaction(async () => {
|
|
428
|
-
await this.db.
|
|
477
|
+
await this.db.sourceTables(this.replicationStreamId).updateMany({ _id: { $in: ids } }, {
|
|
429
478
|
$set: {
|
|
430
479
|
snapshot_done: true
|
|
431
480
|
},
|
|
@@ -433,10 +482,10 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
433
482
|
snapshot_status: 1
|
|
434
483
|
}
|
|
435
484
|
}, { session });
|
|
436
|
-
if (no_checkpoint_before_lsn != null) {
|
|
485
|
+
if (no_checkpoint_before_lsn != null && syncConfigIds.length > 0) {
|
|
437
486
|
await this.db.sync_rules.updateOne({
|
|
438
|
-
_id: this.
|
|
439
|
-
'sync_configs._id':
|
|
487
|
+
_id: this.replicationStreamId,
|
|
488
|
+
'sync_configs._id': { $in: syncConfigIds }
|
|
440
489
|
}, {
|
|
441
490
|
$set: {
|
|
442
491
|
last_keepalive_ts: new Date()
|
|
@@ -446,7 +495,8 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
446
495
|
}
|
|
447
496
|
}, {
|
|
448
497
|
session: this.session,
|
|
449
|
-
|
|
498
|
+
// Only set for sync configs that use this table
|
|
499
|
+
arrayFilters: [{ 'config._id': { $in: syncConfigIds } }]
|
|
450
500
|
});
|
|
451
501
|
}
|
|
452
502
|
});
|