@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +129 -7
- package/dist/storage/MongoBucketStorage.d.ts +42 -5
- package/dist/storage/MongoBucketStorage.js +404 -215
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
- package/dist/storage/implementation/MongoBucketBatch.js +169 -98
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
- package/dist/storage/implementation/MongoCompactor.js +27 -238
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
- package/dist/storage/implementation/MongoIdSequence.js +14 -1
- package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
- package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
- package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +8 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
- package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
- package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
- package/dist/storage/implementation/MongoWriteBatch.js +149 -0
- package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
- package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
- package/dist/storage/implementation/common/PersistedBatch.js +57 -14
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +9 -1
- package/dist/storage/implementation/db.js +28 -1
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +35 -0
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
- package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
- package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
- package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
- package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.js +2 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
- package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
- package/dist/storage/implementation/v3/compact-utils.js +223 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +90 -6
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
- package/dist/storage/implementation/v3/source-table-utils.js +49 -57
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
- package/dist/storage/storage-index.d.ts +2 -1
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +28 -1
- package/dist/types/types.js +28 -7
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +4 -2
- package/dist/utils/test-utils.js +7 -4
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.js +3 -1
- package/dist/utils/util.js.map +1 -1
- package/package.json +10 -9
- package/src/storage/MongoBucketStorage.ts +545 -277
- package/src/storage/implementation/MongoBucketBatch.ts +258 -124
- package/src/storage/implementation/MongoCompactor.ts +37 -346
- package/src/storage/implementation/MongoIdSequence.ts +21 -1
- package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
- package/src/storage/implementation/MongoStorageProvider.ts +16 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
- package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
- package/src/storage/implementation/MongoWriteBatch.ts +172 -0
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
- package/src/storage/implementation/common/PersistedBatch.ts +76 -20
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
- package/src/storage/implementation/db.ts +36 -2
- package/src/storage/implementation/models.ts +39 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
- package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
- package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
- package/src/storage/implementation/v1/models.ts +1 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
- package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
- package/src/storage/implementation/v3/bucket-format.ts +2 -0
- package/src/storage/implementation/v3/compact-utils.ts +329 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +97 -4
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
- package/src/storage/implementation/v3/source-table-utils.ts +82 -76
- package/src/storage/storage-index.ts +2 -0
- package/src/types/types.ts +42 -7
- package/src/utils/test-utils.ts +9 -5
- package/src/utils/util.ts +3 -1
- package/test/src/__snapshots__/storage.test.ts.snap +25 -0
- package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
- package/test/src/chunk_compaction_config.test.ts +38 -0
- package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
- package/test/src/compact-utils.test.ts +251 -0
- package/test/src/concurrent_writers.test.ts +599 -0
- package/test/src/connection_config.test.ts +159 -0
- package/test/src/helpers/MemoryObjectStorage.ts +14 -6
- package/test/src/helpers/concurrentWriter.mjs +29 -0
- package/test/src/helpers/s3TestFactory.ts +12 -12
- package/test/src/object_storage_usage.test.ts +268 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/storage.test.ts +280 -24
- package/test/src/storage_compacting.test.ts +1097 -253
- package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
- package/test/src/storage_s3_checksums.test.ts +6 -7
- package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
- package/test/src/storage_s3_reading.test.ts +294 -6
- package/test/src/storage_s3_writing.test.ts +4 -3
- package/test/src/storage_sync.test.ts +412 -80
- package/test/src/storeCurrentData.test.ts +5 -5
- package/test/src/util.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
2
|
+
import { ReplicationAssertionError, ServiceAssertionError } from '@powersync/lib-services-framework';
|
|
3
3
|
import { storage } from '@powersync/service-core';
|
|
4
4
|
import * as bson from 'bson';
|
|
5
5
|
import { mongoTableId } from '../../../utils/util.js';
|
|
6
6
|
import { canCheckpointState } from '../CheckpointState.js';
|
|
7
7
|
import { MongoBucketBatch } from '../MongoBucketBatch.js';
|
|
8
|
-
import {
|
|
8
|
+
import { stopReplicationStreamPipeline } from '../SyncRuleStateUpdate.js';
|
|
9
9
|
import { PersistedBatchV3 } from './PersistedBatchV3.js';
|
|
10
10
|
import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js';
|
|
11
|
-
import { createNewSourceTable,
|
|
11
|
+
import { createNewSourceTable, overlappingSourceTableFilter, planSourceTableReconciliation, sourceTableDesiredResolution, sourceTableFromDocument } from './source-table-utils.js';
|
|
12
12
|
export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
13
13
|
store;
|
|
14
|
+
/** Avoid repeating idempotent createIndexes calls for every checkpoint flush. */
|
|
15
|
+
initializedCustomCheckpointEventIds = new Set();
|
|
16
|
+
// Used only to validate supplied checkpoint ids; it does not initialize collections.
|
|
17
|
+
knownEventIds;
|
|
14
18
|
syncConfigIds;
|
|
15
19
|
needsActivationV3 = true;
|
|
16
20
|
lastWaitingLogThrottledV3 = 0;
|
|
@@ -21,29 +25,35 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
21
25
|
throw new ReplicationAssertionError('Missing sync config id for v3 batch');
|
|
22
26
|
}
|
|
23
27
|
this.syncConfigIds = syncConfigIds;
|
|
28
|
+
this.knownEventIds = new Set(this.mapping.allEventDefinitionIds());
|
|
24
29
|
this.store = new SourceRecordStoreV3(this.db, this.replicationStreamId, this.mapping);
|
|
25
30
|
}
|
|
26
31
|
createPersistedBatch(writtenSize) {
|
|
27
32
|
return new PersistedBatchV3(this.db, this.replicationStreamId, this.mapping, writtenSize, {
|
|
28
33
|
logger: this.logger,
|
|
29
34
|
objectStorage: this.options.objectStorage,
|
|
30
|
-
inlineThresholdBytes: this.options.inlineThresholdBytes
|
|
35
|
+
inlineThresholdBytes: this.options.inlineThresholdBytes,
|
|
36
|
+
signal: this.options.signal,
|
|
37
|
+
objectStorageUsageWriterId: this.objectStorageUsageWriterId
|
|
31
38
|
});
|
|
32
39
|
}
|
|
33
40
|
get sourceRecordStore() {
|
|
34
41
|
return this.store;
|
|
35
42
|
}
|
|
36
|
-
|
|
43
|
+
persistedOpHead(stream) {
|
|
44
|
+
return stream.last_persisted_op ?? 0n;
|
|
45
|
+
}
|
|
46
|
+
onReplicationTransactionFlush(writes, lastOp) {
|
|
37
47
|
// Durably advance the stream-level head of persisted ops within the flush transaction.
|
|
38
48
|
// This ensures a checkpoint created later (even by an empty commit, or by a freshly-appended
|
|
39
49
|
// config that replicates nothing) covers all ops persisted before a potential crash.
|
|
40
|
-
|
|
50
|
+
writes.updateOne(this.db.sync_rules, {
|
|
41
51
|
_id: this.replicationStreamId
|
|
42
52
|
}, {
|
|
43
53
|
$max: {
|
|
44
54
|
last_persisted_op: lastOp
|
|
45
55
|
}
|
|
46
|
-
}
|
|
56
|
+
});
|
|
47
57
|
}
|
|
48
58
|
selectedSyncConfigObjectIds(syncConfigIds) {
|
|
49
59
|
const selectedIds = new Set(this.syncConfigIds.map((id) => id.toHexString()));
|
|
@@ -54,7 +64,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
54
64
|
relevantSyncConfigIds(table) {
|
|
55
65
|
const bucketDataSourceIds = table.bucketDataSources.map((source) => this.mapping.bucketSourceId(source));
|
|
56
66
|
const parameterLookupSourceIds = table.parameterLookupSources.map((source) => this.mapping.parameterLookupId(source));
|
|
57
|
-
return this.selectedSyncConfigObjectIds(this.mapping.syncConfigIdsForSourceTable(this.syncConfigIds.map((id) => id.toHexString()), table.ref, bucketDataSourceIds, parameterLookupSourceIds));
|
|
67
|
+
return this.selectedSyncConfigObjectIds(this.mapping.syncConfigIdsForSourceTable(this.syncConfigIds.map((id) => id.toHexString()), table.ref, bucketDataSourceIds, parameterLookupSourceIds, [...table.eventDefinitionIds]));
|
|
58
68
|
}
|
|
59
69
|
relevantSyncConfigIdsForTables(tables) {
|
|
60
70
|
const ids = new Map();
|
|
@@ -87,7 +97,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
87
97
|
.sourceRecords(this.replicationStreamId, mongoTableId(table.id))
|
|
88
98
|
.drop()
|
|
89
99
|
.catch((error) => {
|
|
90
|
-
if (lib_mongo.
|
|
100
|
+
if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
|
|
91
101
|
return;
|
|
92
102
|
}
|
|
93
103
|
throw error;
|
|
@@ -100,37 +110,48 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
100
110
|
const parsedOverride = options.parsedSyncConfig;
|
|
101
111
|
const syncConfig = parsedOverride?.hydratedSyncConfig ?? this.sync_rules;
|
|
102
112
|
const mapping = parsedOverride?.mapping ?? this.mapping;
|
|
113
|
+
const eventById = (parsedOverride ?? this.options.parsedSyncConfig).eventById;
|
|
103
114
|
const { connection_id, source } = options;
|
|
104
|
-
const
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
type_oid: column.typeId
|
|
115
|
-
}))
|
|
116
|
-
},
|
|
117
|
-
storeCurrentData: source.sendsCompleteRows !== true,
|
|
118
|
-
syncConfig,
|
|
119
|
-
mapping,
|
|
120
|
-
desired: sourceTableDesiredResolution(syncConfig, source, mapping)
|
|
115
|
+
const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
|
|
116
|
+
const identity = {
|
|
117
|
+
schema: source.schema,
|
|
118
|
+
name: source.name,
|
|
119
|
+
objectId: source.objectId,
|
|
120
|
+
replicaIdColumns: source.replicaIdColumns.map((column) => ({
|
|
121
|
+
name: column.name,
|
|
122
|
+
type: column.type,
|
|
123
|
+
type_oid: column.typeId
|
|
124
|
+
}))
|
|
121
125
|
};
|
|
122
126
|
let result = null;
|
|
123
127
|
const session = this.db.client.startSession();
|
|
124
128
|
await using _ = { [Symbol.asyncDispose]: () => session.endSession() };
|
|
125
|
-
await
|
|
129
|
+
await this.withTransaction(async () => {
|
|
126
130
|
const col = this.db.sourceTables(this.replicationStreamId);
|
|
127
|
-
//
|
|
128
|
-
// Exact-identity docs are candidates for reuse; non-exact overlaps are possible drops.
|
|
131
|
+
// Find records that overlap by name or relation id.
|
|
129
132
|
const candidateDocs = await col
|
|
130
|
-
.find(overlappingSourceTableFilter(connection_id,
|
|
133
|
+
.find(overlappingSourceTableFilter(connection_id, identity), { session })
|
|
131
134
|
.toArray();
|
|
132
|
-
|
|
133
|
-
|
|
135
|
+
const candidateTables = candidateDocs.map((doc) => sourceTableFromDocument(doc, source.connectionTag, syncConfig, mapping, eventById));
|
|
136
|
+
const candidates = candidateTables.map((table) => table.clone());
|
|
137
|
+
const resolution = await reconcile({ source, candidates });
|
|
138
|
+
storage.validateSourceTableCandidateResolution(candidates, resolution);
|
|
139
|
+
// Persist metadata from the reconciler without mutating the queried documents.
|
|
140
|
+
for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
|
|
141
|
+
await col.updateOne({ _id: mongoTableId(id) }, { $set: { source_metadata: sourceMetadata } }, { session });
|
|
142
|
+
}
|
|
143
|
+
const context = {
|
|
144
|
+
connectionId: connection_id,
|
|
145
|
+
connectionTag: source.connectionTag,
|
|
146
|
+
identity,
|
|
147
|
+
storeCurrentData: source.sendsCompleteRows !== true,
|
|
148
|
+
syncConfig,
|
|
149
|
+
mapping,
|
|
150
|
+
desired: sourceTableDesiredResolution(syncConfig, source, mapping, eventById),
|
|
151
|
+
sourceCompatibleTables: resolution.compatibleTables,
|
|
152
|
+
newTableSourceMetadata: resolution.newTableValues.sourceMetadata
|
|
153
|
+
};
|
|
154
|
+
// Plan record reuse, membership changes, creation, and removal.
|
|
134
155
|
const plan = planSourceTableReconciliation(candidateDocs, context);
|
|
135
156
|
// Persist narrowing for incomplete snapshots only. Snapshot-complete docs keep stale
|
|
136
157
|
// coverage ids so compatible future configs can reuse already-snapshotted data.
|
|
@@ -140,7 +161,8 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
140
161
|
await col.updateOne({ _id: update.id }, {
|
|
141
162
|
$set: {
|
|
142
163
|
bucket_data_source_ids: update.memberships.bucketDataSourceIds,
|
|
143
|
-
parameter_lookup_source_ids: update.memberships.parameterLookupSourceIds
|
|
164
|
+
parameter_lookup_source_ids: update.memberships.parameterLookupSourceIds,
|
|
165
|
+
event_definition_ids: update.memberships.eventDefinitionIds
|
|
144
166
|
}
|
|
145
167
|
}, { session });
|
|
146
168
|
}
|
|
@@ -153,13 +175,11 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
153
175
|
await this.db.initializeSourceRecordsCollection(this.replicationStreamId, doc._id, session);
|
|
154
176
|
plan.tables.push(table);
|
|
155
177
|
}
|
|
156
|
-
// If memberships are split across multiple source tables, only one may fire events.
|
|
157
|
-
designateEventCarrier(plan.tables, context.desired.triggersEvent);
|
|
158
178
|
result = {
|
|
159
179
|
tables: plan.tables,
|
|
160
|
-
dropTables: plan.dropDocs.map((doc) => sourceTableFromDocument(doc, context.connectionTag, syncConfig, mapping))
|
|
180
|
+
dropTables: plan.dropDocs.map((doc) => sourceTableFromDocument(doc, context.connectionTag, syncConfig, mapping, eventById))
|
|
161
181
|
};
|
|
162
|
-
});
|
|
182
|
+
}, session);
|
|
163
183
|
return result;
|
|
164
184
|
}
|
|
165
185
|
async getSourceTableStatus(table) {
|
|
@@ -169,127 +189,110 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
169
189
|
if (doc == null) {
|
|
170
190
|
return null;
|
|
171
191
|
}
|
|
172
|
-
|
|
173
|
-
// The event-carrier designation is decided per resolveTables result and not persisted -
|
|
174
|
-
// preserve the caller's designation instead of recomputing it from the ref, so that
|
|
175
|
-
// refreshing a non-carrier table does not make it fire events.
|
|
176
|
-
refreshed.syncEvent = table.syncEvent;
|
|
177
|
-
return refreshed;
|
|
192
|
+
return sourceTableFromDocument(doc, table.ref.connectionTag, this.sync_rules, this.mapping, this.options.parsedSyncConfig.eventById);
|
|
178
193
|
}
|
|
179
194
|
async commit(lsn, options) {
|
|
180
195
|
const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
|
|
181
|
-
await this.flush(options);
|
|
182
196
|
using _ = this.tracer.span('storage', 'commit');
|
|
183
|
-
const
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
const preUpdateDocument = (await this.db.sync_rules.findOne({
|
|
195
|
-
_id: this.replicationStreamId,
|
|
196
|
-
'sync_configs._id': { $in: this.syncConfigIds }
|
|
197
|
-
}, {
|
|
198
|
-
session: this.session,
|
|
199
|
-
projection: {
|
|
200
|
-
sync_configs: 1,
|
|
201
|
-
last_persisted_op: 1
|
|
197
|
+
const checkpoint = async (stream, lastOp) => {
|
|
198
|
+
const now = new Date();
|
|
199
|
+
const preUpdateDocument = stream;
|
|
200
|
+
const writes = this.db.createWriteBatch(this.session, { ordered: false });
|
|
201
|
+
writes.updateMany(this.db.write_checkpoints, { processed_at_lsn: null, 'lsns.1': { $lte: lsn } }, { $set: { processed_at_lsn: lsn } });
|
|
202
|
+
const states = preUpdateDocument?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ??
|
|
203
|
+
[];
|
|
204
|
+
if (states.length == 0) {
|
|
205
|
+
throw new ReplicationAssertionError(`Failed to update checkpoint - no matching sync_config for _id: ${this.replicationStreamId}/${this.syncConfigIds
|
|
206
|
+
.map((id) => id.toHexString())
|
|
207
|
+
.join(',')}`);
|
|
202
208
|
}
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
.
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
// The replication job / MongoBucketBatch must be constructed with all replicating (PROCESSING / ACTIVE)
|
|
211
|
-
// sync configs in the stream, otherwise we'll get inconsistencies. Configs in other states (e.g. STOP)
|
|
212
|
-
// remain embedded in the document and are ignored here.
|
|
213
|
-
const missingSyncConfig = preUpdateDocument.sync_configs.find((config) => [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE].includes(config.state) &&
|
|
214
|
-
!this.syncConfigIds.some((id) => id.equals(config._id)));
|
|
215
|
-
if (missingSyncConfig != null) {
|
|
216
|
-
throw new ReplicationAssertionError(`Replication job not configured for sync config ${missingSyncConfig._id}`);
|
|
217
|
-
}
|
|
218
|
-
// Effective head of the stream's op sequence.
|
|
219
|
-
// last_persisted_op is $max-advanced durably in the same transaction as every flush, and this
|
|
220
|
-
// read uses the same session, so it covers all ops persisted by this batch.
|
|
221
|
-
const newCheckpoint = preUpdateDocument?.last_persisted_op == null ? 0n : BigInt(preUpdateDocument.last_persisted_op);
|
|
222
|
-
let checkpointBlocked = false;
|
|
223
|
-
let checkpointCreated = false;
|
|
224
|
-
let checkpointLogState = null;
|
|
225
|
-
const unblockedConfigIds = [];
|
|
226
|
-
for (const state of states) {
|
|
227
|
-
if (state.last_checkpoint != null && state.last_checkpoint > newCheckpoint) {
|
|
228
|
-
// last_persisted_op is $max-advanced durably in the same transaction as every flush, and
|
|
229
|
-
// checkpoints are only ever created at that head, so a checkpoint past the head means the
|
|
230
|
-
// op sequence or the stored state is corrupt.
|
|
231
|
-
throw new ReplicationAssertionError(`Invariant violation: sync config ${state._id} has last_checkpoint ${state.last_checkpoint} > stream head ${newCheckpoint}`);
|
|
209
|
+
// The replication job / MongoBucketBatch must be constructed with all replicating (PROCESSING / ACTIVE)
|
|
210
|
+
// sync configs in the stream, otherwise we'll get inconsistencies. Configs in other states (e.g. STOP)
|
|
211
|
+
// remain embedded in the document and are ignored here.
|
|
212
|
+
const missingSyncConfig = preUpdateDocument.sync_configs.find((config) => [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE].includes(config.state) &&
|
|
213
|
+
!this.syncConfigIds.some((id) => id.equals(config._id)));
|
|
214
|
+
if (missingSyncConfig != null) {
|
|
215
|
+
throw new ReplicationAssertionError(`Replication job not configured for sync config ${missingSyncConfig._id}`);
|
|
232
216
|
}
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
217
|
+
// Effective head of the stream's op sequence.
|
|
218
|
+
// A combined flush supplies its new head; an empty commit uses the fenced persisted head.
|
|
219
|
+
const newCheckpoint = lastOp ?? this.persistedOpHead(preUpdateDocument);
|
|
220
|
+
let checkpointBlocked = false;
|
|
221
|
+
let checkpointCreated = false;
|
|
222
|
+
let checkpointLogState = null;
|
|
223
|
+
const unblockedConfigIds = [];
|
|
224
|
+
for (const state of states) {
|
|
225
|
+
if (state.last_checkpoint != null && state.last_checkpoint > newCheckpoint) {
|
|
226
|
+
// last_persisted_op is $max-advanced durably in the same transaction as every flush, and
|
|
227
|
+
// checkpoints are only ever created at that head, so a checkpoint past the head means the
|
|
228
|
+
// op sequence or the stored state is corrupt.
|
|
229
|
+
throw new ReplicationAssertionError(`Invariant violation: sync config ${state._id} has last_checkpoint ${state.last_checkpoint} > stream head ${newCheckpoint}`);
|
|
230
|
+
}
|
|
231
|
+
const canCheckpoint = canCheckpointState(lsn, {
|
|
232
|
+
snapshotDone: state.snapshot_done === true,
|
|
233
|
+
lastCheckpointLsn: state.last_checkpoint_lsn,
|
|
234
|
+
noCheckpointBefore: state.no_checkpoint_before
|
|
235
|
+
});
|
|
236
|
+
if (!canCheckpoint) {
|
|
237
|
+
checkpointBlocked = true;
|
|
238
|
+
// Log the first blocked config's state.
|
|
239
|
+
checkpointLogState ??= {
|
|
240
|
+
snapshot_done: state.snapshot_done,
|
|
241
|
+
last_checkpoint_lsn: state.last_checkpoint_lsn,
|
|
242
|
+
no_checkpoint_before: state.no_checkpoint_before
|
|
243
|
+
};
|
|
244
|
+
continue;
|
|
245
|
+
}
|
|
246
|
+
checkpointCreated ||= createEmptyCheckpoints || state.last_checkpoint !== newCheckpoint;
|
|
247
|
+
unblockedConfigIds.push(state._id);
|
|
247
248
|
}
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
const resumeLsnUpdate = { resume_lsn: lsn };
|
|
256
|
-
if (unblockedConfigIds.length > 0) {
|
|
257
|
-
// All unblocked configs get the SAME new value, so we apply it with a single updateOne
|
|
258
|
-
// (single-document atomicity).
|
|
259
|
-
const updateSet = {
|
|
260
|
-
last_keepalive_ts: now,
|
|
261
|
-
last_fatal_error: null,
|
|
262
|
-
last_fatal_error_ts: null
|
|
249
|
+
// Every commit advances the stream's resume position: commit() flushes first, so all
|
|
250
|
+
// source changes up to this lsn have been persisted, even when checkpoints are blocked.
|
|
251
|
+
// In the future we could also advance this on flush, when the connector provides the
|
|
252
|
+
// current position (see setResumeLsn, which connectors may already call after flushing).
|
|
253
|
+
const resumeLsnUpdate = {
|
|
254
|
+
resume_lsn: lsn,
|
|
255
|
+
...(lastOp == null ? {} : { last_persisted_op: lastOp })
|
|
263
256
|
};
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
updateSet
|
|
268
|
-
updateSet['sync_configs.$[config].last_checkpoint_lsn'] = lsn;
|
|
269
|
-
updateSet['last_checkpoint_ts'] = now;
|
|
270
|
-
}
|
|
271
|
-
await this.db.sync_rules.updateOne({
|
|
272
|
-
_id: this.replicationStreamId,
|
|
273
|
-
'sync_configs._id': { $in: unblockedConfigIds }
|
|
274
|
-
}, { $set: updateSet, $max: resumeLsnUpdate }, {
|
|
275
|
-
session: this.session,
|
|
276
|
-
arrayFilters: checkpointCreated ? [{ 'config._id': { $in: unblockedConfigIds } }] : undefined
|
|
277
|
-
});
|
|
278
|
-
}
|
|
279
|
-
else {
|
|
280
|
-
// All selected configs are blocked - only update keepalive/error tracking and the
|
|
281
|
-
// resume position.
|
|
282
|
-
await this.db.sync_rules.updateOne({
|
|
283
|
-
_id: this.replicationStreamId
|
|
284
|
-
}, {
|
|
285
|
-
$set: {
|
|
286
|
-
last_keepalive_ts: now,
|
|
257
|
+
if (unblockedConfigIds.length > 0) {
|
|
258
|
+
// All unblocked configs get the SAME new value, so we apply it with a single updateOne
|
|
259
|
+
// (single-document atomicity).
|
|
260
|
+
const updateSet = {
|
|
287
261
|
last_fatal_error: null,
|
|
288
262
|
last_fatal_error_ts: null
|
|
289
|
-
}
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
263
|
+
};
|
|
264
|
+
// Only advance checkpoint fields when an actual (non-empty) checkpoint is created, matching
|
|
265
|
+
// the previous per-config / v1 behaviour.
|
|
266
|
+
if (checkpointCreated) {
|
|
267
|
+
updateSet['sync_configs.$[config].last_checkpoint'] = newCheckpoint;
|
|
268
|
+
updateSet['sync_configs.$[config].last_checkpoint_lsn'] = lsn;
|
|
269
|
+
updateSet['last_checkpoint_ts'] = now;
|
|
270
|
+
}
|
|
271
|
+
writes.updateOne(this.db.sync_rules, {
|
|
272
|
+
_id: this.replicationStreamId,
|
|
273
|
+
'sync_configs._id': { $in: unblockedConfigIds }
|
|
274
|
+
}, { $set: updateSet, $max: resumeLsnUpdate }, {
|
|
275
|
+
arrayFilters: checkpointCreated ? [{ 'config._id': { $in: unblockedConfigIds } }] : undefined
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
else {
|
|
279
|
+
// All selected configs are blocked - only update keepalive/error tracking and the
|
|
280
|
+
// resume position.
|
|
281
|
+
writes.updateOne(this.db.sync_rules, {
|
|
282
|
+
_id: this.replicationStreamId
|
|
283
|
+
}, {
|
|
284
|
+
$set: {
|
|
285
|
+
last_fatal_error: null,
|
|
286
|
+
last_fatal_error_ts: null
|
|
287
|
+
},
|
|
288
|
+
$max: resumeLsnUpdate
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
await writes.execute();
|
|
292
|
+
return { checkpointBlocked, checkpointCreated, checkpointLogState, newCheckpoint };
|
|
293
|
+
};
|
|
294
|
+
const projection = { sync_configs: 1, last_persisted_op: 1 };
|
|
295
|
+
const { checkpointBlocked, checkpointCreated, checkpointLogState, newCheckpoint } = await this.flushAndCommit(checkpoint, () => this.withFencedTransaction(() => this.fence(this.session, projection), checkpoint), options, projection);
|
|
293
296
|
if (checkpointBlocked) {
|
|
294
297
|
if (Date.now() - this.lastWaitingLogThrottledV3 > 5_000) {
|
|
295
298
|
this.logger.info(`Waiting before creating checkpoint, currently at ${lsn}. Current state: ${JSON.stringify(checkpointLogState)}`);
|
|
@@ -317,19 +320,9 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
317
320
|
}
|
|
318
321
|
async setResumeLsn(lsn) {
|
|
319
322
|
using _ = this.tracer.span('storage', 'set_resume_lsn');
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
}, {
|
|
323
|
-
$set: {
|
|
324
|
-
resume_lsn: lsn
|
|
325
|
-
}
|
|
326
|
-
}, {
|
|
327
|
-
session: this.session,
|
|
328
|
-
// Losing occasional resume LSN is fine. That may mean reprocessing
|
|
329
|
-
// some source changes in some edge cases, which is not an issue since
|
|
330
|
-
// changes are processed in an idempotent way.
|
|
331
|
-
writeConcern: { w: 1 }
|
|
332
|
-
});
|
|
323
|
+
// Losing occasional resume LSN would only reprocess source changes.
|
|
324
|
+
// Keep the lease check atomic, but do not wait for majority replication of this resume hint.
|
|
325
|
+
await this.updateStreamMetadata({ resume_lsn: { $literal: lsn } }, {}, { w: 1 });
|
|
333
326
|
}
|
|
334
327
|
async autoActivateV3(lsn) {
|
|
335
328
|
if (!this.needsActivationV3) {
|
|
@@ -338,7 +331,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
338
331
|
const session = this.session;
|
|
339
332
|
let activated = false;
|
|
340
333
|
let needsFutureActivationCheck = true;
|
|
341
|
-
await
|
|
334
|
+
await this.withTransaction(async () => {
|
|
342
335
|
// Reset on transaction retries.
|
|
343
336
|
needsFutureActivationCheck = true;
|
|
344
337
|
activated = false;
|
|
@@ -375,7 +368,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
375
368
|
await this.db.sync_rules.updateMany({
|
|
376
369
|
_id: { $ne: this.replicationStreamId },
|
|
377
370
|
state: { $in: [storage.SyncRuleState.ACTIVE, storage.SyncRuleState.ERRORED] }
|
|
378
|
-
},
|
|
371
|
+
}, stopReplicationStreamPipeline(), { session });
|
|
379
372
|
activated = true;
|
|
380
373
|
}
|
|
381
374
|
else if (doc.state == storage.SyncRuleState.ACTIVE &&
|
|
@@ -404,7 +397,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
404
397
|
else if (doc.state == storage.SyncRuleState.ACTIVE && processingStates.length == 0) {
|
|
405
398
|
needsFutureActivationCheck = false;
|
|
406
399
|
}
|
|
407
|
-
});
|
|
400
|
+
}, session);
|
|
408
401
|
if (activated) {
|
|
409
402
|
this.logger.info(`Activated new replication stream at ${lsn}`);
|
|
410
403
|
await this.db.notifyCheckpoint();
|
|
@@ -414,21 +407,32 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
414
407
|
this.needsActivationV3 = false;
|
|
415
408
|
}
|
|
416
409
|
}
|
|
410
|
+
syncConfigMetadataUpdate(syncConfigIds, set) {
|
|
411
|
+
return {
|
|
412
|
+
sync_configs: {
|
|
413
|
+
$map: {
|
|
414
|
+
input: '$sync_configs',
|
|
415
|
+
as: 'config',
|
|
416
|
+
in: {
|
|
417
|
+
$cond: [
|
|
418
|
+
{ $in: ['$$config._id', { $literal: syncConfigIds }] },
|
|
419
|
+
{ $mergeObjects: ['$$config', set] },
|
|
420
|
+
'$$config'
|
|
421
|
+
]
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
};
|
|
426
|
+
}
|
|
427
|
+
snapshotDoneUpdate(no_checkpoint_before_lsn) {
|
|
428
|
+
return this.syncConfigMetadataUpdate(this.syncConfigIds, {
|
|
429
|
+
snapshot_done: true,
|
|
430
|
+
no_checkpoint_before: { $max: ['$$config.no_checkpoint_before', { $literal: no_checkpoint_before_lsn }] }
|
|
431
|
+
});
|
|
432
|
+
}
|
|
417
433
|
async markAllSnapshotDone(no_checkpoint_before_lsn) {
|
|
418
|
-
await this.
|
|
419
|
-
_id: this.replicationStreamId,
|
|
434
|
+
await this.updateStreamMetadata(this.snapshotDoneUpdate(no_checkpoint_before_lsn), {
|
|
420
435
|
'sync_configs._id': { $in: this.syncConfigIds }
|
|
421
|
-
}, {
|
|
422
|
-
$set: {
|
|
423
|
-
'sync_configs.$[config].snapshot_done': true,
|
|
424
|
-
last_keepalive_ts: new Date()
|
|
425
|
-
},
|
|
426
|
-
$max: {
|
|
427
|
-
'sync_configs.$[config].no_checkpoint_before': no_checkpoint_before_lsn
|
|
428
|
-
}
|
|
429
|
-
}, {
|
|
430
|
-
session: this.session,
|
|
431
|
-
arrayFilters: [{ 'config._id': { $in: this.syncConfigIds } }]
|
|
432
436
|
});
|
|
433
437
|
}
|
|
434
438
|
async markSnapshotDone(no_checkpoint_before_lsn, options) {
|
|
@@ -449,7 +453,9 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
449
453
|
return;
|
|
450
454
|
}
|
|
451
455
|
}
|
|
452
|
-
await this.
|
|
456
|
+
await this.updateStreamMetadataInTransaction(this.snapshotDoneUpdate(no_checkpoint_before_lsn), {
|
|
457
|
+
'sync_configs._id': { $in: this.syncConfigIds }
|
|
458
|
+
});
|
|
453
459
|
});
|
|
454
460
|
}
|
|
455
461
|
async markTableSnapshotRequired(table) {
|
|
@@ -457,16 +463,8 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
457
463
|
if (syncConfigIds.length == 0) {
|
|
458
464
|
return;
|
|
459
465
|
}
|
|
460
|
-
await this.
|
|
461
|
-
_id: this.replicationStreamId,
|
|
466
|
+
await this.updateStreamMetadata(this.syncConfigMetadataUpdate(syncConfigIds, { snapshot_done: false }), {
|
|
462
467
|
'sync_configs._id': { $in: syncConfigIds }
|
|
463
|
-
}, {
|
|
464
|
-
$set: {
|
|
465
|
-
'sync_configs.$[config].snapshot_done': false
|
|
466
|
-
}
|
|
467
|
-
}, {
|
|
468
|
-
session: this.session,
|
|
469
|
-
arrayFilters: [{ 'config._id': { $in: syncConfigIds } }]
|
|
470
468
|
});
|
|
471
469
|
}
|
|
472
470
|
async markTableSnapshotDone(tables, no_checkpoint_before_lsn) {
|
|
@@ -474,31 +472,29 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
474
472
|
const ids = tables.map((table) => mongoTableId(table.id));
|
|
475
473
|
const syncConfigIds = this.relevantSyncConfigIdsForTables(tables);
|
|
476
474
|
await this.withTransaction(async () => {
|
|
477
|
-
|
|
475
|
+
const writes = this.db.createWriteBatch(session, { ordered: false });
|
|
476
|
+
writes.updateMany(this.db.sourceTables(this.replicationStreamId), { _id: { $in: ids } }, {
|
|
478
477
|
$set: {
|
|
479
478
|
snapshot_done: true
|
|
480
479
|
},
|
|
481
480
|
$unset: {
|
|
482
481
|
snapshot_status: 1
|
|
483
482
|
}
|
|
484
|
-
}
|
|
483
|
+
});
|
|
485
484
|
if (no_checkpoint_before_lsn != null && syncConfigIds.length > 0) {
|
|
486
|
-
|
|
485
|
+
writes.updateOne(this.db.sync_rules, {
|
|
487
486
|
_id: this.replicationStreamId,
|
|
488
487
|
'sync_configs._id': { $in: syncConfigIds }
|
|
489
488
|
}, {
|
|
490
|
-
$set: {
|
|
491
|
-
last_keepalive_ts: new Date()
|
|
492
|
-
},
|
|
493
489
|
$max: {
|
|
494
490
|
'sync_configs.$[config].no_checkpoint_before': no_checkpoint_before_lsn
|
|
495
491
|
}
|
|
496
492
|
}, {
|
|
497
|
-
session: this.session,
|
|
498
493
|
// Only set for sync configs that use this table
|
|
499
494
|
arrayFilters: [{ 'config._id': { $in: syncConfigIds } }]
|
|
500
495
|
});
|
|
501
496
|
}
|
|
497
|
+
await writes.execute();
|
|
502
498
|
});
|
|
503
499
|
return tables.map((table) => {
|
|
504
500
|
const copy = table.clone();
|
|
@@ -506,5 +502,76 @@ export class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
506
502
|
return copy;
|
|
507
503
|
});
|
|
508
504
|
}
|
|
505
|
+
async batchCreateCustomWriteCheckpoints(session, opId) {
|
|
506
|
+
if (this.write_checkpoint_batch.length == 0) {
|
|
507
|
+
return;
|
|
508
|
+
}
|
|
509
|
+
const checkpointsByEvent = Map.groupBy(this.write_checkpoint_batch, (checkpoint) => this.validateCustomCheckpointEventId(checkpoint.event_id));
|
|
510
|
+
const writes = this.db.createWriteBatch(session, { ordered: false });
|
|
511
|
+
for (const [eventId, checkpoints] of checkpointsByEvent) {
|
|
512
|
+
this.batchCreateEventCustomWriteCheckpoints(writes, opId, eventId, checkpoints);
|
|
513
|
+
}
|
|
514
|
+
await writes.execute();
|
|
515
|
+
}
|
|
516
|
+
async prepareCustomWriteCheckpoints() {
|
|
517
|
+
// Most events never produce custom checkpoints, so create collections lazily
|
|
518
|
+
// only for the event ids present in this batch. This hook runs before the
|
|
519
|
+
// replication transaction because MongoDB cannot create indexes within it.
|
|
520
|
+
const eventIds = new Set(this.write_checkpoint_batch.map((checkpoint) => this.validateCustomCheckpointEventId(checkpoint.event_id)));
|
|
521
|
+
for (const eventId of eventIds) {
|
|
522
|
+
if (this.initializedCustomCheckpointEventIds.has(eventId)) {
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
await this.db.initializeCustomCheckpointRequestsCollection({
|
|
526
|
+
replicationStreamId: this.replicationStreamId,
|
|
527
|
+
eventId
|
|
528
|
+
});
|
|
529
|
+
this.initializedCustomCheckpointEventIds.add(eventId);
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
validateCustomCheckpointEventId(eventId) {
|
|
533
|
+
if (eventId == null) {
|
|
534
|
+
throw new ServiceAssertionError('V3 custom checkpoints require an event definition id');
|
|
535
|
+
}
|
|
536
|
+
if (!this.knownEventIds.has(eventId)) {
|
|
537
|
+
throw new ServiceAssertionError(`Unknown custom checkpoint event definition ${eventId}`);
|
|
538
|
+
}
|
|
539
|
+
return eventId;
|
|
540
|
+
}
|
|
541
|
+
batchCreateEventCustomWriteCheckpoints(writes, opId, eventId, checkpoints) {
|
|
542
|
+
// A repeated user within an event replaces the complete checkpoint state.
|
|
543
|
+
const uniqueCheckpoints = new Map(checkpoints.map((checkpoint) => [checkpoint.user_id, checkpoint]));
|
|
544
|
+
writes.bulkWriteUnordered(this.db.customCheckpointRequests({
|
|
545
|
+
eventId,
|
|
546
|
+
replicationStreamId: this.replicationStreamId
|
|
547
|
+
}), [...uniqueCheckpoints.values()].map((checkpointOptions) => {
|
|
548
|
+
const set = {
|
|
549
|
+
user_id: checkpointOptions.user_id,
|
|
550
|
+
checkpoint: checkpointOptions.checkpoint,
|
|
551
|
+
op_id: opId
|
|
552
|
+
};
|
|
553
|
+
if (checkpointOptions.checkpoint_requested_at != null) {
|
|
554
|
+
set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
|
|
555
|
+
}
|
|
556
|
+
return {
|
|
557
|
+
updateOne: {
|
|
558
|
+
filter: {
|
|
559
|
+
user_id: checkpointOptions.user_id
|
|
560
|
+
},
|
|
561
|
+
update: {
|
|
562
|
+
$set: set,
|
|
563
|
+
...(checkpointOptions.checkpoint_requested_at == null
|
|
564
|
+
? {
|
|
565
|
+
$unset: {
|
|
566
|
+
checkpoint_requested_at: 1
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
: {})
|
|
570
|
+
},
|
|
571
|
+
upsert: true
|
|
572
|
+
}
|
|
573
|
+
};
|
|
574
|
+
}));
|
|
575
|
+
}
|
|
509
576
|
}
|
|
510
577
|
//# sourceMappingURL=MongoBucketBatchV3.js.map
|