@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +129 -7
- package/dist/storage/MongoBucketStorage.d.ts +42 -5
- package/dist/storage/MongoBucketStorage.js +404 -215
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
- package/dist/storage/implementation/MongoBucketBatch.js +169 -98
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
- package/dist/storage/implementation/MongoCompactor.js +27 -238
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
- package/dist/storage/implementation/MongoIdSequence.js +14 -1
- package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
- package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
- package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +8 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
- package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
- package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
- package/dist/storage/implementation/MongoWriteBatch.js +149 -0
- package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
- package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
- package/dist/storage/implementation/common/PersistedBatch.js +57 -14
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +9 -1
- package/dist/storage/implementation/db.js +28 -1
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +35 -0
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
- package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
- package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
- package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
- package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.js +2 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
- package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
- package/dist/storage/implementation/v3/compact-utils.js +223 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +90 -6
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
- package/dist/storage/implementation/v3/source-table-utils.js +49 -57
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
- package/dist/storage/storage-index.d.ts +2 -1
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +28 -1
- package/dist/types/types.js +28 -7
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +4 -2
- package/dist/utils/test-utils.js +7 -4
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.js +3 -1
- package/dist/utils/util.js.map +1 -1
- package/package.json +10 -9
- package/src/storage/MongoBucketStorage.ts +545 -277
- package/src/storage/implementation/MongoBucketBatch.ts +258 -124
- package/src/storage/implementation/MongoCompactor.ts +37 -346
- package/src/storage/implementation/MongoIdSequence.ts +21 -1
- package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
- package/src/storage/implementation/MongoStorageProvider.ts +16 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
- package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
- package/src/storage/implementation/MongoWriteBatch.ts +172 -0
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
- package/src/storage/implementation/common/PersistedBatch.ts +76 -20
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
- package/src/storage/implementation/db.ts +36 -2
- package/src/storage/implementation/models.ts +39 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
- package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
- package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
- package/src/storage/implementation/v1/models.ts +1 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
- package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
- package/src/storage/implementation/v3/bucket-format.ts +2 -0
- package/src/storage/implementation/v3/compact-utils.ts +329 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +97 -4
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
- package/src/storage/implementation/v3/source-table-utils.ts +82 -76
- package/src/storage/storage-index.ts +2 -0
- package/src/types/types.ts +42 -7
- package/src/utils/test-utils.ts +9 -5
- package/src/utils/util.ts +3 -1
- package/test/src/__snapshots__/storage.test.ts.snap +25 -0
- package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
- package/test/src/chunk_compaction_config.test.ts +38 -0
- package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
- package/test/src/compact-utils.test.ts +251 -0
- package/test/src/concurrent_writers.test.ts +599 -0
- package/test/src/connection_config.test.ts +159 -0
- package/test/src/helpers/MemoryObjectStorage.ts +14 -6
- package/test/src/helpers/concurrentWriter.mjs +29 -0
- package/test/src/helpers/s3TestFactory.ts +12 -12
- package/test/src/object_storage_usage.test.ts +268 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/storage.test.ts +280 -24
- package/test/src/storage_compacting.test.ts +1097 -253
- package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
- package/test/src/storage_s3_checksums.test.ts +6 -7
- package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
- package/test/src/storage_s3_reading.test.ts +294 -6
- package/test/src/storage_s3_writing.test.ts +4 -3
- package/test/src/storage_sync.test.ts +412 -80
- package/test/src/storeCurrentData.test.ts +5 -5
- package/test/src/util.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,15 +1,20 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
1
2
|
import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
2
3
|
import { ColumnDescriptor, InternalOpId, SourceTable, storage } from '@powersync/service-core';
|
|
4
|
+
import { HydratedSyncConfig } from '@powersync/service-sync-rules';
|
|
3
5
|
import * as bson from 'bson';
|
|
4
6
|
import { mongoTableId } from '../../../utils/util.js';
|
|
5
7
|
import { calculateCheckpointState } from '../CheckpointState.js';
|
|
6
8
|
import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
|
|
9
|
+
import { MongoSyncRulesLock } from '../MongoSyncRulesLock.js';
|
|
10
|
+
import { MongoWriteBatch } from '../MongoWriteBatch.js';
|
|
7
11
|
import { PersistedBatch } from '../common/PersistedBatch.js';
|
|
8
12
|
import { SourceRecordStore } from '../common/SourceRecordStore.js';
|
|
13
|
+
import { SyncRuleDocumentBase } from '../models.js';
|
|
9
14
|
import { PersistedBatchV1 } from './PersistedBatchV1.js';
|
|
10
15
|
import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
|
|
11
16
|
import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
|
|
12
|
-
import { SyncRuleDocumentV1 } from './models.js';
|
|
17
|
+
import { SourceTableDocumentV1, SyncRuleDocumentV1 } from './models.js';
|
|
13
18
|
|
|
14
19
|
export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
15
20
|
declare readonly db: VersionedPowerSyncMongoV1;
|
|
@@ -32,6 +37,27 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
32
37
|
this.store = new SourceRecordStoreV1(this.db, this.replicationStreamId);
|
|
33
38
|
}
|
|
34
39
|
|
|
40
|
+
protected override persistedOpHead(stream: SyncRuleDocumentBase): InternalOpId {
|
|
41
|
+
const legacy = stream as SyncRuleDocumentV1;
|
|
42
|
+
const checkpoint = legacy.last_checkpoint ?? 0n;
|
|
43
|
+
const keepalive = legacy.keepalive_op == null ? 0n : BigInt(legacy.keepalive_op);
|
|
44
|
+
return checkpoint > keepalive ? checkpoint : keepalive;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
protected override onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: InternalOpId): void {
|
|
48
|
+
// Keep flushed operations recoverable before a checkpoint can be published.
|
|
49
|
+
// keepalive_op remains a decimal string for compatibility with legacy readers.
|
|
50
|
+
writes.updateOne(this.db.sync_rules, { _id: this.replicationStreamId }, [
|
|
51
|
+
{
|
|
52
|
+
$set: {
|
|
53
|
+
keepalive_op: {
|
|
54
|
+
$toString: { $max: [{ $toLong: '$keepalive_op' }, '$last_checkpoint', lastOp] }
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
]);
|
|
59
|
+
}
|
|
60
|
+
|
|
35
61
|
protected override recordPersistedOp(lastOp: InternalOpId): void {
|
|
36
62
|
this.persisted_op = lastOp;
|
|
37
63
|
}
|
|
@@ -49,8 +75,46 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
49
75
|
// No-op for V1: source records live in a shared collection.
|
|
50
76
|
}
|
|
51
77
|
|
|
78
|
+
protected override async batchCreateCustomWriteCheckpoints(
|
|
79
|
+
session: mongo.ClientSession,
|
|
80
|
+
opId: InternalOpId
|
|
81
|
+
): Promise<void> {
|
|
82
|
+
// Only the final checkpoint per user/stream is visible after this transaction.
|
|
83
|
+
const checkpoints = new Map(
|
|
84
|
+
this.write_checkpoint_batch.map((checkpoint) => [
|
|
85
|
+
JSON.stringify([checkpoint.sync_rules_id, checkpoint.user_id]),
|
|
86
|
+
checkpoint
|
|
87
|
+
])
|
|
88
|
+
);
|
|
89
|
+
await this.db.custom_write_checkpoints.bulkWrite(
|
|
90
|
+
[...checkpoints.values()].map((checkpointOptions) => {
|
|
91
|
+
const set: Record<string, unknown> = {
|
|
92
|
+
checkpoint: checkpointOptions.checkpoint,
|
|
93
|
+
sync_rules_id: checkpointOptions.sync_rules_id,
|
|
94
|
+
op_id: opId
|
|
95
|
+
};
|
|
96
|
+
if (checkpointOptions.checkpoint_requested_at != null) {
|
|
97
|
+
set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
return {
|
|
101
|
+
updateOne: {
|
|
102
|
+
filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
|
|
103
|
+
update: {
|
|
104
|
+
$set: set,
|
|
105
|
+
...(checkpointOptions.checkpoint_requested_at == null ? { $unset: { checkpoint_requested_at: 1 } } : {})
|
|
106
|
+
},
|
|
107
|
+
upsert: true
|
|
108
|
+
}
|
|
109
|
+
};
|
|
110
|
+
}),
|
|
111
|
+
{ session, ordered: false }
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
|
|
52
115
|
async resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult> {
|
|
53
116
|
const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
|
|
117
|
+
const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
|
|
54
118
|
const { connection_id, source } = options;
|
|
55
119
|
const { schema, name, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
|
|
56
120
|
|
|
@@ -61,22 +125,41 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
61
125
|
}));
|
|
62
126
|
|
|
63
127
|
let result: storage.ResolveTablesResult | null = null;
|
|
64
|
-
|
|
128
|
+
const session = this.session;
|
|
129
|
+
await this.withTransaction(async () => {
|
|
65
130
|
const col = this.db.sourceTablesV1(this.replicationStreamId);
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
schema_name: schema,
|
|
70
|
-
table_name: name,
|
|
71
|
-
replica_id_columns2: normalizedReplicaIdColumns
|
|
72
|
-
};
|
|
131
|
+
|
|
132
|
+
// Find records that overlap by name or relation id.
|
|
133
|
+
const orClauses: Record<string, unknown>[] = [{ schema_name: schema, table_name: name }];
|
|
73
134
|
if (objectId != null) {
|
|
74
|
-
|
|
135
|
+
orClauses.push({ relation_id: objectId });
|
|
75
136
|
}
|
|
137
|
+
const candidateDocs = await col
|
|
138
|
+
.find({ group_id: this.replicationStreamId, connection_id, $or: orClauses }, { session })
|
|
139
|
+
.toArray();
|
|
76
140
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
141
|
+
const candidateTables = candidateDocs.map((doc) =>
|
|
142
|
+
this.hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows)
|
|
143
|
+
);
|
|
144
|
+
const candidates = candidateTables.map((table) => table.clone());
|
|
145
|
+
const resolution = await reconcile({ source, candidates });
|
|
146
|
+
storage.validateSourceTableCandidateResolution(candidates, resolution);
|
|
147
|
+
|
|
148
|
+
for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
|
|
149
|
+
await col.updateOne({ _id: mongoTableId(id) }, { $set: { source_metadata: sourceMetadata } }, { session });
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
|
|
153
|
+
const compatibleTables = materializedResolution.compatibleTables;
|
|
154
|
+
|
|
155
|
+
// V1 keeps one record, preferring one that has already been snapshotted.
|
|
156
|
+
const reuse = compatibleTables.find((table) => table.snapshotComplete) ?? compatibleTables[0] ?? null;
|
|
157
|
+
|
|
158
|
+
let sourceTable: storage.SourceTable;
|
|
159
|
+
if (reuse != null) {
|
|
160
|
+
sourceTable = reuse;
|
|
161
|
+
} else {
|
|
162
|
+
const doc: SourceTableDocumentV1 = {
|
|
80
163
|
_id: options.idGenerator ? (options.idGenerator() as bson.ObjectId) : new bson.ObjectId(),
|
|
81
164
|
group_id: this.replicationStreamId,
|
|
82
165
|
connection_id,
|
|
@@ -86,69 +169,18 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
86
169
|
replica_id_columns: null,
|
|
87
170
|
replica_id_columns2: normalizedReplicaIdColumns,
|
|
88
171
|
snapshot_done: false,
|
|
89
|
-
snapshot_status: undefined
|
|
172
|
+
snapshot_status: undefined,
|
|
173
|
+
source_metadata: resolution.newTableValues.sourceMetadata
|
|
90
174
|
};
|
|
91
175
|
await col.insertOne(doc, { session });
|
|
176
|
+
sourceTable = this.hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows);
|
|
92
177
|
}
|
|
93
178
|
|
|
94
|
-
const
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
snapshotComplete: doc.snapshot_done ?? true,
|
|
100
|
-
...syncRules.getMatchingSources(source)
|
|
101
|
-
});
|
|
102
|
-
sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
|
|
103
|
-
sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
|
|
104
|
-
sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
|
|
105
|
-
sourceTable.storeCurrentData = sendsCompleteRows !== true;
|
|
106
|
-
sourceTable.snapshotStatus =
|
|
107
|
-
doc.snapshot_status == null
|
|
108
|
-
? undefined
|
|
109
|
-
: {
|
|
110
|
-
lastKey: doc.snapshot_status.last_key?.buffer ?? null,
|
|
111
|
-
totalEstimatedCount: doc.snapshot_status.total_estimated_count,
|
|
112
|
-
replicatedCount: doc.snapshot_status.replicated_count
|
|
113
|
-
};
|
|
114
|
-
|
|
115
|
-
const truncateFilter = [{ schema_name: schema, table_name: name }] as any[];
|
|
116
|
-
if (objectId != null) {
|
|
117
|
-
truncateFilter.push({ relation_id: objectId });
|
|
118
|
-
}
|
|
119
|
-
const truncate = await col
|
|
120
|
-
.find(
|
|
121
|
-
{
|
|
122
|
-
group_id: this.replicationStreamId,
|
|
123
|
-
connection_id,
|
|
124
|
-
_id: { $ne: doc._id },
|
|
125
|
-
$or: truncateFilter
|
|
126
|
-
},
|
|
127
|
-
{ session }
|
|
128
|
-
)
|
|
129
|
-
.toArray();
|
|
130
|
-
const dropTables = truncate.map((dropDoc) => {
|
|
131
|
-
const ref = {
|
|
132
|
-
connectionTag,
|
|
133
|
-
schema: dropDoc.schema_name,
|
|
134
|
-
name: dropDoc.table_name
|
|
135
|
-
};
|
|
136
|
-
const table = new storage.SourceTable({
|
|
137
|
-
id: dropDoc._id,
|
|
138
|
-
ref,
|
|
139
|
-
objectId: dropDoc.relation_id,
|
|
140
|
-
replicaIdColumns:
|
|
141
|
-
dropDoc.replica_id_columns2?.map(
|
|
142
|
-
(c) => ({ name: c.name, typeId: c.type_oid, type: c.type }) satisfies ColumnDescriptor
|
|
143
|
-
) ?? [],
|
|
144
|
-
snapshotComplete: dropDoc.snapshot_done ?? true,
|
|
145
|
-
...syncRules.getMatchingSources(ref)
|
|
146
|
-
});
|
|
147
|
-
table.syncEvent = syncRules.tableTriggersEvent(ref);
|
|
148
|
-
table.syncData = table.bucketDataSources.length > 0;
|
|
149
|
-
table.syncParameters = table.parameterLookupSources.length > 0;
|
|
150
|
-
return table;
|
|
151
|
-
});
|
|
179
|
+
const dropTables = [
|
|
180
|
+
...materializedResolution.incompatibleTables,
|
|
181
|
+
// V1 only keeps one SourceTable per physical table.
|
|
182
|
+
...compatibleTables.filter((table) => !storage.sourceTableIdEquals(table.id, sourceTable.id))
|
|
183
|
+
];
|
|
152
184
|
|
|
153
185
|
result = { tables: [sourceTable], dropTables };
|
|
154
186
|
});
|
|
@@ -156,38 +188,33 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
156
188
|
return result!;
|
|
157
189
|
}
|
|
158
190
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
const ref = {
|
|
172
|
-
connectionTag: table.ref.connectionTag,
|
|
173
|
-
schema: doc.schema_name,
|
|
174
|
-
name: doc.table_name
|
|
175
|
-
};
|
|
176
|
-
const sourceTable = new storage.SourceTable({
|
|
191
|
+
/**
|
|
192
|
+
* Build a SourceTable from its persisted v1 record.
|
|
193
|
+
*/
|
|
194
|
+
private hydrateSourceTable(
|
|
195
|
+
doc: SourceTableDocumentV1,
|
|
196
|
+
connectionTag: string,
|
|
197
|
+
syncRules: HydratedSyncConfig,
|
|
198
|
+
sendsCompleteRows: boolean | undefined
|
|
199
|
+
): storage.SourceTable {
|
|
200
|
+
const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
|
|
201
|
+
const table = new storage.SourceTable({
|
|
177
202
|
id: doc._id,
|
|
178
203
|
ref,
|
|
179
204
|
objectId: doc.relation_id,
|
|
180
205
|
replicaIdColumns:
|
|
181
206
|
doc.replica_id_columns2?.map(
|
|
182
|
-
(
|
|
207
|
+
(column) => ({ name: column.name, typeId: column.type_oid, type: column.type }) satisfies ColumnDescriptor
|
|
183
208
|
) ?? [],
|
|
184
209
|
snapshotComplete: doc.snapshot_done ?? true,
|
|
185
|
-
|
|
210
|
+
sourceMetadata: doc.source_metadata ?? null,
|
|
211
|
+
...syncRules.getMatchingSources(ref)
|
|
186
212
|
});
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
213
|
+
table.syncEvent = syncRules.tableTriggersEvent(ref);
|
|
214
|
+
table.syncData = table.bucketDataSources.length > 0;
|
|
215
|
+
table.syncParameters = table.parameterLookupSources.length > 0;
|
|
216
|
+
table.storeCurrentData = sendsCompleteRows !== true;
|
|
217
|
+
table.snapshotStatus =
|
|
191
218
|
doc.snapshot_status == null
|
|
192
219
|
? undefined
|
|
193
220
|
: {
|
|
@@ -195,135 +222,158 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
195
222
|
totalEstimatedCount: doc.snapshot_status.total_estimated_count,
|
|
196
223
|
replicatedCount: doc.snapshot_status.replicated_count
|
|
197
224
|
};
|
|
198
|
-
return
|
|
225
|
+
return table;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
async getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null> {
|
|
229
|
+
const doc = await this.db.sourceTablesV1(this.replicationStreamId).findOne(
|
|
230
|
+
{
|
|
231
|
+
group_id: this.replicationStreamId,
|
|
232
|
+
_id: mongoTableId(table.id)
|
|
233
|
+
},
|
|
234
|
+
{ session: this.session }
|
|
235
|
+
);
|
|
236
|
+
if (doc == null) {
|
|
237
|
+
return null;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
// Use the default storeCurrentData value when reading outside resolution.
|
|
241
|
+
return this.hydrateSourceTable(doc, table.ref.connectionTag, this.sync_rules, undefined);
|
|
199
242
|
}
|
|
200
243
|
|
|
201
244
|
async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult> {
|
|
202
245
|
const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
|
|
203
246
|
|
|
204
|
-
await this.flush(options);
|
|
205
|
-
|
|
206
247
|
using _ = this.tracer.span('storage', 'commit');
|
|
207
248
|
|
|
208
|
-
const
|
|
249
|
+
const updateCheckpoint = async (persistedOp = this.persisted_op) => {
|
|
250
|
+
const now = new Date();
|
|
209
251
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
session: this.session
|
|
222
|
-
}
|
|
223
|
-
);
|
|
252
|
+
const can_checkpoint = {
|
|
253
|
+
$and: [
|
|
254
|
+
{ $eq: ['$snapshot_done', true] },
|
|
255
|
+
{
|
|
256
|
+
$or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
|
|
257
|
+
},
|
|
258
|
+
{
|
|
259
|
+
$or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
|
|
260
|
+
}
|
|
261
|
+
]
|
|
262
|
+
};
|
|
224
263
|
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
264
|
+
const new_keepalive_op = {
|
|
265
|
+
$cond: [
|
|
266
|
+
can_checkpoint,
|
|
267
|
+
{ $literal: null },
|
|
268
|
+
{
|
|
269
|
+
$toString: {
|
|
270
|
+
$max: [{ $toLong: '$keepalive_op' }, { $literal: persistedOp }, 0n]
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
]
|
|
274
|
+
};
|
|
236
275
|
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
276
|
+
const new_last_checkpoint = {
|
|
277
|
+
$cond: [
|
|
278
|
+
can_checkpoint,
|
|
279
|
+
{
|
|
280
|
+
$max: ['$last_checkpoint', { $literal: persistedOp }, { $toLong: '$keepalive_op' }, 0n]
|
|
281
|
+
},
|
|
282
|
+
'$last_checkpoint'
|
|
283
|
+
]
|
|
284
|
+
};
|
|
285
|
+
|
|
286
|
+
const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate(
|
|
287
|
+
MongoSyncRulesLock.ownerFilter(this.replicationStreamId, this.options.replicationLock),
|
|
288
|
+
[
|
|
289
|
+
{
|
|
290
|
+
$set: {
|
|
291
|
+
_can_checkpoint: can_checkpoint,
|
|
292
|
+
_not_empty: createEmptyCheckpoints
|
|
293
|
+
? true
|
|
294
|
+
: {
|
|
295
|
+
$or: [
|
|
296
|
+
{ $literal: createEmptyCheckpoints },
|
|
297
|
+
{ $ne: ['$keepalive_op', new_keepalive_op] },
|
|
298
|
+
{ $ne: ['$last_checkpoint', new_last_checkpoint] }
|
|
299
|
+
]
|
|
300
|
+
}
|
|
301
|
+
}
|
|
302
|
+
},
|
|
303
|
+
{
|
|
304
|
+
$set: {
|
|
305
|
+
...MongoSyncRulesLock.heartbeatUpdate(),
|
|
306
|
+
last_checkpoint_lsn: {
|
|
307
|
+
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
|
|
308
|
+
},
|
|
309
|
+
last_checkpoint_ts: {
|
|
310
|
+
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
|
|
311
|
+
},
|
|
312
|
+
|
|
313
|
+
last_fatal_error: { $literal: null },
|
|
314
|
+
last_fatal_error_ts: { $literal: null },
|
|
315
|
+
keepalive_op: new_keepalive_op,
|
|
316
|
+
last_checkpoint: new_last_checkpoint,
|
|
317
|
+
snapshot_lsn: {
|
|
318
|
+
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
},
|
|
322
|
+
{
|
|
323
|
+
$unset: ['_can_checkpoint', '_not_empty']
|
|
324
|
+
}
|
|
325
|
+
],
|
|
241
326
|
{
|
|
242
|
-
|
|
243
|
-
|
|
327
|
+
session: this.session,
|
|
328
|
+
returnDocument: 'before',
|
|
329
|
+
projection: {
|
|
330
|
+
snapshot_done: 1,
|
|
331
|
+
last_checkpoint_lsn: 1,
|
|
332
|
+
no_checkpoint_before: 1,
|
|
333
|
+
keepalive_op: 1,
|
|
334
|
+
last_checkpoint: 1
|
|
244
335
|
}
|
|
245
336
|
}
|
|
246
|
-
|
|
247
|
-
};
|
|
337
|
+
)) as SyncRuleDocumentV1;
|
|
248
338
|
|
|
249
|
-
|
|
250
|
-
$cond: [
|
|
251
|
-
can_checkpoint,
|
|
252
|
-
{
|
|
253
|
-
$max: ['$last_checkpoint', { $literal: this.persisted_op }, { $toLong: '$keepalive_op' }, 0n]
|
|
254
|
-
},
|
|
255
|
-
'$last_checkpoint'
|
|
256
|
-
]
|
|
339
|
+
return MongoSyncRulesLock.assertOwned(preUpdateDocument);
|
|
257
340
|
};
|
|
258
341
|
|
|
259
|
-
const
|
|
260
|
-
|
|
261
|
-
[
|
|
342
|
+
const finishCheckpoint = async (preUpdateDocument: SyncRuleDocumentV1, lastOp?: InternalOpId) => {
|
|
343
|
+
await this.db.write_checkpoints.updateMany(
|
|
262
344
|
{
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
_not_empty: createEmptyCheckpoints
|
|
266
|
-
? true
|
|
267
|
-
: {
|
|
268
|
-
$or: [
|
|
269
|
-
{ $literal: createEmptyCheckpoints },
|
|
270
|
-
{ $ne: ['$keepalive_op', new_keepalive_op] },
|
|
271
|
-
{ $ne: ['$last_checkpoint', new_last_checkpoint] }
|
|
272
|
-
]
|
|
273
|
-
}
|
|
274
|
-
}
|
|
345
|
+
processed_at_lsn: null,
|
|
346
|
+
'lsns.1': { $lte: lsn }
|
|
275
347
|
},
|
|
276
348
|
{
|
|
277
349
|
$set: {
|
|
278
|
-
|
|
279
|
-
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
|
|
280
|
-
},
|
|
281
|
-
last_checkpoint_ts: {
|
|
282
|
-
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
|
|
283
|
-
},
|
|
284
|
-
last_keepalive_ts: { $literal: now },
|
|
285
|
-
last_fatal_error: { $literal: null },
|
|
286
|
-
last_fatal_error_ts: { $literal: null },
|
|
287
|
-
keepalive_op: new_keepalive_op,
|
|
288
|
-
last_checkpoint: new_last_checkpoint,
|
|
289
|
-
snapshot_lsn: {
|
|
290
|
-
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
|
|
291
|
-
}
|
|
350
|
+
processed_at_lsn: lsn
|
|
292
351
|
}
|
|
293
352
|
},
|
|
294
353
|
{
|
|
295
|
-
|
|
354
|
+
session: this.session
|
|
296
355
|
}
|
|
297
|
-
],
|
|
298
|
-
{
|
|
299
|
-
session: this.session,
|
|
300
|
-
returnDocument: 'before',
|
|
301
|
-
projection: {
|
|
302
|
-
snapshot_done: 1,
|
|
303
|
-
last_checkpoint_lsn: 1,
|
|
304
|
-
no_checkpoint_before: 1,
|
|
305
|
-
keepalive_op: 1,
|
|
306
|
-
last_checkpoint: 1
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
)) as SyncRuleDocumentV1;
|
|
310
|
-
|
|
311
|
-
if (preUpdateDocument == null) {
|
|
312
|
-
throw new ReplicationAssertionError(
|
|
313
|
-
'Failed to update checkpoint - no matching sync_rules document for _id: ' + this.replicationStreamId
|
|
314
356
|
);
|
|
315
|
-
}
|
|
316
357
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
358
|
+
const checkpointState = calculateCheckpointState({
|
|
359
|
+
lsn,
|
|
360
|
+
snapshotDone: preUpdateDocument.snapshot_done === true,
|
|
361
|
+
lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
|
|
362
|
+
noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
|
|
363
|
+
keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
|
|
364
|
+
lastCheckpoint: preUpdateDocument.last_checkpoint,
|
|
365
|
+
persistedOp: lastOp ?? this.persistedOpHead(preUpdateDocument),
|
|
366
|
+
createEmptyCheckpoints
|
|
367
|
+
});
|
|
368
|
+
return { checkpointState, preUpdateDocument };
|
|
369
|
+
};
|
|
370
|
+
|
|
371
|
+
const { checkpointState, preUpdateDocument } = await this.flushAndCommit(
|
|
372
|
+
async (_stream, lastOp) => finishCheckpoint(await updateCheckpoint(lastOp), lastOp),
|
|
373
|
+
// Without a flush, the checkpoint update itself fences the transaction.
|
|
374
|
+
() => this.withFencedTransaction(updateCheckpoint, finishCheckpoint),
|
|
375
|
+
options
|
|
376
|
+
);
|
|
327
377
|
if (checkpointState.checkpointBlocked) {
|
|
328
378
|
if (Date.now() - this.lastWaitingLogThrottled > 5_000) {
|
|
329
379
|
this.logger.info(
|
|
@@ -361,41 +411,20 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
361
411
|
|
|
362
412
|
async setResumeLsn(lsn: string): Promise<void> {
|
|
363
413
|
using _ = this.tracer.span('storage', 'set_resume_lsn');
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
},
|
|
368
|
-
{
|
|
369
|
-
$set: {
|
|
370
|
-
snapshot_lsn: lsn
|
|
371
|
-
}
|
|
372
|
-
},
|
|
373
|
-
{
|
|
374
|
-
session: this.session,
|
|
375
|
-
// Losing occasional resume LSN is fine. That may mean reprocessing
|
|
376
|
-
// some source changes in some edge cases, which is not an issue since
|
|
377
|
-
// changes are processed in an idempotent way.
|
|
378
|
-
writeConcern: { w: 1 }
|
|
379
|
-
}
|
|
380
|
-
);
|
|
414
|
+
// Losing occasional resume LSN would only reprocess source changes.
|
|
415
|
+
// Keep the lease check atomic, but do not wait for majority replication of this resume hint.
|
|
416
|
+
await this.updateStreamMetadata({ snapshot_lsn: { $literal: lsn } }, {}, { w: 1 });
|
|
381
417
|
}
|
|
382
418
|
|
|
383
419
|
async markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void> {
|
|
384
|
-
await this.
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
},
|
|
393
|
-
$max: {
|
|
394
|
-
no_checkpoint_before: no_checkpoint_before_lsn
|
|
395
|
-
}
|
|
396
|
-
},
|
|
397
|
-
{ session: this.session }
|
|
398
|
-
);
|
|
420
|
+
await this.updateStreamMetadata(this.snapshotDoneUpdate(no_checkpoint_before_lsn));
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
private snapshotDoneUpdate(no_checkpoint_before_lsn: string) {
|
|
424
|
+
return {
|
|
425
|
+
snapshot_done: true,
|
|
426
|
+
no_checkpoint_before: { $max: ['$no_checkpoint_before', { $literal: no_checkpoint_before_lsn }] }
|
|
427
|
+
};
|
|
399
428
|
}
|
|
400
429
|
|
|
401
430
|
async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise<void> {
|
|
@@ -425,22 +454,12 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
425
454
|
}
|
|
426
455
|
}
|
|
427
456
|
|
|
428
|
-
await this.
|
|
457
|
+
await this.updateStreamMetadataInTransaction(this.snapshotDoneUpdate(no_checkpoint_before_lsn));
|
|
429
458
|
});
|
|
430
459
|
}
|
|
431
460
|
|
|
432
461
|
async markTableSnapshotRequired(_table: storage.SourceTable): Promise<void> {
|
|
433
|
-
await this.
|
|
434
|
-
{
|
|
435
|
-
_id: this.replicationStreamId
|
|
436
|
-
},
|
|
437
|
-
{
|
|
438
|
-
$set: {
|
|
439
|
-
snapshot_done: false
|
|
440
|
-
}
|
|
441
|
-
},
|
|
442
|
-
{ session: this.session }
|
|
443
|
-
);
|
|
462
|
+
await this.updateStreamMetadata({ snapshot_done: false });
|
|
444
463
|
}
|
|
445
464
|
|
|
446
465
|
async markTableSnapshotDone(
|
|
@@ -451,7 +470,9 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
451
470
|
const ids = tables.map((table) => mongoTableId(table.id));
|
|
452
471
|
|
|
453
472
|
await this.withTransaction(async () => {
|
|
454
|
-
|
|
473
|
+
const writes = this.db.createWriteBatch(session, { ordered: false });
|
|
474
|
+
writes.updateMany(
|
|
475
|
+
this.db.sourceTablesV1(this.replicationStreamId),
|
|
455
476
|
{ _id: { $in: ids } },
|
|
456
477
|
{
|
|
457
478
|
$set: {
|
|
@@ -460,26 +481,23 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
460
481
|
$unset: {
|
|
461
482
|
snapshot_status: 1
|
|
462
483
|
}
|
|
463
|
-
}
|
|
464
|
-
{ session }
|
|
484
|
+
}
|
|
465
485
|
);
|
|
466
486
|
|
|
467
487
|
if (no_checkpoint_before_lsn != null) {
|
|
468
|
-
|
|
488
|
+
writes.updateOne(
|
|
489
|
+
this.db.sync_rules,
|
|
469
490
|
{
|
|
470
491
|
_id: this.replicationStreamId
|
|
471
492
|
},
|
|
472
493
|
{
|
|
473
|
-
$set: {
|
|
474
|
-
last_keepalive_ts: new Date()
|
|
475
|
-
},
|
|
476
494
|
$max: {
|
|
477
495
|
no_checkpoint_before: no_checkpoint_before_lsn
|
|
478
496
|
}
|
|
479
|
-
}
|
|
480
|
-
{ session: this.session }
|
|
497
|
+
}
|
|
481
498
|
);
|
|
482
499
|
}
|
|
500
|
+
await writes.execute();
|
|
483
501
|
});
|
|
484
502
|
return tables.map((table) => {
|
|
485
503
|
const copy = table.clone();
|
|
@@ -496,7 +514,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
496
514
|
const session = this.session;
|
|
497
515
|
let activated = false;
|
|
498
516
|
let needsFutureActivationCheck = true;
|
|
499
|
-
await
|
|
517
|
+
await this.withTransaction(async () => {
|
|
500
518
|
// Reset on transaction retries.
|
|
501
519
|
activated = false;
|
|
502
520
|
needsFutureActivationCheck = true;
|
|
@@ -534,7 +552,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
534
552
|
} else if (doc?.state != storage.SyncRuleState.PROCESSING) {
|
|
535
553
|
needsFutureActivationCheck = false;
|
|
536
554
|
}
|
|
537
|
-
});
|
|
555
|
+
}, session);
|
|
538
556
|
if (activated) {
|
|
539
557
|
this.logger.info(`Activated new replication stream at ${lsn}`);
|
|
540
558
|
await this.db.notifyCheckpoint();
|