@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +129 -7
- package/dist/storage/MongoBucketStorage.d.ts +42 -5
- package/dist/storage/MongoBucketStorage.js +404 -215
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
- package/dist/storage/implementation/MongoBucketBatch.js +169 -98
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
- package/dist/storage/implementation/MongoCompactor.js +27 -238
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
- package/dist/storage/implementation/MongoIdSequence.js +14 -1
- package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
- package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
- package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +8 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
- package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
- package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
- package/dist/storage/implementation/MongoWriteBatch.js +149 -0
- package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
- package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
- package/dist/storage/implementation/common/PersistedBatch.js +57 -14
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +9 -1
- package/dist/storage/implementation/db.js +28 -1
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +35 -0
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
- package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
- package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
- package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
- package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.js +2 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
- package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
- package/dist/storage/implementation/v3/compact-utils.js +223 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +90 -6
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
- package/dist/storage/implementation/v3/source-table-utils.js +49 -57
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
- package/dist/storage/storage-index.d.ts +2 -1
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +28 -1
- package/dist/types/types.js +28 -7
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +4 -2
- package/dist/utils/test-utils.js +7 -4
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.js +3 -1
- package/dist/utils/util.js.map +1 -1
- package/package.json +10 -9
- package/src/storage/MongoBucketStorage.ts +545 -277
- package/src/storage/implementation/MongoBucketBatch.ts +258 -124
- package/src/storage/implementation/MongoCompactor.ts +37 -346
- package/src/storage/implementation/MongoIdSequence.ts +21 -1
- package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
- package/src/storage/implementation/MongoStorageProvider.ts +16 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
- package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
- package/src/storage/implementation/MongoWriteBatch.ts +172 -0
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
- package/src/storage/implementation/common/PersistedBatch.ts +76 -20
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
- package/src/storage/implementation/db.ts +36 -2
- package/src/storage/implementation/models.ts +39 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
- package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
- package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
- package/src/storage/implementation/v1/models.ts +1 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
- package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
- package/src/storage/implementation/v3/bucket-format.ts +2 -0
- package/src/storage/implementation/v3/compact-utils.ts +329 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +97 -4
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
- package/src/storage/implementation/v3/source-table-utils.ts +82 -76
- package/src/storage/storage-index.ts +2 -0
- package/src/types/types.ts +42 -7
- package/src/utils/test-utils.ts +9 -5
- package/src/utils/util.ts +3 -1
- package/test/src/__snapshots__/storage.test.ts.snap +25 -0
- package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
- package/test/src/chunk_compaction_config.test.ts +38 -0
- package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
- package/test/src/compact-utils.test.ts +251 -0
- package/test/src/concurrent_writers.test.ts +599 -0
- package/test/src/connection_config.test.ts +159 -0
- package/test/src/helpers/MemoryObjectStorage.ts +14 -6
- package/test/src/helpers/concurrentWriter.mjs +29 -0
- package/test/src/helpers/s3TestFactory.ts +12 -12
- package/test/src/object_storage_usage.test.ts +268 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/storage.test.ts +280 -24
- package/test/src/storage_compacting.test.ts +1097 -253
- package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
- package/test/src/storage_s3_checksums.test.ts +6 -7
- package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
- package/test/src/storage_s3_reading.test.ts +294 -6
- package/test/src/storage_s3_writing.test.ts +4 -3
- package/test/src/storage_sync.test.ts +412 -80
- package/test/src/storeCurrentData.test.ts +5 -5
- package/test/src/util.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
1
2
|
import { InternalOpId, SourceTable, storage } from '@powersync/service-core';
|
|
2
3
|
import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
|
|
4
|
+
import { MongoWriteBatch } from '../MongoWriteBatch.js';
|
|
3
5
|
import { PersistedBatch } from '../common/PersistedBatch.js';
|
|
4
6
|
import { SourceRecordStore } from '../common/SourceRecordStore.js';
|
|
7
|
+
import { SyncRuleDocumentBase } from '../models.js';
|
|
5
8
|
import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
|
|
6
9
|
export declare class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
7
10
|
readonly db: VersionedPowerSyncMongoV1;
|
|
@@ -16,16 +19,24 @@ export declare class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
16
19
|
*/
|
|
17
20
|
private persisted_op;
|
|
18
21
|
constructor(options: MongoBucketBatchOptions);
|
|
22
|
+
protected persistedOpHead(stream: SyncRuleDocumentBase): InternalOpId;
|
|
23
|
+
protected onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: InternalOpId): void;
|
|
19
24
|
protected recordPersistedOp(lastOp: InternalOpId): void;
|
|
20
25
|
protected createPersistedBatch(writtenSize: number): PersistedBatch;
|
|
21
26
|
protected get sourceRecordStore(): SourceRecordStore;
|
|
22
27
|
protected cleanupDroppedSourceTables(_tables: SourceTable[]): Promise<void>;
|
|
28
|
+
protected batchCreateCustomWriteCheckpoints(session: mongo.ClientSession, opId: InternalOpId): Promise<void>;
|
|
23
29
|
resolveTables(options: storage.ResolveTablesOptions): Promise<storage.ResolveTablesResult>;
|
|
30
|
+
/**
|
|
31
|
+
* Build a SourceTable from its persisted v1 record.
|
|
32
|
+
*/
|
|
33
|
+
private hydrateSourceTable;
|
|
24
34
|
getSourceTableStatus(table: storage.SourceTable): Promise<storage.SourceTable | null>;
|
|
25
35
|
commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise<storage.CheckpointResult>;
|
|
26
36
|
keepalive(lsn: string): Promise<storage.CheckpointResult>;
|
|
27
37
|
setResumeLsn(lsn: string): Promise<void>;
|
|
28
38
|
markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
|
|
39
|
+
private snapshotDoneUpdate;
|
|
29
40
|
markSnapshotDone(no_checkpoint_before_lsn: string, options?: {
|
|
30
41
|
throwOnConflict?: boolean;
|
|
31
42
|
}): Promise<void>;
|
|
@@ -4,6 +4,7 @@ import * as bson from 'bson';
|
|
|
4
4
|
import { mongoTableId } from '../../../utils/util.js';
|
|
5
5
|
import { calculateCheckpointState } from '../CheckpointState.js';
|
|
6
6
|
import { MongoBucketBatch } from '../MongoBucketBatch.js';
|
|
7
|
+
import { MongoSyncRulesLock } from '../MongoSyncRulesLock.js';
|
|
7
8
|
import { PersistedBatchV1 } from './PersistedBatchV1.js';
|
|
8
9
|
import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
|
|
9
10
|
export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
@@ -22,6 +23,25 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
22
23
|
this.persisted_op = options.keepaliveOp ?? null;
|
|
23
24
|
this.store = new SourceRecordStoreV1(this.db, this.replicationStreamId);
|
|
24
25
|
}
|
|
26
|
+
persistedOpHead(stream) {
|
|
27
|
+
const legacy = stream;
|
|
28
|
+
const checkpoint = legacy.last_checkpoint ?? 0n;
|
|
29
|
+
const keepalive = legacy.keepalive_op == null ? 0n : BigInt(legacy.keepalive_op);
|
|
30
|
+
return checkpoint > keepalive ? checkpoint : keepalive;
|
|
31
|
+
}
|
|
32
|
+
onReplicationTransactionFlush(writes, lastOp) {
|
|
33
|
+
// Keep flushed operations recoverable before a checkpoint can be published.
|
|
34
|
+
// keepalive_op remains a decimal string for compatibility with legacy readers.
|
|
35
|
+
writes.updateOne(this.db.sync_rules, { _id: this.replicationStreamId }, [
|
|
36
|
+
{
|
|
37
|
+
$set: {
|
|
38
|
+
keepalive_op: {
|
|
39
|
+
$toString: { $max: [{ $toLong: '$keepalive_op' }, '$last_checkpoint', lastOp] }
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
]);
|
|
44
|
+
}
|
|
25
45
|
recordPersistedOp(lastOp) {
|
|
26
46
|
this.persisted_op = lastOp;
|
|
27
47
|
}
|
|
@@ -36,8 +56,36 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
36
56
|
async cleanupDroppedSourceTables(_tables) {
|
|
37
57
|
// No-op for V1: source records live in a shared collection.
|
|
38
58
|
}
|
|
59
|
+
async batchCreateCustomWriteCheckpoints(session, opId) {
|
|
60
|
+
// Only the final checkpoint per user/stream is visible after this transaction.
|
|
61
|
+
const checkpoints = new Map(this.write_checkpoint_batch.map((checkpoint) => [
|
|
62
|
+
JSON.stringify([checkpoint.sync_rules_id, checkpoint.user_id]),
|
|
63
|
+
checkpoint
|
|
64
|
+
]));
|
|
65
|
+
await this.db.custom_write_checkpoints.bulkWrite([...checkpoints.values()].map((checkpointOptions) => {
|
|
66
|
+
const set = {
|
|
67
|
+
checkpoint: checkpointOptions.checkpoint,
|
|
68
|
+
sync_rules_id: checkpointOptions.sync_rules_id,
|
|
69
|
+
op_id: opId
|
|
70
|
+
};
|
|
71
|
+
if (checkpointOptions.checkpoint_requested_at != null) {
|
|
72
|
+
set.checkpoint_requested_at = checkpointOptions.checkpoint_requested_at;
|
|
73
|
+
}
|
|
74
|
+
return {
|
|
75
|
+
updateOne: {
|
|
76
|
+
filter: { user_id: checkpointOptions.user_id, sync_rules_id: checkpointOptions.sync_rules_id },
|
|
77
|
+
update: {
|
|
78
|
+
$set: set,
|
|
79
|
+
...(checkpointOptions.checkpoint_requested_at == null ? { $unset: { checkpoint_requested_at: 1 } } : {})
|
|
80
|
+
},
|
|
81
|
+
upsert: true
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
}), { session, ordered: false });
|
|
85
|
+
}
|
|
39
86
|
async resolveTables(options) {
|
|
40
87
|
const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
|
|
88
|
+
const reconcile = options.reconcileSourceTables ?? storage.defaultSourceTableReconciler;
|
|
41
89
|
const { connection_id, source } = options;
|
|
42
90
|
const { schema, name, objectId, replicaIdColumns, connectionTag, sendsCompleteRows } = source;
|
|
43
91
|
const normalizedReplicaIdColumns = replicaIdColumns.map((column) => ({
|
|
@@ -46,21 +94,34 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
46
94
|
type_oid: column.typeId
|
|
47
95
|
}));
|
|
48
96
|
let result = null;
|
|
49
|
-
|
|
97
|
+
const session = this.session;
|
|
98
|
+
await this.withTransaction(async () => {
|
|
50
99
|
const col = this.db.sourceTablesV1(this.replicationStreamId);
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
connection_id,
|
|
54
|
-
schema_name: schema,
|
|
55
|
-
table_name: name,
|
|
56
|
-
replica_id_columns2: normalizedReplicaIdColumns
|
|
57
|
-
};
|
|
100
|
+
// Find records that overlap by name or relation id.
|
|
101
|
+
const orClauses = [{ schema_name: schema, table_name: name }];
|
|
58
102
|
if (objectId != null) {
|
|
59
|
-
|
|
103
|
+
orClauses.push({ relation_id: objectId });
|
|
60
104
|
}
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
105
|
+
const candidateDocs = await col
|
|
106
|
+
.find({ group_id: this.replicationStreamId, connection_id, $or: orClauses }, { session })
|
|
107
|
+
.toArray();
|
|
108
|
+
const candidateTables = candidateDocs.map((doc) => this.hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows));
|
|
109
|
+
const candidates = candidateTables.map((table) => table.clone());
|
|
110
|
+
const resolution = await reconcile({ source, candidates });
|
|
111
|
+
storage.validateSourceTableCandidateResolution(candidates, resolution);
|
|
112
|
+
for (const { id, sourceMetadata } of storage.diffSourceTableUpdates(candidateTables, resolution)) {
|
|
113
|
+
await col.updateOne({ _id: mongoTableId(id) }, { $set: { source_metadata: sourceMetadata } }, { session });
|
|
114
|
+
}
|
|
115
|
+
const materializedResolution = storage.materializeSourceTableResolution(candidateTables, resolution);
|
|
116
|
+
const compatibleTables = materializedResolution.compatibleTables;
|
|
117
|
+
// V1 keeps one record, preferring one that has already been snapshotted.
|
|
118
|
+
const reuse = compatibleTables.find((table) => table.snapshotComplete) ?? compatibleTables[0] ?? null;
|
|
119
|
+
let sourceTable;
|
|
120
|
+
if (reuse != null) {
|
|
121
|
+
sourceTable = reuse;
|
|
122
|
+
}
|
|
123
|
+
else {
|
|
124
|
+
const doc = {
|
|
64
125
|
_id: options.idGenerator ? options.idGenerator() : new bson.ObjectId(),
|
|
65
126
|
group_id: this.replicationStreamId,
|
|
66
127
|
connection_id,
|
|
@@ -70,90 +131,40 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
70
131
|
replica_id_columns: null,
|
|
71
132
|
replica_id_columns2: normalizedReplicaIdColumns,
|
|
72
133
|
snapshot_done: false,
|
|
73
|
-
snapshot_status: undefined
|
|
134
|
+
snapshot_status: undefined,
|
|
135
|
+
source_metadata: resolution.newTableValues.sourceMetadata
|
|
74
136
|
};
|
|
75
137
|
await col.insertOne(doc, { session });
|
|
138
|
+
sourceTable = this.hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows);
|
|
76
139
|
}
|
|
77
|
-
const
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
snapshotComplete: doc.snapshot_done ?? true,
|
|
83
|
-
...syncRules.getMatchingSources(source)
|
|
84
|
-
});
|
|
85
|
-
sourceTable.syncEvent = syncRules.tableTriggersEvent(source);
|
|
86
|
-
sourceTable.syncData = sourceTable.bucketDataSources.length > 0;
|
|
87
|
-
sourceTable.syncParameters = sourceTable.parameterLookupSources.length > 0;
|
|
88
|
-
sourceTable.storeCurrentData = sendsCompleteRows !== true;
|
|
89
|
-
sourceTable.snapshotStatus =
|
|
90
|
-
doc.snapshot_status == null
|
|
91
|
-
? undefined
|
|
92
|
-
: {
|
|
93
|
-
lastKey: doc.snapshot_status.last_key?.buffer ?? null,
|
|
94
|
-
totalEstimatedCount: doc.snapshot_status.total_estimated_count,
|
|
95
|
-
replicatedCount: doc.snapshot_status.replicated_count
|
|
96
|
-
};
|
|
97
|
-
const truncateFilter = [{ schema_name: schema, table_name: name }];
|
|
98
|
-
if (objectId != null) {
|
|
99
|
-
truncateFilter.push({ relation_id: objectId });
|
|
100
|
-
}
|
|
101
|
-
const truncate = await col
|
|
102
|
-
.find({
|
|
103
|
-
group_id: this.replicationStreamId,
|
|
104
|
-
connection_id,
|
|
105
|
-
_id: { $ne: doc._id },
|
|
106
|
-
$or: truncateFilter
|
|
107
|
-
}, { session })
|
|
108
|
-
.toArray();
|
|
109
|
-
const dropTables = truncate.map((dropDoc) => {
|
|
110
|
-
const ref = {
|
|
111
|
-
connectionTag,
|
|
112
|
-
schema: dropDoc.schema_name,
|
|
113
|
-
name: dropDoc.table_name
|
|
114
|
-
};
|
|
115
|
-
const table = new storage.SourceTable({
|
|
116
|
-
id: dropDoc._id,
|
|
117
|
-
ref,
|
|
118
|
-
objectId: dropDoc.relation_id,
|
|
119
|
-
replicaIdColumns: dropDoc.replica_id_columns2?.map((c) => ({ name: c.name, typeId: c.type_oid, type: c.type })) ?? [],
|
|
120
|
-
snapshotComplete: dropDoc.snapshot_done ?? true,
|
|
121
|
-
...syncRules.getMatchingSources(ref)
|
|
122
|
-
});
|
|
123
|
-
table.syncEvent = syncRules.tableTriggersEvent(ref);
|
|
124
|
-
table.syncData = table.bucketDataSources.length > 0;
|
|
125
|
-
table.syncParameters = table.parameterLookupSources.length > 0;
|
|
126
|
-
return table;
|
|
127
|
-
});
|
|
140
|
+
const dropTables = [
|
|
141
|
+
...materializedResolution.incompatibleTables,
|
|
142
|
+
// V1 only keeps one SourceTable per physical table.
|
|
143
|
+
...compatibleTables.filter((table) => !storage.sourceTableIdEquals(table.id, sourceTable.id))
|
|
144
|
+
];
|
|
128
145
|
result = { tables: [sourceTable], dropTables };
|
|
129
146
|
});
|
|
130
147
|
return result;
|
|
131
148
|
}
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
return null;
|
|
139
|
-
}
|
|
140
|
-
const ref = {
|
|
141
|
-
connectionTag: table.ref.connectionTag,
|
|
142
|
-
schema: doc.schema_name,
|
|
143
|
-
name: doc.table_name
|
|
144
|
-
};
|
|
145
|
-
const sourceTable = new storage.SourceTable({
|
|
149
|
+
/**
|
|
150
|
+
* Build a SourceTable from its persisted v1 record.
|
|
151
|
+
*/
|
|
152
|
+
hydrateSourceTable(doc, connectionTag, syncRules, sendsCompleteRows) {
|
|
153
|
+
const ref = { connectionTag, schema: doc.schema_name, name: doc.table_name };
|
|
154
|
+
const table = new storage.SourceTable({
|
|
146
155
|
id: doc._id,
|
|
147
156
|
ref,
|
|
148
157
|
objectId: doc.relation_id,
|
|
149
|
-
replicaIdColumns: doc.replica_id_columns2?.map((
|
|
158
|
+
replicaIdColumns: doc.replica_id_columns2?.map((column) => ({ name: column.name, typeId: column.type_oid, type: column.type })) ?? [],
|
|
150
159
|
snapshotComplete: doc.snapshot_done ?? true,
|
|
151
|
-
|
|
160
|
+
sourceMetadata: doc.source_metadata ?? null,
|
|
161
|
+
...syncRules.getMatchingSources(ref)
|
|
152
162
|
});
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
163
|
+
table.syncEvent = syncRules.tableTriggersEvent(ref);
|
|
164
|
+
table.syncData = table.bucketDataSources.length > 0;
|
|
165
|
+
table.syncParameters = table.parameterLookupSources.length > 0;
|
|
166
|
+
table.storeCurrentData = sendsCompleteRows !== true;
|
|
167
|
+
table.snapshotStatus =
|
|
157
168
|
doc.snapshot_status == null
|
|
158
169
|
? undefined
|
|
159
170
|
: {
|
|
@@ -161,114 +172,130 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
161
172
|
totalEstimatedCount: doc.snapshot_status.total_estimated_count,
|
|
162
173
|
replicatedCount: doc.snapshot_status.replicated_count
|
|
163
174
|
};
|
|
164
|
-
return
|
|
175
|
+
return table;
|
|
176
|
+
}
|
|
177
|
+
async getSourceTableStatus(table) {
|
|
178
|
+
const doc = await this.db.sourceTablesV1(this.replicationStreamId).findOne({
|
|
179
|
+
group_id: this.replicationStreamId,
|
|
180
|
+
_id: mongoTableId(table.id)
|
|
181
|
+
}, { session: this.session });
|
|
182
|
+
if (doc == null) {
|
|
183
|
+
return null;
|
|
184
|
+
}
|
|
185
|
+
// Use the default storeCurrentData value when reading outside resolution.
|
|
186
|
+
return this.hydrateSourceTable(doc, table.ref.connectionTag, this.sync_rules, undefined);
|
|
165
187
|
}
|
|
166
188
|
async commit(lsn, options) {
|
|
167
189
|
const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
|
|
168
|
-
await this.flush(options);
|
|
169
190
|
using _ = this.tracer.span('storage', 'commit');
|
|
170
|
-
const
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
191
|
+
const updateCheckpoint = async (persistedOp = this.persisted_op) => {
|
|
192
|
+
const now = new Date();
|
|
193
|
+
const can_checkpoint = {
|
|
194
|
+
$and: [
|
|
195
|
+
{ $eq: ['$snapshot_done', true] },
|
|
196
|
+
{
|
|
197
|
+
$or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
$or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
|
|
201
|
+
}
|
|
202
|
+
]
|
|
203
|
+
};
|
|
204
|
+
const new_keepalive_op = {
|
|
205
|
+
$cond: [
|
|
206
|
+
can_checkpoint,
|
|
207
|
+
{ $literal: null },
|
|
208
|
+
{
|
|
209
|
+
$toString: {
|
|
210
|
+
$max: [{ $toLong: '$keepalive_op' }, { $literal: persistedOp }, 0n]
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
]
|
|
214
|
+
};
|
|
215
|
+
const new_last_checkpoint = {
|
|
216
|
+
$cond: [
|
|
217
|
+
can_checkpoint,
|
|
218
|
+
{
|
|
219
|
+
$max: ['$last_checkpoint', { $literal: persistedOp }, { $toLong: '$keepalive_op' }, 0n]
|
|
220
|
+
},
|
|
221
|
+
'$last_checkpoint'
|
|
222
|
+
]
|
|
223
|
+
};
|
|
224
|
+
const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate(MongoSyncRulesLock.ownerFilter(this.replicationStreamId, this.options.replicationLock), [
|
|
184
225
|
{
|
|
185
|
-
$
|
|
226
|
+
$set: {
|
|
227
|
+
_can_checkpoint: can_checkpoint,
|
|
228
|
+
_not_empty: createEmptyCheckpoints
|
|
229
|
+
? true
|
|
230
|
+
: {
|
|
231
|
+
$or: [
|
|
232
|
+
{ $literal: createEmptyCheckpoints },
|
|
233
|
+
{ $ne: ['$keepalive_op', new_keepalive_op] },
|
|
234
|
+
{ $ne: ['$last_checkpoint', new_last_checkpoint] }
|
|
235
|
+
]
|
|
236
|
+
}
|
|
237
|
+
}
|
|
186
238
|
},
|
|
187
239
|
{
|
|
188
|
-
$
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
240
|
+
$set: {
|
|
241
|
+
...MongoSyncRulesLock.heartbeatUpdate(),
|
|
242
|
+
last_checkpoint_lsn: {
|
|
243
|
+
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
|
|
244
|
+
},
|
|
245
|
+
last_checkpoint_ts: {
|
|
246
|
+
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
|
|
247
|
+
},
|
|
248
|
+
last_fatal_error: { $literal: null },
|
|
249
|
+
last_fatal_error_ts: { $literal: null },
|
|
250
|
+
keepalive_op: new_keepalive_op,
|
|
251
|
+
last_checkpoint: new_last_checkpoint,
|
|
252
|
+
snapshot_lsn: {
|
|
253
|
+
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
|
|
254
|
+
}
|
|
199
255
|
}
|
|
200
|
-
}
|
|
201
|
-
]
|
|
202
|
-
};
|
|
203
|
-
const new_last_checkpoint = {
|
|
204
|
-
$cond: [
|
|
205
|
-
can_checkpoint,
|
|
206
|
-
{
|
|
207
|
-
$max: ['$last_checkpoint', { $literal: this.persisted_op }, { $toLong: '$keepalive_op' }, 0n]
|
|
208
256
|
},
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
};
|
|
212
|
-
const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate({ _id: this.replicationStreamId }, [
|
|
213
|
-
{
|
|
214
|
-
$set: {
|
|
215
|
-
_can_checkpoint: can_checkpoint,
|
|
216
|
-
_not_empty: createEmptyCheckpoints
|
|
217
|
-
? true
|
|
218
|
-
: {
|
|
219
|
-
$or: [
|
|
220
|
-
{ $literal: createEmptyCheckpoints },
|
|
221
|
-
{ $ne: ['$keepalive_op', new_keepalive_op] },
|
|
222
|
-
{ $ne: ['$last_checkpoint', new_last_checkpoint] }
|
|
223
|
-
]
|
|
224
|
-
}
|
|
257
|
+
{
|
|
258
|
+
$unset: ['_can_checkpoint', '_not_empty']
|
|
225
259
|
}
|
|
226
|
-
|
|
227
|
-
|
|
260
|
+
], {
|
|
261
|
+
session: this.session,
|
|
262
|
+
returnDocument: 'before',
|
|
263
|
+
projection: {
|
|
264
|
+
snapshot_done: 1,
|
|
265
|
+
last_checkpoint_lsn: 1,
|
|
266
|
+
no_checkpoint_before: 1,
|
|
267
|
+
keepalive_op: 1,
|
|
268
|
+
last_checkpoint: 1
|
|
269
|
+
}
|
|
270
|
+
}));
|
|
271
|
+
return MongoSyncRulesLock.assertOwned(preUpdateDocument);
|
|
272
|
+
};
|
|
273
|
+
const finishCheckpoint = async (preUpdateDocument, lastOp) => {
|
|
274
|
+
await this.db.write_checkpoints.updateMany({
|
|
275
|
+
processed_at_lsn: null,
|
|
276
|
+
'lsns.1': { $lte: lsn }
|
|
277
|
+
}, {
|
|
228
278
|
$set: {
|
|
229
|
-
|
|
230
|
-
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
|
|
231
|
-
},
|
|
232
|
-
last_checkpoint_ts: {
|
|
233
|
-
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
|
|
234
|
-
},
|
|
235
|
-
last_keepalive_ts: { $literal: now },
|
|
236
|
-
last_fatal_error: { $literal: null },
|
|
237
|
-
last_fatal_error_ts: { $literal: null },
|
|
238
|
-
keepalive_op: new_keepalive_op,
|
|
239
|
-
last_checkpoint: new_last_checkpoint,
|
|
240
|
-
snapshot_lsn: {
|
|
241
|
-
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
|
|
242
|
-
}
|
|
279
|
+
processed_at_lsn: lsn
|
|
243
280
|
}
|
|
244
|
-
},
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
const checkpointState = calculateCheckpointState({
|
|
263
|
-
lsn,
|
|
264
|
-
snapshotDone: preUpdateDocument.snapshot_done === true,
|
|
265
|
-
lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
|
|
266
|
-
noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
|
|
267
|
-
keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
|
|
268
|
-
lastCheckpoint: preUpdateDocument.last_checkpoint,
|
|
269
|
-
persistedOp: this.persisted_op,
|
|
270
|
-
createEmptyCheckpoints
|
|
271
|
-
});
|
|
281
|
+
}, {
|
|
282
|
+
session: this.session
|
|
283
|
+
});
|
|
284
|
+
const checkpointState = calculateCheckpointState({
|
|
285
|
+
lsn,
|
|
286
|
+
snapshotDone: preUpdateDocument.snapshot_done === true,
|
|
287
|
+
lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
|
|
288
|
+
noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
|
|
289
|
+
keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
|
|
290
|
+
lastCheckpoint: preUpdateDocument.last_checkpoint,
|
|
291
|
+
persistedOp: lastOp ?? this.persistedOpHead(preUpdateDocument),
|
|
292
|
+
createEmptyCheckpoints
|
|
293
|
+
});
|
|
294
|
+
return { checkpointState, preUpdateDocument };
|
|
295
|
+
};
|
|
296
|
+
const { checkpointState, preUpdateDocument } = await this.flushAndCommit(async (_stream, lastOp) => finishCheckpoint(await updateCheckpoint(lastOp), lastOp),
|
|
297
|
+
// Without a flush, the checkpoint update itself fences the transaction.
|
|
298
|
+
() => this.withFencedTransaction(updateCheckpoint, finishCheckpoint), options);
|
|
272
299
|
if (checkpointState.checkpointBlocked) {
|
|
273
300
|
if (Date.now() - this.lastWaitingLogThrottled > 5_000) {
|
|
274
301
|
this.logger.info(`Waiting before creating checkpoint, currently at ${lsn} / ${checkpointState.newKeepaliveOp}. Current state: ${JSON.stringify({
|
|
@@ -300,32 +327,18 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
300
327
|
}
|
|
301
328
|
async setResumeLsn(lsn) {
|
|
302
329
|
using _ = this.tracer.span('storage', 'set_resume_lsn');
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
}, {
|
|
306
|
-
$set: {
|
|
307
|
-
snapshot_lsn: lsn
|
|
308
|
-
}
|
|
309
|
-
}, {
|
|
310
|
-
session: this.session,
|
|
311
|
-
// Losing occasional resume LSN is fine. That may mean reprocessing
|
|
312
|
-
// some source changes in some edge cases, which is not an issue since
|
|
313
|
-
// changes are processed in an idempotent way.
|
|
314
|
-
writeConcern: { w: 1 }
|
|
315
|
-
});
|
|
330
|
+
// Losing occasional resume LSN would only reprocess source changes.
|
|
331
|
+
// Keep the lease check atomic, but do not wait for majority replication of this resume hint.
|
|
332
|
+
await this.updateStreamMetadata({ snapshot_lsn: { $literal: lsn } }, {}, { w: 1 });
|
|
316
333
|
}
|
|
317
334
|
async markAllSnapshotDone(no_checkpoint_before_lsn) {
|
|
318
|
-
await this.
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
$max: {
|
|
326
|
-
no_checkpoint_before: no_checkpoint_before_lsn
|
|
327
|
-
}
|
|
328
|
-
}, { session: this.session });
|
|
335
|
+
await this.updateStreamMetadata(this.snapshotDoneUpdate(no_checkpoint_before_lsn));
|
|
336
|
+
}
|
|
337
|
+
snapshotDoneUpdate(no_checkpoint_before_lsn) {
|
|
338
|
+
return {
|
|
339
|
+
snapshot_done: true,
|
|
340
|
+
no_checkpoint_before: { $max: ['$no_checkpoint_before', { $literal: no_checkpoint_before_lsn }] }
|
|
341
|
+
};
|
|
329
342
|
}
|
|
330
343
|
async markSnapshotDone(no_checkpoint_before_lsn, options) {
|
|
331
344
|
await this.withTransaction(async () => {
|
|
@@ -348,42 +361,35 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
348
361
|
return;
|
|
349
362
|
}
|
|
350
363
|
}
|
|
351
|
-
await this.
|
|
364
|
+
await this.updateStreamMetadataInTransaction(this.snapshotDoneUpdate(no_checkpoint_before_lsn));
|
|
352
365
|
});
|
|
353
366
|
}
|
|
354
367
|
async markTableSnapshotRequired(_table) {
|
|
355
|
-
await this.
|
|
356
|
-
_id: this.replicationStreamId
|
|
357
|
-
}, {
|
|
358
|
-
$set: {
|
|
359
|
-
snapshot_done: false
|
|
360
|
-
}
|
|
361
|
-
}, { session: this.session });
|
|
368
|
+
await this.updateStreamMetadata({ snapshot_done: false });
|
|
362
369
|
}
|
|
363
370
|
async markTableSnapshotDone(tables, no_checkpoint_before_lsn) {
|
|
364
371
|
const session = this.session;
|
|
365
372
|
const ids = tables.map((table) => mongoTableId(table.id));
|
|
366
373
|
await this.withTransaction(async () => {
|
|
367
|
-
|
|
374
|
+
const writes = this.db.createWriteBatch(session, { ordered: false });
|
|
375
|
+
writes.updateMany(this.db.sourceTablesV1(this.replicationStreamId), { _id: { $in: ids } }, {
|
|
368
376
|
$set: {
|
|
369
377
|
snapshot_done: true
|
|
370
378
|
},
|
|
371
379
|
$unset: {
|
|
372
380
|
snapshot_status: 1
|
|
373
381
|
}
|
|
374
|
-
}
|
|
382
|
+
});
|
|
375
383
|
if (no_checkpoint_before_lsn != null) {
|
|
376
|
-
|
|
384
|
+
writes.updateOne(this.db.sync_rules, {
|
|
377
385
|
_id: this.replicationStreamId
|
|
378
386
|
}, {
|
|
379
|
-
$set: {
|
|
380
|
-
last_keepalive_ts: new Date()
|
|
381
|
-
},
|
|
382
387
|
$max: {
|
|
383
388
|
no_checkpoint_before: no_checkpoint_before_lsn
|
|
384
389
|
}
|
|
385
|
-
}
|
|
390
|
+
});
|
|
386
391
|
}
|
|
392
|
+
await writes.execute();
|
|
387
393
|
});
|
|
388
394
|
return tables.map((table) => {
|
|
389
395
|
const copy = table.clone();
|
|
@@ -398,7 +404,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
398
404
|
const session = this.session;
|
|
399
405
|
let activated = false;
|
|
400
406
|
let needsFutureActivationCheck = true;
|
|
401
|
-
await
|
|
407
|
+
await this.withTransaction(async () => {
|
|
402
408
|
// Reset on transaction retries.
|
|
403
409
|
activated = false;
|
|
404
410
|
needsFutureActivationCheck = true;
|
|
@@ -424,7 +430,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
424
430
|
else if (doc?.state != storage.SyncRuleState.PROCESSING) {
|
|
425
431
|
needsFutureActivationCheck = false;
|
|
426
432
|
}
|
|
427
|
-
});
|
|
433
|
+
}, session);
|
|
428
434
|
if (activated) {
|
|
429
435
|
this.logger.info(`Activated new replication stream at ${lsn}`);
|
|
430
436
|
await this.db.notifyCheckpoint();
|