@powersync/service-module-mongodb-storage 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +62 -0
- package/dist/storage/MongoBucketStorage.d.ts +36 -3
- package/dist/storage/MongoBucketStorage.js +372 -207
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +17 -5
- package/dist/storage/implementation/MongoBucketBatch.js +44 -17
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +19 -79
- package/dist/storage/implementation/MongoCompactor.js +27 -238
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
- package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +29 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +4 -2
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +7 -3
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +27 -11
- package/dist/storage/implementation/MongoSyncBucketStorage.js +78 -31
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +21 -9
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
- package/dist/storage/implementation/MongoWriteBatch.js +149 -0
- package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
- package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
- package/dist/storage/implementation/common/PersistedBatch.js +49 -6
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +9 -1
- package/dist/storage/implementation/db.js +27 -1
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +31 -0
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +2 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +33 -4
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
- package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +65 -16
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
- package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +10 -2
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +100 -26
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +82 -25
- package/dist/storage/implementation/v3/MongoCompactorV3.js +698 -159
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +91 -31
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
- package/dist/storage/implementation/v3/PersistedBatchV3.js +76 -42
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.js +2 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
- package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
- package/dist/storage/implementation/v3/compact-utils.js +216 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +84 -4
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +6 -6
- package/dist/storage/implementation/v3/source-table-utils.js +30 -37
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
- package/dist/storage/storage-index.d.ts +2 -1
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +28 -1
- package/dist/types/types.js +28 -7
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.js +1 -0
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.js +3 -1
- package/dist/utils/util.js.map +1 -1
- package/package.json +10 -9
- package/src/storage/MongoBucketStorage.ts +507 -269
- package/src/storage/implementation/MongoBucketBatch.ts +67 -24
- package/src/storage/implementation/MongoCompactor.ts +35 -346
- package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +34 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +4 -2
- package/src/storage/implementation/MongoStorageProvider.ts +15 -3
- package/src/storage/implementation/MongoSyncBucketStorage.ts +114 -40
- package/src/storage/implementation/MongoSyncRulesLock.ts +22 -9
- package/src/storage/implementation/MongoWriteBatch.ts +172 -0
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
- package/src/storage/implementation/common/PersistedBatch.ts +66 -10
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
- package/src/storage/implementation/db.ts +35 -2
- package/src/storage/implementation/models.ts +34 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +46 -6
- package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +87 -27
- package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
- package/src/storage/implementation/v1/models.ts +1 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +138 -34
- package/src/storage/implementation/v3/MongoCompactorV3.ts +958 -225
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +111 -41
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +93 -48
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
- package/src/storage/implementation/v3/bucket-format.ts +2 -0
- package/src/storage/implementation/v3/compact-utils.ts +320 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +91 -3
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
- package/src/storage/implementation/v3/source-table-utils.ts +39 -41
- package/src/storage/storage-index.ts +2 -0
- package/src/types/types.ts +42 -7
- package/src/utils/test-utils.ts +1 -0
- package/src/utils/util.ts +3 -1
- package/test/src/__snapshots__/storage.test.ts.snap +25 -0
- package/test/src/__snapshots__/storage_sync.test.ts.snap +605 -16
- package/test/src/chunk_compaction_config.test.ts +38 -0
- package/test/src/cleanup-stopped-sync-configs.test.ts +40 -13
- package/test/src/compact-utils.test.ts +226 -0
- package/test/src/helpers/MemoryObjectStorage.ts +14 -6
- package/test/src/object_storage_usage.test.ts +267 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/storage.test.ts +273 -22
- package/test/src/storage_compacting.test.ts +968 -246
- package/test/src/storage_s3_checksums.test.ts +3 -4
- package/test/src/storage_s3_compaction_lifecycle.test.ts +138 -12
- package/test/src/storage_s3_reading.test.ts +290 -2
- package/test/src/storage_s3_writing.test.ts +1 -0
- package/test/src/storage_sync.test.ts +336 -41
- package/test/src/util.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
|
|
3
3
|
import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
MongoStorageConfig,
|
|
6
|
+
normalizeChunkCompactionConcurrency,
|
|
7
|
+
normalizeClearBatchThrottleRate
|
|
8
|
+
} from '../../types/types.js';
|
|
5
9
|
import { MongoBucketStorage } from '../MongoBucketStorage.js';
|
|
6
10
|
import { MongoReportStorage } from '../MongoReportStorage.js';
|
|
7
11
|
import { PowerSyncMongo } from './db.js';
|
|
@@ -24,6 +28,10 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
24
28
|
}
|
|
25
29
|
|
|
26
30
|
const decodedConfig = MongoStorageConfig.decode(storage as any);
|
|
31
|
+
const chunkCompactionConcurrency = normalizeChunkCompactionConcurrency(
|
|
32
|
+
decodedConfig.chunk_compaction_concurrency,
|
|
33
|
+
decodedConfig.object_storage != null
|
|
34
|
+
);
|
|
27
35
|
|
|
28
36
|
let objectStorage: ObjectStorage | undefined;
|
|
29
37
|
if (decodedConfig.object_storage?.type === 's3') {
|
|
@@ -37,7 +45,8 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
37
45
|
forcePathStyle: decodedConfig.object_storage.force_path_style,
|
|
38
46
|
accessKeyId: decodedConfig.object_storage.access_key_id,
|
|
39
47
|
secretAccessKey: decodedConfig.object_storage.secret_access_key,
|
|
40
|
-
concurrencyLimit: decodedConfig.object_storage.concurrency_limit
|
|
48
|
+
concurrencyLimit: decodedConfig.object_storage.concurrency_limit,
|
|
49
|
+
defaultsMode: decodedConfig.object_storage.defaults_mode
|
|
41
50
|
});
|
|
42
51
|
}
|
|
43
52
|
|
|
@@ -46,6 +55,8 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
46
55
|
maxPoolSize: resolvedConfig.storage.max_pool_size ?? 8
|
|
47
56
|
});
|
|
48
57
|
|
|
58
|
+
const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
|
|
59
|
+
|
|
49
60
|
let shuttingDown = false;
|
|
50
61
|
|
|
51
62
|
// Explicitly connect on startup.
|
|
@@ -55,7 +66,6 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
55
66
|
// Errors here will cause the process to exit.
|
|
56
67
|
await client.connect();
|
|
57
68
|
|
|
58
|
-
const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
|
|
59
69
|
const readPreference =
|
|
60
70
|
decodedConfig.bulk_read_preference == null
|
|
61
71
|
? undefined
|
|
@@ -65,10 +75,12 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
65
75
|
maxStalenessSeconds: decodedConfig.bulk_read_preference == 'primary' ? undefined : 90
|
|
66
76
|
});
|
|
67
77
|
const syncStorageFactory = new MongoBucketStorage(database, {
|
|
78
|
+
chunkCompactionConcurrency,
|
|
68
79
|
replicationStreamNamePrefix: resolvedConfig.slot_name_prefix,
|
|
69
80
|
readPreference,
|
|
70
81
|
clearBatchThrottleRate: normalizeClearBatchThrottleRate(decodedConfig.clear_batch_throttle_rate),
|
|
71
82
|
checksumCacheTtlMs: resolvedConfig.api_parameters.bucket_count_cache_ttl_minutes * 60_000,
|
|
83
|
+
defaultStorageVersion: decodedConfig.default_storage_version,
|
|
72
84
|
// Right now, only MongoDB source databases supports incremental reprocessing.
|
|
73
85
|
// Remove this filter when we support it for other source databases.
|
|
74
86
|
// This assumes a single source connection - revisit if we ever support multiple connections.
|
|
@@ -2,6 +2,7 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
|
2
2
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
3
3
|
import {
|
|
4
4
|
BaseObserver,
|
|
5
|
+
logger as defaultLogger,
|
|
5
6
|
DO_NOT_LOG,
|
|
6
7
|
Logger,
|
|
7
8
|
ReplicationAbortedError,
|
|
@@ -11,14 +12,15 @@ import {
|
|
|
11
12
|
BroadcastIterable,
|
|
12
13
|
CHECKPOINT_INVALIDATE_ALL,
|
|
13
14
|
CheckpointChanges,
|
|
15
|
+
CompactInitialReplicationOptions,
|
|
16
|
+
CompactInitialReplicationResults,
|
|
14
17
|
GetCheckpointChangesOptions,
|
|
15
18
|
InternalOpId,
|
|
16
19
|
mergeAsyncIterables,
|
|
17
|
-
PopulateChecksumCacheOptions,
|
|
18
|
-
PopulateChecksumCacheResults,
|
|
19
20
|
ReplicationCheckpoint,
|
|
20
21
|
ReplicationStreamStorageIds,
|
|
21
22
|
storage,
|
|
23
|
+
SyncRuleState,
|
|
22
24
|
utils,
|
|
23
25
|
WatchWriteCheckpointOptions
|
|
24
26
|
} from '@powersync/service-core';
|
|
@@ -28,6 +30,10 @@ import { LRUCache } from 'lru-cache';
|
|
|
28
30
|
import * as timers from 'timers/promises';
|
|
29
31
|
import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../../types/types.js';
|
|
30
32
|
import { MongoBucketStorage } from '../MongoBucketStorage.js';
|
|
33
|
+
import {
|
|
34
|
+
MongoGetCheckpointChangesOptions,
|
|
35
|
+
MongoSyncBucketStorageCheckpoint
|
|
36
|
+
} from './common/MongoSyncBucketStorageCheckpoint.js';
|
|
31
37
|
import { DEFAULT_INLINE_THRESHOLD_BYTES } from './common/PersistedBatch.js';
|
|
32
38
|
import type { VersionedPowerSyncMongo } from './db.js';
|
|
33
39
|
import { StorageConfig } from './models.js';
|
|
@@ -37,7 +43,7 @@ import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js';
|
|
|
37
43
|
import { MongoParameterCompactor } from './MongoParameterCompactor.js';
|
|
38
44
|
import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
|
|
39
45
|
import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js';
|
|
40
|
-
import { MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
|
|
46
|
+
import { MongoCheckpointAPIOptions, MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
|
|
41
47
|
import { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
|
|
42
48
|
|
|
43
49
|
export interface MongoSyncBucketStorageOptions {
|
|
@@ -50,6 +56,16 @@ export interface MongoSyncBucketStorageOptions {
|
|
|
50
56
|
inlineThresholdBytes?: number;
|
|
51
57
|
}
|
|
52
58
|
|
|
59
|
+
/**
|
|
60
|
+
* The stream state read for a checkpoint. All fields must come from the same snapshot.
|
|
61
|
+
*/
|
|
62
|
+
export interface MongoCheckpointState {
|
|
63
|
+
checkpoint: InternalOpId;
|
|
64
|
+
lsn: string | null;
|
|
65
|
+
/** See {@link MongoSyncBucketStorageCheckpoint.parameterChangesInvalidBefore}. */
|
|
66
|
+
parameterChangesInvalidBefore: InternalOpId;
|
|
67
|
+
}
|
|
68
|
+
|
|
53
69
|
interface InternalCheckpointChanges extends CheckpointChanges {
|
|
54
70
|
updatedWriteCheckpoints: Map<string, bigint>;
|
|
55
71
|
invalidateWriteCheckpoints: boolean;
|
|
@@ -112,12 +128,27 @@ export abstract class MongoSyncBucketStorage
|
|
|
112
128
|
this.clearBatchThrottleRate = options.clearBatchThrottleRate ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;
|
|
113
129
|
this.db = factory.db.versioned(this.storageConfig);
|
|
114
130
|
this.checksums = this.createMongoChecksums(options);
|
|
115
|
-
this.writeCheckpointAPI =
|
|
131
|
+
this.writeCheckpointAPI = this.createWriteCheckpointAPI({
|
|
116
132
|
db: this.db,
|
|
117
|
-
mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED,
|
|
118
|
-
|
|
133
|
+
writeCheckpointMode: { mode: writeCheckpointMode ?? storage.WriteCheckpointMode.MANAGED },
|
|
134
|
+
replicationStreamId
|
|
119
135
|
});
|
|
120
|
-
|
|
136
|
+
|
|
137
|
+
// Include both replication stream name and sync config version labels in log prefix.
|
|
138
|
+
// Examples:
|
|
139
|
+
// No sync config with a label: [stream]
|
|
140
|
+
// All sync configs have a label: [stream][v5,v6]
|
|
141
|
+
// Mixed: [stream][?,v6]
|
|
142
|
+
const hasVersionLabel = replicationStream.syncConfigContent.some((config) => config.version_label != null);
|
|
143
|
+
const versionLabels = hasVersionLabel
|
|
144
|
+
? replicationStream.syncConfigContent.map((config) => config.version_label ?? '?')
|
|
145
|
+
: [];
|
|
146
|
+
this.logger =
|
|
147
|
+
versionLabels.length == 0
|
|
148
|
+
? replicationStream.logger
|
|
149
|
+
: defaultLogger.child({
|
|
150
|
+
prefix: `[${replicationStreamName}][${versionLabels.join(',')}] `
|
|
151
|
+
});
|
|
121
152
|
}
|
|
122
153
|
|
|
123
154
|
/**
|
|
@@ -128,11 +159,20 @@ export abstract class MongoSyncBucketStorage
|
|
|
128
159
|
abstract createMongoCompactor(options: MongoCompactOptions): MongoCompactor;
|
|
129
160
|
|
|
130
161
|
protected abstract createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums;
|
|
162
|
+
protected createWriteCheckpointAPI(options: MongoCheckpointAPIOptions): MongoWriteCheckpointAPI {
|
|
163
|
+
return new MongoWriteCheckpointAPI(options);
|
|
164
|
+
}
|
|
165
|
+
|
|
131
166
|
protected abstract createMongoParameterCompactor(
|
|
132
167
|
checkpoint: InternalOpId,
|
|
133
168
|
options: storage.CompactOptions
|
|
134
169
|
): MongoParameterCompactor;
|
|
135
170
|
|
|
171
|
+
/** MongoDB parameter compaction uses a persisted operation-id cursor. */
|
|
172
|
+
public supportsIncrementalParameterCompaction(): boolean {
|
|
173
|
+
return true;
|
|
174
|
+
}
|
|
175
|
+
|
|
136
176
|
get writeCheckpointMode() {
|
|
137
177
|
return this.writeCheckpointAPI.writeCheckpointMode;
|
|
138
178
|
}
|
|
@@ -144,8 +184,8 @@ export abstract class MongoSyncBucketStorage
|
|
|
144
184
|
return this.replicationStream.storageIds;
|
|
145
185
|
}
|
|
146
186
|
|
|
147
|
-
setWriteCheckpointMode(
|
|
148
|
-
this.writeCheckpointAPI.setWriteCheckpointMode(
|
|
187
|
+
setWriteCheckpointMode(config: storage.WriteCheckpointModeConfig): void {
|
|
188
|
+
this.writeCheckpointAPI.setWriteCheckpointMode(config);
|
|
149
189
|
}
|
|
150
190
|
|
|
151
191
|
createManagedWriteCheckpoints(
|
|
@@ -178,9 +218,7 @@ export abstract class MongoSyncBucketStorage
|
|
|
178
218
|
return (await this.getCheckpointInternal()) ?? new EmptyReplicationCheckpoint();
|
|
179
219
|
}
|
|
180
220
|
|
|
181
|
-
protected abstract fetchCheckpointState(
|
|
182
|
-
session: mongo.ClientSession
|
|
183
|
-
): Promise<{ checkpoint: bigint; lsn: string | null } | null>;
|
|
221
|
+
protected abstract fetchCheckpointState(session: mongo.ClientSession): Promise<MongoCheckpointState | null>;
|
|
184
222
|
|
|
185
223
|
async getCheckpointInternal(): Promise<storage.ReplicationCheckpoint | null> {
|
|
186
224
|
return await this.db.client.withSession({ snapshot: true }, async (session) => {
|
|
@@ -197,7 +235,14 @@ export abstract class MongoSyncBucketStorage
|
|
|
197
235
|
if (clusterTime == null) {
|
|
198
236
|
throw new ServiceAssertionError('Missing clusterTime in getCheckpoint()');
|
|
199
237
|
}
|
|
200
|
-
return new MongoReplicationCheckpoint(
|
|
238
|
+
return new MongoReplicationCheckpoint(
|
|
239
|
+
this,
|
|
240
|
+
state.checkpoint,
|
|
241
|
+
state.lsn,
|
|
242
|
+
snapshotTime,
|
|
243
|
+
clusterTime,
|
|
244
|
+
state.parameterChangesInvalidBefore
|
|
245
|
+
);
|
|
201
246
|
});
|
|
202
247
|
}
|
|
203
248
|
|
|
@@ -235,6 +280,7 @@ export abstract class MongoSyncBucketStorage
|
|
|
235
280
|
markRecordUnavailable: options.markRecordUnavailable,
|
|
236
281
|
hooks: options.hooks,
|
|
237
282
|
tracer: options.tracer,
|
|
283
|
+
signal: options.signal,
|
|
238
284
|
objectStorage: this.objectStorage,
|
|
239
285
|
inlineThresholdBytes: this.inlineThresholdBytes
|
|
240
286
|
};
|
|
@@ -329,6 +375,7 @@ export abstract class MongoSyncBucketStorage
|
|
|
329
375
|
protected abstract clearBucketState(signal?: AbortSignal): Promise<void>;
|
|
330
376
|
|
|
331
377
|
protected abstract clearSourceTables(signal?: AbortSignal): Promise<void>;
|
|
378
|
+
protected async clearCustomCheckpointRequests(_signal?: AbortSignal): Promise<void> {}
|
|
332
379
|
protected abstract clearSyncRuleState(): Promise<void>;
|
|
333
380
|
|
|
334
381
|
async clear(options?: storage.ClearStorageOptions): Promise<void> {
|
|
@@ -345,6 +392,7 @@ export abstract class MongoSyncBucketStorage
|
|
|
345
392
|
await this.clearSourceRecords(signal);
|
|
346
393
|
await this.clearBucketState(signal);
|
|
347
394
|
await this.clearSourceTables(signal);
|
|
395
|
+
await this.clearCustomCheckpointRequests(signal);
|
|
348
396
|
|
|
349
397
|
this.#storageInitialized = false;
|
|
350
398
|
}
|
|
@@ -367,43 +415,30 @@ export abstract class MongoSyncBucketStorage
|
|
|
367
415
|
async compact(options?: storage.CompactOptions) {
|
|
368
416
|
let maxOpId = options?.maxOpId;
|
|
369
417
|
if (maxOpId == null) {
|
|
418
|
+
// For PROCESSING streams, this will be undefined.
|
|
370
419
|
const checkpoint = await this.getCheckpointInternal();
|
|
371
420
|
maxOpId = checkpoint?.checkpoint ?? undefined;
|
|
372
421
|
}
|
|
373
422
|
await this.createMongoCompactor({ ...options, maxOpId, logger: this.logger }).compact();
|
|
374
423
|
|
|
375
|
-
if (maxOpId != null && options?.compactParameterData) {
|
|
376
|
-
|
|
424
|
+
if (maxOpId != null && options?.compactParameterData && this.replicationStream.state == SyncRuleState.ACTIVE) {
|
|
425
|
+
// Use the stream-scoped logger, matching bucket compaction above.
|
|
426
|
+
await this.createMongoParameterCompactor(maxOpId, { ...options, logger: this.logger }).compact();
|
|
377
427
|
}
|
|
378
428
|
}
|
|
379
429
|
|
|
430
|
+
abstract compactInitialReplication(
|
|
431
|
+
options: CompactInitialReplicationOptions
|
|
432
|
+
): Promise<CompactInitialReplicationResults>;
|
|
433
|
+
|
|
380
434
|
/**
|
|
381
435
|
* The highest op id persisted for this stream, whether or not covered by a checkpoint.
|
|
382
436
|
*
|
|
383
|
-
* Used as the default `maxOpId` for {@link
|
|
437
|
+
* Used as the default `maxOpId` for {@link compactInitialReplication}, which runs after
|
|
384
438
|
* initial replication but before the first checkpoint exists.
|
|
385
439
|
*/
|
|
386
440
|
protected abstract fetchPersistedOpHead(): Promise<InternalOpId | null>;
|
|
387
441
|
|
|
388
|
-
async populatePersistentChecksumCache(options: PopulateChecksumCacheOptions): Promise<PopulateChecksumCacheResults> {
|
|
389
|
-
this.logger.info(`Populating persistent checksum cache...`);
|
|
390
|
-
const start = Date.now();
|
|
391
|
-
const maxOpId = options.maxOpId ?? (await this.fetchPersistedOpHead()) ?? undefined;
|
|
392
|
-
const compactor = this.createMongoCompactor({
|
|
393
|
-
...options,
|
|
394
|
-
maxOpId,
|
|
395
|
-
memoryLimitMB: 0,
|
|
396
|
-
logger: this.logger
|
|
397
|
-
});
|
|
398
|
-
|
|
399
|
-
const result = await compactor.populateChecksums({
|
|
400
|
-
minBucketChanges: options.minBucketChanges ?? 10
|
|
401
|
-
});
|
|
402
|
-
const duration = Date.now() - start;
|
|
403
|
-
this.logger.info(`Populated persistent checksum cache in ${(duration / 1000).toFixed(1)}s`);
|
|
404
|
-
return result;
|
|
405
|
-
}
|
|
406
|
-
|
|
407
442
|
private async *watchActiveCheckpoint(signal: AbortSignal): AsyncIterable<ReplicationCheckpoint> {
|
|
408
443
|
if (signal.aborted) {
|
|
409
444
|
return;
|
|
@@ -580,13 +615,31 @@ export abstract class MongoSyncBucketStorage
|
|
|
580
615
|
}
|
|
581
616
|
|
|
582
617
|
protected abstract getParameterBucketChangesImpl(
|
|
583
|
-
options:
|
|
618
|
+
options: MongoGetCheckpointChangesOptions
|
|
584
619
|
): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>>;
|
|
585
620
|
|
|
586
621
|
private async getParameterBucketChanges(
|
|
587
622
|
options: GetCheckpointChangesOptions
|
|
588
623
|
): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
|
|
589
|
-
|
|
624
|
+
const nextCheckpoint = requireMongoCheckpoint(options.nextCheckpoint);
|
|
625
|
+
if (options.lastCheckpoint.checkpoint < nextCheckpoint.parameterChangesInvalidBefore) {
|
|
626
|
+
// Parameter compaction may have deleted parameter entries in the range we'd have to query
|
|
627
|
+
// to find the individual changed lookups. Invalidate all parameter buckets instead.
|
|
628
|
+
//
|
|
629
|
+
// The fence is committed before the first delete of a compaction pass, and captured in the
|
|
630
|
+
// same snapshot as the checkpoint, so a checkpoint that could miss a deleted entry always
|
|
631
|
+
// observes the fence as well.
|
|
632
|
+
return {
|
|
633
|
+
invalidateParameterBuckets: true,
|
|
634
|
+
updatedParameterLookups: new Set<string>()
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
// The query below runs at the checkpoint snapshot, which still sees entries deleted by a
|
|
638
|
+
// compaction pass that started after the snapshot.
|
|
639
|
+
return this.getParameterBucketChangesImpl({
|
|
640
|
+
lastCheckpoint: options.lastCheckpoint,
|
|
641
|
+
nextCheckpoint
|
|
642
|
+
});
|
|
590
643
|
}
|
|
591
644
|
|
|
592
645
|
private checkpointChangesCache = new LRUCache<
|
|
@@ -611,7 +664,10 @@ export abstract class MongoSyncBucketStorage
|
|
|
611
664
|
});
|
|
612
665
|
|
|
613
666
|
async getCheckpointChanges(options: GetCheckpointChangesOptions): Promise<InternalCheckpointChanges> {
|
|
614
|
-
|
|
667
|
+
// The invalidation fence is part of the identity: the same checkpoint pair read before and
|
|
668
|
+
// after a compaction pass produces different results (specific lookups vs. invalidate-all).
|
|
669
|
+
const fence = requireMongoCheckpoint(options.nextCheckpoint).parameterChangesInvalidBefore;
|
|
670
|
+
const key = `${options.lastCheckpoint.checkpoint}_${options.lastCheckpoint.lsn}__${options.nextCheckpoint.checkpoint}_${options.nextCheckpoint.lsn}_${fence}`;
|
|
615
671
|
const result = await this.checkpointChangesCache.fetch(key, { context: { options } });
|
|
616
672
|
return result!;
|
|
617
673
|
}
|
|
@@ -629,7 +685,19 @@ export abstract class MongoSyncBucketStorage
|
|
|
629
685
|
}
|
|
630
686
|
}
|
|
631
687
|
|
|
632
|
-
|
|
688
|
+
/**
|
|
689
|
+
* We don't support any other constructions of ReplicationCheckpoint.
|
|
690
|
+
*/
|
|
691
|
+
function requireMongoCheckpoint(checkpoint: ReplicationCheckpoint): MongoReplicationCheckpoint {
|
|
692
|
+
if (!(checkpoint instanceof MongoReplicationCheckpoint)) {
|
|
693
|
+
throw new ServiceAssertionError(
|
|
694
|
+
`Checkpoint changes require a checkpoint from getCheckpointInternal(), got ${checkpoint.constructor.name}`
|
|
695
|
+
);
|
|
696
|
+
}
|
|
697
|
+
return checkpoint;
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
class MongoReplicationCheckpoint implements MongoSyncBucketStorageCheckpoint {
|
|
633
701
|
#storage: MongoSyncBucketStorage;
|
|
634
702
|
|
|
635
703
|
constructor(
|
|
@@ -637,7 +705,9 @@ class MongoReplicationCheckpoint implements ReplicationCheckpoint {
|
|
|
637
705
|
public readonly checkpoint: InternalOpId,
|
|
638
706
|
public readonly lsn: string | null,
|
|
639
707
|
public snapshotTime: mongo.Timestamp,
|
|
640
|
-
public clusterTime: mongo.ClusterTime
|
|
708
|
+
public clusterTime: mongo.ClusterTime,
|
|
709
|
+
/** Captured in the same snapshot as the checkpoint. */
|
|
710
|
+
public readonly parameterChangesInvalidBefore: InternalOpId
|
|
641
711
|
) {
|
|
642
712
|
this.#storage = storage;
|
|
643
713
|
}
|
|
@@ -647,6 +717,10 @@ class MongoReplicationCheckpoint implements ReplicationCheckpoint {
|
|
|
647
717
|
}
|
|
648
718
|
}
|
|
649
719
|
|
|
720
|
+
/**
|
|
721
|
+
* Used when no checkpoint has been persisted yet. This has no snapshot or invalidation fence, so
|
|
722
|
+
* it cannot be used for checkpoint change detection - see {@link requireMongoCheckpoint}.
|
|
723
|
+
*/
|
|
650
724
|
class EmptyReplicationCheckpoint implements ReplicationCheckpoint {
|
|
651
725
|
readonly checkpoint: InternalOpId = 0n;
|
|
652
726
|
readonly lsn: string | null = null;
|
|
@@ -5,6 +5,8 @@ import { ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framewo
|
|
|
5
5
|
import { storage } from '@powersync/service-core';
|
|
6
6
|
import { VersionedPowerSyncMongo } from './db.js';
|
|
7
7
|
|
|
8
|
+
const LOCK_DURATION_MS = 60 * 1000;
|
|
9
|
+
|
|
8
10
|
/**
|
|
9
11
|
* Manages a lock on a replication stream document, so that only one process
|
|
10
12
|
* processes that replication stream at a time.
|
|
@@ -22,15 +24,20 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
|
|
|
22
24
|
): Promise<MongoSyncRulesLock> {
|
|
23
25
|
const lockId = crypto.randomBytes(8).toString('hex');
|
|
24
26
|
const doc = await db.sync_rules.findOneAndUpdate(
|
|
25
|
-
{ _id: sync_rules.replicationStreamId, $or: [{ lock: null }, { 'lock.expires_at': { $lt: new Date() } }] },
|
|
26
27
|
{
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
28
|
+
_id: sync_rules.replicationStreamId,
|
|
29
|
+
$or: [{ lock: null }, { $expr: { $lt: ['$lock.expires_at', '$$NOW'] } }]
|
|
30
|
+
},
|
|
31
|
+
[
|
|
32
|
+
{
|
|
33
|
+
$set: {
|
|
34
|
+
lock: {
|
|
35
|
+
id: lockId,
|
|
36
|
+
expires_at: { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: LOCK_DURATION_MS } }
|
|
37
|
+
}
|
|
31
38
|
}
|
|
32
39
|
}
|
|
33
|
-
|
|
40
|
+
],
|
|
34
41
|
{
|
|
35
42
|
projection: { lock: 1 },
|
|
36
43
|
returnDocument: 'before',
|
|
@@ -96,9 +103,15 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
|
|
|
96
103
|
_id: this.sync_rules_id,
|
|
97
104
|
'lock.id': this.lock_id
|
|
98
105
|
},
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
106
|
+
[
|
|
107
|
+
{
|
|
108
|
+
$set: {
|
|
109
|
+
'lock.expires_at': {
|
|
110
|
+
$dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: LOCK_DURATION_MS }
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
],
|
|
102
115
|
{ returnDocument: 'after' }
|
|
103
116
|
);
|
|
104
117
|
if (result == null) {
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Explicitly groups writes with no intervening reads or result dependencies.
|
|
5
|
+
* Callers retain responsibility for transaction and ordering boundaries.
|
|
6
|
+
*
|
|
7
|
+
* On MongoDB 8.0+, this uses the client-level bulkWrite API.
|
|
8
|
+
*
|
|
9
|
+
* On MongoDB earlier versions, this uses the the collection-level bulkWrite API, or individual operations.
|
|
10
|
+
* This is less efficient, but produces the same results.
|
|
11
|
+
*/
|
|
12
|
+
export class MongoWriteBatch {
|
|
13
|
+
private readonly operations: mongo.AnyClientBulkWriteModel<mongo.Document>[] = [];
|
|
14
|
+
private readonly deleteChecks: { index: number; check: (count: number) => void }[] = [];
|
|
15
|
+
private readonly fallback: (() => Promise<unknown>)[] = [];
|
|
16
|
+
|
|
17
|
+
private readonly ordered: boolean;
|
|
18
|
+
|
|
19
|
+
constructor(
|
|
20
|
+
private readonly client: mongo.MongoClient,
|
|
21
|
+
private readonly supportsClientBulkWrite: boolean,
|
|
22
|
+
private readonly session: mongo.ClientSession | undefined,
|
|
23
|
+
options: { ordered: boolean }
|
|
24
|
+
) {
|
|
25
|
+
this.ordered = options.ordered;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Perform a collection-level bulkWrite.
|
|
30
|
+
*
|
|
31
|
+
* On MongoDB 8.0+, these operations join the client-level bulkWrite with the batch's ordering.
|
|
32
|
+
* The driver may split commands to respect server batch limits.
|
|
33
|
+
*
|
|
34
|
+
* On earlier MongoDB versions, this always uses an unordered collection-level bulkWrite.
|
|
35
|
+
* Operations within this collection bulk must therefore be independent of their order.
|
|
36
|
+
*/
|
|
37
|
+
bulkWriteUnordered<T extends mongo.Document>(
|
|
38
|
+
collection: mongo.Collection<T>,
|
|
39
|
+
operations: mongo.AnyBulkWriteOperation<T>[]
|
|
40
|
+
): void {
|
|
41
|
+
if (operations.length === 0) return;
|
|
42
|
+
if (!this.supportsClientBulkWrite) {
|
|
43
|
+
this.fallback.push(() => collection.bulkWrite(operations, { ordered: false, session: this.session }));
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
for (const operation of operations) {
|
|
47
|
+
// Collection and client models contain the same operation fields, with a
|
|
48
|
+
// different envelope. Keep the schema checked at the collection boundary.
|
|
49
|
+
const [name, model] = Object.entries(operation)[0];
|
|
50
|
+
this.operations.push({ ...model, name, namespace: collection.namespace });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
insertOne<T extends mongo.Document>(collection: mongo.Collection<T>, document: mongo.OptionalUnlessRequiredId<T>) {
|
|
55
|
+
if (!this.supportsClientBulkWrite) {
|
|
56
|
+
this.fallback.push(() => collection.insertOne(document, { session: this.session }));
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
this.addModel<T>({ name: 'insertOne', namespace: collection.namespace, document: document as mongo.OptionalId<T> });
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
insertMany<T extends mongo.Document>(
|
|
63
|
+
collection: mongo.Collection<T>,
|
|
64
|
+
documents: mongo.OptionalUnlessRequiredId<T>[]
|
|
65
|
+
) {
|
|
66
|
+
if (documents.length === 0) return;
|
|
67
|
+
if (!this.supportsClientBulkWrite) {
|
|
68
|
+
this.fallback.push(() => collection.insertMany(documents, { session: this.session }));
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
for (const document of documents) {
|
|
72
|
+
this.addModel<T>({
|
|
73
|
+
name: 'insertOne',
|
|
74
|
+
namespace: collection.namespace,
|
|
75
|
+
document: document as mongo.OptionalId<T>
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
deleteMany<T extends mongo.Document>(
|
|
81
|
+
collection: mongo.Collection<T>,
|
|
82
|
+
filter: mongo.Filter<T>,
|
|
83
|
+
checkDeletedCount?: (count: number) => void
|
|
84
|
+
) {
|
|
85
|
+
if (checkDeletedCount) {
|
|
86
|
+
// Client-bulk results are checked after all writes execute. A failed
|
|
87
|
+
// invariant must roll back the entire batch, not leave later writes visible.
|
|
88
|
+
if (!this.session?.inTransaction()) throw new Error('Checked deletes require a transaction');
|
|
89
|
+
}
|
|
90
|
+
if (!this.supportsClientBulkWrite) {
|
|
91
|
+
this.fallback.push(async () => {
|
|
92
|
+
const result = await collection.deleteMany(filter, { session: this.session });
|
|
93
|
+
checkDeletedCount?.(result.deletedCount);
|
|
94
|
+
});
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
if (checkDeletedCount) {
|
|
98
|
+
this.deleteChecks.push({ index: this.operations.length, check: checkDeletedCount });
|
|
99
|
+
}
|
|
100
|
+
this.addModel<T>({ name: 'deleteMany', namespace: collection.namespace, filter });
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
updateOne<T extends mongo.Document>(
|
|
104
|
+
collection: mongo.Collection<T>,
|
|
105
|
+
filter: mongo.Filter<T>,
|
|
106
|
+
update: mongo.UpdateFilter<T> | mongo.Document[],
|
|
107
|
+
options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
|
|
108
|
+
) {
|
|
109
|
+
if (!this.supportsClientBulkWrite) {
|
|
110
|
+
this.fallback.push(() => collection.updateOne(filter, update, { ...options, session: this.session }));
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
this.addModel({ name: 'updateOne', namespace: collection.namespace, filter, update, ...options });
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
updateMany<T extends mongo.Document>(
|
|
117
|
+
collection: mongo.Collection<T>,
|
|
118
|
+
filter: mongo.Filter<T>,
|
|
119
|
+
update: mongo.UpdateFilter<T> | mongo.Document[],
|
|
120
|
+
options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
|
|
121
|
+
) {
|
|
122
|
+
if (!this.supportsClientBulkWrite) {
|
|
123
|
+
this.fallback.push(() => collection.updateMany(filter, update, { ...options, session: this.session }));
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
this.addModel({ name: 'updateMany', namespace: collection.namespace, filter, update, ...options });
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
private addModel<T extends mongo.Document>(operation: mongo.AnyClientBulkWriteModel<T>) {
|
|
130
|
+
// The collection-specific methods validate the schema before erasing it here.
|
|
131
|
+
this.operations.push(operation as unknown as mongo.AnyClientBulkWriteModel<mongo.Document>);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Execute the queued writes.
|
|
136
|
+
*
|
|
137
|
+
* On MongoDB 8.0+, this uses the client-level bulkWrite API, typically resulting in a single command.
|
|
138
|
+
* The command may be split if there are a large number of operations.
|
|
139
|
+
*
|
|
140
|
+
* On earlier MongoDB versions, this executes each queued operation individually.
|
|
141
|
+
*
|
|
142
|
+
* If no operations were queued, this is a no-op.
|
|
143
|
+
*/
|
|
144
|
+
async execute(): Promise<void> {
|
|
145
|
+
if (!this.supportsClientBulkWrite) {
|
|
146
|
+
// Sessions cannot execute concurrent operations within a transaction.
|
|
147
|
+
for (const write of this.fallback) await write();
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (this.operations.length === 0) return;
|
|
151
|
+
|
|
152
|
+
try {
|
|
153
|
+
const result = await this.client.bulkWrite(this.operations, {
|
|
154
|
+
session: this.session,
|
|
155
|
+
ordered: this.ordered,
|
|
156
|
+
verboseResults: this.deleteChecks.length > 0
|
|
157
|
+
});
|
|
158
|
+
for (const { index, check } of this.deleteChecks) {
|
|
159
|
+
const deleted = result.deleteResults?.get(index);
|
|
160
|
+
if (deleted == null) throw new Error('Missing client bulk delete result');
|
|
161
|
+
check(deleted.deletedCount);
|
|
162
|
+
}
|
|
163
|
+
} catch (error) {
|
|
164
|
+
// The driver wraps command failures without copying their error labels.
|
|
165
|
+
// Preserve withTransaction's retry handling for e.g. write conflicts.
|
|
166
|
+
if (error instanceof mongo.MongoClientBulkWriteError && error.cause instanceof mongo.MongoError) {
|
|
167
|
+
throw error.cause;
|
|
168
|
+
}
|
|
169
|
+
throw error;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|