@powersync/service-module-mongodb-storage 0.0.0-dev-20260909133214 → 0.0.0-dev-20260929095716
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -6
- package/dist/storage/MongoBucketStorage.d.ts +11 -2
- package/dist/storage/MongoBucketStorage.js +43 -13
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +24 -10
- package/dist/storage/implementation/MongoBucketBatch.js +134 -89
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +2 -0
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
- package/dist/storage/implementation/MongoIdSequence.js +14 -1
- package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
- package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +6 -2
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +6 -6
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +5 -3
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +5 -0
- package/dist/storage/implementation/MongoSyncBucketStorage.js +17 -0
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
- package/dist/storage/implementation/MongoSyncRulesLock.js +38 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
- package/dist/storage/implementation/MongoWriteBatch.js +149 -0
- package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +2 -2
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/PersistedBatch.d.ts +5 -4
- package/dist/storage/implementation/common/PersistedBatch.js +6 -4
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +5 -0
- package/dist/storage/implementation/db.js +14 -0
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +5 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +149 -139
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +2 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.js +23 -17
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
- package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +6 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +148 -162
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +10 -4
- package/dist/storage/implementation/v3/MongoCompactorV3.js +175 -76
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +12 -8
- package/dist/storage/implementation/v3/PersistedBatchV3.js +25 -32
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/compact-utils.d.ts +3 -2
- package/dist/storage/implementation/v3/compact-utils.js +12 -5
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -1
- package/dist/storage/implementation/v3/models.d.ts +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +8 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +3 -3
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +7 -9
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +3 -2
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +4 -7
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +21 -3
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
- package/dist/types/types.d.ts +5 -0
- package/dist/types/types.js +12 -0
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +4 -2
- package/dist/utils/test-utils.js +7 -4
- package/dist/utils/test-utils.js.map +1 -1
- package/package.json +6 -6
- package/src/storage/MongoBucketStorage.ts +54 -13
- package/src/storage/implementation/MongoBucketBatch.ts +205 -111
- package/src/storage/implementation/MongoCompactor.ts +2 -0
- package/src/storage/implementation/MongoIdSequence.ts +21 -1
- package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +12 -1
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -31
- package/src/storage/implementation/MongoStorageProvider.ts +13 -3
- package/src/storage/implementation/MongoSyncBucketStorage.ts +19 -0
- package/src/storage/implementation/MongoSyncRulesLock.ts +56 -3
- package/src/storage/implementation/MongoWriteBatch.ts +172 -0
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +4 -2
- package/src/storage/implementation/common/PersistedBatch.ts +12 -8
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
- package/src/storage/implementation/db.ts +15 -0
- package/src/storage/implementation/models.ts +1 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +167 -160
- package/src/storage/implementation/v1/MongoCompactorV1.ts +29 -26
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +1 -1
- package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +217 -242
- package/src/storage/implementation/v3/MongoCompactorV3.ts +284 -108
- package/src/storage/implementation/v3/PersistedBatchV3.ts +36 -36
- package/src/storage/implementation/v3/compact-utils.ts +16 -7
- package/src/storage/implementation/v3/models.ts +1 -1
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +8 -1
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +11 -10
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +8 -9
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +23 -3
- package/src/types/types.ts +18 -0
- package/src/utils/test-utils.ts +9 -5
- package/test/src/__snapshots__/storage_sync.test.ts.snap +12 -12
- package/test/src/chunk_compaction_config.test.ts +38 -0
- package/test/src/cleanup-stopped-sync-configs.test.ts +19 -13
- package/test/src/compact-utils.test.ts +33 -8
- package/test/src/concurrent_writers.test.ts +599 -0
- package/test/src/connection_config.test.ts +159 -0
- package/test/src/helpers/concurrentWriter.mjs +29 -0
- package/test/src/helpers/s3TestFactory.ts +12 -12
- package/test/src/object_storage_usage.test.ts +18 -5
- package/test/src/parameter_compacting_v1.test.ts +2 -2
- package/test/src/parameter_compacting_v3.test.ts +2 -2
- package/test/src/parameter_compaction_fence.test.ts +3 -3
- package/test/src/storage.test.ts +15 -10
- package/test/src/storage_compacting.test.ts +397 -23
- package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
- package/test/src/storage_s3_checksums.test.ts +4 -4
- package/test/src/storage_s3_compaction_lifecycle.test.ts +8 -8
- package/test/src/storage_s3_reading.test.ts +51 -5
- package/test/src/storage_s3_writing.test.ts +3 -3
- package/test/src/storage_sync.test.ts +55 -47
- package/test/src/storeCurrentData.test.ts +5 -5
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -20,13 +20,17 @@ export abstract class MongoPersistedSyncConfigContentBase extends storage.Persis
|
|
|
20
20
|
options: Omit<storage.PersistedSyncConfigContentData, 'syncConfigId'> & {
|
|
21
21
|
mapping: SingleSyncConfigBucketDefinitionMapping;
|
|
22
22
|
syncConfigId: bson.ObjectId | null;
|
|
23
|
-
}
|
|
23
|
+
},
|
|
24
|
+
syncConfigParser: storage.SyncConfigParser
|
|
24
25
|
) {
|
|
25
26
|
const { mapping, syncConfigId, ...base } = options;
|
|
26
|
-
super(
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
27
|
+
super(
|
|
28
|
+
{
|
|
29
|
+
...base,
|
|
30
|
+
syncConfigId: syncConfigId?.toHexString() ?? null
|
|
31
|
+
},
|
|
32
|
+
syncConfigParser
|
|
33
|
+
);
|
|
30
34
|
this.mapping = mapping;
|
|
31
35
|
this.syncConfigObjectId = syncConfigId;
|
|
32
36
|
}
|
|
@@ -60,19 +64,23 @@ export abstract class MongoPersistedSyncConfigContentBase extends storage.Persis
|
|
|
60
64
|
}
|
|
61
65
|
}
|
|
62
66
|
export class MongoPersistedSyncConfigContentV1 extends MongoPersistedSyncConfigContentBase {
|
|
63
|
-
constructor(db: PowerSyncMongo, doc: SyncRuleDocumentV1) {
|
|
64
|
-
super(
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
67
|
+
constructor(db: PowerSyncMongo, doc: SyncRuleDocumentV1, syncConfigParser: storage.SyncConfigParser) {
|
|
68
|
+
super(
|
|
69
|
+
db,
|
|
70
|
+
{
|
|
71
|
+
replicationStreamId: doc._id,
|
|
72
|
+
sync_rules_content: doc.content,
|
|
73
|
+
compiled_plan: doc.serialized_plan ?? null,
|
|
74
|
+
// Handle legacy values
|
|
75
|
+
replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
|
|
76
|
+
storageVersion: doc.storage_version ?? storage.LEGACY_STORAGE_VERSION,
|
|
77
|
+
mapping: new SingleSyncConfigBucketDefinitionMapping(),
|
|
78
|
+
syncConfigId: null,
|
|
79
|
+
syncConfigState: doc.state,
|
|
80
|
+
version_label: doc.version_label ?? undefined
|
|
81
|
+
},
|
|
82
|
+
syncConfigParser
|
|
83
|
+
);
|
|
76
84
|
}
|
|
77
85
|
|
|
78
86
|
async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
|
|
@@ -87,23 +95,32 @@ export class MongoPersistedSyncConfigContentV1 extends MongoPersistedSyncConfigC
|
|
|
87
95
|
export class MongoPersistedSyncConfigContentV3 extends MongoPersistedSyncConfigContentBase {
|
|
88
96
|
declare public readonly syncConfigObjectId: bson.ObjectId;
|
|
89
97
|
|
|
90
|
-
constructor(
|
|
98
|
+
constructor(
|
|
99
|
+
db: PowerSyncMongo,
|
|
100
|
+
doc: ReplicationStreamDocumentV3,
|
|
101
|
+
config: SyncConfigDefinition,
|
|
102
|
+
syncConfigParser: storage.SyncConfigParser
|
|
103
|
+
) {
|
|
91
104
|
const state = doc.sync_configs.find((c) => c._id.equals(config._id));
|
|
92
105
|
if (state == null) {
|
|
93
106
|
throw new ServiceAssertionError(`Cannot find sync config ${config._id} in replication stream ${doc._id}`);
|
|
94
107
|
}
|
|
95
|
-
super(
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
108
|
+
super(
|
|
109
|
+
db,
|
|
110
|
+
{
|
|
111
|
+
replicationStreamId: doc._id,
|
|
112
|
+
sync_rules_content: config.content,
|
|
113
|
+
compiled_plan: config.serialized_plan ?? null,
|
|
114
|
+
|
|
115
|
+
replicationStreamName: doc.slot_name ?? `powersync_${doc._id}`,
|
|
116
|
+
storageVersion: doc.storage_version,
|
|
117
|
+
mapping: SingleSyncConfigBucketDefinitionMapping.fromPersistedMapping(config.rule_mapping),
|
|
118
|
+
syncConfigId: config._id,
|
|
119
|
+
syncConfigState: state.state,
|
|
120
|
+
version_label: config.version_label
|
|
121
|
+
},
|
|
122
|
+
syncConfigParser
|
|
123
|
+
);
|
|
107
124
|
}
|
|
108
125
|
|
|
109
126
|
async getSyncConfigStatus(): Promise<storage.PersistedSyncConfigStatus | null> {
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
|
|
3
3
|
import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
MongoStorageConfig,
|
|
6
|
+
normalizeChunkCompactionConcurrency,
|
|
7
|
+
normalizeClearBatchThrottleRate
|
|
8
|
+
} from '../../types/types.js';
|
|
5
9
|
import { MongoBucketStorage } from '../MongoBucketStorage.js';
|
|
6
10
|
import { MongoReportStorage } from '../MongoReportStorage.js';
|
|
7
11
|
import { PowerSyncMongo } from './db.js';
|
|
@@ -24,6 +28,10 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
24
28
|
}
|
|
25
29
|
|
|
26
30
|
const decodedConfig = MongoStorageConfig.decode(storage as any);
|
|
31
|
+
const chunkCompactionConcurrency = normalizeChunkCompactionConcurrency(
|
|
32
|
+
decodedConfig.chunk_compaction_concurrency,
|
|
33
|
+
decodedConfig.object_storage != null
|
|
34
|
+
);
|
|
27
35
|
|
|
28
36
|
let objectStorage: ObjectStorage | undefined;
|
|
29
37
|
if (decodedConfig.object_storage?.type === 's3') {
|
|
@@ -47,6 +55,8 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
47
55
|
maxPoolSize: resolvedConfig.storage.max_pool_size ?? 8
|
|
48
56
|
});
|
|
49
57
|
|
|
58
|
+
const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
|
|
59
|
+
|
|
50
60
|
let shuttingDown = false;
|
|
51
61
|
|
|
52
62
|
// Explicitly connect on startup.
|
|
@@ -56,7 +66,6 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
56
66
|
// Errors here will cause the process to exit.
|
|
57
67
|
await client.connect();
|
|
58
68
|
|
|
59
|
-
const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
|
|
60
69
|
const readPreference =
|
|
61
70
|
decodedConfig.bulk_read_preference == null
|
|
62
71
|
? undefined
|
|
@@ -65,7 +74,8 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
65
74
|
// 90 is the minimum value.
|
|
66
75
|
maxStalenessSeconds: decodedConfig.bulk_read_preference == 'primary' ? undefined : 90
|
|
67
76
|
});
|
|
68
|
-
const syncStorageFactory = new MongoBucketStorage(database, {
|
|
77
|
+
const syncStorageFactory = new MongoBucketStorage(database, options.syncConfigParser, {
|
|
78
|
+
chunkCompactionConcurrency,
|
|
69
79
|
replicationStreamNamePrefix: resolvedConfig.slot_name_prefix,
|
|
70
80
|
readPreference,
|
|
71
81
|
clearBatchThrottleRate: normalizeClearBatchThrottleRate(decodedConfig.clear_batch_throttle_rate),
|
|
@@ -43,6 +43,7 @@ import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js';
|
|
|
43
43
|
import { MongoParameterCompactor } from './MongoParameterCompactor.js';
|
|
44
44
|
import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
|
|
45
45
|
import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js';
|
|
46
|
+
import { MongoSyncRulesLock } from './MongoSyncRulesLock.js';
|
|
46
47
|
import { MongoCheckpointAPIOptions, MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js';
|
|
47
48
|
import { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
|
|
48
49
|
|
|
@@ -109,6 +110,10 @@ export abstract class MongoSyncBucketStorage
|
|
|
109
110
|
public readonly readPreference: mongo.ReadPreference | undefined;
|
|
110
111
|
public readonly clearBatchThrottleRate: number;
|
|
111
112
|
#storageInitialized = false;
|
|
113
|
+
/**
|
|
114
|
+
* Required for writing, optional for reading.
|
|
115
|
+
*/
|
|
116
|
+
private readonly replicationLock: MongoSyncRulesLock | null;
|
|
112
117
|
|
|
113
118
|
constructor(
|
|
114
119
|
public readonly factory: MongoBucketStorage,
|
|
@@ -119,6 +124,7 @@ export abstract class MongoSyncBucketStorage
|
|
|
119
124
|
options: MongoSyncBucketStorageOptions
|
|
120
125
|
) {
|
|
121
126
|
super();
|
|
127
|
+
this.replicationLock = replicationStream.current_lock;
|
|
122
128
|
this.storageConfig = options.storageConfig;
|
|
123
129
|
this.objectStorage = options.objectStorage;
|
|
124
130
|
// Keep small chunks inline in MongoDB rather than offloading them to S3.
|
|
@@ -269,12 +275,15 @@ export abstract class MongoSyncBucketStorage
|
|
|
269
275
|
* The version-independent part of the batch options.
|
|
270
276
|
*/
|
|
271
277
|
protected writerBatchOptions(options: storage.CreateWriterOptions): Omit<MongoBucketBatchOptions, 'resumeFromLsn'> {
|
|
278
|
+
const replicationLock = this.requireReplicationLock();
|
|
272
279
|
return {
|
|
273
280
|
logger: options.logger ?? this.logger,
|
|
274
281
|
db: this.db,
|
|
275
282
|
parsedSyncConfig: this.getParsedSyncConfigSet(options),
|
|
276
283
|
replicationStreamId: this.replicationStreamId,
|
|
277
284
|
replicationStreamName: this.replicationStreamName,
|
|
285
|
+
replicationLock,
|
|
286
|
+
opIdAllocator: this.factory.getOpIdAllocator(this.replicationStream, replicationLock),
|
|
278
287
|
storeCurrentData: options.storeCurrentData,
|
|
279
288
|
skipExistingRows: options.skipExistingRows ?? false,
|
|
280
289
|
markRecordUnavailable: options.markRecordUnavailable,
|
|
@@ -286,7 +295,16 @@ export abstract class MongoSyncBucketStorage
|
|
|
286
295
|
};
|
|
287
296
|
}
|
|
288
297
|
|
|
298
|
+
private requireReplicationLock(): MongoSyncRulesLock {
|
|
299
|
+
if (this.replicationLock == null) {
|
|
300
|
+
throw new ServiceAssertionError('A replication lease is required to create a writer');
|
|
301
|
+
}
|
|
302
|
+
this.replicationLock.throwIfAborted();
|
|
303
|
+
return this.replicationLock;
|
|
304
|
+
}
|
|
305
|
+
|
|
289
306
|
async createWriter(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
|
|
307
|
+
this.requireReplicationLock();
|
|
290
308
|
await this.initializeStorage();
|
|
291
309
|
|
|
292
310
|
const writer = await this.createWriterImpl(options);
|
|
@@ -385,6 +403,7 @@ export abstract class MongoSyncBucketStorage
|
|
|
385
403
|
throw new ReplicationAbortedError('Aborted clearing data', signal.reason);
|
|
386
404
|
}
|
|
387
405
|
|
|
406
|
+
this.factory.discardOpIdAllocator(this.replicationStreamId);
|
|
388
407
|
await this.clearSyncRuleState();
|
|
389
408
|
|
|
390
409
|
await this.clearBucketData(signal);
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
import crypto from 'crypto';
|
|
2
2
|
|
|
3
3
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
4
|
-
import { ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framework';
|
|
5
|
-
import { storage } from '@powersync/service-core';
|
|
4
|
+
import { ErrorCode, Logger, ReplicationAbortedError, ServiceError } from '@powersync/lib-services-framework';
|
|
5
|
+
import { storage, utils } from '@powersync/service-core';
|
|
6
6
|
import { VersionedPowerSyncMongo } from './db.js';
|
|
7
7
|
|
|
8
8
|
const LOCK_DURATION_MS = 60 * 1000;
|
|
@@ -12,6 +12,12 @@ const LOCK_DURATION_MS = 60 * 1000;
|
|
|
12
12
|
* processes that replication stream at a time.
|
|
13
13
|
*/
|
|
14
14
|
export class MongoSyncRulesLock implements storage.ReplicationLock {
|
|
15
|
+
/** FIFO admission shared by this lease's writers; database fencing still enforces ownership. */
|
|
16
|
+
readonly writerMutex = new utils.Mutex();
|
|
17
|
+
|
|
18
|
+
private readonly abort = new AbortController();
|
|
19
|
+
readonly signal = this.abort.signal;
|
|
20
|
+
|
|
15
21
|
private readonly refreshInterval: NodeJS.Timeout;
|
|
16
22
|
|
|
17
23
|
/**
|
|
@@ -67,13 +73,14 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
|
|
|
67
73
|
constructor(
|
|
68
74
|
private db: VersionedPowerSyncMongo,
|
|
69
75
|
public sync_rules_id: number,
|
|
70
|
-
|
|
76
|
+
public readonly lock_id: string,
|
|
71
77
|
private logger: Logger
|
|
72
78
|
) {
|
|
73
79
|
this.refreshInterval = setInterval(async () => {
|
|
74
80
|
try {
|
|
75
81
|
await this.refresh();
|
|
76
82
|
} catch (e) {
|
|
83
|
+
this.abort.abort(e);
|
|
77
84
|
this.logger.error('Failed to refresh lock', e);
|
|
78
85
|
clearInterval(this.refreshInterval);
|
|
79
86
|
}
|
|
@@ -81,6 +88,7 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
|
|
|
81
88
|
}
|
|
82
89
|
|
|
83
90
|
async release(): Promise<void> {
|
|
91
|
+
this.abort.abort(new Error('Replication lock released'));
|
|
84
92
|
clearInterval(this.refreshInterval);
|
|
85
93
|
const result = await this.db.sync_rules.updateOne(
|
|
86
94
|
{
|
|
@@ -97,6 +105,51 @@ export class MongoSyncRulesLock implements storage.ReplicationLock {
|
|
|
97
105
|
}
|
|
98
106
|
}
|
|
99
107
|
|
|
108
|
+
throwIfAborted(): void {
|
|
109
|
+
this.signal.throwIfAborted();
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
static ownerFilter(streamId: number, lock: MongoSyncRulesLock) {
|
|
113
|
+
lock.throwIfAborted();
|
|
114
|
+
return { _id: streamId, 'lock.id': lock.lock_id };
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
static heartbeatUpdate() {
|
|
118
|
+
// Always change the existing heartbeat, even for writes in the same
|
|
119
|
+
// millisecond. A no-op update is not sufficient for a transactional fence.
|
|
120
|
+
return {
|
|
121
|
+
last_keepalive_ts: {
|
|
122
|
+
$max: ['$$NOW', { $add: [{ $ifNull: ['$last_keepalive_ts', new Date(0)] }, 1] }]
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
static assertOwned<T>(document: T | null): T {
|
|
128
|
+
if (document == null) {
|
|
129
|
+
throw new ReplicationAbortedError('Replication writer no longer owns the stream');
|
|
130
|
+
}
|
|
131
|
+
return document;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* A write, not just an ownership read: takeover conflicts with every transaction
|
|
136
|
+
* that publishes under this owner. Every writer must hold a stream lease.
|
|
137
|
+
*/
|
|
138
|
+
static async fence(
|
|
139
|
+
db: VersionedPowerSyncMongo,
|
|
140
|
+
streamId: number,
|
|
141
|
+
lock: MongoSyncRulesLock,
|
|
142
|
+
session: mongo.ClientSession,
|
|
143
|
+
projection: mongo.Document = { last_persisted_op: 1, last_checkpoint: 1, keepalive_op: 1 }
|
|
144
|
+
) {
|
|
145
|
+
const doc = await db.sync_rules.findOneAndUpdate(
|
|
146
|
+
this.ownerFilter(streamId, lock),
|
|
147
|
+
[{ $set: this.heartbeatUpdate() }],
|
|
148
|
+
{ session, returnDocument: 'after', projection }
|
|
149
|
+
);
|
|
150
|
+
return this.assertOwned(doc);
|
|
151
|
+
}
|
|
152
|
+
|
|
100
153
|
private async refresh(): Promise<void> {
|
|
101
154
|
const result = await this.db.sync_rules.findOneAndUpdate(
|
|
102
155
|
{
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Explicitly groups writes with no intervening reads or result dependencies.
|
|
5
|
+
* Callers retain responsibility for transaction and ordering boundaries.
|
|
6
|
+
*
|
|
7
|
+
* On MongoDB 8.0+, this uses the client-level bulkWrite API.
|
|
8
|
+
*
|
|
9
|
+
* On MongoDB earlier versions, this uses the the collection-level bulkWrite API, or individual operations.
|
|
10
|
+
* This is less efficient, but produces the same results.
|
|
11
|
+
*/
|
|
12
|
+
export class MongoWriteBatch {
|
|
13
|
+
private readonly operations: mongo.AnyClientBulkWriteModel<mongo.Document>[] = [];
|
|
14
|
+
private readonly deleteChecks: { index: number; check: (count: number) => void }[] = [];
|
|
15
|
+
private readonly fallback: (() => Promise<unknown>)[] = [];
|
|
16
|
+
|
|
17
|
+
private readonly ordered: boolean;
|
|
18
|
+
|
|
19
|
+
constructor(
|
|
20
|
+
private readonly client: mongo.MongoClient,
|
|
21
|
+
private readonly supportsClientBulkWrite: boolean,
|
|
22
|
+
private readonly session: mongo.ClientSession | undefined,
|
|
23
|
+
options: { ordered: boolean }
|
|
24
|
+
) {
|
|
25
|
+
this.ordered = options.ordered;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Perform a collection-level bulkWrite.
|
|
30
|
+
*
|
|
31
|
+
* On MongoDB 8.0+, these operations join the client-level bulkWrite with the batch's ordering.
|
|
32
|
+
* The driver may split commands to respect server batch limits.
|
|
33
|
+
*
|
|
34
|
+
* On earlier MongoDB versions, this always uses an unordered collection-level bulkWrite.
|
|
35
|
+
* Operations within this collection bulk must therefore be independent of their order.
|
|
36
|
+
*/
|
|
37
|
+
bulkWriteUnordered<T extends mongo.Document>(
|
|
38
|
+
collection: mongo.Collection<T>,
|
|
39
|
+
operations: mongo.AnyBulkWriteOperation<T>[]
|
|
40
|
+
): void {
|
|
41
|
+
if (operations.length === 0) return;
|
|
42
|
+
if (!this.supportsClientBulkWrite) {
|
|
43
|
+
this.fallback.push(() => collection.bulkWrite(operations, { ordered: false, session: this.session }));
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
for (const operation of operations) {
|
|
47
|
+
// Collection and client models contain the same operation fields, with a
|
|
48
|
+
// different envelope. Keep the schema checked at the collection boundary.
|
|
49
|
+
const [name, model] = Object.entries(operation)[0];
|
|
50
|
+
this.operations.push({ ...model, name, namespace: collection.namespace });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
insertOne<T extends mongo.Document>(collection: mongo.Collection<T>, document: mongo.OptionalUnlessRequiredId<T>) {
|
|
55
|
+
if (!this.supportsClientBulkWrite) {
|
|
56
|
+
this.fallback.push(() => collection.insertOne(document, { session: this.session }));
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
this.addModel<T>({ name: 'insertOne', namespace: collection.namespace, document: document as mongo.OptionalId<T> });
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
insertMany<T extends mongo.Document>(
|
|
63
|
+
collection: mongo.Collection<T>,
|
|
64
|
+
documents: mongo.OptionalUnlessRequiredId<T>[]
|
|
65
|
+
) {
|
|
66
|
+
if (documents.length === 0) return;
|
|
67
|
+
if (!this.supportsClientBulkWrite) {
|
|
68
|
+
this.fallback.push(() => collection.insertMany(documents, { session: this.session }));
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
for (const document of documents) {
|
|
72
|
+
this.addModel<T>({
|
|
73
|
+
name: 'insertOne',
|
|
74
|
+
namespace: collection.namespace,
|
|
75
|
+
document: document as mongo.OptionalId<T>
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
deleteMany<T extends mongo.Document>(
|
|
81
|
+
collection: mongo.Collection<T>,
|
|
82
|
+
filter: mongo.Filter<T>,
|
|
83
|
+
checkDeletedCount?: (count: number) => void
|
|
84
|
+
) {
|
|
85
|
+
if (checkDeletedCount) {
|
|
86
|
+
// Client-bulk results are checked after all writes execute. A failed
|
|
87
|
+
// invariant must roll back the entire batch, not leave later writes visible.
|
|
88
|
+
if (!this.session?.inTransaction()) throw new Error('Checked deletes require a transaction');
|
|
89
|
+
}
|
|
90
|
+
if (!this.supportsClientBulkWrite) {
|
|
91
|
+
this.fallback.push(async () => {
|
|
92
|
+
const result = await collection.deleteMany(filter, { session: this.session });
|
|
93
|
+
checkDeletedCount?.(result.deletedCount);
|
|
94
|
+
});
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
if (checkDeletedCount) {
|
|
98
|
+
this.deleteChecks.push({ index: this.operations.length, check: checkDeletedCount });
|
|
99
|
+
}
|
|
100
|
+
this.addModel<T>({ name: 'deleteMany', namespace: collection.namespace, filter });
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
updateOne<T extends mongo.Document>(
|
|
104
|
+
collection: mongo.Collection<T>,
|
|
105
|
+
filter: mongo.Filter<T>,
|
|
106
|
+
update: mongo.UpdateFilter<T> | mongo.Document[],
|
|
107
|
+
options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
|
|
108
|
+
) {
|
|
109
|
+
if (!this.supportsClientBulkWrite) {
|
|
110
|
+
this.fallback.push(() => collection.updateOne(filter, update, { ...options, session: this.session }));
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
this.addModel({ name: 'updateOne', namespace: collection.namespace, filter, update, ...options });
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
updateMany<T extends mongo.Document>(
|
|
117
|
+
collection: mongo.Collection<T>,
|
|
118
|
+
filter: mongo.Filter<T>,
|
|
119
|
+
update: mongo.UpdateFilter<T> | mongo.Document[],
|
|
120
|
+
options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
|
|
121
|
+
) {
|
|
122
|
+
if (!this.supportsClientBulkWrite) {
|
|
123
|
+
this.fallback.push(() => collection.updateMany(filter, update, { ...options, session: this.session }));
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
this.addModel({ name: 'updateMany', namespace: collection.namespace, filter, update, ...options });
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
private addModel<T extends mongo.Document>(operation: mongo.AnyClientBulkWriteModel<T>) {
|
|
130
|
+
// The collection-specific methods validate the schema before erasing it here.
|
|
131
|
+
this.operations.push(operation as unknown as mongo.AnyClientBulkWriteModel<mongo.Document>);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Execute the queued writes.
|
|
136
|
+
*
|
|
137
|
+
* On MongoDB 8.0+, this uses the client-level bulkWrite API, typically resulting in a single command.
|
|
138
|
+
* The command may be split if there are a large number of operations.
|
|
139
|
+
*
|
|
140
|
+
* On earlier MongoDB versions, this executes each queued operation individually.
|
|
141
|
+
*
|
|
142
|
+
* If no operations were queued, this is a no-op.
|
|
143
|
+
*/
|
|
144
|
+
async execute(): Promise<void> {
|
|
145
|
+
if (!this.supportsClientBulkWrite) {
|
|
146
|
+
// Sessions cannot execute concurrent operations within a transaction.
|
|
147
|
+
for (const write of this.fallback) await write();
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (this.operations.length === 0) return;
|
|
151
|
+
|
|
152
|
+
try {
|
|
153
|
+
const result = await this.client.bulkWrite(this.operations, {
|
|
154
|
+
session: this.session,
|
|
155
|
+
ordered: this.ordered,
|
|
156
|
+
verboseResults: this.deleteChecks.length > 0
|
|
157
|
+
});
|
|
158
|
+
for (const { index, check } of this.deleteChecks) {
|
|
159
|
+
const deleted = result.deleteResults?.get(index);
|
|
160
|
+
if (deleted == null) throw new Error('Missing client bulk delete result');
|
|
161
|
+
check(deleted.deletedCount);
|
|
162
|
+
}
|
|
163
|
+
} catch (error) {
|
|
164
|
+
// The driver wraps command failures without copying their error labels.
|
|
165
|
+
// Preserve withTransaction's retry handling for e.g. write conflicts.
|
|
166
|
+
if (error instanceof mongo.MongoClientBulkWriteError && error.cause instanceof mongo.MongoError) {
|
|
167
|
+
throw error.cause;
|
|
168
|
+
}
|
|
169
|
+
throw error;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
@@ -95,7 +95,8 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
95
95
|
},
|
|
96
96
|
upsert: true
|
|
97
97
|
}
|
|
98
|
-
}))
|
|
98
|
+
})),
|
|
99
|
+
{ ordered: false }
|
|
99
100
|
);
|
|
100
101
|
|
|
101
102
|
const userIds = generatedCheckpoints.map((checkpoint) => checkpoint.user_id);
|
|
@@ -180,7 +181,8 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
180
181
|
upsert: true
|
|
181
182
|
}
|
|
182
183
|
};
|
|
183
|
-
})
|
|
184
|
+
}),
|
|
185
|
+
{ ordered: false }
|
|
184
186
|
);
|
|
185
187
|
|
|
186
188
|
// Fetch the final ids separately so stale requests can still return the
|
|
@@ -8,6 +8,7 @@ import { JSONBig } from '@powersync/service-jsonbig';
|
|
|
8
8
|
import { mongoTableId, replicaIdToSubkey } from '../../../utils/util.js';
|
|
9
9
|
import { currentBucketKey, MAX_ROW_SIZE } from '../MongoBucketBatchShared.js';
|
|
10
10
|
import { MongoIdSequence } from '../MongoIdSequence.js';
|
|
11
|
+
import { MongoWriteBatch } from '../MongoWriteBatch.js';
|
|
11
12
|
import type { VersionedPowerSyncMongo } from '../db.js';
|
|
12
13
|
import { TaggedBucketParameterDocument } from '../models.js';
|
|
13
14
|
import { ObjectStorage } from '../v3/object-storage/ObjectStorage.js';
|
|
@@ -216,13 +217,13 @@ export abstract class PersistedBatch {
|
|
|
216
217
|
|
|
217
218
|
protected abstract get currentDataCount(): number;
|
|
218
219
|
|
|
219
|
-
protected abstract
|
|
220
|
+
protected abstract queueBucketData(writes: MongoWriteBatch): Promise<void>;
|
|
220
221
|
|
|
221
|
-
protected abstract
|
|
222
|
+
protected abstract queueBucketParameters(writes: MongoWriteBatch): void;
|
|
222
223
|
|
|
223
|
-
protected abstract
|
|
224
|
+
protected abstract queueCurrentData(writes: MongoWriteBatch): void;
|
|
224
225
|
|
|
225
|
-
protected abstract
|
|
226
|
+
protected abstract queueBucketStates(writes: MongoWriteBatch): void;
|
|
226
227
|
|
|
227
228
|
protected abstract resetCurrentData(): void;
|
|
228
229
|
|
|
@@ -348,24 +349,27 @@ export abstract class PersistedBatch {
|
|
|
348
349
|
async flush(session: mongo.ClientSession, options?: storage.BucketBatchCommitOptions) {
|
|
349
350
|
const startAt = performance.now();
|
|
350
351
|
let flushedSomething = false;
|
|
352
|
+
const writes = this.db.createWriteBatch(session, { ordered: false });
|
|
351
353
|
if (this.bucketDataCount > 0) {
|
|
352
354
|
flushedSomething = true;
|
|
353
|
-
await this.
|
|
355
|
+
await this.queueBucketData(writes);
|
|
354
356
|
}
|
|
355
357
|
if (this.bucketParameters.length > 0) {
|
|
356
358
|
flushedSomething = true;
|
|
357
|
-
|
|
359
|
+
this.queueBucketParameters(writes);
|
|
358
360
|
}
|
|
359
361
|
if (this.currentDataCount > 0) {
|
|
360
362
|
flushedSomething = true;
|
|
361
|
-
|
|
363
|
+
this.queueCurrentData(writes);
|
|
362
364
|
}
|
|
363
365
|
|
|
364
366
|
if (this.bucketStates.size > 0) {
|
|
365
367
|
flushedSomething = true;
|
|
366
|
-
|
|
368
|
+
this.queueBucketStates(writes);
|
|
367
369
|
}
|
|
368
370
|
|
|
371
|
+
await writes.execute();
|
|
372
|
+
|
|
369
373
|
if (flushedSomething) {
|
|
370
374
|
const duration = Math.round(performance.now() - startAt);
|
|
371
375
|
if (options?.oldestUncommittedChange != null) {
|
|
@@ -58,6 +58,10 @@ export abstract class BaseVersionedPowerSyncMongo {
|
|
|
58
58
|
return this.upstream.notifyCheckpoint();
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
createWriteBatch(session: mongo.ClientSession | undefined, options: { ordered: boolean }) {
|
|
62
|
+
return this.upstream.createWriteBatch(session, options);
|
|
63
|
+
}
|
|
64
|
+
|
|
61
65
|
protected sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId) {
|
|
62
66
|
return this.upstream.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
|
|
63
67
|
}
|
|
@@ -17,6 +17,7 @@ import {
|
|
|
17
17
|
SyncRuleDocumentBase,
|
|
18
18
|
WriteCheckpointDocument
|
|
19
19
|
} from './models.js';
|
|
20
|
+
import { MongoWriteBatch } from './MongoWriteBatch.js';
|
|
20
21
|
import {
|
|
21
22
|
BucketDataDocumentV1,
|
|
22
23
|
BucketParameterDocument,
|
|
@@ -54,6 +55,19 @@ export class PowerSyncMongo {
|
|
|
54
55
|
|
|
55
56
|
readonly client: mongo.MongoClient;
|
|
56
57
|
readonly db: mongo.Db;
|
|
58
|
+
private get clientBulkWriteSupport(): boolean {
|
|
59
|
+
// The driver exposes topology at runtime, but omits this internal property
|
|
60
|
+
// from its public types. Reuse its handshake results without another command.
|
|
61
|
+
const client = this.client as mongo.MongoClient & { topology?: { description: mongo.TopologyDescription } };
|
|
62
|
+
const servers = [...(client.topology?.description.servers.values() ?? [])];
|
|
63
|
+
// MongoDB 8.0 introduced client bulkWrite at wire version 25. Unknown / mixed-version
|
|
64
|
+
// topologies use collection writes until all known servers support it.
|
|
65
|
+
return servers.length > 0 && servers.every((server) => server.maxWireVersion >= 25);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
createWriteBatch(session: mongo.ClientSession | undefined, options: { ordered: boolean }): MongoWriteBatch {
|
|
69
|
+
return new MongoWriteBatch(this.client, this.clientBulkWriteSupport, session, options);
|
|
70
|
+
}
|
|
57
71
|
|
|
58
72
|
constructor(client: mongo.MongoClient, options?: PowerSyncMongoOptions) {
|
|
59
73
|
this.client = client;
|
|
@@ -67,6 +81,7 @@ export class PowerSyncMongo {
|
|
|
67
81
|
this.bucket_data = db.collection('bucket_data');
|
|
68
82
|
this.bucket_parameters = db.collection('bucket_parameters');
|
|
69
83
|
this.op_id_sequence = db.collection('op_id_sequence');
|
|
84
|
+
// Uses legacy terminology instead of "sync config" for backwards-compatibility
|
|
70
85
|
this.sync_rules = db.collection('sync_rules');
|
|
71
86
|
this.source_tables = db.collection('source_tables');
|
|
72
87
|
this.custom_write_checkpoints = db.collection('custom_write_checkpoints');
|
|
@@ -308,6 +308,7 @@ export interface CustomWriteCheckpointDocument {
|
|
|
308
308
|
_id: bson.ObjectId;
|
|
309
309
|
user_id: string;
|
|
310
310
|
checkpoint: bigint;
|
|
311
|
+
// Uses the legacy field name "sync_rules_id" for backwards-compatibility. See AGENTS.md.
|
|
311
312
|
sync_rules_id: number;
|
|
312
313
|
/**
|
|
313
314
|
* Unlike managed write checkpoints, custom write checkpoints are flushed together with
|