@powersync/service-module-mongodb-storage 0.21.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +43 -0
- package/dist/storage/MongoBucketStorage.d.ts +10 -1
- package/dist/storage/MongoBucketStorage.js +37 -7
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +24 -10
- package/dist/storage/implementation/MongoBucketBatch.js +134 -89
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +2 -0
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
- package/dist/storage/implementation/MongoIdSequence.js +14 -1
- package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
- package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +4 -2
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +5 -0
- package/dist/storage/implementation/MongoSyncBucketStorage.js +17 -0
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
- package/dist/storage/implementation/MongoSyncRulesLock.js +38 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
- package/dist/storage/implementation/MongoWriteBatch.js +149 -0
- package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +2 -2
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/PersistedBatch.d.ts +5 -4
- package/dist/storage/implementation/common/PersistedBatch.js +6 -4
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +5 -0
- package/dist/storage/implementation/db.js +14 -0
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +5 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +149 -139
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +2 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.js +23 -17
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
- package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +6 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +148 -162
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +10 -4
- package/dist/storage/implementation/v3/MongoCompactorV3.js +175 -76
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +12 -8
- package/dist/storage/implementation/v3/PersistedBatchV3.js +25 -32
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/compact-utils.d.ts +3 -2
- package/dist/storage/implementation/v3/compact-utils.js +12 -5
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -1
- package/dist/storage/implementation/v3/models.d.ts +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +8 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +3 -3
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +7 -9
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +3 -2
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +4 -7
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +21 -3
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
- package/dist/types/types.d.ts +5 -0
- package/dist/types/types.js +12 -0
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +2 -1
- package/dist/utils/test-utils.js +5 -3
- package/dist/utils/test-utils.js.map +1 -1
- package/package.json +7 -7
- package/src/storage/MongoBucketStorage.ts +47 -8
- package/src/storage/implementation/MongoBucketBatch.ts +205 -111
- package/src/storage/implementation/MongoCompactor.ts +2 -0
- package/src/storage/implementation/MongoIdSequence.ts +21 -1
- package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
- package/src/storage/implementation/MongoStorageProvider.ts +12 -2
- package/src/storage/implementation/MongoSyncBucketStorage.ts +19 -0
- package/src/storage/implementation/MongoSyncRulesLock.ts +56 -3
- package/src/storage/implementation/MongoWriteBatch.ts +172 -0
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +4 -2
- package/src/storage/implementation/common/PersistedBatch.ts +12 -8
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
- package/src/storage/implementation/db.ts +15 -0
- package/src/storage/implementation/models.ts +1 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +167 -160
- package/src/storage/implementation/v1/MongoCompactorV1.ts +29 -26
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +1 -1
- package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +217 -242
- package/src/storage/implementation/v3/MongoCompactorV3.ts +284 -108
- package/src/storage/implementation/v3/PersistedBatchV3.ts +36 -36
- package/src/storage/implementation/v3/compact-utils.ts +16 -7
- package/src/storage/implementation/v3/models.ts +1 -1
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +8 -1
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +11 -10
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +8 -9
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +23 -3
- package/src/types/types.ts +18 -0
- package/src/utils/test-utils.ts +6 -3
- package/test/src/__snapshots__/storage.test.ts.snap +102 -2
- package/test/src/__snapshots__/storage_compacting.test.ts.snap +2 -2
- package/test/src/__snapshots__/storage_sync.test.ts.snap +12 -12
- package/test/src/chunk_compaction_config.test.ts +38 -0
- package/test/src/cleanup-stopped-sync-configs.test.ts +19 -13
- package/test/src/compact-utils.test.ts +33 -8
- package/test/src/concurrent_writers.test.ts +599 -0
- package/test/src/helpers/concurrentWriter.mjs +29 -0
- package/test/src/helpers/s3TestFactory.ts +12 -12
- package/test/src/object_storage_usage.test.ts +18 -5
- package/test/src/parameter_compacting_v1.test.ts +2 -2
- package/test/src/parameter_compacting_v3.test.ts +2 -2
- package/test/src/parameter_compaction_fence.test.ts +3 -3
- package/test/src/storage.test.ts +32 -33
- package/test/src/storage_compacting.test.ts +402 -24
- package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
- package/test/src/storage_s3_checksums.test.ts +4 -4
- package/test/src/storage_s3_compaction_lifecycle.test.ts +8 -8
- package/test/src/storage_s3_reading.test.ts +51 -5
- package/test/src/storage_s3_writing.test.ts +3 -3
- package/test/src/storage_sync.test.ts +52 -46
- package/test/src/storeCurrentData.test.ts +5 -5
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -4,6 +4,7 @@ import * as bson from 'bson';
|
|
|
4
4
|
import { Logger } from '@powersync/lib-services-framework';
|
|
5
5
|
import { BucketDefinitionMapping, InternalOpId, storage } from '@powersync/service-core';
|
|
6
6
|
import { MongoIdSequence } from '../MongoIdSequence.js';
|
|
7
|
+
import { MongoWriteBatch } from '../MongoWriteBatch.js';
|
|
7
8
|
import type { VersionedPowerSyncMongo } from '../db.js';
|
|
8
9
|
import { TaggedBucketParameterDocument } from '../models.js';
|
|
9
10
|
import { ObjectStorage } from '../v3/object-storage/ObjectStorage.js';
|
|
@@ -74,10 +75,10 @@ export declare abstract class PersistedBatch {
|
|
|
74
75
|
abstract softDeleteCurrentData(sourceTableId: bson.ObjectId, replicaId: storage.ReplicaId, checkpointGreaterThan: bigint): void;
|
|
75
76
|
abstract upsertCurrentData(values: UpsertCurrentDataOptions): void;
|
|
76
77
|
protected abstract get currentDataCount(): number;
|
|
77
|
-
protected abstract
|
|
78
|
-
protected abstract
|
|
79
|
-
protected abstract
|
|
80
|
-
protected abstract
|
|
78
|
+
protected abstract queueBucketData(writes: MongoWriteBatch): Promise<void>;
|
|
79
|
+
protected abstract queueBucketParameters(writes: MongoWriteBatch): void;
|
|
80
|
+
protected abstract queueCurrentData(writes: MongoWriteBatch): void;
|
|
81
|
+
protected abstract queueBucketStates(writes: MongoWriteBatch): void;
|
|
81
82
|
protected abstract resetCurrentData(): void;
|
|
82
83
|
protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId;
|
|
83
84
|
protected abstract getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId;
|
|
@@ -229,22 +229,24 @@ export class PersistedBatch {
|
|
|
229
229
|
async flush(session, options) {
|
|
230
230
|
const startAt = performance.now();
|
|
231
231
|
let flushedSomething = false;
|
|
232
|
+
const writes = this.db.createWriteBatch(session, { ordered: false });
|
|
232
233
|
if (this.bucketDataCount > 0) {
|
|
233
234
|
flushedSomething = true;
|
|
234
|
-
await this.
|
|
235
|
+
await this.queueBucketData(writes);
|
|
235
236
|
}
|
|
236
237
|
if (this.bucketParameters.length > 0) {
|
|
237
238
|
flushedSomething = true;
|
|
238
|
-
|
|
239
|
+
this.queueBucketParameters(writes);
|
|
239
240
|
}
|
|
240
241
|
if (this.currentDataCount > 0) {
|
|
241
242
|
flushedSomething = true;
|
|
242
|
-
|
|
243
|
+
this.queueCurrentData(writes);
|
|
243
244
|
}
|
|
244
245
|
if (this.bucketStates.size > 0) {
|
|
245
246
|
flushedSomething = true;
|
|
246
|
-
|
|
247
|
+
this.queueBucketStates(writes);
|
|
247
248
|
}
|
|
249
|
+
await writes.execute();
|
|
248
250
|
if (flushedSomething) {
|
|
249
251
|
const duration = Math.round(performance.now() - startAt);
|
|
250
252
|
if (options?.oldestUncommittedChange != null) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/PersistedBatch.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAU,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAkD,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChG,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;
|
|
1
|
+
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/PersistedBatch.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAU,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAkD,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChG,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAS9E;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,GAAG,IAAI,CAAC;AAqCxD;;;;;GAKG;AACH,MAAM,OAAgB,cAAc;IAsBb;IACA;IACA;IAvBrB,MAAM,CAAS;IACf,UAAU,GAAoB,EAAE,CAAC;IACjC,gBAAgB,GAAoC,EAAE,CAAC;IACvD,YAAY,GAAmC,IAAI,GAAG,EAAE,CAAC;IAEtC,aAAa,CAAiB;IAC9B,oBAAoB,GAAW,8BAA8B,CAAC;IAC9D,MAAM,CAAe;IACrB,0BAA0B,CAAU;IAEvD;;OAEG;IACH,aAAa,GAAwB,IAAI,CAAC;IAE1C;;OAEG;IACH,WAAW,GAAG,CAAC,CAAC;IAEhB,YACqB,EAA2B,EAC3B,QAAgB,EAChB,OAAgC,EACnD,WAAmB,EACnB,OAA+B;QAJZ,OAAE,GAAF,EAAE,CAAyB;QAC3B,aAAQ,GAAR,QAAQ,CAAQ;QAChB,YAAO,GAAP,OAAO,CAAyB;QAInD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,aAAa,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC9B,IAAI,CAAC,0BAA0B,GAAG,OAAO,EAAE,0BAA0B,CAAC;QACtE,IAAI,OAAO,EAAE,oBAAoB,IAAI,IAAI,EAAE,CAAC;YAC1C,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,cAAc,CAAC,OAA8B;QAC3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAmC,CAAC;QACrE,KAAK,IAAI,MAAM,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAC1C,MAAM,MAAM,GAA4B;gBACtC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC;gBACzD,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;YACF,IAAI,OAAO,CAAC,KAAK,CAAC,mBAAmB,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,YAAa,CAAC,EAAE,CAAC;gBAC9G,+CAA+C;gBAC/C,yGAAyG;gBACzG,SAAS;YACX,CAAC;YACD,iBAAiB,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QAEnD,KAAK,MAAM,SAAS,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAClE,MAAM,GAAG,GAAG,gBAAgB,CAAC;gBAC3B,YAAY,EAAE,YAAY;gBAC1B,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,EAAE,EAAE,SAAS,CAAC,EAAE;aACjB,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3E,IAAI,UAAU,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;gBACrC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,eAAe,UAAU,CAAC,MAAM,mBAAmB,CAAC,CAAC;gBACjF,SAAS;YACX,CAAC;YAED,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAC7C,IAAI,CAAC,WAAW,IAAI,YAAY,CAAC;YAEjC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,gBAAgB,CAAC;gBACpB,SAAS,EAAE;oBACT,MAAM,EAAE,SAAS,CAAC,MAAM;oBACxB,YAAY,EAAE,YAAY;oBAC1B,mBAAmB,EAAE,IAAI,CAAC,QAAQ;iBACnC;gBACD,KAAK;gBACL,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM;gBACN,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,KAAK,EAAE,SAAS,CAAC,EAAE;gBACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;gBAC1B,IAAI,EAAE,UAAU;aACjB,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;QAC5E,CAAC;QAED,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAa,CAAC;YAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,mBAAmB,CAAC;gBACvB,SAAS,EAAE;oBACT,mBAAmB,EAAE,IAAI,CAAC,QAAQ;oBAClC,YAAY;oBACZ,MAAM,EAAE,MAAM,CAAC,MAAM;iBACtB;gBACD,KAAK;gBACL,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE,MAAM,CAAC,EAAE;gBAChB,QAAQ,EAAE,SAAS;aACpB,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;YACxB,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IA6BD;;;OAGG;IACO,eAAe,CAAC,KAA4B,EAAE,GAAsB;QAC5E,OAAO,iBAAiB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACvC,CAAC;IAED,IAAc,eAAe;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAES,eAAe,CAAC,YAAgC,EAAE,MAAc,EAAE,KAAmB,EAAE,KAAa;QAC5G,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;QAC9C,IAAI,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,aAAa,EAAE,CAAC;YAClB,aAAa,CAAC,MAAM,GAAG,KAAK,CAAC;YAC7B,aAAa,CAAC,cAAc,IAAI,CAAC,CAAC;YAClC,aAAa,CAAC,cAAc,IAAI,KAAK,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE;gBACzB,YAAY;gBACZ,MAAM;gBACN,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,CAAC;gBACjB,cAAc,EAAE,KAAK;gBACrB,eAAe,EAAE,CAAC;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;OAIG;IACO,6BAA6B,CAAC,YAAgC,EAAE,MAAc,EAAE,KAAa;QACrG,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;YAC1B,aAAa,CAAC,eAAe,IAAI,CAAC,CAAC;YACnC,aAAa,CAAC,cAAc,IAAI,KAAK,CAAC;QACxC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACO,yBAAyB,CAAC,YAAgC,EAAE,MAAc;QAClF,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC,CAAC;QACvE,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,KAAK,CAAC,cAAc,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAES,gBAAgB,CAAC,OAW1B;QACC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,CAAC,EAAE,OAAO,CAAC,KAAK;YAChB,EAAE,EAAE,KAAK;YACT,YAAY,EAAE,YAAY,CAAC,OAAO,CAAC,aAAa,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,OAAO,CAAC,IAAI;SACnB,CAAC,CAAC;IACL,CAAC;IAES,mBAAmB,CAAC,OAS7B;QACC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,CAAC,EAAE,OAAO,CAAC,KAAK;YAChB,EAAE,EAAE,QAAQ;YACZ,YAAY,EAAE,YAAY,CAAC,OAAO,CAAC,aAAa,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,IAAI;SACX,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB;QACpB,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,eAAe,IAAI,yBAAyB;YACjD,IAAI,CAAC,gBAAgB,IAAI,yBAAyB;YAClD,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAA4B,EAAE,OAA0C;QAClF,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QACrE,IAAI,IAAI,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;YAC7B,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,gBAAgB,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;YAC9B,gBAAgB,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC/B,gBAAgB,GAAG,IAAI,CAAC;YACxB,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QAEvB,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;YACzD,IAAI,OAAO,EAAE,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBAC7C,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBAEnG,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,IAAI,CAAC,eAAe,SAAS,IAAI,CAAC,gBAAgB,CAAC,MAAM,mBACnE,IAAI,CAAC,gBACP,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,sBAAsB,cAAc,GAAG,EAC7I;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,gBAAgB,EAAE,IAAI,CAAC,eAAe;wBACtC,uBAAuB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBACrD,oBAAoB,EAAE,IAAI,CAAC,gBAAgB;wBAC3C,uBAAuB,EAAE,cAAc;qBACxC;iBACF,CACF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,IAAI,CAAC,eAAe,SAAS,IAAI,CAAC,gBAAgB,CAAC,MAAM,mBACnE,IAAI,CAAC,gBACP,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,EAAE,EACxG;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,gBAAgB,EAAE,IAAI,CAAC,eAAe;wBACtC,uBAAuB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBACrD,oBAAoB,EAAE,IAAI,CAAC,gBAAgB;qBAC5C;iBACF,CACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG;YACZ,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;YAChD,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,UAAU,EAAE,gBAAgB;SAC7B,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAE1B,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}
|
|
@@ -19,6 +19,9 @@ export declare abstract class BaseVersionedPowerSyncMongo {
|
|
|
19
19
|
get checkpoint_events(): mongo.Collection<import("../models.js").CheckpointEventDocument>;
|
|
20
20
|
get connection_report_events(): mongo.Collection<import("../models.js").ClientConnectionDocument>;
|
|
21
21
|
notifyCheckpoint(): Promise<void>;
|
|
22
|
+
createWriteBatch(session: mongo.ClientSession | undefined, options: {
|
|
23
|
+
ordered: boolean;
|
|
24
|
+
}): import("../MongoWriteBatch.js").MongoWriteBatch;
|
|
22
25
|
protected sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId): string;
|
|
23
26
|
protected sourceTableCollectionName(replicationStreamId: number): string;
|
|
24
27
|
protected listCollectionsByPrefix<T extends mongo.Document>(prefix: string): Promise<mongo.Collection<T>[]>;
|
|
@@ -41,6 +41,9 @@ export class BaseVersionedPowerSyncMongo {
|
|
|
41
41
|
notifyCheckpoint() {
|
|
42
42
|
return this.upstream.notifyCheckpoint();
|
|
43
43
|
}
|
|
44
|
+
createWriteBatch(session, options) {
|
|
45
|
+
return this.upstream.createWriteBatch(session, options);
|
|
46
|
+
}
|
|
44
47
|
sourceRecordsCollectionName(replicationStreamId, sourceTableId) {
|
|
45
48
|
return this.upstream.sourceRecordsCollectionName(replicationStreamId, sourceTableId);
|
|
46
49
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"VersionedPowerSyncMongoBase.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/VersionedPowerSyncMongoBase.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAI/D,MAAM,OAAgB,2BAA2B;IAO1B;IANZ,MAAM,CAAoB;IAC1B,EAAE,CAAW;IACb,aAAa,CAAgB;IACtC,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAEpB,YACqB,QAAwB,EAC3C,aAA4B;QADT,aAAQ,GAAR,QAAQ,CAAgB;QAG3C,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACtB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;IACtC,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;IAClC,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACzC,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC7B,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACzC,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAChD,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;IAC1C,CAAC;IAES,2BAA2B,CAAC,mBAA2B,EAAE,aAA6B;QAC9F,OAAO,IAAI,CAAC,QAAQ,CAAC,2BAA2B,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;IACvF,CAAC;IAES,yBAAyB,CAAC,mBAA2B;QAC7D,OAAO,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;IACtE,CAAC;IAES,KAAK,CAAC,uBAAuB,CAA2B,MAAc;QAC9E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAEpH,OAAO,WAAW;aACf,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,CAAC;CAKF"}
|
|
1
|
+
{"version":3,"file":"VersionedPowerSyncMongoBase.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/VersionedPowerSyncMongoBase.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAI/D,MAAM,OAAgB,2BAA2B;IAO1B;IANZ,MAAM,CAAoB;IAC1B,EAAE,CAAW;IACb,aAAa,CAAgB;IACtC,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAEpB,YACqB,QAAwB,EAC3C,aAA4B;QADT,aAAQ,GAAR,QAAQ,CAAgB;QAG3C,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACtB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;IACtC,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;IAClC,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACzC,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC7B,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACzC,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAChD,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;IAC1C,CAAC;IAED,gBAAgB,CAAC,OAAwC,EAAE,OAA6B;QACtF,OAAO,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC1D,CAAC;IAES,2BAA2B,CAAC,mBAA2B,EAAE,aAA6B;QAC9F,OAAO,IAAI,CAAC,QAAQ,CAAC,2BAA2B,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;IACvF,CAAC;IAES,yBAAyB,CAAC,mBAA2B;QAC7D,OAAO,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;IACtE,CAAC;IAES,KAAK,CAAC,uBAAuB,CAA2B,MAAc;QAC9E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAEpH,OAAO,WAAW;aACf,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,CAAC;CAKF"}
|
|
@@ -4,6 +4,7 @@ import { DO_NOT_LOG } from '@powersync/lib-services-framework';
|
|
|
4
4
|
import { MongoStorageConfig } from '../../types/types.js';
|
|
5
5
|
import { BaseVersionedPowerSyncMongo } from './common/VersionedPowerSyncMongoBase.js';
|
|
6
6
|
import { CheckpointEventDocument, ClientConnectionDocument, CommonSourceTableDocument, CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, SourceTableDocument, StorageConfig, SyncRuleDocumentBase, WriteCheckpointDocument } from './models.js';
|
|
7
|
+
import { MongoWriteBatch } from './MongoWriteBatch.js';
|
|
7
8
|
import { BucketDataDocumentV1, BucketParameterDocument, BucketStateDocumentV1, CurrentDataDocument } from './v1/models.js';
|
|
8
9
|
import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js';
|
|
9
10
|
import { BucketDataDocumentV3, CustomCheckpointRequestDocumentV3 } from './v3/models.js';
|
|
@@ -32,6 +33,10 @@ export declare class PowerSyncMongo {
|
|
|
32
33
|
readonly connection_report_events: mongo.Collection<ClientConnectionDocument>;
|
|
33
34
|
readonly client: mongo.MongoClient;
|
|
34
35
|
readonly db: mongo.Db;
|
|
36
|
+
private get clientBulkWriteSupport();
|
|
37
|
+
createWriteBatch(session: mongo.ClientSession | undefined, options: {
|
|
38
|
+
ordered: boolean;
|
|
39
|
+
}): MongoWriteBatch;
|
|
35
40
|
constructor(client: mongo.MongoClient, options?: PowerSyncMongoOptions);
|
|
36
41
|
versioned(storageConfig: StorageConfig & {
|
|
37
42
|
incrementalReprocessing: true;
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
|
|
3
3
|
import { DO_NOT_LOG } from '@powersync/lib-services-framework';
|
|
4
|
+
import { MongoWriteBatch } from './MongoWriteBatch.js';
|
|
4
5
|
import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js';
|
|
5
6
|
import { OBJECT_STORAGE_USAGE_COLLECTION, VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
|
|
6
7
|
export class PowerSyncMongo {
|
|
@@ -20,6 +21,18 @@ export class PowerSyncMongo {
|
|
|
20
21
|
connection_report_events;
|
|
21
22
|
client;
|
|
22
23
|
db;
|
|
24
|
+
get clientBulkWriteSupport() {
|
|
25
|
+
// The driver exposes topology at runtime, but omits this internal property
|
|
26
|
+
// from its public types. Reuse its handshake results without another command.
|
|
27
|
+
const client = this.client;
|
|
28
|
+
const servers = [...(client.topology?.description.servers.values() ?? [])];
|
|
29
|
+
// MongoDB 8.0 introduced client bulkWrite at wire version 25. Unknown / mixed-version
|
|
30
|
+
// topologies use collection writes until all known servers support it.
|
|
31
|
+
return servers.length > 0 && servers.every((server) => server.maxWireVersion >= 25);
|
|
32
|
+
}
|
|
33
|
+
createWriteBatch(session, options) {
|
|
34
|
+
return new MongoWriteBatch(this.client, this.clientBulkWriteSupport, session, options);
|
|
35
|
+
}
|
|
23
36
|
constructor(client, options) {
|
|
24
37
|
this.client = client;
|
|
25
38
|
const db = client.db(options?.database, {
|
|
@@ -30,6 +43,7 @@ export class PowerSyncMongo {
|
|
|
30
43
|
this.bucket_data = db.collection('bucket_data');
|
|
31
44
|
this.bucket_parameters = db.collection('bucket_parameters');
|
|
32
45
|
this.op_id_sequence = db.collection('op_id_sequence');
|
|
46
|
+
// Uses legacy terminology instead of "sync config" for backwards-compatibility
|
|
33
47
|
this.sync_rules = db.collection('sync_rules');
|
|
34
48
|
this.source_tables = db.collection('source_tables');
|
|
35
49
|
this.custom_write_checkpoints = db.collection('custom_write_checkpoints');
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAe/D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAOvD,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAG9E,OAAO,EAAE,+BAA+B,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAS/G,MAAM,OAAO,cAAc;IACzB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAEX,YAAY,CAAwC;IACpD,WAAW,CAAyC;IACpD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAyC;IACnD,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAC9C,YAAY,CAA0C;IACtD,iBAAiB,CAA4C;IAC7D,wBAAwB,CAA6C;IAErE,MAAM,CAAoB;IAC1B,EAAE,CAAW;IACtB,IAAY,sBAAsB;QAChC,2EAA2E;QAC3E,8EAA8E;QAC9E,MAAM,MAAM,GAAG,IAAI,CAAC,MAAuF,CAAC;QAC5G,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QAC3E,sFAAsF;QACtF,uEAAuE;QACvE,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,cAAc,IAAI,EAAE,CAAC,CAAC;IACtF,CAAC;IAED,gBAAgB,CAAC,OAAwC,EAAE,OAA6B;QACtF,OAAO,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,sBAAsB,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACzF,CAAC;IAED,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,+EAA+E;QAC/E,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QACjE,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;IACjF,CAAC;IAKD,SAAS,CAAC,aAA4B;QACpC,IAAI,aAAa,CAAC,OAAO,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;YACvD,OAAO,IAAI,yBAAyB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,yBAAyB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAC5D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,2BAA2B,CAAC,OAAgB;QAChD,MAAM,MAAM,GAAG,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,OAAO,GAAG,CAAC;QAC5E,OAAO,IAAI,CAAC,mBAAmB,CAAuB,MAAM,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAmC,MAAc;QAChF,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAEpH,OAAO,WAAW;aACf,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kCAAkC;QACtC,OAAO,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,CAAC;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gCAAgC;QACpC,OAAO,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,+BAA+B;QACnC,OAAO,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IACnD,CAAC;IAED,2BAA2B,CAAC,mBAA2B,EAAE,aAA6B;QACpF,OAAO,kBAAkB,mBAAmB,IAAI,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC;IAChF,CAAC;IAED,yBAAyB,CAAC,mBAA2B;QACnD,OAAO,gBAAgB,mBAAmB,EAAE,CAAC;IAC/C,CAAC;IAED,qCAAqC,CAAC,OAAmD;QACvF,OAAO,8BAA8B,OAAO,CAAC,mBAAmB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IACxF,CAAC;IAED,KAAK,CAAC,sCAAsC,CAC1C,mBAA4B;QAE5B,MAAM,MAAM,GACV,mBAAmB,IAAI,IAAI;YACzB,CAAC,CAAC,6BAA6B;YAC/B,CAAC,CAAC,8BAA8B,mBAAmB,GAAG,CAAC;QAC3D,OAAO,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,0BAA0B,CAC9B,mBAA4B;QAE5B,IAAI,mBAAmB,IAAI,IAAI,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;QACnD,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QACjE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAC1F,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAA4B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACvF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,gCAAgC,EAAE,EAAE,CAAC;YACvE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,2BAA2B,EAAE,EAAE,CAAC;YAClE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,kCAAkC,EAAE,EAAE,CAAC;YACzE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,+BAA+B,EAAE,EAAE,CAAC;YACtE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,0BAA0B,EAAE,EAAE,CAAC;YACjE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,+BAA+B,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzE,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,sCAAsC,EAAE,EAAE,CAAC;YAC7E,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAS,EAAE;YAChD,mBAAmB,EAAE,IAAI;YACzB,8DAA8D;YAC9D,qGAAqG;YACrG,YAAY,EAAE;gBACZ,CAAC,EAAE,CAAC;aACL;SACF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gCAAgC;QACpC,6FAA6F;QAC7F,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aACnE,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBAChC,oEAAoE;gBACpE,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,iDAAiD;gBACjD,OAAO;YACT,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,EAAE;YAClD,MAAM,EAAE,IAAI;YACZ,sFAAsF;YACtF,mFAAmF;YACnF,iFAAiF;YACjF,4BAA4B;YAC5B,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,gBAAgB;YACjC,GAAG,EAAE,EAAE,CAAC,0BAA0B;SACnC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mCAAmC;QACvC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aAC1E,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QACD,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,0BAA0B,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QAC1B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,CACzC,CAAC;IACJ,CAAC;IACD;;OAEG;IACH,KAAK,CAAC,uBAAuB;QAC3B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,8BAA8B,EAAE,CAAC,CAAC;SACnC,EACD,EAAE,IAAI,EAAE,aAAa,EAAE,CACxB,CAAC;IACJ,CAAC;CACF;AAOD,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,OAA0C;IACzG,OAAO,IAAI,cAAc,CACvB,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAClC,gBAAgB,EAAE,iBAAiB;QACnC,GAAG,OAAO;KACX,CAAC,EACF,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAC9B,CAAC;AACJ,CAAC"}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { InternalOpId, SourceTable, storage } from '@powersync/service-core';
|
|
3
3
|
import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
|
|
4
|
+
import { MongoWriteBatch } from '../MongoWriteBatch.js';
|
|
4
5
|
import { PersistedBatch } from '../common/PersistedBatch.js';
|
|
5
6
|
import { SourceRecordStore } from '../common/SourceRecordStore.js';
|
|
7
|
+
import { SyncRuleDocumentBase } from '../models.js';
|
|
6
8
|
import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
|
|
7
9
|
export declare class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
8
10
|
readonly db: VersionedPowerSyncMongoV1;
|
|
@@ -17,6 +19,8 @@ export declare class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
17
19
|
*/
|
|
18
20
|
private persisted_op;
|
|
19
21
|
constructor(options: MongoBucketBatchOptions);
|
|
22
|
+
protected persistedOpHead(stream: SyncRuleDocumentBase): InternalOpId;
|
|
23
|
+
protected onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: InternalOpId): void;
|
|
20
24
|
protected recordPersistedOp(lastOp: InternalOpId): void;
|
|
21
25
|
protected createPersistedBatch(writtenSize: number): PersistedBatch;
|
|
22
26
|
protected get sourceRecordStore(): SourceRecordStore;
|
|
@@ -32,6 +36,7 @@ export declare class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
32
36
|
keepalive(lsn: string): Promise<storage.CheckpointResult>;
|
|
33
37
|
setResumeLsn(lsn: string): Promise<void>;
|
|
34
38
|
markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
|
|
39
|
+
private snapshotDoneUpdate;
|
|
35
40
|
markSnapshotDone(no_checkpoint_before_lsn: string, options?: {
|
|
36
41
|
throwOnConflict?: boolean;
|
|
37
42
|
}): Promise<void>;
|
|
@@ -4,6 +4,7 @@ import * as bson from 'bson';
|
|
|
4
4
|
import { mongoTableId } from '../../../utils/util.js';
|
|
5
5
|
import { calculateCheckpointState } from '../CheckpointState.js';
|
|
6
6
|
import { MongoBucketBatch } from '../MongoBucketBatch.js';
|
|
7
|
+
import { MongoSyncRulesLock } from '../MongoSyncRulesLock.js';
|
|
7
8
|
import { PersistedBatchV1 } from './PersistedBatchV1.js';
|
|
8
9
|
import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js';
|
|
9
10
|
export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
@@ -22,6 +23,25 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
22
23
|
this.persisted_op = options.keepaliveOp ?? null;
|
|
23
24
|
this.store = new SourceRecordStoreV1(this.db, this.replicationStreamId);
|
|
24
25
|
}
|
|
26
|
+
persistedOpHead(stream) {
|
|
27
|
+
const legacy = stream;
|
|
28
|
+
const checkpoint = legacy.last_checkpoint ?? 0n;
|
|
29
|
+
const keepalive = legacy.keepalive_op == null ? 0n : BigInt(legacy.keepalive_op);
|
|
30
|
+
return checkpoint > keepalive ? checkpoint : keepalive;
|
|
31
|
+
}
|
|
32
|
+
onReplicationTransactionFlush(writes, lastOp) {
|
|
33
|
+
// Keep flushed operations recoverable before a checkpoint can be published.
|
|
34
|
+
// keepalive_op remains a decimal string for compatibility with legacy readers.
|
|
35
|
+
writes.updateOne(this.db.sync_rules, { _id: this.replicationStreamId }, [
|
|
36
|
+
{
|
|
37
|
+
$set: {
|
|
38
|
+
keepalive_op: {
|
|
39
|
+
$toString: { $max: [{ $toLong: '$keepalive_op' }, '$last_checkpoint', lastOp] }
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
]);
|
|
44
|
+
}
|
|
25
45
|
recordPersistedOp(lastOp) {
|
|
26
46
|
this.persisted_op = lastOp;
|
|
27
47
|
}
|
|
@@ -37,7 +57,12 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
37
57
|
// No-op for V1: source records live in a shared collection.
|
|
38
58
|
}
|
|
39
59
|
async batchCreateCustomWriteCheckpoints(session, opId) {
|
|
40
|
-
|
|
60
|
+
// Only the final checkpoint per user/stream is visible after this transaction.
|
|
61
|
+
const checkpoints = new Map(this.write_checkpoint_batch.map((checkpoint) => [
|
|
62
|
+
JSON.stringify([checkpoint.sync_rules_id, checkpoint.user_id]),
|
|
63
|
+
checkpoint
|
|
64
|
+
]));
|
|
65
|
+
await this.db.custom_write_checkpoints.bulkWrite([...checkpoints.values()].map((checkpointOptions) => {
|
|
41
66
|
const set = {
|
|
42
67
|
checkpoint: checkpointOptions.checkpoint,
|
|
43
68
|
sync_rules_id: checkpointOptions.sync_rules_id,
|
|
@@ -56,7 +81,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
56
81
|
upsert: true
|
|
57
82
|
}
|
|
58
83
|
};
|
|
59
|
-
}), { session });
|
|
84
|
+
}), { session, ordered: false });
|
|
60
85
|
}
|
|
61
86
|
async resolveTables(options) {
|
|
62
87
|
const syncRules = options.parsedSyncConfig?.hydratedSyncConfig ?? this.sync_rules;
|
|
@@ -69,7 +94,8 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
69
94
|
type_oid: column.typeId
|
|
70
95
|
}));
|
|
71
96
|
let result = null;
|
|
72
|
-
|
|
97
|
+
const session = this.session;
|
|
98
|
+
await this.withTransaction(async () => {
|
|
73
99
|
const col = this.db.sourceTablesV1(this.replicationStreamId);
|
|
74
100
|
// Find records that overlap by name or relation id.
|
|
75
101
|
const orClauses = [{ schema_name: schema, table_name: name }];
|
|
@@ -161,110 +187,115 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
161
187
|
}
|
|
162
188
|
async commit(lsn, options) {
|
|
163
189
|
const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options };
|
|
164
|
-
await this.flush(options);
|
|
165
190
|
using _ = this.tracer.span('storage', 'commit');
|
|
166
|
-
const
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
191
|
+
const updateCheckpoint = async (persistedOp = this.persisted_op) => {
|
|
192
|
+
const now = new Date();
|
|
193
|
+
const can_checkpoint = {
|
|
194
|
+
$and: [
|
|
195
|
+
{ $eq: ['$snapshot_done', true] },
|
|
196
|
+
{
|
|
197
|
+
$or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }]
|
|
198
|
+
},
|
|
199
|
+
{
|
|
200
|
+
$or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }]
|
|
201
|
+
}
|
|
202
|
+
]
|
|
203
|
+
};
|
|
204
|
+
const new_keepalive_op = {
|
|
205
|
+
$cond: [
|
|
206
|
+
can_checkpoint,
|
|
207
|
+
{ $literal: null },
|
|
208
|
+
{
|
|
209
|
+
$toString: {
|
|
210
|
+
$max: [{ $toLong: '$keepalive_op' }, { $literal: persistedOp }, 0n]
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
]
|
|
214
|
+
};
|
|
215
|
+
const new_last_checkpoint = {
|
|
216
|
+
$cond: [
|
|
217
|
+
can_checkpoint,
|
|
218
|
+
{
|
|
219
|
+
$max: ['$last_checkpoint', { $literal: persistedOp }, { $toLong: '$keepalive_op' }, 0n]
|
|
220
|
+
},
|
|
221
|
+
'$last_checkpoint'
|
|
222
|
+
]
|
|
223
|
+
};
|
|
224
|
+
const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate(MongoSyncRulesLock.ownerFilter(this.replicationStreamId, this.options.replicationLock), [
|
|
180
225
|
{
|
|
181
|
-
$
|
|
226
|
+
$set: {
|
|
227
|
+
_can_checkpoint: can_checkpoint,
|
|
228
|
+
_not_empty: createEmptyCheckpoints
|
|
229
|
+
? true
|
|
230
|
+
: {
|
|
231
|
+
$or: [
|
|
232
|
+
{ $literal: createEmptyCheckpoints },
|
|
233
|
+
{ $ne: ['$keepalive_op', new_keepalive_op] },
|
|
234
|
+
{ $ne: ['$last_checkpoint', new_last_checkpoint] }
|
|
235
|
+
]
|
|
236
|
+
}
|
|
237
|
+
}
|
|
182
238
|
},
|
|
183
239
|
{
|
|
184
|
-
$
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
240
|
+
$set: {
|
|
241
|
+
...MongoSyncRulesLock.heartbeatUpdate(),
|
|
242
|
+
last_checkpoint_lsn: {
|
|
243
|
+
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
|
|
244
|
+
},
|
|
245
|
+
last_checkpoint_ts: {
|
|
246
|
+
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
|
|
247
|
+
},
|
|
248
|
+
last_fatal_error: { $literal: null },
|
|
249
|
+
last_fatal_error_ts: { $literal: null },
|
|
250
|
+
keepalive_op: new_keepalive_op,
|
|
251
|
+
last_checkpoint: new_last_checkpoint,
|
|
252
|
+
snapshot_lsn: {
|
|
253
|
+
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
|
|
254
|
+
}
|
|
195
255
|
}
|
|
196
|
-
}
|
|
197
|
-
]
|
|
198
|
-
};
|
|
199
|
-
const new_last_checkpoint = {
|
|
200
|
-
$cond: [
|
|
201
|
-
can_checkpoint,
|
|
202
|
-
{
|
|
203
|
-
$max: ['$last_checkpoint', { $literal: this.persisted_op }, { $toLong: '$keepalive_op' }, 0n]
|
|
204
256
|
},
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
};
|
|
208
|
-
const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate({ _id: this.replicationStreamId }, [
|
|
209
|
-
{
|
|
210
|
-
$set: {
|
|
211
|
-
_can_checkpoint: can_checkpoint,
|
|
212
|
-
_not_empty: createEmptyCheckpoints
|
|
213
|
-
? true
|
|
214
|
-
: {
|
|
215
|
-
$or: [
|
|
216
|
-
{ $literal: createEmptyCheckpoints },
|
|
217
|
-
{ $ne: ['$keepalive_op', new_keepalive_op] },
|
|
218
|
-
{ $ne: ['$last_checkpoint', new_last_checkpoint] }
|
|
219
|
-
]
|
|
220
|
-
}
|
|
257
|
+
{
|
|
258
|
+
$unset: ['_can_checkpoint', '_not_empty']
|
|
221
259
|
}
|
|
222
|
-
|
|
223
|
-
|
|
260
|
+
], {
|
|
261
|
+
session: this.session,
|
|
262
|
+
returnDocument: 'before',
|
|
263
|
+
projection: {
|
|
264
|
+
snapshot_done: 1,
|
|
265
|
+
last_checkpoint_lsn: 1,
|
|
266
|
+
no_checkpoint_before: 1,
|
|
267
|
+
keepalive_op: 1,
|
|
268
|
+
last_checkpoint: 1
|
|
269
|
+
}
|
|
270
|
+
}));
|
|
271
|
+
return MongoSyncRulesLock.assertOwned(preUpdateDocument);
|
|
272
|
+
};
|
|
273
|
+
const finishCheckpoint = async (preUpdateDocument, lastOp) => {
|
|
274
|
+
await this.db.write_checkpoints.updateMany({
|
|
275
|
+
processed_at_lsn: null,
|
|
276
|
+
'lsns.1': { $lte: lsn }
|
|
277
|
+
}, {
|
|
224
278
|
$set: {
|
|
225
|
-
|
|
226
|
-
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn']
|
|
227
|
-
},
|
|
228
|
-
last_checkpoint_ts: {
|
|
229
|
-
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts']
|
|
230
|
-
},
|
|
231
|
-
last_keepalive_ts: { $literal: now },
|
|
232
|
-
last_fatal_error: { $literal: null },
|
|
233
|
-
last_fatal_error_ts: { $literal: null },
|
|
234
|
-
keepalive_op: new_keepalive_op,
|
|
235
|
-
last_checkpoint: new_last_checkpoint,
|
|
236
|
-
snapshot_lsn: {
|
|
237
|
-
$cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn']
|
|
238
|
-
}
|
|
279
|
+
processed_at_lsn: lsn
|
|
239
280
|
}
|
|
240
|
-
},
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
}
|
|
254
|
-
}
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
const checkpointState = calculateCheckpointState({
|
|
259
|
-
lsn,
|
|
260
|
-
snapshotDone: preUpdateDocument.snapshot_done === true,
|
|
261
|
-
lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
|
|
262
|
-
noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
|
|
263
|
-
keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
|
|
264
|
-
lastCheckpoint: preUpdateDocument.last_checkpoint,
|
|
265
|
-
persistedOp: this.persisted_op,
|
|
266
|
-
createEmptyCheckpoints
|
|
267
|
-
});
|
|
281
|
+
}, {
|
|
282
|
+
session: this.session
|
|
283
|
+
});
|
|
284
|
+
const checkpointState = calculateCheckpointState({
|
|
285
|
+
lsn,
|
|
286
|
+
snapshotDone: preUpdateDocument.snapshot_done === true,
|
|
287
|
+
lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn,
|
|
288
|
+
noCheckpointBefore: preUpdateDocument.no_checkpoint_before,
|
|
289
|
+
keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op),
|
|
290
|
+
lastCheckpoint: preUpdateDocument.last_checkpoint,
|
|
291
|
+
persistedOp: lastOp ?? this.persistedOpHead(preUpdateDocument),
|
|
292
|
+
createEmptyCheckpoints
|
|
293
|
+
});
|
|
294
|
+
return { checkpointState, preUpdateDocument };
|
|
295
|
+
};
|
|
296
|
+
const { checkpointState, preUpdateDocument } = await this.flushAndCommit(async (_stream, lastOp) => finishCheckpoint(await updateCheckpoint(lastOp), lastOp),
|
|
297
|
+
// Without a flush, the checkpoint update itself fences the transaction.
|
|
298
|
+
() => this.withFencedTransaction(updateCheckpoint, finishCheckpoint), options);
|
|
268
299
|
if (checkpointState.checkpointBlocked) {
|
|
269
300
|
if (Date.now() - this.lastWaitingLogThrottled > 5_000) {
|
|
270
301
|
this.logger.info(`Waiting before creating checkpoint, currently at ${lsn} / ${checkpointState.newKeepaliveOp}. Current state: ${JSON.stringify({
|
|
@@ -296,32 +327,18 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
296
327
|
}
|
|
297
328
|
async setResumeLsn(lsn) {
|
|
298
329
|
using _ = this.tracer.span('storage', 'set_resume_lsn');
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
}, {
|
|
302
|
-
$set: {
|
|
303
|
-
snapshot_lsn: lsn
|
|
304
|
-
}
|
|
305
|
-
}, {
|
|
306
|
-
session: this.session,
|
|
307
|
-
// Losing occasional resume LSN is fine. That may mean reprocessing
|
|
308
|
-
// some source changes in some edge cases, which is not an issue since
|
|
309
|
-
// changes are processed in an idempotent way.
|
|
310
|
-
writeConcern: { w: 1 }
|
|
311
|
-
});
|
|
330
|
+
// Losing occasional resume LSN would only reprocess source changes.
|
|
331
|
+
// Keep the lease check atomic, but do not wait for majority replication of this resume hint.
|
|
332
|
+
await this.updateStreamMetadata({ snapshot_lsn: { $literal: lsn } }, {}, { w: 1 });
|
|
312
333
|
}
|
|
313
334
|
async markAllSnapshotDone(no_checkpoint_before_lsn) {
|
|
314
|
-
await this.
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
$max: {
|
|
322
|
-
no_checkpoint_before: no_checkpoint_before_lsn
|
|
323
|
-
}
|
|
324
|
-
}, { session: this.session });
|
|
335
|
+
await this.updateStreamMetadata(this.snapshotDoneUpdate(no_checkpoint_before_lsn));
|
|
336
|
+
}
|
|
337
|
+
snapshotDoneUpdate(no_checkpoint_before_lsn) {
|
|
338
|
+
return {
|
|
339
|
+
snapshot_done: true,
|
|
340
|
+
no_checkpoint_before: { $max: ['$no_checkpoint_before', { $literal: no_checkpoint_before_lsn }] }
|
|
341
|
+
};
|
|
325
342
|
}
|
|
326
343
|
async markSnapshotDone(no_checkpoint_before_lsn, options) {
|
|
327
344
|
await this.withTransaction(async () => {
|
|
@@ -344,42 +361,35 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
344
361
|
return;
|
|
345
362
|
}
|
|
346
363
|
}
|
|
347
|
-
await this.
|
|
364
|
+
await this.updateStreamMetadataInTransaction(this.snapshotDoneUpdate(no_checkpoint_before_lsn));
|
|
348
365
|
});
|
|
349
366
|
}
|
|
350
367
|
async markTableSnapshotRequired(_table) {
|
|
351
|
-
await this.
|
|
352
|
-
_id: this.replicationStreamId
|
|
353
|
-
}, {
|
|
354
|
-
$set: {
|
|
355
|
-
snapshot_done: false
|
|
356
|
-
}
|
|
357
|
-
}, { session: this.session });
|
|
368
|
+
await this.updateStreamMetadata({ snapshot_done: false });
|
|
358
369
|
}
|
|
359
370
|
async markTableSnapshotDone(tables, no_checkpoint_before_lsn) {
|
|
360
371
|
const session = this.session;
|
|
361
372
|
const ids = tables.map((table) => mongoTableId(table.id));
|
|
362
373
|
await this.withTransaction(async () => {
|
|
363
|
-
|
|
374
|
+
const writes = this.db.createWriteBatch(session, { ordered: false });
|
|
375
|
+
writes.updateMany(this.db.sourceTablesV1(this.replicationStreamId), { _id: { $in: ids } }, {
|
|
364
376
|
$set: {
|
|
365
377
|
snapshot_done: true
|
|
366
378
|
},
|
|
367
379
|
$unset: {
|
|
368
380
|
snapshot_status: 1
|
|
369
381
|
}
|
|
370
|
-
}
|
|
382
|
+
});
|
|
371
383
|
if (no_checkpoint_before_lsn != null) {
|
|
372
|
-
|
|
384
|
+
writes.updateOne(this.db.sync_rules, {
|
|
373
385
|
_id: this.replicationStreamId
|
|
374
386
|
}, {
|
|
375
|
-
$set: {
|
|
376
|
-
last_keepalive_ts: new Date()
|
|
377
|
-
},
|
|
378
387
|
$max: {
|
|
379
388
|
no_checkpoint_before: no_checkpoint_before_lsn
|
|
380
389
|
}
|
|
381
|
-
}
|
|
390
|
+
});
|
|
382
391
|
}
|
|
392
|
+
await writes.execute();
|
|
383
393
|
});
|
|
384
394
|
return tables.map((table) => {
|
|
385
395
|
const copy = table.clone();
|
|
@@ -394,7 +404,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
394
404
|
const session = this.session;
|
|
395
405
|
let activated = false;
|
|
396
406
|
let needsFutureActivationCheck = true;
|
|
397
|
-
await
|
|
407
|
+
await this.withTransaction(async () => {
|
|
398
408
|
// Reset on transaction retries.
|
|
399
409
|
activated = false;
|
|
400
410
|
needsFutureActivationCheck = true;
|
|
@@ -420,7 +430,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch {
|
|
|
420
430
|
else if (doc?.state != storage.SyncRuleState.PROCESSING) {
|
|
421
431
|
needsFutureActivationCheck = false;
|
|
422
432
|
}
|
|
423
|
-
});
|
|
433
|
+
}, session);
|
|
424
434
|
if (activated) {
|
|
425
435
|
this.logger.info(`Activated new replication stream at ${lsn}`);
|
|
426
436
|
await this.db.notifyCheckpoint();
|