@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +177 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +71 -18
- package/dist/storage/MongoBucketStorage.js +516 -202
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
- package/dist/storage/implementation/MongoBucketBatch.js +140 -66
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
- package/dist/storage/implementation/MongoCompactor.js +87 -481
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
- package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +41 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
- package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
- package/dist/storage/implementation/common/PersistedBatch.js +63 -10
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +4 -1
- package/dist/storage/implementation/db.js +31 -16
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +55 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
- package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
- package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +79 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
- package/dist/storage/implementation/v3/compact-utils.js +216 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +187 -23
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
- package/dist/storage/implementation/v3/source-table-utils.js +206 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +5 -4
- package/dist/storage/storage-index.js +5 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +71 -3
- package/dist/types/types.js +58 -2
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +131 -5
- package/dist/utils/util.js.map +1 -1
- package/package.json +13 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +728 -242
- package/src/storage/implementation/MongoBucketBatch.ts +196 -91
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +106 -631
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
- package/src/storage/implementation/MongoStorageProvider.ts +48 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
- package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
- package/src/storage/implementation/common/PersistedBatch.ts +83 -15
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +40 -22
- package/src/storage/implementation/models.ts +58 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +12 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
- package/src/storage/implementation/v3/bucket-format.ts +99 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/compact-utils.ts +320 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +228 -58
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
- package/src/storage/implementation/v3/source-table-utils.ts +398 -0
- package/src/storage/storage-index.ts +14 -4
- package/src/types/types.ts +76 -2
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +188 -7
- package/test/src/__snapshots__/storage.test.ts.snap +4 -54
- package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
- package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
- package/test/src/compact-utils.test.ts +226 -0
- package/test/src/helpers/MemoryObjectStorage.ts +61 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/object_storage_usage.test.ts +255 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +478 -13
- package/test/src/storage_compacting.test.ts +2516 -107
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +190 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
- package/test/src/storage_s3_reading.test.ts +768 -0
- package/test/src/storage_s3_writing.test.ts +136 -0
- package/test/src/storage_sync.test.ts +1507 -76
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +3 -2
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -1,58 +1,50 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { InternalOpId } from '@powersync/service-core';
|
|
3
3
|
import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
|
|
4
|
-
import {
|
|
5
|
-
BucketDataDocumentGeneric,
|
|
6
|
-
BucketDataDocumentGenericId,
|
|
7
|
-
SingleBucketStore
|
|
8
|
-
} from '../common/SingleBucketStore.js';
|
|
9
4
|
import { BucketDataProperties } from '../models.js';
|
|
10
5
|
import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
|
|
11
6
|
import { BucketDataDocumentV1, BucketDataKeyV1, serializeBucketDataV1 } from './models.js';
|
|
12
7
|
|
|
13
|
-
export class SingleBucketStoreV1
|
|
14
|
-
public readonly collection: mongo.Collection<
|
|
8
|
+
export class SingleBucketStoreV1 {
|
|
9
|
+
public readonly collection: mongo.Collection<BucketDataDocumentV1>;
|
|
15
10
|
|
|
16
11
|
constructor(
|
|
17
|
-
|
|
12
|
+
db: VersionedPowerSyncMongoV1,
|
|
18
13
|
public readonly key: BucketKey
|
|
19
14
|
) {
|
|
20
|
-
this.collection = db.bucketDataV1
|
|
15
|
+
this.collection = db.bucketDataV1;
|
|
21
16
|
}
|
|
22
17
|
|
|
23
|
-
docId(o: InternalOpId):
|
|
24
|
-
// `satisfies BucketDataKeyV1` checks that we use the correct type for V1 storage
|
|
25
|
-
// `as anyt` is to allow casting to the interface virtual type
|
|
18
|
+
docId(o: InternalOpId): BucketDataKeyV1 {
|
|
26
19
|
return {
|
|
27
20
|
g: this.key.replicationStreamId,
|
|
28
21
|
b: this.key.bucket,
|
|
29
22
|
o
|
|
30
|
-
}
|
|
23
|
+
};
|
|
31
24
|
}
|
|
32
25
|
|
|
33
|
-
get minId():
|
|
26
|
+
get minId(): BucketDataKeyV1 {
|
|
34
27
|
return {
|
|
35
28
|
g: this.key.replicationStreamId,
|
|
36
29
|
b: this.key.bucket,
|
|
37
|
-
o: new mongo.MinKey()
|
|
38
|
-
}
|
|
30
|
+
o: new mongo.MinKey() as any
|
|
31
|
+
};
|
|
39
32
|
}
|
|
40
33
|
|
|
41
|
-
get maxId():
|
|
34
|
+
get maxId(): BucketDataKeyV1 {
|
|
42
35
|
return {
|
|
43
36
|
g: this.key.replicationStreamId,
|
|
44
37
|
b: this.key.bucket,
|
|
45
|
-
o: new mongo.MaxKey()
|
|
46
|
-
}
|
|
38
|
+
o: new mongo.MaxKey() as any
|
|
39
|
+
};
|
|
47
40
|
}
|
|
48
41
|
|
|
49
|
-
toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>):
|
|
50
|
-
return serializeBucketDataV1({ bucketKey: this.key, ...source })
|
|
42
|
+
toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentV1 {
|
|
43
|
+
return serializeBucketDataV1({ bucketKey: this.key, ...source });
|
|
51
44
|
}
|
|
52
45
|
|
|
53
|
-
fromPersistedDocument(doc:
|
|
54
|
-
const
|
|
55
|
-
const { _id, ...rest } = document;
|
|
46
|
+
fromPersistedDocument(doc: BucketDataDocumentV1): BucketDataDoc {
|
|
47
|
+
const { _id, ...rest } = doc;
|
|
56
48
|
return {
|
|
57
49
|
bucketKey: this.key,
|
|
58
50
|
o: _id.o,
|
|
@@ -61,10 +53,9 @@ export class SingleBucketStoreV1 implements SingleBucketStore {
|
|
|
61
53
|
}
|
|
62
54
|
|
|
63
55
|
fromPartialPersistedDocument<T extends keyof BucketDataProperties>(
|
|
64
|
-
doc: Pick<
|
|
56
|
+
doc: Pick<BucketDataDocumentV1, '_id' | T>
|
|
65
57
|
): Pick<BucketDataDoc, 'bucketKey' | 'o' | T> {
|
|
66
|
-
const
|
|
67
|
-
const { _id, ...rest } = document;
|
|
58
|
+
const { _id, ...rest } = doc;
|
|
68
59
|
return {
|
|
69
60
|
bucketKey: this.key,
|
|
70
61
|
o: _id.o,
|
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
|
|
3
3
|
import { CommonSourceTableDocument } from '../models.js';
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
BucketDataDocumentV1,
|
|
6
|
+
BucketParameterDocument,
|
|
7
|
+
BucketStateDocumentV1,
|
|
8
|
+
CurrentDataDocument,
|
|
9
|
+
SourceTableDocumentV1
|
|
10
|
+
} from './models.js';
|
|
5
11
|
|
|
6
12
|
export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
|
|
7
13
|
get sourceRecordsV1(): mongo.Collection<CurrentDataDocument> {
|
|
@@ -16,6 +22,10 @@ export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
|
|
|
16
22
|
return this.upstream.source_tables as any as mongo.Collection<CommonSourceTableDocument>;
|
|
17
23
|
}
|
|
18
24
|
|
|
25
|
+
sourceTablesV1(_replicationStreamId: number): mongo.Collection<SourceTableDocumentV1> {
|
|
26
|
+
return this.upstream.source_tables as any as mongo.Collection<SourceTableDocumentV1>;
|
|
27
|
+
}
|
|
28
|
+
|
|
19
29
|
async initializeStreamStorage(_replicationStreamId: number): Promise<void> {}
|
|
20
30
|
|
|
21
31
|
get bucketDataV1(): mongo.Collection<BucketDataDocumentV1> {
|
|
@@ -48,6 +48,7 @@ export function serializeBucketDataV1(document: BucketDataDoc): BucketDataDocume
|
|
|
48
48
|
op: document.op,
|
|
49
49
|
source_table: document.source_table,
|
|
50
50
|
source_key: document.source_key,
|
|
51
|
+
subkey: document.subkey,
|
|
51
52
|
table: document.table,
|
|
52
53
|
row_id: document.row_id,
|
|
53
54
|
checksum: document.checksum,
|
|
@@ -78,7 +79,7 @@ export interface SourceTableDocumentV1 extends SourceTableDocument {
|
|
|
78
79
|
group_id: number;
|
|
79
80
|
}
|
|
80
81
|
|
|
81
|
-
export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckpointFields
|
|
82
|
+
export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckpointFields {
|
|
82
83
|
content: string;
|
|
83
84
|
serialized_plan?: SerializedSyncPlan | null;
|
|
84
85
|
|
|
@@ -88,6 +89,16 @@ export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckp
|
|
|
88
89
|
* Can only be false if state == PROCESSING.
|
|
89
90
|
*/
|
|
90
91
|
snapshot_done: boolean;
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Goes together with no_checkpoint_before.
|
|
95
|
+
*
|
|
96
|
+
* If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
|
|
97
|
+
* then the checkpoint must be equal to this keepalive_op.
|
|
98
|
+
*
|
|
99
|
+
* string for legacy reasons - it's always a number.
|
|
100
|
+
*/
|
|
101
|
+
keepalive_op: string | null;
|
|
91
102
|
}
|
|
92
103
|
|
|
93
104
|
export interface BucketStateDocumentV1 extends BucketStateDocumentBase {
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { InternalOpId } from '@powersync/service-core';
|
|
3
|
+
import { BucketKey } from '../common/BucketDataDoc.js';
|
|
4
|
+
import { BucketDataKey } from '../models.js';
|
|
5
|
+
import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
|
|
6
|
+
import { BucketDataDocumentV3 } from './models.js';
|
|
7
|
+
|
|
8
|
+
export class BucketDataContextV3 {
|
|
9
|
+
public readonly collection: mongo.Collection<BucketDataDocumentV3>;
|
|
10
|
+
|
|
11
|
+
constructor(
|
|
12
|
+
db: VersionedPowerSyncMongoV3,
|
|
13
|
+
public readonly key: BucketKey
|
|
14
|
+
) {
|
|
15
|
+
this.collection = db.bucketData(key.replicationStreamId, key.definitionId);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
docId(o: InternalOpId): BucketDataKey {
|
|
19
|
+
return {
|
|
20
|
+
b: this.key.bucket,
|
|
21
|
+
o
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
get minId(): BucketDataKey {
|
|
26
|
+
// MongoDB's MinKey sentinel does not match the bigint type for _id.o.
|
|
27
|
+
return {
|
|
28
|
+
b: this.key.bucket,
|
|
29
|
+
o: new mongo.MinKey()
|
|
30
|
+
} as unknown as BucketDataKey;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { InternalOpId } from '@powersync/service-core';
|
|
3
|
+
import { CompactionLeaseLostError } from '../MongoCompactor.js';
|
|
4
|
+
import { BucketStateDocumentV3 } from './models.js';
|
|
5
|
+
|
|
6
|
+
const LEASE_RENEW_INTERVAL_MS = 60 * 1000;
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Filters for buckets that have no lease or an expired lease.
|
|
10
|
+
*/
|
|
11
|
+
export const AVAILABLE_LEASE_EXPR = {
|
|
12
|
+
// $$NOW is evaluated by MongoDB, avoiding lease expiry races due
|
|
13
|
+
// to clocks on separate compact workers.
|
|
14
|
+
$expr: {
|
|
15
|
+
$or: [{ $eq: [{ $type: '$compact_lease' }, 'missing'] }, { $lte: ['$compact_lease.expires_at', '$$NOW'] }]
|
|
16
|
+
}
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Owns one V3 bucket-compaction lease, including its server-time renewal and
|
|
21
|
+
* the owner-fenced operations which release it.
|
|
22
|
+
*
|
|
23
|
+
* This is intended as a way to reduce redundant work if multiple jobs attempt to
|
|
24
|
+
* compact the same bucket concurrently, but it's not an absolute safety guarantee.
|
|
25
|
+
* The individual operations must still be designed to be safe with concurrent compacting.
|
|
26
|
+
*/
|
|
27
|
+
export class CompactionLease implements AsyncDisposable {
|
|
28
|
+
readonly startedAt: Date;
|
|
29
|
+
readonly lastOp: InternalOpId;
|
|
30
|
+
|
|
31
|
+
private timer: NodeJS.Timeout | undefined;
|
|
32
|
+
private renewalInFlight = false;
|
|
33
|
+
private renewalError: unknown;
|
|
34
|
+
private finalizing = false;
|
|
35
|
+
private finished = false;
|
|
36
|
+
|
|
37
|
+
private constructor(
|
|
38
|
+
private readonly collection: mongo.Collection<BucketStateDocumentV3>,
|
|
39
|
+
readonly state: BucketStateDocumentV3,
|
|
40
|
+
readonly id: mongo.ObjectId,
|
|
41
|
+
private readonly durationMs: number
|
|
42
|
+
) {
|
|
43
|
+
this.startedAt = new Date(state.compact_lease!.expires_at.getTime() - durationMs);
|
|
44
|
+
this.lastOp = state.last_op;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
static async claim(
|
|
48
|
+
collection: mongo.Collection<BucketStateDocumentV3>,
|
|
49
|
+
filter: mongo.Filter<BucketStateDocumentV3>,
|
|
50
|
+
sort: mongo.Sort | undefined,
|
|
51
|
+
durationMs: number
|
|
52
|
+
): Promise<CompactionLease | null> {
|
|
53
|
+
const id = new mongo.ObjectId();
|
|
54
|
+
const state = await collection.findOneAndUpdate(
|
|
55
|
+
{
|
|
56
|
+
$and: [filter, AVAILABLE_LEASE_EXPR]
|
|
57
|
+
},
|
|
58
|
+
[
|
|
59
|
+
{
|
|
60
|
+
$set: {
|
|
61
|
+
compact_lease: {
|
|
62
|
+
id,
|
|
63
|
+
expires_at: { $dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: durationMs } }
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
],
|
|
68
|
+
{ sort, returnDocument: 'after' }
|
|
69
|
+
);
|
|
70
|
+
return state == null ? null : new CompactionLease(collection, state, id, durationMs);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
startRenewal() {
|
|
74
|
+
const interval = Math.max(1, Math.min(LEASE_RENEW_INTERVAL_MS, Math.floor(this.durationMs / 2)));
|
|
75
|
+
this.timer = setInterval(() => {
|
|
76
|
+
if (this.renewalInFlight) {
|
|
77
|
+
return;
|
|
78
|
+
}
|
|
79
|
+
this.renewalInFlight = true;
|
|
80
|
+
void this.renew()
|
|
81
|
+
.catch((error) => {
|
|
82
|
+
this.renewalError = error;
|
|
83
|
+
})
|
|
84
|
+
.finally(() => {
|
|
85
|
+
this.renewalInFlight = false;
|
|
86
|
+
});
|
|
87
|
+
}, interval);
|
|
88
|
+
this.timer.unref();
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
async throwIfLost() {
|
|
92
|
+
if (this.renewalError != null) {
|
|
93
|
+
throw this.renewalError;
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async reschedule(nextCompactCheck: mongo.Document) {
|
|
98
|
+
await this.finish([{ $set: { next_compact_check: nextCompactCheck } }, { $unset: 'compact_lease' }]);
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async finalize(update: mongo.Document) {
|
|
102
|
+
await this.finish([{ $set: update }, { $unset: 'compact_lease' }]);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async [Symbol.asyncDispose]() {
|
|
106
|
+
if (this.timer != null) {
|
|
107
|
+
clearInterval(this.timer);
|
|
108
|
+
this.timer = undefined;
|
|
109
|
+
}
|
|
110
|
+
if (this.finished) {
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
this.finalizing = true;
|
|
114
|
+
await this.collection.updateOne(this.filter, { $unset: { compact_lease: '' } });
|
|
115
|
+
this.finished = true;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
private get filter(): mongo.Filter<BucketStateDocumentV3> {
|
|
119
|
+
return { _id: this.state._id, 'compact_lease.id': this.id };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
private async renew() {
|
|
123
|
+
const result = await this.collection.updateOne(this.filter, [
|
|
124
|
+
{
|
|
125
|
+
$set: {
|
|
126
|
+
'compact_lease.expires_at': {
|
|
127
|
+
$dateAdd: { startDate: '$$NOW', unit: 'millisecond', amount: this.durationMs }
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
]);
|
|
132
|
+
// A finalization in progress owns the authoritative fenced result.
|
|
133
|
+
if (result.matchedCount != 1 && !this.finalizing) {
|
|
134
|
+
throw new CompactionLeaseLostError(`Lost compaction lease for bucket ${this.state._id.b}`);
|
|
135
|
+
}
|
|
136
|
+
// A successful renewal confirms that a transient failure has passed. A
|
|
137
|
+
// lease-loss error remains sticky: it means another worker may own it.
|
|
138
|
+
if (!(this.renewalError instanceof CompactionLeaseLostError)) {
|
|
139
|
+
this.renewalError = undefined;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
private async finish(update: mongo.Document[]) {
|
|
144
|
+
await this.throwIfLost();
|
|
145
|
+
this.finalizing = true;
|
|
146
|
+
const result = await this.collection.updateOne(this.filter, update);
|
|
147
|
+
if (result.matchedCount != 1) {
|
|
148
|
+
throw new CompactionLeaseLostError(`Lost compaction lease for bucket ${this.state._id.b}`);
|
|
149
|
+
}
|
|
150
|
+
this.finished = true;
|
|
151
|
+
}
|
|
152
|
+
}
|