@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +177 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +71 -18
- package/dist/storage/MongoBucketStorage.js +516 -202
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
- package/dist/storage/implementation/MongoBucketBatch.js +140 -66
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
- package/dist/storage/implementation/MongoCompactor.js +87 -481
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
- package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +41 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
- package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
- package/dist/storage/implementation/common/PersistedBatch.js +63 -10
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +4 -1
- package/dist/storage/implementation/db.js +31 -16
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +55 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
- package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
- package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +79 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
- package/dist/storage/implementation/v3/compact-utils.js +216 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +187 -23
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
- package/dist/storage/implementation/v3/source-table-utils.js +206 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +5 -4
- package/dist/storage/storage-index.js +5 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +71 -3
- package/dist/types/types.js +58 -2
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +131 -5
- package/dist/utils/util.js.map +1 -1
- package/package.json +13 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +728 -242
- package/src/storage/implementation/MongoBucketBatch.ts +196 -91
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +106 -631
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
- package/src/storage/implementation/MongoStorageProvider.ts +48 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
- package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
- package/src/storage/implementation/common/PersistedBatch.ts +83 -15
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +40 -22
- package/src/storage/implementation/models.ts +58 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +12 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
- package/src/storage/implementation/v3/bucket-format.ts +99 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/compact-utils.ts +320 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +228 -58
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
- package/src/storage/implementation/v3/source-table-utils.ts +398 -0
- package/src/storage/storage-index.ts +14 -4
- package/src/types/types.ts +76 -2
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +188 -7
- package/test/src/__snapshots__/storage.test.ts.snap +4 -54
- package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
- package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
- package/test/src/compact-utils.test.ts +226 -0
- package/test/src/helpers/MemoryObjectStorage.ts +61 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/object_storage_usage.test.ts +255 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +478 -13
- package/test/src/storage_compacting.test.ts +2516 -107
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +190 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
- package/test/src/storage_s3_reading.test.ts +768 -0
- package/test/src/storage_s3_writing.test.ts +136 -0
- package/test/src/storage_sync.test.ts +1507 -76
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +3 -2
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -2,13 +2,14 @@ import { mongo } from '@powersync/lib-service-mongodb';
|
|
|
2
2
|
import { BucketDataSource, BucketDefinitionId, EvaluatedParameters, EvaluatedRow } from '@powersync/service-sync-rules';
|
|
3
3
|
import * as bson from 'bson';
|
|
4
4
|
import { Logger } from '@powersync/lib-services-framework';
|
|
5
|
-
import { InternalOpId, storage } from '@powersync/service-core';
|
|
6
|
-
import { BucketDefinitionMapping } from '../BucketDefinitionMapping.js';
|
|
5
|
+
import { BucketDefinitionMapping, InternalOpId, storage } from '@powersync/service-core';
|
|
7
6
|
import { MongoIdSequence } from '../MongoIdSequence.js';
|
|
8
7
|
import type { VersionedPowerSyncMongo } from '../db.js';
|
|
9
8
|
import { TaggedBucketParameterDocument } from '../models.js';
|
|
9
|
+
import { ObjectStorage } from '../v3/object-storage/ObjectStorage.js';
|
|
10
10
|
import { BucketDataDoc, BucketKey } from './BucketDataDoc.js';
|
|
11
11
|
import { SourceRecordBucketState, SourceRecordLookupState } from './SourceRecordStore.js';
|
|
12
|
+
export declare const DEFAULT_INLINE_THRESHOLD_BYTES: number;
|
|
12
13
|
export interface SaveBucketDataOptions {
|
|
13
14
|
op_seq: MongoIdSequence;
|
|
14
15
|
sourceKey: storage.ReplicaId;
|
|
@@ -32,6 +33,13 @@ export interface UpsertCurrentDataOptions {
|
|
|
32
33
|
}
|
|
33
34
|
export interface PersistedBatchOptions {
|
|
34
35
|
logger?: Logger;
|
|
36
|
+
objectStorage?: ObjectStorage;
|
|
37
|
+
inlineThresholdBytes?: number;
|
|
38
|
+
/**
|
|
39
|
+
* Aborts in-flight object storage uploads when replication stops.
|
|
40
|
+
*/
|
|
41
|
+
signal?: AbortSignal;
|
|
42
|
+
objectStorageUsageWriterId?: string;
|
|
35
43
|
}
|
|
36
44
|
/**
|
|
37
45
|
* Keeps track of bulkwrite operations within a transaction.
|
|
@@ -47,6 +55,10 @@ export declare abstract class PersistedBatch {
|
|
|
47
55
|
bucketData: BucketDataDoc[];
|
|
48
56
|
bucketParameters: TaggedBucketParameterDocument[];
|
|
49
57
|
bucketStates: Map<string, BucketStateUpdate>;
|
|
58
|
+
protected readonly objectStorage?: ObjectStorage;
|
|
59
|
+
protected readonly inlineThresholdBytes: number;
|
|
60
|
+
protected readonly signal?: AbortSignal;
|
|
61
|
+
protected readonly objectStorageUsageWriterId?: string;
|
|
50
62
|
/**
|
|
51
63
|
* For debug logging only.
|
|
52
64
|
*/
|
|
@@ -69,14 +81,32 @@ export declare abstract class PersistedBatch {
|
|
|
69
81
|
protected abstract resetCurrentData(): void;
|
|
70
82
|
protected abstract checkDefinitionId(definitionId: BucketDefinitionId | null): BucketDefinitionId;
|
|
71
83
|
protected abstract getBucketDefinitionId(bucketSource: BucketDataSource): BucketDefinitionId;
|
|
84
|
+
/**
|
|
85
|
+
* New bucket operations persist protocol subkeys during replication. Older
|
|
86
|
+
* V1 documents may not have one, so sync reads retain a fallback.
|
|
87
|
+
*/
|
|
88
|
+
protected persistedSubkey(table: storage.SourceTableId, key: storage.ReplicaId): string;
|
|
72
89
|
protected get bucketDataCount(): number;
|
|
73
90
|
protected incrementBucket(definitionId: BucketDefinitionId, bucket: string, op_id: InternalOpId, bytes: number): void;
|
|
91
|
+
/**
|
|
92
|
+
* V3 persists operations in chunks. Keep this separate from incrementBucket:
|
|
93
|
+
* operation counts are known while evaluating rows, while chunk counts and
|
|
94
|
+
* exact persisted bytes are only known after the writer has chunked a flush.
|
|
95
|
+
*/
|
|
96
|
+
protected incrementBucketPersistedChunk(definitionId: BucketDefinitionId, bucket: string, bytes: number): void;
|
|
97
|
+
/**
|
|
98
|
+
* V3's compactor calculates byte deltas from persisted chunk metadata.
|
|
99
|
+
* Replace the writer's per-operation estimate with those exact sizes before
|
|
100
|
+
* flushing the corresponding bucket state.
|
|
101
|
+
*/
|
|
102
|
+
protected resetBucketPersistedBytes(definitionId: BucketDefinitionId, bucket: string): void;
|
|
74
103
|
protected addBucketDataPut(options: {
|
|
75
104
|
op_id: InternalOpId;
|
|
76
105
|
bucketKey: BucketKey;
|
|
77
106
|
bucket: string;
|
|
78
107
|
sourceTableId: storage.SourceTable['id'];
|
|
79
108
|
sourceKey: storage.ReplicaId;
|
|
109
|
+
subkey: string;
|
|
80
110
|
table: string;
|
|
81
111
|
rowId: string;
|
|
82
112
|
checksum: bigint;
|
|
@@ -87,6 +117,7 @@ export declare abstract class PersistedBatch {
|
|
|
87
117
|
bucketKey: BucketKey;
|
|
88
118
|
sourceTableId: storage.SourceTable['id'];
|
|
89
119
|
sourceKey: storage.ReplicaId;
|
|
120
|
+
subkey: string;
|
|
90
121
|
table: string;
|
|
91
122
|
rowId: string;
|
|
92
123
|
checksum: bigint;
|
|
@@ -105,4 +136,5 @@ export interface BucketStateUpdate {
|
|
|
105
136
|
lastOp: InternalOpId;
|
|
106
137
|
incrementCount: number;
|
|
107
138
|
incrementBytes: number;
|
|
139
|
+
incrementChunks: number;
|
|
108
140
|
}
|
|
@@ -22,6 +22,7 @@ const MAX_TRANSACTION_BATCH_SIZE = 30_000_000;
|
|
|
22
22
|
* This has an effect on error message size in some cases.
|
|
23
23
|
*/
|
|
24
24
|
const MAX_TRANSACTION_DOC_COUNT = 2_000;
|
|
25
|
+
export const DEFAULT_INLINE_THRESHOLD_BYTES = 16 * 1024;
|
|
25
26
|
/**
|
|
26
27
|
* Keeps track of bulkwrite operations within a transaction.
|
|
27
28
|
*
|
|
@@ -36,6 +37,10 @@ export class PersistedBatch {
|
|
|
36
37
|
bucketData = [];
|
|
37
38
|
bucketParameters = [];
|
|
38
39
|
bucketStates = new Map();
|
|
40
|
+
objectStorage;
|
|
41
|
+
inlineThresholdBytes = DEFAULT_INLINE_THRESHOLD_BYTES;
|
|
42
|
+
signal;
|
|
43
|
+
objectStorageUsageWriterId;
|
|
39
44
|
/**
|
|
40
45
|
* For debug logging only.
|
|
41
46
|
*/
|
|
@@ -50,6 +55,12 @@ export class PersistedBatch {
|
|
|
50
55
|
this.mapping = mapping;
|
|
51
56
|
this.currentSize = writtenSize;
|
|
52
57
|
this.logger = options?.logger ?? defaultLogger;
|
|
58
|
+
this.objectStorage = options?.objectStorage;
|
|
59
|
+
this.signal = options?.signal;
|
|
60
|
+
this.objectStorageUsageWriterId = options?.objectStorageUsageWriterId;
|
|
61
|
+
if (options?.inlineThresholdBytes != null) {
|
|
62
|
+
this.inlineThresholdBytes = options.inlineThresholdBytes;
|
|
63
|
+
}
|
|
53
64
|
}
|
|
54
65
|
saveBucketData(options) {
|
|
55
66
|
const remaining_buckets = new Map();
|
|
@@ -60,9 +71,15 @@ export class PersistedBatch {
|
|
|
60
71
|
id: bucket.id,
|
|
61
72
|
table: bucket.table
|
|
62
73
|
};
|
|
74
|
+
if (options.table.bucketDataSourceIds != null && !options.table.bucketDataSourceIds.has(mapped.definitionId)) {
|
|
75
|
+
// The bucket definition is not active anymore.
|
|
76
|
+
// We don't cleanup these references upfront, but do need to ignore them after the definition is removed.
|
|
77
|
+
continue;
|
|
78
|
+
}
|
|
63
79
|
remaining_buckets.set(currentBucketKey(mapped), mapped);
|
|
64
80
|
}
|
|
65
|
-
const
|
|
81
|
+
const subkey = this.persistedSubkey(options.table.id, options.sourceKey);
|
|
82
|
+
const dchecksum = BigInt(utils.hashDelete(subkey));
|
|
66
83
|
for (const evaluated of options.evaluated) {
|
|
67
84
|
const definitionId = this.getBucketDefinitionId(evaluated.source);
|
|
68
85
|
const key = currentBucketKey({
|
|
@@ -92,6 +109,7 @@ export class PersistedBatch {
|
|
|
92
109
|
bucket: evaluated.bucket,
|
|
93
110
|
sourceTableId: options.table.id,
|
|
94
111
|
sourceKey: options.sourceKey,
|
|
112
|
+
subkey,
|
|
95
113
|
table: evaluated.table,
|
|
96
114
|
rowId: evaluated.id,
|
|
97
115
|
checksum: BigInt(checksum),
|
|
@@ -112,6 +130,7 @@ export class PersistedBatch {
|
|
|
112
130
|
op_id,
|
|
113
131
|
sourceTableId: options.table.id,
|
|
114
132
|
sourceKey: options.sourceKey,
|
|
133
|
+
subkey,
|
|
115
134
|
table: bucket.table,
|
|
116
135
|
rowId: bucket.id,
|
|
117
136
|
checksum: dchecksum
|
|
@@ -120,6 +139,13 @@ export class PersistedBatch {
|
|
|
120
139
|
this.incrementBucket(definitionId, bucket.bucket, op_id, 200);
|
|
121
140
|
}
|
|
122
141
|
}
|
|
142
|
+
/**
|
|
143
|
+
* New bucket operations persist protocol subkeys during replication. Older
|
|
144
|
+
* V1 documents may not have one, so sync reads retain a fallback.
|
|
145
|
+
*/
|
|
146
|
+
persistedSubkey(table, key) {
|
|
147
|
+
return replicaIdToSubkey(table, key);
|
|
148
|
+
}
|
|
123
149
|
get bucketDataCount() {
|
|
124
150
|
return this.bucketData.length;
|
|
125
151
|
}
|
|
@@ -137,10 +163,35 @@ export class PersistedBatch {
|
|
|
137
163
|
bucket,
|
|
138
164
|
lastOp: op_id,
|
|
139
165
|
incrementCount: 1,
|
|
140
|
-
incrementBytes: bytes
|
|
166
|
+
incrementBytes: bytes,
|
|
167
|
+
incrementChunks: 0
|
|
141
168
|
});
|
|
142
169
|
}
|
|
143
170
|
}
|
|
171
|
+
/**
|
|
172
|
+
* V3 persists operations in chunks. Keep this separate from incrementBucket:
|
|
173
|
+
* operation counts are known while evaluating rows, while chunk counts and
|
|
174
|
+
* exact persisted bytes are only known after the writer has chunked a flush.
|
|
175
|
+
*/
|
|
176
|
+
incrementBucketPersistedChunk(definitionId, bucket, bytes) {
|
|
177
|
+
const key = `${definitionId ?? ''}:${bucket}`;
|
|
178
|
+
const existingState = this.bucketStates.get(key);
|
|
179
|
+
if (existingState != null) {
|
|
180
|
+
existingState.incrementChunks += 1;
|
|
181
|
+
existingState.incrementBytes += bytes;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* V3's compactor calculates byte deltas from persisted chunk metadata.
|
|
186
|
+
* Replace the writer's per-operation estimate with those exact sizes before
|
|
187
|
+
* flushing the corresponding bucket state.
|
|
188
|
+
*/
|
|
189
|
+
resetBucketPersistedBytes(definitionId, bucket) {
|
|
190
|
+
const state = this.bucketStates.get(`${definitionId ?? ''}:${bucket}`);
|
|
191
|
+
if (state != null) {
|
|
192
|
+
state.incrementBytes = 0;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
144
195
|
addBucketDataPut(options) {
|
|
145
196
|
this.bucketData.push({
|
|
146
197
|
bucketKey: options.bucketKey,
|
|
@@ -148,6 +199,7 @@ export class PersistedBatch {
|
|
|
148
199
|
op: 'PUT',
|
|
149
200
|
source_table: mongoTableId(options.sourceTableId),
|
|
150
201
|
source_key: options.sourceKey,
|
|
202
|
+
subkey: options.subkey,
|
|
151
203
|
table: options.table,
|
|
152
204
|
row_id: options.rowId,
|
|
153
205
|
checksum: options.checksum,
|
|
@@ -161,6 +213,7 @@ export class PersistedBatch {
|
|
|
161
213
|
op: 'REMOVE',
|
|
162
214
|
source_table: mongoTableId(options.sourceTableId),
|
|
163
215
|
source_key: options.sourceKey,
|
|
216
|
+
subkey: options.subkey,
|
|
164
217
|
table: options.table,
|
|
165
218
|
row_id: options.rowId,
|
|
166
219
|
checksum: options.checksum,
|
|
@@ -196,25 +249,25 @@ export class PersistedBatch {
|
|
|
196
249
|
const duration = Math.round(performance.now() - startAt);
|
|
197
250
|
if (options?.oldestUncommittedChange != null) {
|
|
198
251
|
const replicationLag = Math.round((Date.now() - options.oldestUncommittedChange.getTime()) / 1000);
|
|
199
|
-
this.logger.info(`Flushed ${this.bucketDataCount}
|
|
252
|
+
this.logger.info(`Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${this.currentDataCount} records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}. Replication lag: ${replicationLag}s`, {
|
|
200
253
|
flushed: {
|
|
201
254
|
duration: duration,
|
|
202
255
|
size: this.currentSize,
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
256
|
+
bucket_ops_count: this.bucketDataCount,
|
|
257
|
+
parameter_indexes_count: this.bucketParameters.length,
|
|
258
|
+
source_records_count: this.currentDataCount,
|
|
206
259
|
replication_lag_seconds: replicationLag
|
|
207
260
|
}
|
|
208
261
|
});
|
|
209
262
|
}
|
|
210
263
|
else {
|
|
211
|
-
this.logger.info(`Flushed ${this.bucketDataCount}
|
|
264
|
+
this.logger.info(`Flushed: ${this.bucketDataCount} ops, ${this.bucketParameters.length} index entries, ${this.currentDataCount} records. ${Math.round(this.currentSize / 1024)}kb in ${duration}ms. Last op_id: ${this.debugLastOpId}`, {
|
|
212
265
|
flushed: {
|
|
213
266
|
duration: duration,
|
|
214
267
|
size: this.currentSize,
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
268
|
+
bucket_ops_count: this.bucketDataCount,
|
|
269
|
+
parameter_indexes_count: this.bucketParameters.length,
|
|
270
|
+
source_records_count: this.currentDataCount
|
|
218
271
|
}
|
|
219
272
|
});
|
|
220
273
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/PersistedBatch.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAU,MAAM,mCAAmC,CAAC;AACpF,OAAO,
|
|
1
|
+
{"version":3,"file":"PersistedBatch.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/PersistedBatch.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,IAAI,aAAa,EAAU,MAAM,mCAAmC,CAAC;AACpF,OAAO,EAAkD,KAAK,EAAE,MAAM,yBAAyB,CAAC;AAChG,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,8BAA8B,CAAC;AAQ9E;;;;;;;;;;;GAWG;AACH,MAAM,0BAA0B,GAAG,UAAU,CAAC;AAE9C;;;;GAIG;AACH,MAAM,yBAAyB,GAAG,KAAK,CAAC;AAExC,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,GAAG,IAAI,CAAC;AAqCxD;;;;;GAKG;AACH,MAAM,OAAgB,cAAc;IAsBb;IACA;IACA;IAvBrB,MAAM,CAAS;IACf,UAAU,GAAoB,EAAE,CAAC;IACjC,gBAAgB,GAAoC,EAAE,CAAC;IACvD,YAAY,GAAmC,IAAI,GAAG,EAAE,CAAC;IAEtC,aAAa,CAAiB;IAC9B,oBAAoB,GAAW,8BAA8B,CAAC;IAC9D,MAAM,CAAe;IACrB,0BAA0B,CAAU;IAEvD;;OAEG;IACH,aAAa,GAAwB,IAAI,CAAC;IAE1C;;OAEG;IACH,WAAW,GAAG,CAAC,CAAC;IAEhB,YACqB,EAA2B,EAC3B,QAAgB,EAChB,OAAgC,EACnD,WAAmB,EACnB,OAA+B;QAJZ,OAAE,GAAF,EAAE,CAAyB;QAC3B,aAAQ,GAAR,QAAQ,CAAQ;QAChB,YAAO,GAAP,OAAO,CAAyB;QAInD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,IAAI,aAAa,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;QAC5C,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC9B,IAAI,CAAC,0BAA0B,GAAG,OAAO,EAAE,0BAA0B,CAAC;QACtE,IAAI,OAAO,EAAE,oBAAoB,IAAI,IAAI,EAAE,CAAC;YAC1C,IAAI,CAAC,oBAAoB,GAAG,OAAO,CAAC,oBAAoB,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,cAAc,CAAC,OAA8B;QAC3C,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAmC,CAAC;QACrE,KAAK,IAAI,MAAM,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;YAC1C,MAAM,MAAM,GAA4B;gBACtC,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,YAAY,EAAE,IAAI,CAAC,iBAAiB,CAAC,MAAM,CAAC,YAAY,CAAC;gBACzD,EAAE,EAAE,MAAM,CAAC,EAAE;gBACb,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;YACF,IAAI,OAAO,CAAC,KAAK,CAAC,mBAAmB,IAAI,IAAI,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,mBAAmB,CAAC,GAAG,CAAC,MAAM,CAAC,YAAa,CAAC,EAAE,CAAC;gBAC9G,+CAA+C;gBAC/C,yGAAyG;gBACzG,SAAS;YACX,CAAC;YACD,iBAAiB,CAAC,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QAEnD,KAAK,MAAM,SAAS,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAClE,MAAM,GAAG,GAAG,gBAAgB,CAAC;gBAC3B,YAAY,EAAE,YAAY;gBAC1B,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,EAAE,EAAE,SAAS,CAAC,EAAE;aACjB,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YACrD,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC;YAC3E,IAAI,UAAU,CAAC,MAAM,GAAG,YAAY,EAAE,CAAC;gBACrC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,eAAe,UAAU,CAAC,MAAM,mBAAmB,CAAC,CAAC;gBACjF,SAAS;YACX,CAAC;YAED,iBAAiB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAC9B,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC;YAC7C,IAAI,CAAC,WAAW,IAAI,YAAY,CAAC;YAEjC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,gBAAgB,CAAC;gBACpB,SAAS,EAAE;oBACT,MAAM,EAAE,SAAS,CAAC,MAAM;oBACxB,YAAY,EAAE,YAAY;oBAC1B,mBAAmB,EAAE,IAAI,CAAC,QAAQ;iBACnC;gBACD,KAAK;gBACL,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM;gBACN,KAAK,EAAE,SAAS,CAAC,KAAK;gBACtB,KAAK,EAAE,SAAS,CAAC,EAAE;gBACnB,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC;gBAC1B,IAAI,EAAE,UAAU;aACjB,CAAC,CAAC;YACH,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,YAAY,CAAC,CAAC;QAC5E,CAAC;QAED,KAAK,IAAI,MAAM,IAAI,iBAAiB,CAAC,MAAM,EAAE,EAAE,CAAC;YAC9C,MAAM,YAAY,GAAG,MAAM,CAAC,YAAa,CAAC;YAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACpC,IAAI,CAAC,aAAa,GAAG,KAAK,CAAC;YAE3B,IAAI,CAAC,mBAAmB,CAAC;gBACvB,SAAS,EAAE;oBACT,mBAAmB,EAAE,IAAI,CAAC,QAAQ;oBAClC,YAAY;oBACZ,MAAM,EAAE,MAAM,CAAC,MAAM;iBACtB;gBACD,KAAK;gBACL,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;gBAC/B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,MAAM;gBACN,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE,MAAM,CAAC,EAAE;gBAChB,QAAQ,EAAE,SAAS;aACpB,CAAC,CAAC;YACH,IAAI,CAAC,WAAW,IAAI,GAAG,CAAC;YACxB,IAAI,CAAC,eAAe,CAAC,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IA6BD;;;OAGG;IACO,eAAe,CAAC,KAA4B,EAAE,GAAsB;QAC5E,OAAO,iBAAiB,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IACvC,CAAC;IAED,IAAc,eAAe;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAChC,CAAC;IAES,eAAe,CAAC,YAAgC,EAAE,MAAc,EAAE,KAAmB,EAAE,KAAa;QAC5G,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;QAC9C,IAAI,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,aAAa,EAAE,CAAC;YAClB,aAAa,CAAC,MAAM,GAAG,KAAK,CAAC;YAC7B,aAAa,CAAC,cAAc,IAAI,CAAC,CAAC;YAClC,aAAa,CAAC,cAAc,IAAI,KAAK,CAAC;QACxC,CAAC;aAAM,CAAC;YACN,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,EAAE;gBACzB,YAAY;gBACZ,MAAM;gBACN,MAAM,EAAE,KAAK;gBACb,cAAc,EAAE,CAAC;gBACjB,cAAc,EAAE,KAAK;gBACrB,eAAe,EAAE,CAAC;aACnB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;;;OAIG;IACO,6BAA6B,CAAC,YAAgC,EAAE,MAAc,EAAE,KAAa;QACrG,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC;QAC9C,MAAM,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjD,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;YAC1B,aAAa,CAAC,eAAe,IAAI,CAAC,CAAC;YACnC,aAAa,CAAC,cAAc,IAAI,KAAK,CAAC;QACxC,CAAC;IACH,CAAC;IAED;;;;OAIG;IACO,yBAAyB,CAAC,YAAgC,EAAE,MAAc;QAClF,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,YAAY,IAAI,EAAE,IAAI,MAAM,EAAE,CAAC,CAAC;QACvE,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;YAClB,KAAK,CAAC,cAAc,GAAG,CAAC,CAAC;QAC3B,CAAC;IACH,CAAC;IAES,gBAAgB,CAAC,OAW1B;QACC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,CAAC,EAAE,OAAO,CAAC,KAAK;YAChB,EAAE,EAAE,KAAK;YACT,YAAY,EAAE,YAAY,CAAC,OAAO,CAAC,aAAa,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,OAAO,CAAC,IAAI;SACnB,CAAC,CAAC;IACL,CAAC;IAES,mBAAmB,CAAC,OAS7B;QACC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;YACnB,SAAS,EAAE,OAAO,CAAC,SAAS;YAC5B,CAAC,EAAE,OAAO,CAAC,KAAK;YAChB,EAAE,EAAE,QAAQ;YACZ,YAAY,EAAE,YAAY,CAAC,OAAO,CAAC,aAAa,CAAC;YACjD,UAAU,EAAE,OAAO,CAAC,SAAS;YAC7B,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,OAAO,CAAC,KAAK;YACrB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,IAAI,EAAE,IAAI;SACX,CAAC,CAAC;IACL,CAAC;IAED,sBAAsB;QACpB,OAAO,CACL,IAAI,CAAC,WAAW,IAAI,0BAA0B;YAC9C,IAAI,CAAC,eAAe,IAAI,yBAAyB;YACjD,IAAI,CAAC,gBAAgB,IAAI,yBAAyB;YAClD,IAAI,CAAC,gBAAgB,CAAC,MAAM,IAAI,yBAAyB,CAC1D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,OAA4B,EAAE,OAA0C;QAClF,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,IAAI,gBAAgB,GAAG,KAAK,CAAC;QAC7B,IAAI,IAAI,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;YAC7B,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC;QACtC,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACrC,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC;QAC5C,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,GAAG,CAAC,EAAE,CAAC;YAC9B,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACvC,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC/B,gBAAgB,GAAG,IAAI,CAAC;YACxB,MAAM,IAAI,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACxC,CAAC;QAED,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;YACzD,IAAI,OAAO,EAAE,uBAAuB,IAAI,IAAI,EAAE,CAAC;gBAC7C,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,uBAAuB,CAAC,OAAO,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;gBAEnG,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,IAAI,CAAC,eAAe,SAAS,IAAI,CAAC,gBAAgB,CAAC,MAAM,mBACnE,IAAI,CAAC,gBACP,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,sBAAsB,cAAc,GAAG,EAC7I;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,gBAAgB,EAAE,IAAI,CAAC,eAAe;wBACtC,uBAAuB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBACrD,oBAAoB,EAAE,IAAI,CAAC,gBAAgB;wBAC3C,uBAAuB,EAAE,cAAc;qBACxC;iBACF,CACF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,YAAY,IAAI,CAAC,eAAe,SAAS,IAAI,CAAC,gBAAgB,CAAC,MAAM,mBACnE,IAAI,CAAC,gBACP,aAAa,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,QAAQ,mBAAmB,IAAI,CAAC,aAAa,EAAE,EACxG;oBACE,OAAO,EAAE;wBACP,QAAQ,EAAE,QAAQ;wBAClB,IAAI,EAAE,IAAI,CAAC,WAAW;wBACtB,gBAAgB,EAAE,IAAI,CAAC,eAAe;wBACtC,uBAAuB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;wBACrD,oBAAoB,EAAE,IAAI,CAAC,gBAAgB;qBAC5C;iBACF,CACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG;YACZ,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,kBAAkB,EAAE,IAAI,CAAC,gBAAgB,CAAC,MAAM;YAChD,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,UAAU,EAAE,gBAAgB;SAC7B,CAAC;QAEF,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,gBAAgB,GAAG,EAAE,CAAC;QAC3B,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;QAC1B,IAAI,CAAC,WAAW,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAE1B,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}
|
|
@@ -48,7 +48,7 @@ export class BaseVersionedPowerSyncMongo {
|
|
|
48
48
|
return this.upstream.sourceTableCollectionName(replicationStreamId);
|
|
49
49
|
}
|
|
50
50
|
async listCollectionsByPrefix(prefix) {
|
|
51
|
-
const collections = await this.db.listCollections({ name:
|
|
51
|
+
const collections = await this.db.listCollections({ name: { $regex: `^${prefix}` } }, { nameOnly: true }).toArray();
|
|
52
52
|
return collections
|
|
53
53
|
.filter((collection) => collection.name.startsWith(prefix))
|
|
54
54
|
.map((collection) => this.db.collection(collection.name));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"VersionedPowerSyncMongoBase.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/VersionedPowerSyncMongoBase.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAI/D,MAAM,OAAgB,2BAA2B;IAO1B;IANZ,MAAM,CAAoB;IAC1B,EAAE,CAAW;IACb,aAAa,CAAgB;IACtC,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAEpB,YACqB,QAAwB,EAC3C,aAA4B;QADT,aAAQ,GAAR,QAAQ,CAAgB;QAG3C,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACtB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;IACtC,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;IAClC,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACzC,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC7B,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACzC,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAChD,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;IAC1C,CAAC;IAES,2BAA2B,CAAC,mBAA2B,EAAE,aAA6B;QAC9F,OAAO,IAAI,CAAC,QAAQ,CAAC,2BAA2B,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;IACvF,CAAC;IAES,yBAAyB,CAAC,mBAA2B;QAC7D,OAAO,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;IACtE,CAAC;IAES,KAAK,CAAC,uBAAuB,CAA2B,MAAc;QAC9E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,
|
|
1
|
+
{"version":3,"file":"VersionedPowerSyncMongoBase.js","sourceRoot":"","sources":["../../../../src/storage/implementation/common/VersionedPowerSyncMongoBase.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAI/D,MAAM,OAAgB,2BAA2B;IAO1B;IANZ,MAAM,CAAoB;IAC1B,EAAE,CAAW;IACb,aAAa,CAAgB;IACtC,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAEpB,YACqB,QAAwB,EAC3C,aAA4B;QADT,aAAQ,GAAR,QAAQ,CAAgB;QAG3C,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC;QAC9B,IAAI,CAAC,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACtB,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;IACrC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC;IACnC,CAAC;IAED,IAAI,cAAc;QAChB,OAAO,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC;IACtC,CAAC;IAED,IAAI,UAAU;QACZ,OAAO,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;IAClC,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAChD,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACzC,CAAC;IAED,IAAI,QAAQ;QACV,OAAO,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC;IAChC,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;IAC7B,CAAC;IAED,IAAI,iBAAiB;QACnB,OAAO,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC;IACzC,CAAC;IAED,IAAI,wBAAwB;QAC1B,OAAO,IAAI,CAAC,QAAQ,CAAC,wBAAwB,CAAC;IAChD,CAAC;IAED,gBAAgB;QACd,OAAO,IAAI,CAAC,QAAQ,CAAC,gBAAgB,EAAE,CAAC;IAC1C,CAAC;IAES,2BAA2B,CAAC,mBAA2B,EAAE,aAA6B;QAC9F,OAAO,IAAI,CAAC,QAAQ,CAAC,2BAA2B,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;IACvF,CAAC;IAES,yBAAyB,CAAC,mBAA2B;QAC7D,OAAO,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;IACtE,CAAC;IAES,KAAK,CAAC,uBAAuB,CAA2B,MAAc;QAC9E,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAEpH,OAAO,WAAW;aACf,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,CAAC;CAKF"}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { storage } from '@powersync/service-core';
|
|
2
2
|
import { MongoBucketStorage } from '../MongoBucketStorage.js';
|
|
3
|
-
import {
|
|
3
|
+
import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js';
|
|
4
4
|
import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
|
|
5
5
|
export { MongoSyncBucketStorageOptions } from './MongoSyncBucketStorage.js';
|
|
6
6
|
export type { MongoSyncBucketStorage };
|
|
7
|
-
export declare function createMongoSyncBucketStorage(factory: MongoBucketStorage,
|
|
7
|
+
export declare function createMongoSyncBucketStorage(factory: MongoBucketStorage, replicationStreamId: number, replicationStream: MongoPersistedReplicationStream, replicationStreamName: string, writeCheckpointMode: storage.WriteCheckpointMode | undefined, options: MongoSyncBucketStorageOptions): MongoSyncBucketStorage;
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import { MongoSyncBucketStorageV1 } from './v1/MongoSyncBucketStorageV1.js';
|
|
2
2
|
import { MongoSyncBucketStorageV3 } from './v3/MongoSyncBucketStorageV3.js';
|
|
3
|
-
export function createMongoSyncBucketStorage(factory,
|
|
4
|
-
if (
|
|
5
|
-
return new MongoSyncBucketStorageV3(factory,
|
|
3
|
+
export function createMongoSyncBucketStorage(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options) {
|
|
4
|
+
if (replicationStream.getStorageConfig().incrementalReprocessing) {
|
|
5
|
+
return new MongoSyncBucketStorageV3(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
|
|
6
6
|
}
|
|
7
|
-
return new MongoSyncBucketStorageV1(factory,
|
|
7
|
+
return new MongoSyncBucketStorageV1(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
|
|
8
8
|
}
|
|
9
9
|
//# sourceMappingURL=createMongoSyncBucketStorage.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"createMongoSyncBucketStorage.js","sourceRoot":"","sources":["../../../src/storage/implementation/createMongoSyncBucketStorage.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAM5E,MAAM,UAAU,4BAA4B,CAC1C,OAA2B,EAC3B,
|
|
1
|
+
{"version":3,"file":"createMongoSyncBucketStorage.js","sourceRoot":"","sources":["../../../src/storage/implementation/createMongoSyncBucketStorage.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAC5E,OAAO,EAAE,wBAAwB,EAAE,MAAM,kCAAkC,CAAC;AAM5E,MAAM,UAAU,4BAA4B,CAC1C,OAA2B,EAC3B,mBAA2B,EAC3B,iBAAkD,EAClD,qBAA6B,EAC7B,mBAA4D,EAC5D,OAAsC;IAEtC,IAAI,iBAAiB,CAAC,gBAAgB,EAAE,CAAC,uBAAuB,EAAE,CAAC;QACjE,OAAO,IAAI,wBAAwB,CACjC,OAAO,EACP,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,mBAAmB,EACnB,OAAO,CACR,CAAC;IACJ,CAAC;IAED,OAAO,IAAI,wBAAwB,CACjC,OAAO,EACP,mBAAmB,EACnB,iBAAiB,EACjB,qBAAqB,EACrB,mBAAmB,EACnB,OAAO,CACR,CAAC;AACJ,CAAC"}
|
|
@@ -6,7 +6,8 @@ import { BaseVersionedPowerSyncMongo } from './common/VersionedPowerSyncMongoBas
|
|
|
6
6
|
import { CheckpointEventDocument, ClientConnectionDocument, CommonSourceTableDocument, CustomWriteCheckpointDocument, IdSequenceDocument, InstanceDocument, SourceTableDocument, StorageConfig, SyncRuleDocumentBase, WriteCheckpointDocument } from './models.js';
|
|
7
7
|
import { BucketDataDocumentV1, BucketParameterDocument, BucketStateDocumentV1, CurrentDataDocument } from './v1/models.js';
|
|
8
8
|
import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js';
|
|
9
|
-
import { BucketDataDocumentV3 } from './v3/models.js';
|
|
9
|
+
import { BucketDataDocumentV3, CustomCheckpointRequestDocumentV3 } from './v3/models.js';
|
|
10
|
+
import type { CustomCheckpointRequestCollectionSpecifier } from './v3/VersionedPowerSyncMongoV3.js';
|
|
10
11
|
import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
|
|
11
12
|
export interface PowerSyncMongoOptions {
|
|
12
13
|
/**
|
|
@@ -62,6 +63,8 @@ export declare class PowerSyncMongo {
|
|
|
62
63
|
listAllBucketStateCollectionsV3(): Promise<mongo.Collection<never>[]>;
|
|
63
64
|
sourceRecordsCollectionName(replicationStreamId: number, sourceTableId: mongo.ObjectId): string;
|
|
64
65
|
sourceTableCollectionName(replicationStreamId: number): string;
|
|
66
|
+
customCheckpointRequestCollectionName(options: CustomCheckpointRequestCollectionSpecifier): string;
|
|
67
|
+
listCustomCheckpointRequestCollections(replicationStreamId?: number): Promise<mongo.Collection<CustomCheckpointRequestDocumentV3>[]>;
|
|
65
68
|
listSourceTableCollections(replicationStreamId?: number): Promise<mongo.Collection<CommonSourceTableDocument>[]>;
|
|
66
69
|
/**
|
|
67
70
|
* Clear all collections.
|
|
@@ -2,7 +2,7 @@ import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
|
2
2
|
import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
|
|
3
3
|
import { DO_NOT_LOG } from '@powersync/lib-services-framework';
|
|
4
4
|
import { VersionedPowerSyncMongoV1 } from './v1/VersionedPowerSyncMongoV1.js';
|
|
5
|
-
import { VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
|
|
5
|
+
import { OBJECT_STORAGE_USAGE_COLLECTION, VersionedPowerSyncMongoV3 } from './v3/VersionedPowerSyncMongoV3.js';
|
|
6
6
|
export class PowerSyncMongo {
|
|
7
7
|
[DO_NOT_LOG] = true;
|
|
8
8
|
current_data;
|
|
@@ -41,7 +41,7 @@ export class PowerSyncMongo {
|
|
|
41
41
|
this.connection_report_events = this.db.collection('connection_report_events');
|
|
42
42
|
}
|
|
43
43
|
versioned(storageConfig) {
|
|
44
|
-
if (storageConfig.
|
|
44
|
+
if (storageConfig.version >= storage.STORAGE_VERSION_3) {
|
|
45
45
|
return new VersionedPowerSyncMongoV3(this, storageConfig);
|
|
46
46
|
}
|
|
47
47
|
return new VersionedPowerSyncMongoV1(this, storageConfig);
|
|
@@ -51,16 +51,13 @@ export class PowerSyncMongo {
|
|
|
51
51
|
*/
|
|
52
52
|
async listBucketDataCollectionsV3(groupId) {
|
|
53
53
|
const prefix = groupId == null ? 'bucket_data_' : `bucket_data_${groupId}_`;
|
|
54
|
-
|
|
55
|
-
return collections
|
|
56
|
-
.filter((collection) => collection.name.startsWith(prefix))
|
|
57
|
-
.map((collection) => this.db.collection(collection.name));
|
|
54
|
+
return this.collectionsByPrefix(prefix);
|
|
58
55
|
}
|
|
59
56
|
/**
|
|
60
57
|
* Not safe for user-provided prefix - only for hardcoded values.
|
|
61
58
|
*/
|
|
62
59
|
async collectionsByPrefix(prefix) {
|
|
63
|
-
const collections = await this.db.listCollections({ name:
|
|
60
|
+
const collections = await this.db.listCollections({ name: { $regex: `^${prefix}` } }, { nameOnly: true }).toArray();
|
|
64
61
|
return collections
|
|
65
62
|
.filter((collection) => collection.name.startsWith(prefix))
|
|
66
63
|
.map((collection) => this.db.collection(collection.name));
|
|
@@ -90,15 +87,22 @@ export class PowerSyncMongo {
|
|
|
90
87
|
sourceTableCollectionName(replicationStreamId) {
|
|
91
88
|
return `source_table_${replicationStreamId}`;
|
|
92
89
|
}
|
|
90
|
+
customCheckpointRequestCollectionName(options) {
|
|
91
|
+
return `custom_checkpoint_requests_${options.replicationStreamId}_${options.eventId}`;
|
|
92
|
+
}
|
|
93
|
+
async listCustomCheckpointRequestCollections(replicationStreamId) {
|
|
94
|
+
const prefix = replicationStreamId == null
|
|
95
|
+
? 'custom_checkpoint_requests_'
|
|
96
|
+
: `custom_checkpoint_requests_${replicationStreamId}_`;
|
|
97
|
+
return this.collectionsByPrefix(prefix);
|
|
98
|
+
}
|
|
93
99
|
async listSourceTableCollections(replicationStreamId) {
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
const
|
|
98
|
-
const collections = await this.db.listCollections(
|
|
99
|
-
return collections
|
|
100
|
-
.filter((collection) => collection.name.startsWith(prefix))
|
|
101
|
-
.map((collection) => this.db.collection(collection.name));
|
|
100
|
+
if (replicationStreamId == null) {
|
|
101
|
+
return this.collectionsByPrefix('source_table_');
|
|
102
|
+
}
|
|
103
|
+
const name = this.sourceTableCollectionName(replicationStreamId);
|
|
104
|
+
const collections = await this.db.listCollections({ name }, { nameOnly: true }).toArray();
|
|
105
|
+
return collections.map((c) => this.db.collection(c.name));
|
|
102
106
|
}
|
|
103
107
|
/**
|
|
104
108
|
* Clear all collections.
|
|
@@ -130,6 +134,10 @@ export class PowerSyncMongo {
|
|
|
130
134
|
await this.locks.deleteMany({});
|
|
131
135
|
await this.bucket_state.deleteMany({});
|
|
132
136
|
await this.custom_write_checkpoints.deleteMany({});
|
|
137
|
+
await this.db.collection(OBJECT_STORAGE_USAGE_COLLECTION).deleteMany({});
|
|
138
|
+
for (const collection of await this.listCustomCheckpointRequestCollections()) {
|
|
139
|
+
await collection.drop();
|
|
140
|
+
}
|
|
133
141
|
}
|
|
134
142
|
/**
|
|
135
143
|
* Drop the entire database.
|
|
@@ -145,7 +153,14 @@ export class PowerSyncMongo {
|
|
|
145
153
|
* This is used in a similar way to the Postgres NOTIFY functionality.
|
|
146
154
|
*/
|
|
147
155
|
async notifyCheckpoint() {
|
|
148
|
-
await this.checkpoint_events.insertOne({}, {
|
|
156
|
+
await this.checkpoint_events.insertOne({}, {
|
|
157
|
+
forceServerObjectId: true,
|
|
158
|
+
// No correctness issue if this write is missed / rolled back.
|
|
159
|
+
// It could cause some sync delays, that will resolve in at most a minute, but no consistency issues.
|
|
160
|
+
writeConcern: {
|
|
161
|
+
w: 1
|
|
162
|
+
}
|
|
163
|
+
});
|
|
149
164
|
}
|
|
150
165
|
/**
|
|
151
166
|
* Only use in migrations and tests.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAqB/D,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;
|
|
1
|
+
{"version":3,"file":"db.js","sourceRoot":"","sources":["../../../src/storage/implementation/db.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,mCAAmC,CAAC;AAqB/D,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAG9E,OAAO,EAAE,+BAA+B,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAS/G,MAAM,OAAO,cAAc;IACzB,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC;IAEX,YAAY,CAAwC;IACpD,WAAW,CAAyC;IACpD,iBAAiB,CAA4C;IAC7D,cAAc,CAAuC;IACrD,UAAU,CAAyC;IACnD,aAAa,CAAwC;IACrD,wBAAwB,CAAkD;IAC1E,iBAAiB,CAA4C;IAC7D,QAAQ,CAAqC;IAC7C,KAAK,CAAyC;IAC9C,YAAY,CAA0C;IACtD,iBAAiB,CAA4C;IAC7D,wBAAwB,CAA6C;IAErE,MAAM,CAAoB;IAC1B,EAAE,CAAW;IAEtB,YAAY,MAAyB,EAAE,OAA+B;QACpE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QAErB,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE;YACtC,GAAG,OAAO,CAAC,iCAAiC;SAC7C,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QAEb,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAClD,IAAI,CAAC,WAAW,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,cAAc,GAAG,EAAE,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC;QACtD,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC9C,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,CAAC;QACpD,IAAI,CAAC,wBAAwB,GAAG,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;QAC1E,IAAI,CAAC,iBAAiB,GAAG,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QAC5D,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QACvD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC,CAAC;QACjE,IAAI,CAAC,wBAAwB,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,0BAA0B,CAAC,CAAC;IACjF,CAAC;IAKD,SAAS,CAAC,aAA4B;QACpC,IAAI,aAAa,CAAC,OAAO,IAAI,OAAO,CAAC,iBAAiB,EAAE,CAAC;YACvD,OAAO,IAAI,yBAAyB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;QAC5D,CAAC;QAED,OAAO,IAAI,yBAAyB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAC;IAC5D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,2BAA2B,CAAC,OAAgB;QAChD,MAAM,MAAM,GAAG,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,eAAe,OAAO,GAAG,CAAC;QAC5E,OAAO,IAAI,CAAC,mBAAmB,CAAuB,MAAM,CAAC,CAAC;IAChE,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,mBAAmB,CAAmC,MAAc;QAChF,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,MAAM,EAAE,EAAE,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAEpH,OAAO,WAAW;aACf,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;aAC1D,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC;IACjE,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,kCAAkC;QACtC,OAAO,IAAI,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,CAAC;IACtD,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gCAAgC;QACpC,OAAO,IAAI,CAAC,mBAAmB,CAAC,iBAAiB,CAAC,CAAC;IACrD,CAAC;IAED,KAAK,CAAC,+BAA+B;QACnC,OAAO,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;IACnD,CAAC;IAED,2BAA2B,CAAC,mBAA2B,EAAE,aAA6B;QACpF,OAAO,kBAAkB,mBAAmB,IAAI,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC;IAChF,CAAC;IAED,yBAAyB,CAAC,mBAA2B;QACnD,OAAO,gBAAgB,mBAAmB,EAAE,CAAC;IAC/C,CAAC;IAED,qCAAqC,CAAC,OAAmD;QACvF,OAAO,8BAA8B,OAAO,CAAC,mBAAmB,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;IACxF,CAAC;IAED,KAAK,CAAC,sCAAsC,CAC1C,mBAA4B;QAE5B,MAAM,MAAM,GACV,mBAAmB,IAAI,IAAI;YACzB,CAAC,CAAC,6BAA6B;YAC/B,CAAC,CAAC,8BAA8B,mBAAmB,GAAG,CAAC;QAC3D,OAAO,IAAI,CAAC,mBAAmB,CAAC,MAAM,CAAC,CAAC;IAC1C,CAAC;IAED,KAAK,CAAC,0BAA0B,CAC9B,mBAA4B;QAE5B,IAAI,mBAAmB,IAAI,IAAI,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,mBAAmB,CAAC,eAAe,CAAC,CAAC;QACnD,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,yBAAyB,CAAC,mBAAmB,CAAC,CAAC;QACjE,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;QAC1F,OAAO,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAA4B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IACvF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACT,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,gCAAgC,EAAE,EAAE,CAAC;YACvE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACtC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,2BAA2B,EAAE,EAAE,CAAC;YAClE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,kCAAkC,EAAE,EAAE,CAAC;YACzE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,+BAA+B,EAAE,EAAE,CAAC;YACtE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,cAAc,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzC,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACrC,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,0BAA0B,EAAE,EAAE,CAAC;YACjE,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;QACD,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACxC,MAAM,IAAI,CAAC,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAC5C,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;QAClC,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAChC,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACvC,MAAM,IAAI,CAAC,wBAAwB,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACnD,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,+BAA+B,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QACzE,KAAK,MAAM,UAAU,IAAI,MAAM,IAAI,CAAC,sCAAsC,EAAE,EAAE,CAAC;YAC7E,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,IAAI;QACR,MAAM,IAAI,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC;IAC/B,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB;QACpB,MAAM,IAAI,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAS,EAAE;YAChD,mBAAmB,EAAE,IAAI;YACzB,8DAA8D;YAC9D,qGAAqG;YACrG,YAAY,EAAE;gBACZ,CAAC,EAAE,CAAC;aACL;SACF,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,gCAAgC;QACpC,6FAA6F;QAC7F,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,mBAAmB,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aACnE,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;gBAChC,oEAAoE;gBACpE,MAAM,IAAI,CAAC,EAAE,CAAC,cAAc,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;iBAAM,CAAC;gBACN,iDAAiD;gBACjD,OAAO;YACT,CAAC;QACH,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,EAAE;YAClD,MAAM,EAAE,IAAI;YACZ,sFAAsF;YACtF,mFAAmF;YACnF,iFAAiF;YACjF,4BAA4B;YAC5B,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,gBAAgB;YACjC,GAAG,EAAE,EAAE,CAAC,0BAA0B;SACnC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,mCAAmC;QACvC,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,EAAE;aACtC,eAAe,CAAC,EAAE,IAAI,EAAE,0BAA0B,EAAE,EAAE,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;aAC1E,OAAO,EAAE,CAAC;QACb,MAAM,UAAU,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;YACvB,OAAO;QACT,CAAC;QACD,MAAM,IAAI,CAAC,EAAE,CAAC,gBAAgB,CAAC,0BAA0B,CAAC,CAAC;IAC7D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,sBAAsB;QAC1B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,OAAO,EAAE,CAAC;SACX,EACD,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE,CACzC,CAAC;IACJ,CAAC;IACD;;OAEG;IACH,KAAK,CAAC,uBAAuB;QAC3B,gEAAgE;QAChE,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,CACjC;YACE,OAAO,EAAE,CAAC;YACV,8BAA8B,EAAE,CAAC,CAAC;SACnC,EACD,EAAE,IAAI,EAAE,aAAa,EAAE,CACxB,CAAC;IACJ,CAAC;CACF;AAOD,MAAM,UAAU,oBAAoB,CAAC,MAA0B,EAAE,OAA0C;IACzG,OAAO,IAAI,cAAc,CACvB,SAAS,CAAC,iBAAiB,CAAC,MAAM,EAAE;QAClC,gBAAgB,EAAE,iBAAiB;QACnC,GAAG,OAAO;KACX,CAAC,EACF,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAC9B,CAAC;AACJ,CAAC"}
|
|
@@ -3,7 +3,7 @@ import { ParameterIndexId, SqliteJsonValue } from '@powersync/service-sync-rules
|
|
|
3
3
|
import { event_types } from '@powersync/service-types';
|
|
4
4
|
import * as bson from 'bson';
|
|
5
5
|
import type { CurrentDataDocument, SourceTableDocumentV1 } from './v1/models.js';
|
|
6
|
-
import type {
|
|
6
|
+
import type { CurrentDataDocumentV3, RecordedLookupV3 } from './v3/models.js';
|
|
7
7
|
/**
|
|
8
8
|
* Replica id uniquely identifying a row on the source database.
|
|
9
9
|
*
|
|
@@ -55,10 +55,21 @@ export interface BucketDataProperties {
|
|
|
55
55
|
op: OpType;
|
|
56
56
|
source_table?: bson.ObjectId;
|
|
57
57
|
source_key?: ReplicaId;
|
|
58
|
+
/**
|
|
59
|
+
* Stable identifier for the source record in the client protocol.
|
|
60
|
+
*
|
|
61
|
+
* This is derived from (source_table, source_key), so persistence is optional but preferred.
|
|
62
|
+
*/
|
|
63
|
+
subkey?: string;
|
|
58
64
|
table?: string;
|
|
59
65
|
row_id?: string;
|
|
60
66
|
checksum: bigint;
|
|
61
67
|
data: string | null;
|
|
68
|
+
/**
|
|
69
|
+
* V1-only.
|
|
70
|
+
*
|
|
71
|
+
* V3 stores this on the BucketDataDocumentV3 instead of the individual ops.
|
|
72
|
+
*/
|
|
62
73
|
target_op?: bigint | null;
|
|
63
74
|
}
|
|
64
75
|
export interface BucketDataDocumentBase extends BucketDataProperties {
|
|
@@ -88,6 +99,10 @@ export interface SourceTableDocument {
|
|
|
88
99
|
}[] | undefined;
|
|
89
100
|
snapshot_done: boolean | undefined;
|
|
90
101
|
snapshot_status: SourceTableDocumentSnapshotStatus | undefined;
|
|
102
|
+
/**
|
|
103
|
+
* Source-specific metadata. Absent for legacy records.
|
|
104
|
+
*/
|
|
105
|
+
source_metadata?: storage.JsonValue;
|
|
91
106
|
}
|
|
92
107
|
export interface SourceTableDocumentSnapshotStatus {
|
|
93
108
|
total_estimated_count: number;
|
|
@@ -175,8 +190,33 @@ export interface SyncRuleDocumentBase {
|
|
|
175
190
|
expires_at: Date;
|
|
176
191
|
} | null;
|
|
177
192
|
storage_version?: number;
|
|
193
|
+
/** Optional operator-supplied label identifying the sync config version. */
|
|
194
|
+
version_label?: string | null;
|
|
195
|
+
/**
|
|
196
|
+
* Incremental parameter compaction state for the replication stream.
|
|
197
|
+
*
|
|
198
|
+
* Operation ids are allocated across all parameter indexes of a stream, so this is
|
|
199
|
+
* stream-level state, shared by all sync configs of the stream.
|
|
200
|
+
*/
|
|
201
|
+
parameter_compaction?: {
|
|
202
|
+
/**
|
|
203
|
+
* The exclusive operation-id boundary through which every parameter index in this stream has
|
|
204
|
+
* been compacted.
|
|
205
|
+
*/
|
|
206
|
+
compacted_before: InternalOpId;
|
|
207
|
+
/**
|
|
208
|
+
* Parameter entries below this boundary may no longer be available for checkpoint change
|
|
209
|
+
* detection, since compaction may have deleted them.
|
|
210
|
+
*
|
|
211
|
+
* This is advanced before the first delete of a compaction pass, while
|
|
212
|
+
* {@link compacted_before} is only advanced after every delete of the pass completed.
|
|
213
|
+
* A checkpoint transition starting below this boundary must conservatively invalidate all
|
|
214
|
+
* parameter buckets, since the individual changes may no longer be available.
|
|
215
|
+
*/
|
|
216
|
+
checkpoint_changes_invalid_before?: InternalOpId;
|
|
217
|
+
};
|
|
178
218
|
}
|
|
179
|
-
export interface SyncRuleCheckpointFields
|
|
219
|
+
export interface SyncRuleCheckpointFields {
|
|
180
220
|
/**
|
|
181
221
|
* The last consistent checkpoint.
|
|
182
222
|
*
|
|
@@ -191,15 +231,6 @@ export interface SyncRuleCheckpointFields<TKeepaliveOp extends string | bigint |
|
|
|
191
231
|
* If set, no new checkpoints may be created < this value.
|
|
192
232
|
*/
|
|
193
233
|
no_checkpoint_before: string | null;
|
|
194
|
-
/**
|
|
195
|
-
* Goes together with no_checkpoint_before.
|
|
196
|
-
*
|
|
197
|
-
* If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
|
|
198
|
-
* then the checkpoint must be equal to this keepalive_op.
|
|
199
|
-
*
|
|
200
|
-
* This is a string in V1, bigint in V3.
|
|
201
|
-
*/
|
|
202
|
-
keepalive_op: TKeepaliveOp;
|
|
203
234
|
}
|
|
204
235
|
export interface StorageConfig extends storage.StorageVersionConfig {
|
|
205
236
|
/**
|
|
@@ -230,6 +261,11 @@ export interface CustomWriteCheckpointDocument {
|
|
|
230
261
|
* This is not unique - multiple write checkpoints can have the same op_id.
|
|
231
262
|
*/
|
|
232
263
|
op_id?: InternalOpId;
|
|
264
|
+
/**
|
|
265
|
+
* Set when this checkpoint was created from a client-supplied checkpoint
|
|
266
|
+
* request rather than a persistent custom write checkpoint.
|
|
267
|
+
*/
|
|
268
|
+
checkpoint_requested_at?: Date | null;
|
|
233
269
|
}
|
|
234
270
|
export interface WriteCheckpointDocument {
|
|
235
271
|
_id: bson.ObjectId;
|
|
@@ -242,6 +278,12 @@ export interface WriteCheckpointDocument {
|
|
|
242
278
|
* between two checkpoints.
|
|
243
279
|
*/
|
|
244
280
|
processed_at_lsn: string | null;
|
|
281
|
+
/**
|
|
282
|
+
* Set when this checkpoint was created from a client-supplied checkpoint request
|
|
283
|
+
* rather than a server-generated id. A missing/null value means this is not a
|
|
284
|
+
* checkpoint request.
|
|
285
|
+
*/
|
|
286
|
+
checkpoint_requested_at?: Date | null;
|
|
245
287
|
}
|
|
246
288
|
export interface InstanceDocument {
|
|
247
289
|
_id: string;
|
|
@@ -249,6 +291,6 @@ export interface InstanceDocument {
|
|
|
249
291
|
export interface ClientConnectionDocument extends event_types.ClientConnection {
|
|
250
292
|
}
|
|
251
293
|
export type CurrentDataDocumentId = CurrentDataDocument['_id'] | CurrentDataDocumentV3['_id'];
|
|
252
|
-
export type CommonCurrentBucket = CurrentBucket
|
|
294
|
+
export type CommonCurrentBucket = CurrentBucket;
|
|
253
295
|
export type CommonCurrentLookup = bson.Binary | RecordedLookupV3;
|
|
254
|
-
export type CommonSourceTableDocument = SourceTableDocumentV1 |
|
|
296
|
+
export type CommonSourceTableDocument = Pick<SourceTableDocumentV1, '_id' | 'snapshot_status'>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../src/storage/implementation/models.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAgB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAwDhE,MAAM,UAAU,+BAA+B,CAC7C,QAA2C,EAC3C,KAAuB;IAEvB,OAAO;QACL,GAAG,QAAQ;QACX,KAAK;KACN,CAAC;AACJ,CAAC;
|
|
1
|
+
{"version":3,"file":"models.js","sourceRoot":"","sources":["../../../src/storage/implementation/models.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,EAAgB,OAAO,EAAE,MAAM,yBAAyB,CAAC;AAwDhE,MAAM,UAAU,+BAA+B,CAC7C,QAA2C,EAC3C,KAAuB;IAEvB,OAAO;QACL,GAAG,QAAQ;QACX,KAAK;KACN,CAAC;AACJ,CAAC;AAiCD;;GAEG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAC;AAEpD;;GAEG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,GAAG,CAAC;AAoLpD,MAAM,8BAA8B,GAAG,CAAC,CAAC;AACzC,MAAM,wCAAwC,GAAG,OAAO,CAAC,iBAAiB,CAAC;AAE3E,MAAM,UAAU,qBAAqB,CAAC,cAAsB;IAC1D,MAAM,UAAU,GAAG,OAAO,CAAC,sBAAsB,CAAC,cAAc,CAAC,CAAC;IAClE,IAAI,UAAU,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,IAAI,YAAY,CAAC,SAAS,CAAC,WAAW,EAAE,+BAA+B,cAAc,EAAE,CAAC,CAAC;IACjG,CAAC;IAED,OAAO;QACL,GAAG,UAAU;QACb,aAAa,EAAE,cAAc,IAAI,8BAA8B;QAC/D,uBAAuB,EAAE,cAAc,IAAI,wCAAwC;KACpF,CAAC;AACJ,CAAC"}
|