@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -16,18 +16,22 @@ import {
|
|
|
16
16
|
import { JSONBig } from '@powersync/service-jsonbig';
|
|
17
17
|
import { ParameterLookupRows, ScopedParameterLookup, SqliteJsonRow } from '@powersync/service-sync-rules';
|
|
18
18
|
import * as bson from 'bson';
|
|
19
|
-
import { idPrefixFilter, mapOpEntry, readSingleBatch, setSessionSnapshotTime } from '../../../utils/util.js';
|
|
20
|
-
import { MongoBucketStorage } from '../../MongoBucketStorage.js';
|
|
21
19
|
import {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
20
|
+
clearCollectionInIdBatches,
|
|
21
|
+
clearCollectionInIdRanges,
|
|
22
|
+
idPrefixFilter,
|
|
23
|
+
mapOpEntry,
|
|
24
|
+
readSingleBatch,
|
|
25
|
+
retryOnMongoMaxTimeMSExpired,
|
|
26
|
+
setSessionSnapshotTime
|
|
27
|
+
} from '../../../utils/util.js';
|
|
28
|
+
import { MongoBucketStorage } from '../../MongoBucketStorage.js';
|
|
29
|
+
import { MongoSyncBucketStorageCheckpoint } from '../common/MongoSyncBucketStorageCheckpoint.js';
|
|
25
30
|
import { SourceKey } from '../models.js';
|
|
26
|
-
import { MongoBucketBatchOptions } from '../MongoBucketBatch.js';
|
|
27
31
|
import { MongoChecksums } from '../MongoChecksums.js';
|
|
28
32
|
import { MongoCompactOptions, MongoCompactor } from '../MongoCompactor.js';
|
|
29
33
|
import { MongoParameterCompactor } from '../MongoParameterCompactor.js';
|
|
30
|
-
import {
|
|
34
|
+
import { MongoPersistedReplicationStream } from '../MongoPersistedReplicationStream.js';
|
|
31
35
|
import { MongoSyncBucketStorage, MongoSyncBucketStorageOptions } from '../MongoSyncBucketStorage.js';
|
|
32
36
|
import {
|
|
33
37
|
BucketDataDocumentV1,
|
|
@@ -42,33 +46,65 @@ import { MongoCompactorV1 } from './MongoCompactorV1.js';
|
|
|
42
46
|
import { MongoParameterCompactorV1 } from './MongoParameterCompactorV1.js';
|
|
43
47
|
import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
|
|
44
48
|
|
|
49
|
+
export interface MongoSyncBucketStorageContextV1 {
|
|
50
|
+
db: VersionedPowerSyncMongoV1;
|
|
51
|
+
replicationStreamId: number;
|
|
52
|
+
readPreference?: mongo.ReadPreference;
|
|
53
|
+
}
|
|
54
|
+
|
|
45
55
|
export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
46
|
-
// Declare types to be more specific
|
|
47
56
|
declare readonly db: VersionedPowerSyncMongoV1;
|
|
48
57
|
declare readonly checksums: MongoChecksumsV1;
|
|
49
58
|
|
|
50
59
|
constructor(
|
|
51
60
|
factory: MongoBucketStorage,
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
61
|
+
replicationStreamId: number,
|
|
62
|
+
replicationStream: MongoPersistedReplicationStream,
|
|
63
|
+
replicationStreamName: string,
|
|
55
64
|
writeCheckpointMode: storage.WriteCheckpointMode | undefined,
|
|
56
65
|
options: MongoSyncBucketStorageOptions
|
|
57
66
|
) {
|
|
58
|
-
super(factory,
|
|
67
|
+
super(factory, replicationStreamId, replicationStream, replicationStreamName, writeCheckpointMode, options);
|
|
59
68
|
}
|
|
60
69
|
|
|
61
70
|
protected async initializeVersionStorage(): Promise<void> {}
|
|
62
71
|
|
|
63
|
-
protected
|
|
64
|
-
|
|
72
|
+
protected async fetchPersistedOpHead(): Promise<InternalOpId | null> {
|
|
73
|
+
const doc = (await this.db.sync_rules.findOne(
|
|
74
|
+
{ _id: this.replicationStreamId },
|
|
75
|
+
{ projection: { keepalive_op: 1, last_checkpoint: 1 } }
|
|
76
|
+
)) as SyncRuleDocumentV1;
|
|
77
|
+
// keepalive_op covers ops not yet in a checkpoint (cleared once checkpointed),
|
|
78
|
+
// so the head is the max of the two.
|
|
79
|
+
const keepaliveOp = doc?.keepalive_op == null ? null : BigInt(doc.keepalive_op);
|
|
80
|
+
const lastCheckpoint = doc?.last_checkpoint ?? null;
|
|
81
|
+
if (keepaliveOp == null && lastCheckpoint == null) {
|
|
82
|
+
return null;
|
|
83
|
+
}
|
|
84
|
+
return (keepaliveOp ?? 0n) > (lastCheckpoint ?? 0n) ? keepaliveOp : lastCheckpoint;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
protected async createWriterImpl(options: storage.CreateWriterOptions): Promise<storage.BucketStorageBatch> {
|
|
88
|
+
const doc = (await this.db.sync_rules.findOne(
|
|
89
|
+
{
|
|
90
|
+
_id: this.replicationStreamId
|
|
91
|
+
},
|
|
92
|
+
{ projection: { last_checkpoint_lsn: 1, keepalive_op: 1, snapshot_lsn: 1 } }
|
|
93
|
+
)) as SyncRuleDocumentV1;
|
|
94
|
+
|
|
95
|
+
return new MongoBucketBatchV1({
|
|
96
|
+
...this.writerBatchOptions(options),
|
|
97
|
+
// Resume from the last consistent checkpoint, or the in-progress snapshot position if it is newer.
|
|
98
|
+
resumeFromLsn: maxLsn(doc?.last_checkpoint_lsn, doc?.snapshot_lsn),
|
|
99
|
+
keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null
|
|
100
|
+
});
|
|
65
101
|
}
|
|
66
102
|
|
|
67
103
|
protected async fetchCheckpointState(
|
|
68
104
|
session: mongo.ClientSession
|
|
69
105
|
): Promise<{ checkpoint: bigint; lsn: string | null } | null> {
|
|
70
106
|
const doc = (await this.db.sync_rules.findOne(
|
|
71
|
-
{ _id: this.
|
|
107
|
+
{ _id: this.replicationStreamId },
|
|
72
108
|
{
|
|
73
109
|
session,
|
|
74
110
|
projection: { _id: 1, state: 1, last_checkpoint: 1, last_checkpoint_lsn: 1, snapshot_done: 1 }
|
|
@@ -83,26 +119,10 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
|
83
119
|
};
|
|
84
120
|
}
|
|
85
121
|
|
|
86
|
-
protected async getWriterSyncState() {
|
|
87
|
-
const doc = (await this.db.sync_rules.findOne(
|
|
88
|
-
{
|
|
89
|
-
_id: this.group_id
|
|
90
|
-
},
|
|
91
|
-
{ projection: { last_checkpoint_lsn: 1, keepalive_op: 1, snapshot_lsn: 1 } }
|
|
92
|
-
)) as SyncRuleDocumentV1;
|
|
93
|
-
const checkpointLsn = doc?.last_checkpoint_lsn ?? null;
|
|
94
|
-
return {
|
|
95
|
-
lastCheckpointLsn: checkpointLsn,
|
|
96
|
-
resumeFromLsn: maxLsn(checkpointLsn, doc?.snapshot_lsn),
|
|
97
|
-
keepaliveOp: doc?.keepalive_op ? BigInt(doc.keepalive_op) : null,
|
|
98
|
-
syncConfigId: null
|
|
99
|
-
};
|
|
100
|
-
}
|
|
101
|
-
|
|
102
122
|
protected async terminateSyncRuleState(): Promise<void> {
|
|
103
123
|
await this.db.sync_rules.updateOne(
|
|
104
124
|
{
|
|
105
|
-
_id: this.
|
|
125
|
+
_id: this.replicationStreamId
|
|
106
126
|
},
|
|
107
127
|
{
|
|
108
128
|
$set: {
|
|
@@ -114,18 +134,16 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
|
114
134
|
);
|
|
115
135
|
}
|
|
116
136
|
|
|
117
|
-
protected async getStatusImpl(): Promise<storage.
|
|
137
|
+
protected async getStatusImpl(): Promise<storage.ReplicationStreamStatus> {
|
|
118
138
|
const doc = (await this.db.sync_rules.findOne(
|
|
119
139
|
{
|
|
120
|
-
_id: this.
|
|
140
|
+
_id: this.replicationStreamId
|
|
121
141
|
},
|
|
122
142
|
{
|
|
123
143
|
projection: {
|
|
124
144
|
snapshot_done: 1,
|
|
125
145
|
last_checkpoint_lsn: 1,
|
|
126
|
-
|
|
127
|
-
snapshot_lsn: 1,
|
|
128
|
-
keepalive_op: 1
|
|
146
|
+
snapshot_lsn: 1
|
|
129
147
|
}
|
|
130
148
|
}
|
|
131
149
|
)) as SyncRuleDocumentV1;
|
|
@@ -134,18 +152,15 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
|
134
152
|
}
|
|
135
153
|
|
|
136
154
|
return {
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
active: doc.state == storage.SyncRuleState.ACTIVE,
|
|
140
|
-
checkpoint_lsn: doc.last_checkpoint_lsn,
|
|
141
|
-
keepalive_op: doc.keepalive_op == null ? null : BigInt(doc.keepalive_op)
|
|
155
|
+
snapshotDone: doc.snapshot_done && doc.last_checkpoint_lsn != null,
|
|
156
|
+
resumeLsn: maxLsn(doc.snapshot_lsn, doc.last_checkpoint_lsn)
|
|
142
157
|
};
|
|
143
158
|
}
|
|
144
159
|
|
|
145
160
|
protected async clearSyncRuleState(): Promise<void> {
|
|
146
161
|
await this.db.sync_rules.updateOne(
|
|
147
162
|
{
|
|
148
|
-
_id: this.
|
|
163
|
+
_id: this.replicationStreamId
|
|
149
164
|
},
|
|
150
165
|
{
|
|
151
166
|
$set: {
|
|
@@ -164,10 +179,10 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
|
164
179
|
}
|
|
165
180
|
|
|
166
181
|
protected createMongoChecksums(options: MongoSyncBucketStorageOptions): MongoChecksums {
|
|
167
|
-
return new MongoChecksumsV1(this.db, this.
|
|
182
|
+
return new MongoChecksumsV1(this.db, this.replicationStreamId, {
|
|
168
183
|
...options.checksumOptions,
|
|
169
|
-
|
|
170
|
-
|
|
184
|
+
checksumCacheTtlMs: options.checksumCacheTtlMs,
|
|
185
|
+
storageConfig: options?.storageConfig
|
|
171
186
|
});
|
|
172
187
|
}
|
|
173
188
|
|
|
@@ -179,14 +194,14 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
|
179
194
|
checkpoint: InternalOpId,
|
|
180
195
|
options: storage.CompactOptions
|
|
181
196
|
): MongoParameterCompactor {
|
|
182
|
-
return new MongoParameterCompactorV1(this.db, this.
|
|
197
|
+
return new MongoParameterCompactorV1(this.db, this.replicationStreamId, checkpoint, options);
|
|
183
198
|
}
|
|
184
199
|
|
|
185
|
-
protected
|
|
200
|
+
protected get versionContext(): MongoSyncBucketStorageContextV1 {
|
|
186
201
|
return {
|
|
187
202
|
db: this.db,
|
|
188
|
-
|
|
189
|
-
|
|
203
|
+
replicationStreamId: this.replicationStreamId,
|
|
204
|
+
readPreference: this.readPreference
|
|
190
205
|
};
|
|
191
206
|
}
|
|
192
207
|
|
|
@@ -199,7 +214,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
|
199
214
|
}
|
|
200
215
|
|
|
201
216
|
protected getBucketDataBatchImpl(
|
|
202
|
-
checkpoint:
|
|
217
|
+
checkpoint: MongoSyncBucketStorageCheckpoint,
|
|
203
218
|
dataBuckets: storage.BucketDataRequest[],
|
|
204
219
|
options?: storage.BucketDataBatchOptions
|
|
205
220
|
): AsyncIterable<storage.SyncBucketDataChunk> {
|
|
@@ -207,72 +222,77 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
|
207
222
|
}
|
|
208
223
|
|
|
209
224
|
protected async clearBucketData(signal?: AbortSignal): Promise<void> {
|
|
210
|
-
await
|
|
225
|
+
await clearCollectionInIdRanges(
|
|
226
|
+
this.logger,
|
|
211
227
|
'bucket data',
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
),
|
|
219
|
-
signal
|
|
228
|
+
this.db.bucket_data,
|
|
229
|
+
{
|
|
230
|
+
_id: idPrefixFilter<BucketDataKeyV1>({ g: this.replicationStreamId }, ['b', 'o'])
|
|
231
|
+
},
|
|
232
|
+
signal,
|
|
233
|
+
this.clearBatchThrottleRate
|
|
220
234
|
);
|
|
221
235
|
}
|
|
222
236
|
|
|
223
237
|
protected async clearParameterIndexes(signal?: AbortSignal): Promise<void> {
|
|
224
|
-
await
|
|
238
|
+
await clearCollectionInIdBatches(
|
|
239
|
+
this.logger,
|
|
225
240
|
'parameter index',
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
),
|
|
233
|
-
signal
|
|
241
|
+
this.db.parameterIndexV1,
|
|
242
|
+
{
|
|
243
|
+
'key.g': this.replicationStreamId
|
|
244
|
+
},
|
|
245
|
+
signal,
|
|
246
|
+
this.clearBatchThrottleRate
|
|
234
247
|
);
|
|
235
248
|
}
|
|
236
249
|
|
|
237
250
|
protected async clearSourceRecords(signal?: AbortSignal): Promise<void> {
|
|
238
|
-
await
|
|
251
|
+
await clearCollectionInIdRanges(
|
|
252
|
+
this.logger,
|
|
239
253
|
'source records',
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
),
|
|
247
|
-
signal
|
|
254
|
+
this.db.sourceRecordsV1,
|
|
255
|
+
{
|
|
256
|
+
_id: idPrefixFilter<SourceKey>({ g: this.replicationStreamId }, ['t', 'k'])
|
|
257
|
+
},
|
|
258
|
+
signal,
|
|
259
|
+
this.clearBatchThrottleRate
|
|
248
260
|
);
|
|
249
261
|
}
|
|
250
262
|
|
|
251
263
|
protected async clearBucketState(signal?: AbortSignal): Promise<void> {
|
|
252
|
-
await
|
|
264
|
+
await clearCollectionInIdRanges(
|
|
265
|
+
this.logger,
|
|
253
266
|
'bucket state',
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
),
|
|
261
|
-
signal
|
|
267
|
+
this.db.bucketStateV1,
|
|
268
|
+
{
|
|
269
|
+
_id: idPrefixFilter<BucketStateDocument['_id']>({ g: this.replicationStreamId }, ['b'])
|
|
270
|
+
},
|
|
271
|
+
signal,
|
|
272
|
+
this.clearBatchThrottleRate
|
|
262
273
|
);
|
|
263
274
|
}
|
|
264
275
|
|
|
265
276
|
protected async clearSourceTables(signal?: AbortSignal): Promise<void> {
|
|
266
|
-
await
|
|
267
|
-
'source tables',
|
|
277
|
+
await retryOnMongoMaxTimeMSExpired(
|
|
268
278
|
() =>
|
|
269
|
-
this.db.
|
|
279
|
+
this.db.sourceTablesV1(this.replicationStreamId).deleteMany(
|
|
270
280
|
{
|
|
271
|
-
group_id: this.
|
|
281
|
+
group_id: this.replicationStreamId
|
|
272
282
|
},
|
|
273
283
|
{ maxTimeMS: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS }
|
|
274
284
|
),
|
|
275
|
-
|
|
285
|
+
{
|
|
286
|
+
signal,
|
|
287
|
+
abortMessage: 'Aborted clearing data',
|
|
288
|
+
// This is a fairly long delay - only expected to hit this when the storage database is under high load.
|
|
289
|
+
retryDelayMs: lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS,
|
|
290
|
+
onRetry: () => {
|
|
291
|
+
this.logger.info(
|
|
292
|
+
`Clearing batch of source tables timed out after ${lib_mongo.db.MONGO_CLEAR_OPERATION_TIMEOUT_MS}ms, retrying...`
|
|
293
|
+
);
|
|
294
|
+
}
|
|
295
|
+
}
|
|
276
296
|
);
|
|
277
297
|
}
|
|
278
298
|
|
|
@@ -290,7 +310,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage {
|
|
|
290
310
|
}
|
|
291
311
|
|
|
292
312
|
export async function getParameterSetsV1(
|
|
293
|
-
ctx:
|
|
313
|
+
ctx: MongoSyncBucketStorageContextV1,
|
|
294
314
|
checkpoint: MongoSyncBucketStorageCheckpoint,
|
|
295
315
|
lookups: ScopedParameterLookup[],
|
|
296
316
|
limit: number
|
|
@@ -306,7 +326,7 @@ export async function getParameterSetsV1(
|
|
|
306
326
|
[
|
|
307
327
|
{
|
|
308
328
|
$match: {
|
|
309
|
-
'key.g': ctx.
|
|
329
|
+
'key.g': ctx.replicationStreamId,
|
|
310
330
|
lookup: { $in: lookupFilter },
|
|
311
331
|
_id: { $lte: checkpoint.checkpoint }
|
|
312
332
|
}
|
|
@@ -364,31 +384,44 @@ export async function getParameterSetsV1(
|
|
|
364
384
|
}
|
|
365
385
|
|
|
366
386
|
export async function* getBucketDataBatchV1(
|
|
367
|
-
ctx:
|
|
368
|
-
checkpoint:
|
|
387
|
+
ctx: MongoSyncBucketStorageContextV1,
|
|
388
|
+
checkpoint: MongoSyncBucketStorageCheckpoint,
|
|
369
389
|
dataBuckets: storage.BucketDataRequest[],
|
|
370
390
|
options?: storage.BucketDataBatchOptions
|
|
371
391
|
): AsyncIterable<storage.SyncBucketDataChunk> {
|
|
372
392
|
if (dataBuckets.length == 0) {
|
|
373
393
|
return;
|
|
374
394
|
}
|
|
395
|
+
const readPreference = options?.requestHint == 'bulk' ? ctx.readPreference : undefined;
|
|
396
|
+
const readConcern = ctx.readPreference == null ? undefined : 'majority';
|
|
397
|
+
const session =
|
|
398
|
+
readPreference == null || checkpoint.snapshotTime == null
|
|
399
|
+
? undefined
|
|
400
|
+
: ctx.db.client.startSession({ causalConsistency: true });
|
|
401
|
+
await using _ = { [Symbol.asyncDispose]: async () => session?.endSession() };
|
|
402
|
+
|
|
403
|
+
if (session != null) {
|
|
404
|
+
session.advanceOperationTime(checkpoint.snapshotTime);
|
|
405
|
+
session.advanceClusterTime(checkpoint.clusterTime);
|
|
406
|
+
}
|
|
407
|
+
|
|
375
408
|
let filters: mongo.Filter<BucketDataDocumentV1>[] = [];
|
|
376
409
|
const bucketMap = new Map(dataBuckets.map((request) => [request.bucket, request.start]));
|
|
377
410
|
|
|
378
|
-
if (checkpoint == null) {
|
|
411
|
+
if (checkpoint.checkpoint == null) {
|
|
379
412
|
throw new Error('checkpoint is null');
|
|
380
413
|
}
|
|
381
|
-
const end = checkpoint;
|
|
414
|
+
const end = checkpoint.checkpoint;
|
|
382
415
|
for (let { bucket: name, start } of dataBuckets) {
|
|
383
416
|
filters.push({
|
|
384
417
|
_id: {
|
|
385
418
|
$gt: {
|
|
386
|
-
g: ctx.
|
|
419
|
+
g: ctx.replicationStreamId,
|
|
387
420
|
b: name,
|
|
388
421
|
o: start
|
|
389
422
|
},
|
|
390
423
|
$lte: {
|
|
391
|
-
g: ctx.
|
|
424
|
+
g: ctx.replicationStreamId,
|
|
392
425
|
b: name,
|
|
393
426
|
o: end as any
|
|
394
427
|
}
|
|
@@ -399,12 +432,16 @@ export async function* getBucketDataBatchV1(
|
|
|
399
432
|
const batchLimit = options?.limit ?? storage.DEFAULT_DOCUMENT_BATCH_LIMIT;
|
|
400
433
|
const chunkSizeLimitBytes = options?.chunkLimitBytes ?? storage.DEFAULT_DOCUMENT_CHUNK_LIMIT_BYTES;
|
|
401
434
|
|
|
435
|
+
// raw: true returns Buffers, but the driver typing doesn't reflect that
|
|
436
|
+
// without an explicit cast to FindCursor<Buffer>.
|
|
402
437
|
const cursor = ctx.db.bucket_data.find(
|
|
403
438
|
{
|
|
404
439
|
$or: filters
|
|
405
440
|
},
|
|
406
441
|
{
|
|
407
|
-
session
|
|
442
|
+
session,
|
|
443
|
+
readPreference,
|
|
444
|
+
readConcern,
|
|
408
445
|
sort: { _id: 1 },
|
|
409
446
|
limit: batchLimit,
|
|
410
447
|
batchSize: batchLimit + 1,
|
|
@@ -481,14 +518,14 @@ export async function* getBucketDataBatchV1(
|
|
|
481
518
|
}
|
|
482
519
|
|
|
483
520
|
export async function getDataBucketChangesV1(
|
|
484
|
-
ctx:
|
|
521
|
+
ctx: MongoSyncBucketStorageContextV1,
|
|
485
522
|
options: GetCheckpointChangesOptions
|
|
486
523
|
): Promise<Pick<CheckpointChanges, 'updatedDataBuckets' | 'invalidateDataBuckets'>> {
|
|
487
524
|
const limit = 1000;
|
|
488
525
|
const bucketStateUpdates = await ctx.db.bucketStateV1
|
|
489
526
|
.find(
|
|
490
527
|
{
|
|
491
|
-
'_id.g': ctx.
|
|
528
|
+
'_id.g': ctx.replicationStreamId,
|
|
492
529
|
last_op: { $gt: options.lastCheckpoint.checkpoint }
|
|
493
530
|
},
|
|
494
531
|
{
|
|
@@ -512,7 +549,7 @@ export async function getDataBucketChangesV1(
|
|
|
512
549
|
}
|
|
513
550
|
|
|
514
551
|
export async function getParameterBucketChangesV1(
|
|
515
|
-
ctx:
|
|
552
|
+
ctx: MongoSyncBucketStorageContextV1,
|
|
516
553
|
options: GetCheckpointChangesOptions
|
|
517
554
|
): Promise<Pick<CheckpointChanges, 'updatedParameterLookups' | 'invalidateParameterBuckets'>> {
|
|
518
555
|
const limit = 1000;
|
|
@@ -520,7 +557,7 @@ export async function getParameterBucketChangesV1(
|
|
|
520
557
|
.find(
|
|
521
558
|
{
|
|
522
559
|
_id: { $gt: options.lastCheckpoint.checkpoint, $lte: options.nextCheckpoint.checkpoint },
|
|
523
|
-
'key.g': ctx.
|
|
560
|
+
'key.g': ctx.replicationStreamId
|
|
524
561
|
},
|
|
525
562
|
{
|
|
526
563
|
projection: {
|
|
@@ -1,58 +1,50 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { InternalOpId } from '@powersync/service-core';
|
|
3
3
|
import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
|
|
4
|
-
import {
|
|
5
|
-
BucketDataDocumentGeneric,
|
|
6
|
-
BucketDataDocumentGenericId,
|
|
7
|
-
SingleBucketStore
|
|
8
|
-
} from '../common/SingleBucketStore.js';
|
|
9
4
|
import { BucketDataProperties } from '../models.js';
|
|
10
5
|
import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js';
|
|
11
6
|
import { BucketDataDocumentV1, BucketDataKeyV1, serializeBucketDataV1 } from './models.js';
|
|
12
7
|
|
|
13
|
-
export class SingleBucketStoreV1
|
|
14
|
-
public readonly collection: mongo.Collection<
|
|
8
|
+
export class SingleBucketStoreV1 {
|
|
9
|
+
public readonly collection: mongo.Collection<BucketDataDocumentV1>;
|
|
15
10
|
|
|
16
11
|
constructor(
|
|
17
|
-
|
|
12
|
+
db: VersionedPowerSyncMongoV1,
|
|
18
13
|
public readonly key: BucketKey
|
|
19
14
|
) {
|
|
20
|
-
this.collection = db.bucketDataV1
|
|
15
|
+
this.collection = db.bucketDataV1;
|
|
21
16
|
}
|
|
22
17
|
|
|
23
|
-
docId(o: InternalOpId):
|
|
24
|
-
// `satisfies BucketDataKeyV1` checks that we use the correct type for V1 storage
|
|
25
|
-
// `as anyt` is to allow casting to the interface virtual type
|
|
18
|
+
docId(o: InternalOpId): BucketDataKeyV1 {
|
|
26
19
|
return {
|
|
27
20
|
g: this.key.replicationStreamId,
|
|
28
21
|
b: this.key.bucket,
|
|
29
22
|
o
|
|
30
|
-
}
|
|
23
|
+
};
|
|
31
24
|
}
|
|
32
25
|
|
|
33
|
-
get minId():
|
|
26
|
+
get minId(): BucketDataKeyV1 {
|
|
34
27
|
return {
|
|
35
28
|
g: this.key.replicationStreamId,
|
|
36
29
|
b: this.key.bucket,
|
|
37
|
-
o: new mongo.MinKey()
|
|
38
|
-
}
|
|
30
|
+
o: new mongo.MinKey() as any
|
|
31
|
+
};
|
|
39
32
|
}
|
|
40
33
|
|
|
41
|
-
get maxId():
|
|
34
|
+
get maxId(): BucketDataKeyV1 {
|
|
42
35
|
return {
|
|
43
36
|
g: this.key.replicationStreamId,
|
|
44
37
|
b: this.key.bucket,
|
|
45
|
-
o: new mongo.MaxKey()
|
|
46
|
-
}
|
|
38
|
+
o: new mongo.MaxKey() as any
|
|
39
|
+
};
|
|
47
40
|
}
|
|
48
41
|
|
|
49
|
-
toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>):
|
|
50
|
-
return serializeBucketDataV1({ bucketKey: this.key, ...source })
|
|
42
|
+
toPersistedDocument(source: Omit<BucketDataDoc, 'bucketKey'>): BucketDataDocumentV1 {
|
|
43
|
+
return serializeBucketDataV1({ bucketKey: this.key, ...source });
|
|
51
44
|
}
|
|
52
45
|
|
|
53
|
-
fromPersistedDocument(doc:
|
|
54
|
-
const
|
|
55
|
-
const { _id, ...rest } = document;
|
|
46
|
+
fromPersistedDocument(doc: BucketDataDocumentV1): BucketDataDoc {
|
|
47
|
+
const { _id, ...rest } = doc;
|
|
56
48
|
return {
|
|
57
49
|
bucketKey: this.key,
|
|
58
50
|
o: _id.o,
|
|
@@ -61,10 +53,9 @@ export class SingleBucketStoreV1 implements SingleBucketStore {
|
|
|
61
53
|
}
|
|
62
54
|
|
|
63
55
|
fromPartialPersistedDocument<T extends keyof BucketDataProperties>(
|
|
64
|
-
doc: Pick<
|
|
56
|
+
doc: Pick<BucketDataDocumentV1, '_id' | T>
|
|
65
57
|
): Pick<BucketDataDoc, 'bucketKey' | 'o' | T> {
|
|
66
|
-
const
|
|
67
|
-
const { _id, ...rest } = document;
|
|
58
|
+
const { _id, ...rest } = doc;
|
|
68
59
|
return {
|
|
69
60
|
bucketKey: this.key,
|
|
70
61
|
o: _id.o,
|
|
@@ -1,7 +1,13 @@
|
|
|
1
1
|
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { BaseVersionedPowerSyncMongo } from '../common/VersionedPowerSyncMongoBase.js';
|
|
3
3
|
import { CommonSourceTableDocument } from '../models.js';
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
BucketDataDocumentV1,
|
|
6
|
+
BucketParameterDocument,
|
|
7
|
+
BucketStateDocumentV1,
|
|
8
|
+
CurrentDataDocument,
|
|
9
|
+
SourceTableDocumentV1
|
|
10
|
+
} from './models.js';
|
|
5
11
|
|
|
6
12
|
export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
|
|
7
13
|
get sourceRecordsV1(): mongo.Collection<CurrentDataDocument> {
|
|
@@ -16,6 +22,10 @@ export class VersionedPowerSyncMongoV1 extends BaseVersionedPowerSyncMongo {
|
|
|
16
22
|
return this.upstream.source_tables as any as mongo.Collection<CommonSourceTableDocument>;
|
|
17
23
|
}
|
|
18
24
|
|
|
25
|
+
sourceTablesV1(_replicationStreamId: number): mongo.Collection<SourceTableDocumentV1> {
|
|
26
|
+
return this.upstream.source_tables as any as mongo.Collection<SourceTableDocumentV1>;
|
|
27
|
+
}
|
|
28
|
+
|
|
19
29
|
async initializeStreamStorage(_replicationStreamId: number): Promise<void> {}
|
|
20
30
|
|
|
21
31
|
get bucketDataV1(): mongo.Collection<BucketDataDocumentV1> {
|
|
@@ -78,7 +78,7 @@ export interface SourceTableDocumentV1 extends SourceTableDocument {
|
|
|
78
78
|
group_id: number;
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
-
export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckpointFields
|
|
81
|
+
export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckpointFields {
|
|
82
82
|
content: string;
|
|
83
83
|
serialized_plan?: SerializedSyncPlan | null;
|
|
84
84
|
|
|
@@ -88,6 +88,16 @@ export interface SyncRuleDocumentV1 extends SyncRuleDocumentBase, SyncRuleCheckp
|
|
|
88
88
|
* Can only be false if state == PROCESSING.
|
|
89
89
|
*/
|
|
90
90
|
snapshot_done: boolean;
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Goes together with no_checkpoint_before.
|
|
94
|
+
*
|
|
95
|
+
* If a keepalive is triggered that creates the checkpoint > no_checkpoint_before,
|
|
96
|
+
* then the checkpoint must be equal to this keepalive_op.
|
|
97
|
+
*
|
|
98
|
+
* string for legacy reasons - it's always a number.
|
|
99
|
+
*/
|
|
100
|
+
keepalive_op: string | null;
|
|
91
101
|
}
|
|
92
102
|
|
|
93
103
|
export interface BucketStateDocumentV1 extends BucketStateDocumentBase {
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { InternalOpId } from '@powersync/service-core';
|
|
3
|
+
import { BucketKey } from '../common/BucketDataDoc.js';
|
|
4
|
+
import { BucketDataKey } from '../models.js';
|
|
5
|
+
import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
|
|
6
|
+
import { BucketDataDocumentV3 } from './models.js';
|
|
7
|
+
|
|
8
|
+
export class BucketDataContextV3 {
|
|
9
|
+
public readonly collection: mongo.Collection<BucketDataDocumentV3>;
|
|
10
|
+
|
|
11
|
+
constructor(
|
|
12
|
+
db: VersionedPowerSyncMongoV3,
|
|
13
|
+
public readonly key: BucketKey
|
|
14
|
+
) {
|
|
15
|
+
this.collection = db.bucketData(key.replicationStreamId, key.definitionId);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
docId(o: InternalOpId): BucketDataKey {
|
|
19
|
+
return {
|
|
20
|
+
b: this.key.bucket,
|
|
21
|
+
o
|
|
22
|
+
};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
get minId(): BucketDataKey {
|
|
26
|
+
// MongoDB's MinKey sentinel does not match the bigint type for _id.o.
|
|
27
|
+
return {
|
|
28
|
+
b: this.key.bucket,
|
|
29
|
+
o: new mongo.MinKey()
|
|
30
|
+
} as unknown as BucketDataKey;
|
|
31
|
+
}
|
|
32
|
+
}
|