@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260803144812
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +41 -16
- package/dist/storage/MongoBucketStorage.js +226 -71
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +56 -25
- package/dist/storage/implementation/MongoBucketBatch.js +110 -62
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +17 -19
- package/dist/storage/implementation/MongoCompactor.js +80 -263
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +7 -6
- package/dist/storage/implementation/MongoParameterCompactor.js +25 -2
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +123 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +39 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +57 -29
- package/dist/storage/implementation/MongoSyncBucketStorage.js +87 -61
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +3 -3
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +2 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +171 -24
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +8 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +7 -2
- package/dist/storage/implementation/common/PersistedBatch.js +12 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.js +17 -15
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +20 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +9 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +61 -29
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +11 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js +251 -2
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -19
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +82 -58
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +8 -4
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +309 -259
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +40 -2
- package/dist/storage/implementation/v3/MongoCompactorV3.js +647 -23
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +67 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +351 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +45 -30
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +272 -140
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +6 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +84 -21
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +15 -13
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +77 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +101 -21
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +21 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +44 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +30 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +70 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +146 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +42 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +285 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +49 -3
- package/dist/types/types.js +48 -1
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +128 -4
- package/dist/utils/util.js.map +1 -1
- package/package.json +12 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +347 -88
- package/src/storage/implementation/MongoBucketBatch.ts +147 -82
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +102 -316
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +25 -6
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +154 -0
- package/src/storage/implementation/MongoStorageProvider.ts +46 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +121 -87
- package/src/storage/implementation/MongoSyncRulesLock.ts +7 -4
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +202 -28
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +9 -0
- package/src/storage/implementation/common/PersistedBatch.ts +19 -3
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +19 -20
- package/src/storage/implementation/models.ts +20 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +75 -37
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +294 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +142 -105
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +11 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +382 -321
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +847 -32
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +482 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +359 -187
- package/src/storage/implementation/v3/PersistedBatchV3.ts +118 -37
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +46 -36
- package/src/storage/implementation/v3/bucket-format.ts +97 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/models.ts +132 -55
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +54 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +29 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +171 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +343 -0
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/types/types.ts +58 -1
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +185 -6
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +35 -617
- package/test/src/cleanup-stopped-sync-configs.test.ts +671 -0
- package/test/src/helpers/MemoryObjectStorage.ts +53 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +227 -13
- package/test/src/storage_compacting.test.ts +1951 -12
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +191 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +417 -0
- package/test/src/storage_s3_reading.test.ts +526 -0
- package/test/src/storage_s3_writing.test.ts +135 -0
- package/test/src/storage_sync.test.ts +1180 -73
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -1,15 +1,25 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
1
|
+
import {
|
|
2
|
+
InternalOpId,
|
|
3
|
+
PersistedDefinitionMapping,
|
|
4
|
+
SerializedSyncPlan,
|
|
5
|
+
SyncRuleState,
|
|
6
|
+
deserializeParameterLookup as deserializeParameterLookupCore
|
|
7
|
+
} from '@powersync/service-core';
|
|
8
|
+
import {
|
|
9
|
+
BucketDefinitionId,
|
|
10
|
+
ParameterIndexId,
|
|
11
|
+
ScopedParameterLookup,
|
|
12
|
+
SqliteJsonValue
|
|
13
|
+
} from '@powersync/service-sync-rules';
|
|
3
14
|
import * as bson from 'bson';
|
|
4
|
-
import { BucketDataDoc, BucketKey } from '../common/BucketDataDoc.js';
|
|
5
15
|
import {
|
|
6
|
-
BucketDataDocumentBase,
|
|
7
16
|
BucketDataKey,
|
|
8
17
|
BucketParameterDocumentBase,
|
|
9
18
|
BucketStateDocumentBase,
|
|
10
19
|
CurrentBucket,
|
|
20
|
+
OpType,
|
|
11
21
|
ReplicaId,
|
|
12
|
-
|
|
22
|
+
SourceTableDocumentSnapshotStatus,
|
|
13
23
|
SourceTableKey,
|
|
14
24
|
SyncRuleCheckpointFields,
|
|
15
25
|
SyncRuleDocumentBase,
|
|
@@ -19,7 +29,7 @@ import {
|
|
|
19
29
|
/**
|
|
20
30
|
* Embedded in sync_rules.sync_configs.
|
|
21
31
|
*/
|
|
22
|
-
export interface SyncRuleConfigStateV3 extends SyncRuleCheckpointFields
|
|
32
|
+
export interface SyncRuleConfigStateV3 extends SyncRuleCheckpointFields {
|
|
23
33
|
_id: bson.ObjectId;
|
|
24
34
|
|
|
25
35
|
/**
|
|
@@ -47,6 +57,31 @@ export interface ReplicationStreamDocumentV3 extends SyncRuleDocumentBase {
|
|
|
47
57
|
* but the model allows multiple configs in any state.
|
|
48
58
|
*/
|
|
49
59
|
sync_configs: SyncRuleConfigStateV3[];
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* The monotonic head of the stream's op sequence: the highest op id persisted to bucket data,
|
|
63
|
+
* whether or not yet covered by a checkpoint.
|
|
64
|
+
*
|
|
65
|
+
* This is shared across all sync configs of the stream (they share the global op sequence).
|
|
66
|
+
* It is never cleared, only `$max`-advanced. A newly-appended config that replicates nothing
|
|
67
|
+
* adopts this value as its checkpoint rather than starting at 0.
|
|
68
|
+
*
|
|
69
|
+
* Stored as a mongo Long, nullable.
|
|
70
|
+
*/
|
|
71
|
+
last_persisted_op?: bigint | null;
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* The stream's replication position: all source changes up to this LSN have been processed,
|
|
75
|
+
* and the resulting ops persisted. Replication resumes from here.
|
|
76
|
+
*
|
|
77
|
+
* Like {@link last_persisted_op}, this is shared across all sync configs of the stream -
|
|
78
|
+
* per-config last_checkpoint_lsn values are consistency markers, not replication positions.
|
|
79
|
+
*
|
|
80
|
+
* Set via setResumeLsn() (snapshot start, and per-batch progress during streaming), and
|
|
81
|
+
* advanced on every commit/keepalive - including checkpoint-blocked ones, since commit
|
|
82
|
+
* flushes first and blocking only delays consistency markers, not data persistence.
|
|
83
|
+
*/
|
|
84
|
+
resume_lsn?: string | null;
|
|
50
85
|
}
|
|
51
86
|
|
|
52
87
|
/**
|
|
@@ -70,16 +105,7 @@ export interface SyncConfigDefinition {
|
|
|
70
105
|
content: string;
|
|
71
106
|
serialized_plan?: SerializedSyncPlan | null;
|
|
72
107
|
|
|
73
|
-
rule_mapping:
|
|
74
|
-
/**
|
|
75
|
-
* Map of uniqueName -> id, unique per replication stream.
|
|
76
|
-
*/
|
|
77
|
-
definitions: Record<string, string>;
|
|
78
|
-
/**
|
|
79
|
-
* Map of (lookupName, queryId) -> id, unique per replication stream.
|
|
80
|
-
*/
|
|
81
|
-
parameter_indexes: Record<string, string>;
|
|
82
|
-
};
|
|
108
|
+
rule_mapping: PersistedDefinitionMapping;
|
|
83
109
|
}
|
|
84
110
|
|
|
85
111
|
export interface CurrentBucketV3 extends CurrentBucket {
|
|
@@ -108,50 +134,26 @@ export interface BucketParameterDocumentV3 extends BucketParameterDocumentBase<S
|
|
|
108
134
|
|
|
109
135
|
export type BucketDataKeyV3 = BucketDataKey;
|
|
110
136
|
|
|
111
|
-
export interface BucketDataDocumentV3 extends BucketDataDocumentBase {
|
|
112
|
-
_id: BucketDataKeyV3;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
export function serializeBucketDataV3(document: BucketDataDoc): BucketDataDocumentV3 {
|
|
116
|
-
const { bucketKey, o } = document;
|
|
117
|
-
return {
|
|
118
|
-
_id: {
|
|
119
|
-
b: bucketKey.bucket,
|
|
120
|
-
o: o
|
|
121
|
-
},
|
|
122
|
-
// List fields directly, so that we don't accidentally persist any unknown fields
|
|
123
|
-
op: document.op,
|
|
124
|
-
source_table: document.source_table,
|
|
125
|
-
source_key: document.source_key,
|
|
126
|
-
table: document.table,
|
|
127
|
-
row_id: document.row_id,
|
|
128
|
-
checksum: document.checksum,
|
|
129
|
-
data: document.data,
|
|
130
|
-
target_op: document.target_op
|
|
131
|
-
};
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
export function loadBucketDataDocumentV3(
|
|
135
|
-
context: Pick<BucketKey, 'replicationStreamId' | 'definitionId'>,
|
|
136
|
-
doc: BucketDataDocumentV3
|
|
137
|
-
): BucketDataDoc {
|
|
138
|
-
const { _id, ...rest } = doc;
|
|
139
|
-
return {
|
|
140
|
-
bucketKey: {
|
|
141
|
-
...context,
|
|
142
|
-
bucket: _id.b
|
|
143
|
-
},
|
|
144
|
-
o: _id.o,
|
|
145
|
-
...rest
|
|
146
|
-
};
|
|
147
|
-
}
|
|
148
|
-
|
|
149
137
|
export function taggedBucketParameterDocumentToV3(document: TaggedBucketParameterDocument): BucketParameterDocumentV3 {
|
|
150
138
|
const { index: _index, ...rest } = document;
|
|
151
139
|
return rest as BucketParameterDocumentV3;
|
|
152
140
|
}
|
|
153
141
|
|
|
154
|
-
export interface
|
|
142
|
+
export interface ReplicaIdColumn {
|
|
143
|
+
name: string;
|
|
144
|
+
type_oid?: number;
|
|
145
|
+
type?: string;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export interface SourceTableDocumentV3 {
|
|
149
|
+
_id: bson.ObjectId;
|
|
150
|
+
connection_id: number;
|
|
151
|
+
relation_id: number | string | undefined;
|
|
152
|
+
schema_name: string;
|
|
153
|
+
table_name: string;
|
|
154
|
+
replica_id_columns: ReplicaIdColumn[];
|
|
155
|
+
snapshot_done: boolean;
|
|
156
|
+
snapshot_status: SourceTableDocumentSnapshotStatus | undefined;
|
|
155
157
|
bucket_data_source_ids: BucketDefinitionId[];
|
|
156
158
|
parameter_lookup_source_ids: ParameterIndexId[];
|
|
157
159
|
latest_pending_delete?: InternalOpId | undefined;
|
|
@@ -162,3 +164,78 @@ export interface BucketStateDocumentV3 extends BucketStateDocumentBase {
|
|
|
162
164
|
d: BucketDefinitionId;
|
|
163
165
|
};
|
|
164
166
|
}
|
|
167
|
+
|
|
168
|
+
export interface BucketOperation {
|
|
169
|
+
o: bigint;
|
|
170
|
+
op: OpType;
|
|
171
|
+
source_table?: bson.ObjectId;
|
|
172
|
+
source_key?: ReplicaId;
|
|
173
|
+
table?: string;
|
|
174
|
+
row_id?: string;
|
|
175
|
+
checksum: bigint;
|
|
176
|
+
data: string | null;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
export interface StorageRef {
|
|
180
|
+
path: string;
|
|
181
|
+
file_size: number;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** An S3 object that may be deleted once its grace period has elapsed. */
|
|
185
|
+
export interface ObjectStorageDeletionMarker {
|
|
186
|
+
_id: bson.ObjectId;
|
|
187
|
+
path: string;
|
|
188
|
+
delete_after: Date;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* A non-empty chunk of a bucket's ordered operation stream.
|
|
193
|
+
*
|
|
194
|
+
* All writers maintain these invariants:
|
|
195
|
+
*
|
|
196
|
+
* - `ops` is non-empty and strictly ordered by `o`.
|
|
197
|
+
* - `min_op` is `ops[0].o` and `_id.o` is `ops[ops.length - 1].o`.
|
|
198
|
+
* - Documents for the same bucket have ordered, disjoint operation ranges,
|
|
199
|
+
* although gaps between ranges are allowed.
|
|
200
|
+
* - `checksum` is the sum of every operation checksum and `count` is
|
|
201
|
+
* `ops.length`.
|
|
202
|
+
*
|
|
203
|
+
* Metadata-only checksum and data queries rely on these invariants to identify
|
|
204
|
+
* boundary-straddling documents without reading `ops`.
|
|
205
|
+
*/
|
|
206
|
+
export interface BucketDataDocumentV3 {
|
|
207
|
+
_id: BucketDataKey;
|
|
208
|
+
min_op: bigint;
|
|
209
|
+
checksum: bigint;
|
|
210
|
+
count: number;
|
|
211
|
+
size: number;
|
|
212
|
+
/**
|
|
213
|
+
* The greatest operation boundary that influenced this document's contents.
|
|
214
|
+
* Bucket-data reads propagate it so serving an earlier checkpoint can be
|
|
215
|
+
* invalidated when compaction changed the operations being returned.
|
|
216
|
+
*/
|
|
217
|
+
target_op?: bigint | null;
|
|
218
|
+
ops?: BucketOperation[];
|
|
219
|
+
storage_ref?: StorageRef;
|
|
220
|
+
/**
|
|
221
|
+
* Present (and always true) when this document contains a CLEAR operation.
|
|
222
|
+
* In that case, this is the first document in the bucket: all preceding
|
|
223
|
+
* documents have been removed.
|
|
224
|
+
*/
|
|
225
|
+
has_clear_op?: true;
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
export function serializeParameterLookup(lookup: ScopedParameterLookup): bson.Binary {
|
|
229
|
+
return new bson.Binary(bson.serialize({ l: lookup.values.slice(2) }));
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
export function deserializeParameterLookup(lookup: bson.Binary, indexId: ParameterIndexId): SqliteJsonValue[] {
|
|
233
|
+
return [indexId, '', ...deserializeParameterLookupCore(lookup)];
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
export function taggedBucketParameterDocumentToTagged(
|
|
237
|
+
document: TaggedBucketParameterDocument
|
|
238
|
+
): BucketParameterDocumentV3 {
|
|
239
|
+
const { index: _index, ...rest } = document;
|
|
240
|
+
return rest as BucketParameterDocumentV3;
|
|
241
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import { storage } from '@powersync/service-core';
|
|
2
|
+
import * as bson from 'bson';
|
|
3
|
+
import { BucketDataDocumentV3, BucketOperation } from '../models.js';
|
|
4
|
+
import { ObjectStorage } from './ObjectStorage.js';
|
|
5
|
+
|
|
6
|
+
export class BucketDataObjectStorage {
|
|
7
|
+
constructor(private readonly storage: ObjectStorage) {}
|
|
8
|
+
|
|
9
|
+
async store(path: string, ops: BucketOperation[]): Promise<{ fileSize: number }> {
|
|
10
|
+
const bsonBuffer = bson.serialize({ ops });
|
|
11
|
+
await this.storage.put(path, bsonBuffer, { contentType: 'application/bson', contentEncoding: null });
|
|
12
|
+
return { fileSize: bsonBuffer.byteLength };
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
async retrieve(path: string, options: { signal?: AbortSignal }): Promise<BucketOperation[]> {
|
|
16
|
+
const { data, metadata } = await this.storage.get(path, { signal: options.signal });
|
|
17
|
+
if (metadata.contentEncoding != null) {
|
|
18
|
+
throw new Error(`Unexpected content encoding: ${metadata.contentEncoding}`);
|
|
19
|
+
}
|
|
20
|
+
if (metadata.contentType !== 'application/bson') {
|
|
21
|
+
throw new Error(`Unexpected content type: ${metadata.contentType}`);
|
|
22
|
+
}
|
|
23
|
+
const wrapper = bson.deserialize(data, storage.BSON_DESERIALIZE_INTERNAL_OPTIONS);
|
|
24
|
+
return wrapper.ops;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async delete(paths: string[]): Promise<void> {
|
|
28
|
+
return this.storage.delete(paths);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Load offloaded operations and patch them onto their MongoDB metadata documents.
|
|
34
|
+
* The object-storage implementation is responsible for limiting request
|
|
35
|
+
* concurrency across all callers.
|
|
36
|
+
*/
|
|
37
|
+
export async function hydrateBucketDataDocuments(
|
|
38
|
+
documents: BucketDataDocumentV3[],
|
|
39
|
+
objectStorage: ObjectStorage | undefined,
|
|
40
|
+
options: { signal?: AbortSignal }
|
|
41
|
+
): Promise<void> {
|
|
42
|
+
if (!objectStorage) {
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
options.signal?.throwIfAborted();
|
|
47
|
+
const store = new BucketDataObjectStorage(objectStorage);
|
|
48
|
+
const storedDocuments = documents.filter((document) => document.storage_ref);
|
|
49
|
+
await Promise.all(
|
|
50
|
+
storedDocuments.map(async (document) => {
|
|
51
|
+
document.ops = await store.retrieve(document.storage_ref!.path, { signal: options.signal });
|
|
52
|
+
})
|
|
53
|
+
);
|
|
54
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
export interface ObjectStoragePutMetadata {
|
|
2
|
+
contentType: string;
|
|
3
|
+
contentEncoding: string | null;
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
export class ObjectStorageError extends Error {
|
|
7
|
+
readonly retryable: boolean;
|
|
8
|
+
|
|
9
|
+
constructor(message: string, options: { cause: unknown; retryable: boolean }) {
|
|
10
|
+
super(message, { cause: options.cause });
|
|
11
|
+
this.name = 'ObjectStorageError';
|
|
12
|
+
this.retryable = options.retryable;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function isRetryableObjectStorageError(error: unknown): error is ObjectStorageError {
|
|
17
|
+
return error instanceof ObjectStorageError && error.retryable;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface ObjectStorage {
|
|
21
|
+
put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata): Promise<void>;
|
|
22
|
+
get(
|
|
23
|
+
path: string,
|
|
24
|
+
options?: { signal?: AbortSignal }
|
|
25
|
+
): Promise<{ data: Uint8Array; metadata: ObjectStoragePutMetadata }>;
|
|
26
|
+
list(prefix: string, options?: { signal?: AbortSignal }): AsyncIterable<string>;
|
|
27
|
+
delete(paths: string[]): Promise<void>;
|
|
28
|
+
deletePrefix(prefix: string, options?: { signal?: AbortSignal }): Promise<{ objectCount: number }>;
|
|
29
|
+
}
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { Logger } from '@powersync/lib-services-framework';
|
|
3
|
+
import * as bson from 'bson';
|
|
4
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
5
|
+
import { ObjectStorageDeletionMarker } from '../models.js';
|
|
6
|
+
import { VersionedPowerSyncMongoV3 } from '../VersionedPowerSyncMongoV3.js';
|
|
7
|
+
import { BucketDataObjectStorage } from './BucketDataObjectStorage.js';
|
|
8
|
+
import { ObjectStorage } from './ObjectStorage.js';
|
|
9
|
+
|
|
10
|
+
export const OBJECT_STORAGE_UPLOAD_LEASE_MS = 60 * 60 * 1000;
|
|
11
|
+
export const OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS = 2 * 60 * 1000;
|
|
12
|
+
export const OBJECT_STORAGE_REFERENCE_GRACE_MS = 15 * 60 * 1000;
|
|
13
|
+
/**
|
|
14
|
+
* Protect against arbitrary-long bucket names and parameters exceeding the S3 key length limits.
|
|
15
|
+
*
|
|
16
|
+
* Any bucket name over this length is hashed.
|
|
17
|
+
*/
|
|
18
|
+
const MAX_READABLE_BUCKET_SEGMENT_BYTES = 128;
|
|
19
|
+
|
|
20
|
+
export interface PreparedObjectStorageUpload {
|
|
21
|
+
markerId: bson.ObjectId;
|
|
22
|
+
path: string;
|
|
23
|
+
deleteAfter: Date;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Coordinates the MongoDB outbox used to make object-storage changes recoverable.
|
|
28
|
+
*
|
|
29
|
+
* A MongoDB collection is used to track files that will be uploaded, or should be deleted.
|
|
30
|
+
* By using a MongoDB collection, we can use the same transaction that creates or removes
|
|
31
|
+
* references to these files, making the operations atomic.
|
|
32
|
+
*
|
|
33
|
+
* These markers indicate which files should be deleted - either by being orphaned during create,
|
|
34
|
+
* or after references to them are removed.
|
|
35
|
+
*/
|
|
36
|
+
export class ObjectStorageLifecycle {
|
|
37
|
+
readonly bucketData: BucketDataObjectStorage;
|
|
38
|
+
|
|
39
|
+
constructor(
|
|
40
|
+
private readonly db: VersionedPowerSyncMongoV3,
|
|
41
|
+
private readonly replicationStreamId: number,
|
|
42
|
+
private readonly objectStorage: ObjectStorage
|
|
43
|
+
) {
|
|
44
|
+
this.bucketData = new BucketDataObjectStorage(objectStorage);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
streamPrefix(): string {
|
|
48
|
+
return `bucket-data/${this.replicationStreamId}/`;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
definitionPrefix(definitionId: string): string {
|
|
52
|
+
return `${this.streamPrefix()}${definitionId}/`;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
allocatePath(definitionId: string, bucket: string, minOp: bigint, maxOp: bigint): string {
|
|
56
|
+
// Preserve readable paths for normal buckets, but bound user-controlled
|
|
57
|
+
// parameter values so they cannot consume the S3 key-length budget.
|
|
58
|
+
const escapedBucket = encodeURIComponent(bucket);
|
|
59
|
+
const bucketSegment =
|
|
60
|
+
Buffer.byteLength(escapedBucket, 'utf8') <= MAX_READABLE_BUCKET_SEGMENT_BYTES
|
|
61
|
+
? escapedBucket
|
|
62
|
+
: `sha256-${createHash('sha256').update(bucket).digest('base64url')}`;
|
|
63
|
+
return `${this.definitionPrefix(definitionId)}${bucketSegment}/${minOp}-${maxOp}-${randomUUID()}.bson`;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* Indicate that files will be uploaded to S3.
|
|
68
|
+
*
|
|
69
|
+
* These markers are expected to be removed using publishUploads(). If that does not happen within
|
|
70
|
+
* the grace period, the files will be deleted.
|
|
71
|
+
*/
|
|
72
|
+
async prepareUploads(paths: string[], now = new Date()): Promise<PreparedObjectStorageUpload[]> {
|
|
73
|
+
const deleteAfter = new Date(now.getTime() + OBJECT_STORAGE_UPLOAD_LEASE_MS);
|
|
74
|
+
const uploads = paths.map((path) => ({ markerId: new bson.ObjectId(), path, deleteAfter }));
|
|
75
|
+
if (uploads.length) {
|
|
76
|
+
await this.db.pendingObjectStorageDeletes(this.replicationStreamId).insertMany(
|
|
77
|
+
uploads.map((upload) => ({
|
|
78
|
+
_id: upload.markerId,
|
|
79
|
+
path: upload.path,
|
|
80
|
+
delete_after: upload.deleteAfter
|
|
81
|
+
}))
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
return uploads;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
canPublish(upload: PreparedObjectStorageUpload, now = new Date()): boolean {
|
|
88
|
+
return now.getTime() < upload.deleteAfter.getTime() - OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Remove markers for files that have finished uploading, and are now referenced from other documents.
|
|
93
|
+
*
|
|
94
|
+
* Call this in the same transaction as the one creating the references.
|
|
95
|
+
*/
|
|
96
|
+
async publishUploads(uploads: PreparedObjectStorageUpload[], session: mongo.ClientSession): Promise<void> {
|
|
97
|
+
for (const upload of uploads) {
|
|
98
|
+
if (!this.canPublish(upload)) {
|
|
99
|
+
throw new Error(`Object storage publication lease expired for ${upload.path}`);
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
if (uploads.length === 0) {
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const result = await this.db
|
|
107
|
+
.pendingObjectStorageDeletes(this.replicationStreamId)
|
|
108
|
+
.deleteMany({ _id: { $in: uploads.map((upload) => upload.markerId) } }, { session });
|
|
109
|
+
if (result.deletedCount !== uploads.length) {
|
|
110
|
+
throw new Error(`Missing object storage publication markers`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Mark old paths for deletion.
|
|
116
|
+
*
|
|
117
|
+
* By marking for deletion instead of immediately deleting:
|
|
118
|
+
* 1. This avoids inconsistencies/orphaned files between MongoDB and S3, by allowing the markers to
|
|
119
|
+
* be persisted transactionally, while the deletes happen asynchronously and can be retried.
|
|
120
|
+
* 2. The deletes are delayed, giving clients a grace period to continue using the files for
|
|
121
|
+
* in-progress requests.
|
|
122
|
+
*
|
|
123
|
+
* Call this in the same transaction as the one removing the references to these files.
|
|
124
|
+
*/
|
|
125
|
+
async retire(paths: Iterable<string>, session: mongo.ClientSession, now = new Date()): Promise<void> {
|
|
126
|
+
const markers: ObjectStorageDeletionMarker[] = Array.from(paths, (path) => ({
|
|
127
|
+
_id: new bson.ObjectId(),
|
|
128
|
+
path,
|
|
129
|
+
delete_after: new Date(now.getTime() + OBJECT_STORAGE_REFERENCE_GRACE_MS)
|
|
130
|
+
}));
|
|
131
|
+
if (markers.length) {
|
|
132
|
+
await this.db.pendingObjectStorageDeletes(this.replicationStreamId).insertMany(markers, { session });
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Discover and delete objects by their storage prefix
|
|
138
|
+
* without touching the database.
|
|
139
|
+
*/
|
|
140
|
+
async deletePrefix(prefix: string, options?: { signal?: AbortSignal }): Promise<{ objectCount: number }> {
|
|
141
|
+
return this.objectStorage.deletePrefix(prefix, { signal: options?.signal });
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
async cleanup(logger: Logger): Promise<void> {
|
|
145
|
+
const markers = this.db.pendingObjectStorageDeletes(this.replicationStreamId);
|
|
146
|
+
const cursor = markers.find({ delete_after: { $lte: new Date() } });
|
|
147
|
+
let batch: ObjectStorageDeletionMarker[] = [];
|
|
148
|
+
|
|
149
|
+
const deleteBatch = async () => {
|
|
150
|
+
if (batch.length === 0) {
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
const deleting = batch;
|
|
154
|
+
batch = [];
|
|
155
|
+
try {
|
|
156
|
+
await this.bucketData.delete(deleting.map((marker) => marker.path));
|
|
157
|
+
await markers.deleteMany({ _id: { $in: deleting.map((marker) => marker._id) } });
|
|
158
|
+
} catch (error) {
|
|
159
|
+
logger.warn('Failed to clean up object storage deletion markers; will retry during the next compaction', error);
|
|
160
|
+
}
|
|
161
|
+
};
|
|
162
|
+
|
|
163
|
+
for await (const marker of cursor) {
|
|
164
|
+
batch.push(marker);
|
|
165
|
+
if (batch.length === 500) {
|
|
166
|
+
await deleteBatch();
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
await deleteBatch();
|
|
170
|
+
}
|
|
171
|
+
}
|