@powersync/service-module-mongodb-storage 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/dist/storage/MongoBucketStorage.d.ts +35 -15
- package/dist/storage/MongoBucketStorage.js +309 -86
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/CheckpointState.d.ts +20 -0
- package/dist/storage/implementation/CheckpointState.js +31 -0
- package/dist/storage/implementation/CheckpointState.js.map +1 -0
- package/dist/storage/implementation/MongoBucketBatch.d.ts +69 -39
- package/dist/storage/implementation/MongoBucketBatch.js +104 -313
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +11 -9
- package/dist/storage/implementation/MongoChecksums.js +3 -111
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +3 -1
- package/dist/storage/implementation/MongoCompactor.js +18 -9
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +6 -5
- package/dist/storage/implementation/MongoParameterCompactor.js +24 -1
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +13 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +64 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +121 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +5 -2
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +50 -21
- package/dist/storage/implementation/MongoSyncBucketStorage.js +62 -199
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +5 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +9 -5
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +14 -0
- package/dist/storage/implementation/SyncRuleStateUpdate.js +36 -0
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -0
- package/dist/storage/implementation/common/BucketDataDoc.d.ts +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +6 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +2 -3
- package/dist/storage/implementation/common/PersistedBatch.js +5 -0
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/SourceRecordStore.d.ts +1 -2
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +10 -2
- package/dist/storage/implementation/db.js +9 -14
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +25 -50
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +23 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +403 -2
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +114 -3
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +22 -15
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +100 -27
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +1 -2
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +21 -1
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +21 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +459 -3
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +27 -7
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +202 -18
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +20 -3
- package/dist/storage/implementation/v3/MongoCompactorV3.js +511 -22
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +62 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +350 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +50 -16
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +292 -75
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +2 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +43 -20
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +52 -16
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +16 -14
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +32 -27
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +4 -0
- package/dist/storage/implementation/v3/bucket-format.js +58 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +28 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +111 -10
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +55 -0
- package/dist/storage/implementation/v3/source-table-utils.js +214 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +4 -4
- package/dist/storage/storage-index.js +4 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +2 -2
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +5 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +1 -1
- package/dist/utils/util.js +3 -2
- package/dist/utils/util.js.map +1 -1
- package/package.json +9 -19
- package/src/migrations/db/migrations/1702295701188-sync-rule-state.ts +1 -1
- package/src/storage/MongoBucketStorage.ts +464 -95
- package/src/storage/implementation/CheckpointState.ts +59 -0
- package/src/storage/implementation/MongoBucketBatch.ts +159 -405
- package/src/storage/implementation/MongoChecksums.ts +7 -138
- package/src/storage/implementation/MongoCompactor.ts +20 -10
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +24 -5
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +88 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +152 -0
- package/src/storage/implementation/MongoStorageProvider.ts +5 -2
- package/src/storage/implementation/MongoSyncBucketStorage.ts +89 -250
- package/src/storage/implementation/MongoSyncRulesLock.ts +14 -5
- package/src/storage/implementation/SyncRuleStateUpdate.ts +38 -0
- package/src/storage/implementation/common/BucketDataDoc.ts +1 -1
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +7 -0
- package/src/storage/implementation/common/PersistedBatch.ts +8 -4
- package/src/storage/implementation/common/SourceRecordStore.ts +1 -2
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +16 -21
- package/src/storage/implementation/models.ts +28 -61
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +503 -5
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +137 -4
- package/src/storage/implementation/v1/MongoCompactorV1.ts +1 -1
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +150 -49
- package/src/storage/implementation/v1/PersistedBatchV1.ts +1 -2
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +25 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +611 -4
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +262 -26
- package/src/storage/implementation/v3/MongoCompactorV3.ts +647 -30
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +480 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +397 -111
- package/src/storage/implementation/v3/PersistedBatchV3.ts +63 -38
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +59 -19
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +48 -38
- package/src/storage/implementation/v3/bucket-format.ts +70 -0
- package/src/storage/implementation/v3/chunking.ts +38 -0
- package/src/storage/implementation/v3/models.ts +150 -44
- package/src/storage/implementation/v3/source-table-utils.ts +392 -0
- package/src/storage/storage-index.ts +12 -4
- package/src/utils/test-utils.ts +6 -3
- package/src/utils/util.ts +3 -2
- package/test/src/__snapshots__/storage.test.ts.snap +0 -75
- package/test/src/__snapshots__/storage_sync.test.ts.snap +48 -48
- package/test/src/cleanup-stopped-sync-configs.test.ts +648 -0
- package/test/src/storage.test.ts +11 -11
- package/test/src/storage_compacting.test.ts +1956 -6
- package/test/src/storage_sync.test.ts +1429 -42
- package/test/src/storeCurrentData.test.ts +221 -0
- package/test/src/util.ts +2 -1
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -17
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -64
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -15
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -42
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +0 -9
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +0 -18
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -5
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -72
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -76
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -56
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +0 -24
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -12
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
export interface CheckpointStateInput {
|
|
2
|
+
lsn: string;
|
|
3
|
+
snapshotDone: boolean;
|
|
4
|
+
lastCheckpointLsn: string | null;
|
|
5
|
+
noCheckpointBefore: string | null;
|
|
6
|
+
keepaliveOp: bigint | null;
|
|
7
|
+
lastCheckpoint: bigint | null;
|
|
8
|
+
persistedOp: bigint | null;
|
|
9
|
+
createEmptyCheckpoints: boolean;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export interface CheckpointStateResult {
|
|
13
|
+
canCheckpoint: boolean;
|
|
14
|
+
checkpointBlocked: boolean;
|
|
15
|
+
checkpointCreated: boolean;
|
|
16
|
+
notEmpty: boolean;
|
|
17
|
+
newKeepaliveOp: bigint | null;
|
|
18
|
+
newLastCheckpoint: bigint | null;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function maxOpId(...values: (bigint | null | undefined)[]): bigint {
|
|
22
|
+
let max = 0n;
|
|
23
|
+
for (const value of values) {
|
|
24
|
+
if (value != null && value > max) {
|
|
25
|
+
max = value;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
return max;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export function canCheckpointState(
|
|
32
|
+
lsn: string,
|
|
33
|
+
state: Pick<CheckpointStateInput, 'snapshotDone' | 'lastCheckpointLsn' | 'noCheckpointBefore'>
|
|
34
|
+
): boolean {
|
|
35
|
+
return (
|
|
36
|
+
state.snapshotDone === true &&
|
|
37
|
+
(state.lastCheckpointLsn == null || state.lastCheckpointLsn <= lsn) &&
|
|
38
|
+
(state.noCheckpointBefore == null || state.noCheckpointBefore <= lsn)
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function calculateCheckpointState(input: CheckpointStateInput): CheckpointStateResult {
|
|
43
|
+
const canCheckpoint = canCheckpointState(input.lsn, input);
|
|
44
|
+
const newKeepaliveOp = canCheckpoint ? null : maxOpId(input.keepaliveOp, input.persistedOp);
|
|
45
|
+
const newLastCheckpoint = canCheckpoint
|
|
46
|
+
? maxOpId(input.lastCheckpoint, input.persistedOp, input.keepaliveOp)
|
|
47
|
+
: input.lastCheckpoint;
|
|
48
|
+
const notEmpty =
|
|
49
|
+
input.createEmptyCheckpoints || input.keepaliveOp !== newKeepaliveOp || input.lastCheckpoint !== newLastCheckpoint;
|
|
50
|
+
|
|
51
|
+
return {
|
|
52
|
+
canCheckpoint,
|
|
53
|
+
checkpointBlocked: !canCheckpoint,
|
|
54
|
+
checkpointCreated: canCheckpoint && notEmpty,
|
|
55
|
+
notEmpty,
|
|
56
|
+
newKeepaliveOp,
|
|
57
|
+
newLastCheckpoint
|
|
58
|
+
};
|
|
59
|
+
}
|