@powersync/service-module-mongodb-storage 0.0.0-dev-20260803144812 → 0.0.0-dev-20260929095716
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +129 -7
- package/dist/storage/MongoBucketStorage.d.ts +42 -5
- package/dist/storage/MongoBucketStorage.js +404 -215
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +39 -14
- package/dist/storage/implementation/MongoBucketBatch.js +169 -98
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +21 -79
- package/dist/storage/implementation/MongoCompactor.js +27 -238
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoIdSequence.d.ts +10 -1
- package/dist/storage/implementation/MongoIdSequence.js +14 -1
- package/dist/storage/implementation/MongoIdSequence.js.map +1 -1
- package/dist/storage/implementation/MongoOpIdAllocator.d.ts +27 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js +97 -0
- package/dist/storage/implementation/MongoOpIdAllocator.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +83 -10
- package/dist/storage/implementation/MongoParameterCompactor.js +309 -96
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +9 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +35 -2
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +2 -1
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +7 -4
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -1
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +3 -3
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +10 -8
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +8 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +32 -11
- package/dist/storage/implementation/MongoSyncBucketStorage.js +95 -31
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +26 -2
- package/dist/storage/implementation/MongoSyncRulesLock.js +59 -10
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
- package/dist/storage/implementation/MongoWriteBatch.js +149 -0
- package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +9 -7
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +9 -30
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +22 -2
- package/dist/storage/implementation/common/PersistedBatch.d.ts +32 -4
- package/dist/storage/implementation/common/PersistedBatch.js +57 -14
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +9 -1
- package/dist/storage/implementation/db.js +28 -1
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +35 -0
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +11 -0
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +227 -221
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +38 -6
- package/dist/storage/implementation/v1/MongoCompactorV1.js +247 -20
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +18 -11
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +66 -17
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
- package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +14 -2
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +261 -194
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +83 -25
- package/dist/storage/implementation/v3/MongoCompactorV3.js +702 -159
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +18 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +23 -0
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +2 -15
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +60 -92
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -1
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +19 -9
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +92 -32
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +14 -8
- package/dist/storage/implementation/v3/PersistedBatchV3.js +77 -45
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +12 -2
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +42 -3
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.js +2 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -1
- package/dist/storage/implementation/v3/compact-utils.d.ts +108 -0
- package/dist/storage/implementation/v3/compact-utils.js +223 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +90 -6
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +7 -8
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +16 -5
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +14 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +6 -8
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +17 -11
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +38 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +164 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +37 -12
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +197 -25
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/source-table-utils.d.ts +15 -7
- package/dist/storage/implementation/v3/source-table-utils.js +49 -57
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -1
- package/dist/storage/storage-index.d.ts +2 -1
- package/dist/storage/storage-index.js +1 -0
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +28 -1
- package/dist/types/types.js +28 -7
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +4 -2
- package/dist/utils/test-utils.js +7 -4
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.js +3 -1
- package/dist/utils/util.js.map +1 -1
- package/package.json +10 -9
- package/src/storage/MongoBucketStorage.ts +545 -277
- package/src/storage/implementation/MongoBucketBatch.ts +258 -124
- package/src/storage/implementation/MongoCompactor.ts +37 -346
- package/src/storage/implementation/MongoIdSequence.ts +21 -1
- package/src/storage/implementation/MongoOpIdAllocator.ts +105 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +388 -102
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +46 -1
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +12 -4
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +48 -29
- package/src/storage/implementation/MongoStorageProvider.ts +16 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +133 -40
- package/src/storage/implementation/MongoSyncRulesLock.ts +78 -12
- package/src/storage/implementation/MongoWriteBatch.ts +172 -0
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +16 -46
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +24 -2
- package/src/storage/implementation/common/PersistedBatch.ts +76 -20
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
- package/src/storage/implementation/db.ts +36 -2
- package/src/storage/implementation/models.ts +39 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +272 -254
- package/src/storage/implementation/v1/MongoCompactorV1.ts +327 -40
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +88 -28
- package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
- package/src/storage/implementation/v1/models.ts +1 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +339 -247
- package/src/storage/implementation/v3/MongoCompactorV3.ts +955 -225
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +34 -0
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +100 -109
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +112 -42
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +94 -51
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +55 -4
- package/src/storage/implementation/v3/bucket-format.ts +2 -0
- package/src/storage/implementation/v3/compact-utils.ts +329 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +97 -4
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +24 -9
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +20 -5
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +26 -14
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +229 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +279 -29
- package/src/storage/implementation/v3/source-table-utils.ts +82 -76
- package/src/storage/storage-index.ts +2 -0
- package/src/types/types.ts +42 -7
- package/src/utils/test-utils.ts +9 -5
- package/src/utils/util.ts +3 -1
- package/test/src/__snapshots__/storage.test.ts.snap +25 -0
- package/test/src/__snapshots__/storage_sync.test.ts.snap +794 -205
- package/test/src/chunk_compaction_config.test.ts +38 -0
- package/test/src/cleanup-stopped-sync-configs.test.ts +59 -26
- package/test/src/compact-utils.test.ts +251 -0
- package/test/src/concurrent_writers.test.ts +599 -0
- package/test/src/connection_config.test.ts +159 -0
- package/test/src/helpers/MemoryObjectStorage.ts +14 -6
- package/test/src/helpers/concurrentWriter.mjs +29 -0
- package/test/src/helpers/s3TestFactory.ts +12 -12
- package/test/src/object_storage_usage.test.ts +268 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/storage.test.ts +280 -24
- package/test/src/storage_compacting.test.ts +1097 -253
- package/test/src/storage_object_storage_inline_threshold.test.ts +3 -3
- package/test/src/storage_s3_checksums.test.ts +6 -7
- package/test/src/storage_s3_compaction_lifecycle.test.ts +143 -17
- package/test/src/storage_s3_reading.test.ts +294 -6
- package/test/src/storage_s3_writing.test.ts +4 -3
- package/test/src/storage_sync.test.ts +412 -80
- package/test/src/storeCurrentData.test.ts +5 -5
- package/test/src/util.ts +1 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1,19 +1,25 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
|
-
import { storage } from '@powersync/service-core';
|
|
2
|
+
import { InternalOpId, storage } from '@powersync/service-core';
|
|
3
3
|
import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js';
|
|
4
|
+
import { MongoWriteBatch } from '../MongoWriteBatch.js';
|
|
4
5
|
import { PersistedBatch } from '../common/PersistedBatch.js';
|
|
5
6
|
import { SourceRecordStore } from '../common/SourceRecordStore.js';
|
|
7
|
+
import { SyncRuleDocumentBase } from '../models.js';
|
|
6
8
|
import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js';
|
|
7
9
|
export declare class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
8
10
|
readonly db: VersionedPowerSyncMongoV3;
|
|
9
11
|
private readonly store;
|
|
12
|
+
/** Avoid repeating idempotent createIndexes calls for every checkpoint flush. */
|
|
13
|
+
private readonly initializedCustomCheckpointEventIds;
|
|
14
|
+
private readonly knownEventIds;
|
|
10
15
|
private readonly syncConfigIds;
|
|
11
16
|
private needsActivationV3;
|
|
12
17
|
private lastWaitingLogThrottledV3;
|
|
13
18
|
constructor(options: MongoBucketBatchOptions);
|
|
14
19
|
protected createPersistedBatch(writtenSize: number): PersistedBatch;
|
|
15
20
|
protected get sourceRecordStore(): SourceRecordStore;
|
|
16
|
-
protected
|
|
21
|
+
protected persistedOpHead(stream: SyncRuleDocumentBase): InternalOpId;
|
|
22
|
+
protected onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: InternalOpId): void;
|
|
17
23
|
private selectedSyncConfigObjectIds;
|
|
18
24
|
private relevantSyncConfigIds;
|
|
19
25
|
private relevantSyncConfigIdsForTables;
|
|
@@ -25,10 +31,16 @@ export declare class MongoBucketBatchV3 extends MongoBucketBatch {
|
|
|
25
31
|
keepalive(lsn: string): Promise<storage.CheckpointResult>;
|
|
26
32
|
setResumeLsn(lsn: string): Promise<void>;
|
|
27
33
|
private autoActivateV3;
|
|
34
|
+
private syncConfigMetadataUpdate;
|
|
35
|
+
private snapshotDoneUpdate;
|
|
28
36
|
markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise<void>;
|
|
29
37
|
markSnapshotDone(no_checkpoint_before_lsn: string, options?: {
|
|
30
38
|
throwOnConflict?: boolean;
|
|
31
39
|
}): Promise<void>;
|
|
32
40
|
markTableSnapshotRequired(table: storage.SourceTable): Promise<void>;
|
|
33
41
|
markTableSnapshotDone(tables: storage.SourceTable[], no_checkpoint_before_lsn?: string): Promise<storage.SourceTable[]>;
|
|
42
|
+
protected batchCreateCustomWriteCheckpoints(session: lib_mongo.mongo.ClientSession, opId: InternalOpId): Promise<void>;
|
|
43
|
+
protected prepareCustomWriteCheckpoints(): Promise<void>;
|
|
44
|
+
private validateCustomCheckpointEventId;
|
|
45
|
+
private batchCreateEventCustomWriteCheckpoints;
|
|
34
46
|
}
|