@powersync/service-module-mongodb-storage 0.0.0-dev-20260602073133 → 0.0.0-dev-20260909133214
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +177 -5
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.d.ts +3 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js +41 -0
- package/dist/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.js.map +1 -0
- package/dist/storage/MongoBucketStorage.d.ts +71 -18
- package/dist/storage/MongoBucketStorage.js +516 -202
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +71 -29
- package/dist/storage/implementation/MongoBucketBatch.js +140 -66
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoChecksums.d.ts +41 -13
- package/dist/storage/implementation/MongoChecksums.js +36 -121
- package/dist/storage/implementation/MongoChecksums.js.map +1 -1
- package/dist/storage/implementation/MongoCompactor.d.ts +28 -90
- package/dist/storage/implementation/MongoCompactor.js +87 -481
- package/dist/storage/implementation/MongoCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoHydrationState.d.ts +11 -0
- package/dist/storage/implementation/MongoHydrationState.js +49 -0
- package/dist/storage/implementation/MongoHydrationState.js.map +1 -0
- package/dist/storage/implementation/MongoParameterCompactor.d.ts +82 -8
- package/dist/storage/implementation/MongoParameterCompactor.js +312 -76
- package/dist/storage/implementation/MongoParameterCompactor.js.map +1 -1
- package/dist/storage/implementation/MongoParsedSyncConfigSet.d.ts +21 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js +93 -0
- package/dist/storage/implementation/MongoParsedSyncConfigSet.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.d.ts +36 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js +90 -0
- package/dist/storage/implementation/MongoPersistedReplicationStream.js.map +1 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.d.ts +25 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js +125 -0
- package/dist/storage/implementation/MongoPersistedSyncConfigContent.js.map +1 -0
- package/dist/storage/implementation/MongoStorageProvider.js +41 -4
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoSyncBucketStorage.d.ts +81 -37
- package/dist/storage/implementation/MongoSyncBucketStorage.js +159 -86
- package/dist/storage/implementation/MongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.d.ts +1 -1
- package/dist/storage/implementation/MongoSyncRulesLock.js +23 -11
- package/dist/storage/implementation/MongoSyncRulesLock.js.map +1 -1
- package/dist/storage/implementation/MongoWriteCheckpointAPI.d.ts +11 -8
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +159 -33
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/SyncRuleStateUpdate.d.ts +11 -5
- package/dist/storage/implementation/SyncRuleStateUpdate.js +14 -6
- package/dist/storage/implementation/SyncRuleStateUpdate.js.map +1 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.d.ts +28 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js +2 -0
- package/dist/storage/implementation/common/MongoSyncBucketStorageCheckpoint.js.map +1 -0
- package/dist/storage/implementation/common/PersistedBatch.d.ts +34 -2
- package/dist/storage/implementation/common/PersistedBatch.js +63 -10
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/createMongoSyncBucketStorage.d.ts +2 -2
- package/dist/storage/implementation/createMongoSyncBucketStorage.js +4 -4
- package/dist/storage/implementation/createMongoSyncBucketStorage.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +4 -1
- package/dist/storage/implementation/db.js +31 -16
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/models.d.ts +55 -13
- package/dist/storage/implementation/models.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.d.ts +15 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +135 -107
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoChecksumsV1.d.ts +4 -4
- package/dist/storage/implementation/v1/MongoChecksumsV1.js +119 -7
- package/dist/storage/implementation/v1/MongoChecksumsV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +47 -7
- package/dist/storage/implementation/v1/MongoCompactorV1.js +478 -8
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.d.ts +16 -2
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js +20 -9
- package/dist/storage/implementation/v1/MongoParameterCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.d.ts +33 -27
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js +146 -73
- package/dist/storage/implementation/v1/MongoSyncBucketStorageV1.js.map +1 -1
- package/dist/storage/implementation/v1/SingleBucketStoreV1.d.ts +9 -10
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js +2 -8
- package/dist/storage/implementation/v1/SingleBucketStoreV1.js.map +1 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.d.ts +2 -1
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js +3 -0
- package/dist/storage/implementation/v1/VersionedPowerSyncMongoV1.js.map +1 -1
- package/dist/storage/implementation/v1/models.d.ts +10 -1
- package/dist/storage/implementation/v1/models.js +1 -0
- package/dist/storage/implementation/v1/models.js.map +1 -1
- package/dist/storage/implementation/v3/BucketDataContextV3.d.ts +13 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js +23 -0
- package/dist/storage/implementation/v3/BucketDataContextV3.js.map +1 -0
- package/dist/storage/implementation/v3/CompactionLease.d.ts +50 -0
- package/dist/storage/implementation/v3/CompactionLease.js +131 -0
- package/dist/storage/implementation/v3/CompactionLease.js.map +1 -0
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +16 -5
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +393 -262
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoChecksumsV3.d.ts +30 -9
- package/dist/storage/implementation/v3/MongoChecksumsV3.js +161 -20
- package/dist/storage/implementation/v3/MongoChecksumsV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +108 -18
- package/dist/storage/implementation/v3/MongoCompactorV3.js +1115 -47
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.d.ts +12 -3
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js +13 -8
- package/dist/storage/implementation/v3/MongoParameterCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.d.ts +54 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js +319 -0
- package/dist/storage/implementation/v3/MongoStoppedSyncConfigCleanup.js.map +1 -0
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.d.ts +61 -36
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js +353 -161
- package/dist/storage/implementation/v3/MongoSyncBucketStorageV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.d.ts +26 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js +82 -0
- package/dist/storage/implementation/v3/MongoWriteCheckpointAPIV3.js.map +1 -0
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +8 -3
- package/dist/storage/implementation/v3/PersistedBatchV3.js +131 -29
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/SourceRecordStoreV3.d.ts +3 -3
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js +13 -11
- package/dist/storage/implementation/v3/SourceRecordStoreV3.js.map +1 -1
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.d.ts +26 -14
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js +73 -29
- package/dist/storage/implementation/v3/VersionedPowerSyncMongoV3.js.map +1 -1
- package/dist/storage/implementation/v3/bucket-format.d.ts +13 -0
- package/dist/storage/implementation/v3/bucket-format.js +79 -0
- package/dist/storage/implementation/v3/bucket-format.js.map +1 -0
- package/dist/storage/implementation/v3/chunking.d.ts +10 -0
- package/dist/storage/implementation/v3/chunking.js +30 -0
- package/dist/storage/implementation/v3/chunking.js.map +1 -0
- package/dist/storage/implementation/v3/compact-utils.d.ts +107 -0
- package/dist/storage/implementation/v3/compact-utils.js +216 -0
- package/dist/storage/implementation/v3/compact-utils.js.map +1 -0
- package/dist/storage/implementation/v3/compaction-constants.d.ts +5 -0
- package/dist/storage/implementation/v3/compaction-constants.js +6 -0
- package/dist/storage/implementation/v3/compaction-constants.js.map +1 -0
- package/dist/storage/implementation/v3/models.d.ts +187 -23
- package/dist/storage/implementation/v3/models.js +11 -28
- package/dist/storage/implementation/v3/models.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.d.ts +20 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +48 -0
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.d.ts +33 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js +12 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +68 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +154 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +37 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +167 -0
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.d.ts +67 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +439 -0
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -0
- package/dist/storage/implementation/v3/source-table-utils.d.ts +63 -0
- package/dist/storage/implementation/v3/source-table-utils.js +206 -0
- package/dist/storage/implementation/v3/source-table-utils.js.map +1 -0
- package/dist/storage/storage-index.d.ts +5 -4
- package/dist/storage/storage-index.js +5 -4
- package/dist/storage/storage-index.js.map +1 -1
- package/dist/types/types.d.ts +71 -3
- package/dist/types/types.js +58 -2
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.d.ts +1 -2
- package/dist/utils/test-utils.js +7 -1
- package/dist/utils/test-utils.js.map +1 -1
- package/dist/utils/util.d.ts +9 -2
- package/dist/utils/util.js +131 -5
- package/dist/utils/util.js.map +1 -1
- package/package.json +13 -19
- package/src/migrations/db/migrations/1782950400000-checkpoint-requested-at-index.ts +55 -0
- package/src/storage/MongoBucketStorage.ts +728 -242
- package/src/storage/implementation/MongoBucketBatch.ts +196 -91
- package/src/storage/implementation/MongoChecksums.ts +93 -153
- package/src/storage/implementation/MongoCompactor.ts +106 -631
- package/src/storage/implementation/MongoHydrationState.ts +61 -0
- package/src/storage/implementation/MongoParameterCompactor.ts +392 -87
- package/src/storage/implementation/MongoParsedSyncConfigSet.ts +122 -0
- package/src/storage/implementation/MongoPersistedReplicationStream.ts +118 -0
- package/src/storage/implementation/MongoPersistedSyncConfigContent.ts +156 -0
- package/src/storage/implementation/MongoStorageProvider.ts +48 -4
- package/src/storage/implementation/MongoSyncBucketStorage.ts +226 -118
- package/src/storage/implementation/MongoSyncRulesLock.ts +28 -12
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +194 -52
- package/src/storage/implementation/SyncRuleStateUpdate.ts +13 -6
- package/src/storage/implementation/common/MongoSyncBucketStorageCheckpoint.ts +31 -0
- package/src/storage/implementation/common/PersistedBatch.ts +83 -15
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +1 -1
- package/src/storage/implementation/createMongoSyncBucketStorage.ts +21 -7
- package/src/storage/implementation/db.ts +40 -22
- package/src/storage/implementation/models.ts +58 -14
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +174 -125
- package/src/storage/implementation/v1/MongoChecksumsV1.ts +154 -12
- package/src/storage/implementation/v1/MongoCompactorV1.ts +594 -25
- package/src/storage/implementation/v1/MongoParameterCompactorV1.ts +25 -9
- package/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +226 -129
- package/src/storage/implementation/v1/SingleBucketStoreV1.ts +18 -27
- package/src/storage/implementation/v1/VersionedPowerSyncMongoV1.ts +11 -1
- package/src/storage/implementation/v1/models.ts +12 -1
- package/src/storage/implementation/v3/BucketDataContextV3.ts +32 -0
- package/src/storage/implementation/v3/CompactionLease.ts +152 -0
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +502 -324
- package/src/storage/implementation/v3/MongoChecksumsV3.ts +250 -32
- package/src/storage/implementation/v3/MongoCompactorV3.ts +1439 -70
- package/src/storage/implementation/v3/MongoParameterCompactorV3.ts +19 -9
- package/src/storage/implementation/v3/MongoStoppedSyncConfigCleanup.ts +473 -0
- package/src/storage/implementation/v3/MongoSyncBucketStorageV3.ts +453 -211
- package/src/storage/implementation/v3/MongoWriteCheckpointAPIV3.ts +111 -0
- package/src/storage/implementation/v3/PersistedBatchV3.ts +169 -45
- package/src/storage/implementation/v3/SourceRecordStoreV3.ts +12 -12
- package/src/storage/implementation/v3/VersionedPowerSyncMongoV3.ts +99 -38
- package/src/storage/implementation/v3/bucket-format.ts +99 -0
- package/src/storage/implementation/v3/chunking.ts +40 -0
- package/src/storage/implementation/v3/compact-utils.ts +320 -0
- package/src/storage/implementation/v3/compaction-constants.ts +5 -0
- package/src/storage/implementation/v3/models.ts +228 -58
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +62 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorage.ts +44 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +182 -0
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +230 -0
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +573 -0
- package/src/storage/implementation/v3/source-table-utils.ts +398 -0
- package/src/storage/storage-index.ts +14 -4
- package/src/types/types.ts +76 -2
- package/src/utils/test-utils.ts +8 -3
- package/src/utils/util.ts +188 -7
- package/test/src/__snapshots__/storage.test.ts.snap +4 -54
- package/test/src/__snapshots__/storage_sync.test.ts.snap +652 -645
- package/test/src/cleanup-stopped-sync-configs.test.ts +698 -0
- package/test/src/compact-utils.test.ts +226 -0
- package/test/src/helpers/MemoryObjectStorage.ts +61 -0
- package/test/src/helpers/s3TestFactory.ts +69 -0
- package/test/src/object_storage_lifecycle.test.ts +59 -0
- package/test/src/object_storage_usage.test.ts +255 -0
- package/test/src/parameter_compacting_v1.test.ts +120 -0
- package/test/src/parameter_compacting_v3.test.ts +192 -0
- package/test/src/parameter_compaction_fence.test.ts +276 -0
- package/test/src/setup.ts +6 -1
- package/test/src/storage.test.ts +478 -13
- package/test/src/storage_compacting.test.ts +2516 -107
- package/test/src/storage_object_storage_inline_threshold.test.ts +116 -0
- package/test/src/storage_s3_checksums.test.ts +190 -0
- package/test/src/storage_s3_compaction_lifecycle.test.ts +543 -0
- package/test/src/storage_s3_reading.test.ts +768 -0
- package/test/src/storage_s3_writing.test.ts +136 -0
- package/test/src/storage_sync.test.ts +1507 -76
- package/test/src/storeCurrentData.test.ts +227 -0
- package/test/src/util.ts +3 -2
- package/tsconfig.json +1 -4
- package/tsconfig.tsbuildinfo +1 -1
- package/dist/storage/implementation/BucketDefinitionMapping.d.ts +0 -15
- package/dist/storage/implementation/BucketDefinitionMapping.js +0 -58
- package/dist/storage/implementation/BucketDefinitionMapping.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRules.d.ts +0 -14
- package/dist/storage/implementation/MongoPersistedSyncRules.js +0 -67
- package/dist/storage/implementation/MongoPersistedSyncRules.js.map +0 -1
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.d.ts +0 -29
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js +0 -76
- package/dist/storage/implementation/MongoPersistedSyncRulesContent.js.map +0 -1
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.d.ts +0 -13
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js +0 -2
- package/dist/storage/implementation/common/MongoSyncBucketStorageContext.js.map +0 -1
- package/dist/storage/implementation/common/SingleBucketStore.d.ts +0 -54
- package/dist/storage/implementation/common/SingleBucketStore.js +0 -3
- package/dist/storage/implementation/common/SingleBucketStore.js.map +0 -1
- package/dist/storage/implementation/v3/MongoParameterLookupV3.d.ts +0 -4
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js +0 -9
- package/dist/storage/implementation/v3/MongoParameterLookupV3.js.map +0 -1
- package/dist/storage/implementation/v3/SingleBucketStoreV3.d.ts +0 -18
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js +0 -48
- package/dist/storage/implementation/v3/SingleBucketStoreV3.js.map +0 -1
- package/src/storage/implementation/BucketDefinitionMapping.ts +0 -75
- package/src/storage/implementation/MongoPersistedSyncRules.ts +0 -82
- package/src/storage/implementation/MongoPersistedSyncRulesContent.ts +0 -101
- package/src/storage/implementation/common/MongoSyncBucketStorageContext.ts +0 -15
- package/src/storage/implementation/common/SingleBucketStore.ts +0 -63
- package/src/storage/implementation/v3/MongoParameterLookupV3.ts +0 -11
- package/src/storage/implementation/v3/SingleBucketStoreV3.ts +0 -68
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
export class ObjectStorageError extends Error {
|
|
2
|
+
retryable;
|
|
3
|
+
constructor(message, options) {
|
|
4
|
+
super(message, { cause: options.cause });
|
|
5
|
+
this.name = 'ObjectStorageError';
|
|
6
|
+
this.retryable = options.retryable;
|
|
7
|
+
}
|
|
8
|
+
}
|
|
9
|
+
export function isRetryableObjectStorageError(error) {
|
|
10
|
+
return error instanceof ObjectStorageError && error.retryable;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=ObjectStorage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ObjectStorage.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/ObjectStorage.ts"],"names":[],"mappings":"AAKA,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IAClC,SAAS,CAAU;IAE5B,YAAY,OAAe,EAAE,OAA+C;QAC1E,KAAK,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,GAAG,oBAAoB,CAAC;QACjC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IACrC,CAAC;CACF;AAED,MAAM,UAAU,6BAA6B,CAAC,KAAc;IAC1D,OAAO,KAAK,YAAY,kBAAkB,IAAI,KAAK,CAAC,SAAS,CAAC;AAChE,CAAC"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { Logger } from '@powersync/lib-services-framework';
|
|
3
|
+
import * as bson from 'bson';
|
|
4
|
+
import { VersionedPowerSyncMongoV3 } from '../VersionedPowerSyncMongoV3.js';
|
|
5
|
+
import { BucketDataObjectStorage } from './BucketDataObjectStorage.js';
|
|
6
|
+
import { ObjectStorage, ObjectStorageOperationOptions } from './ObjectStorage.js';
|
|
7
|
+
export declare const OBJECT_STORAGE_UPLOAD_LEASE_MS: number;
|
|
8
|
+
export declare const OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS: number;
|
|
9
|
+
export declare const OBJECT_STORAGE_REFERENCE_GRACE_MS: number;
|
|
10
|
+
export interface PreparedObjectStorageUpload {
|
|
11
|
+
markerId: bson.ObjectId;
|
|
12
|
+
path: string;
|
|
13
|
+
deleteAfter: Date;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* Coordinates the MongoDB outbox used to make object-storage changes recoverable.
|
|
17
|
+
*
|
|
18
|
+
* A MongoDB collection is used to track files that will be uploaded, or should be deleted.
|
|
19
|
+
* By using a MongoDB collection, we can use the same transaction that creates or removes
|
|
20
|
+
* references to these files, making the operations atomic.
|
|
21
|
+
*
|
|
22
|
+
* These markers indicate which files should be deleted - either by being orphaned during create,
|
|
23
|
+
* or after references to them are removed.
|
|
24
|
+
*/
|
|
25
|
+
export declare class ObjectStorageLifecycle {
|
|
26
|
+
private readonly db;
|
|
27
|
+
private readonly replicationStreamId;
|
|
28
|
+
private readonly objectStorage;
|
|
29
|
+
readonly bucketData: BucketDataObjectStorage;
|
|
30
|
+
constructor(db: VersionedPowerSyncMongoV3, replicationStreamId: number, objectStorage: ObjectStorage);
|
|
31
|
+
streamPrefix(): string;
|
|
32
|
+
definitionPrefix(definitionId: string): string;
|
|
33
|
+
allocatePath(definitionId: string, bucket: string, minOp: bigint, maxOp: bigint): string;
|
|
34
|
+
/**
|
|
35
|
+
* Indicate that files will be uploaded to S3.
|
|
36
|
+
*
|
|
37
|
+
* These markers are expected to be removed using publishUploads(). If that does not happen within
|
|
38
|
+
* the grace period, the files will be deleted.
|
|
39
|
+
*/
|
|
40
|
+
prepareUploads(paths: string[], now?: Date): Promise<PreparedObjectStorageUpload[]>;
|
|
41
|
+
canPublish(upload: PreparedObjectStorageUpload, now?: Date): boolean;
|
|
42
|
+
/**
|
|
43
|
+
* Remove markers for files that have finished uploading, and are now referenced from other documents.
|
|
44
|
+
*
|
|
45
|
+
* Call this in the same transaction as the one creating the references.
|
|
46
|
+
*/
|
|
47
|
+
publishUploads(uploads: PreparedObjectStorageUpload[], session: mongo.ClientSession): Promise<void>;
|
|
48
|
+
/**
|
|
49
|
+
* Mark old paths for deletion.
|
|
50
|
+
*
|
|
51
|
+
* By marking for deletion instead of immediately deleting:
|
|
52
|
+
* 1. This avoids inconsistencies/orphaned files between MongoDB and S3, by allowing the markers to
|
|
53
|
+
* be persisted transactionally, while the deletes happen asynchronously and can be retried.
|
|
54
|
+
* 2. The deletes are delayed, giving clients a grace period to continue using the files for
|
|
55
|
+
* in-progress requests.
|
|
56
|
+
*
|
|
57
|
+
* Call this in the same transaction as the one removing the references to these files.
|
|
58
|
+
*/
|
|
59
|
+
retire(paths: Iterable<string>, session: mongo.ClientSession, now?: Date): Promise<void>;
|
|
60
|
+
/**
|
|
61
|
+
* Discover and delete objects by their storage prefix
|
|
62
|
+
* without touching the database.
|
|
63
|
+
*/
|
|
64
|
+
deletePrefix(prefix: string, options?: ObjectStorageOperationOptions): Promise<{
|
|
65
|
+
objectCount: number;
|
|
66
|
+
}>;
|
|
67
|
+
cleanup(logger: Logger, options?: ObjectStorageOperationOptions): Promise<void>;
|
|
68
|
+
}
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
import * as bson from 'bson';
|
|
2
|
+
import { createHash, randomUUID } from 'node:crypto';
|
|
3
|
+
import { BucketDataObjectStorage } from './BucketDataObjectStorage.js';
|
|
4
|
+
export const OBJECT_STORAGE_UPLOAD_LEASE_MS = 60 * 60 * 1000;
|
|
5
|
+
export const OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS = 2 * 60 * 1000;
|
|
6
|
+
export const OBJECT_STORAGE_REFERENCE_GRACE_MS = 15 * 60 * 1000;
|
|
7
|
+
/**
|
|
8
|
+
* Protect against arbitrary-long bucket names and parameters exceeding the S3 key length limits.
|
|
9
|
+
*
|
|
10
|
+
* Any bucket name over this length is hashed.
|
|
11
|
+
*/
|
|
12
|
+
const MAX_READABLE_BUCKET_SEGMENT_BYTES = 128;
|
|
13
|
+
/**
|
|
14
|
+
* Coordinates the MongoDB outbox used to make object-storage changes recoverable.
|
|
15
|
+
*
|
|
16
|
+
* A MongoDB collection is used to track files that will be uploaded, or should be deleted.
|
|
17
|
+
* By using a MongoDB collection, we can use the same transaction that creates or removes
|
|
18
|
+
* references to these files, making the operations atomic.
|
|
19
|
+
*
|
|
20
|
+
* These markers indicate which files should be deleted - either by being orphaned during create,
|
|
21
|
+
* or after references to them are removed.
|
|
22
|
+
*/
|
|
23
|
+
export class ObjectStorageLifecycle {
|
|
24
|
+
db;
|
|
25
|
+
replicationStreamId;
|
|
26
|
+
objectStorage;
|
|
27
|
+
bucketData;
|
|
28
|
+
constructor(db, replicationStreamId, objectStorage) {
|
|
29
|
+
this.db = db;
|
|
30
|
+
this.replicationStreamId = replicationStreamId;
|
|
31
|
+
this.objectStorage = objectStorage;
|
|
32
|
+
this.bucketData = new BucketDataObjectStorage(objectStorage);
|
|
33
|
+
}
|
|
34
|
+
streamPrefix() {
|
|
35
|
+
return `bucket-data/${this.replicationStreamId}/`;
|
|
36
|
+
}
|
|
37
|
+
definitionPrefix(definitionId) {
|
|
38
|
+
return `${this.streamPrefix()}${definitionId}/`;
|
|
39
|
+
}
|
|
40
|
+
allocatePath(definitionId, bucket, minOp, maxOp) {
|
|
41
|
+
// Preserve readable paths for normal buckets, but bound user-controlled
|
|
42
|
+
// parameter values so they cannot consume the S3 key-length budget.
|
|
43
|
+
const escapedBucket = encodeURIComponent(bucket);
|
|
44
|
+
const bucketSegment = Buffer.byteLength(escapedBucket, 'utf8') <= MAX_READABLE_BUCKET_SEGMENT_BYTES
|
|
45
|
+
? escapedBucket
|
|
46
|
+
: `sha256-${createHash('sha256').update(bucket).digest('base64url')}`;
|
|
47
|
+
return `${this.definitionPrefix(definitionId)}${bucketSegment}/${minOp}-${maxOp}-${randomUUID()}.bson`;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Indicate that files will be uploaded to S3.
|
|
51
|
+
*
|
|
52
|
+
* These markers are expected to be removed using publishUploads(). If that does not happen within
|
|
53
|
+
* the grace period, the files will be deleted.
|
|
54
|
+
*/
|
|
55
|
+
async prepareUploads(paths, now = new Date()) {
|
|
56
|
+
const deleteAfter = new Date(now.getTime() + OBJECT_STORAGE_UPLOAD_LEASE_MS);
|
|
57
|
+
const uploads = paths.map((path) => ({ markerId: new bson.ObjectId(), path, deleteAfter }));
|
|
58
|
+
if (uploads.length) {
|
|
59
|
+
await this.db.pendingObjectStorageDeletes(this.replicationStreamId).insertMany(uploads.map((upload) => ({
|
|
60
|
+
_id: upload.markerId,
|
|
61
|
+
path: upload.path,
|
|
62
|
+
delete_after: upload.deleteAfter
|
|
63
|
+
})));
|
|
64
|
+
}
|
|
65
|
+
return uploads;
|
|
66
|
+
}
|
|
67
|
+
canPublish(upload, now = new Date()) {
|
|
68
|
+
return now.getTime() < upload.deleteAfter.getTime() - OBJECT_STORAGE_PUBLICATION_SAFETY_MARGIN_MS;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Remove markers for files that have finished uploading, and are now referenced from other documents.
|
|
72
|
+
*
|
|
73
|
+
* Call this in the same transaction as the one creating the references.
|
|
74
|
+
*/
|
|
75
|
+
async publishUploads(uploads, session) {
|
|
76
|
+
for (const upload of uploads) {
|
|
77
|
+
if (!this.canPublish(upload)) {
|
|
78
|
+
throw new Error(`Object storage publication lease expired for ${upload.path}`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
if (uploads.length === 0) {
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
const result = await this.db
|
|
85
|
+
.pendingObjectStorageDeletes(this.replicationStreamId)
|
|
86
|
+
.deleteMany({ _id: { $in: uploads.map((upload) => upload.markerId) } }, { session });
|
|
87
|
+
if (result.deletedCount !== uploads.length) {
|
|
88
|
+
throw new Error(`Missing object storage publication markers`);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Mark old paths for deletion.
|
|
93
|
+
*
|
|
94
|
+
* By marking for deletion instead of immediately deleting:
|
|
95
|
+
* 1. This avoids inconsistencies/orphaned files between MongoDB and S3, by allowing the markers to
|
|
96
|
+
* be persisted transactionally, while the deletes happen asynchronously and can be retried.
|
|
97
|
+
* 2. The deletes are delayed, giving clients a grace period to continue using the files for
|
|
98
|
+
* in-progress requests.
|
|
99
|
+
*
|
|
100
|
+
* Call this in the same transaction as the one removing the references to these files.
|
|
101
|
+
*/
|
|
102
|
+
async retire(paths, session, now = new Date()) {
|
|
103
|
+
const markers = Array.from(paths, (path) => ({
|
|
104
|
+
_id: new bson.ObjectId(),
|
|
105
|
+
path,
|
|
106
|
+
delete_after: new Date(now.getTime() + OBJECT_STORAGE_REFERENCE_GRACE_MS)
|
|
107
|
+
}));
|
|
108
|
+
if (markers.length) {
|
|
109
|
+
await this.db.pendingObjectStorageDeletes(this.replicationStreamId).insertMany(markers, { session });
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Discover and delete objects by their storage prefix
|
|
114
|
+
* without touching the database.
|
|
115
|
+
*/
|
|
116
|
+
async deletePrefix(prefix, options) {
|
|
117
|
+
return this.objectStorage.deletePrefix(prefix, { signal: options?.signal });
|
|
118
|
+
}
|
|
119
|
+
async cleanup(logger, options) {
|
|
120
|
+
const signal = options?.signal;
|
|
121
|
+
const markers = this.db.pendingObjectStorageDeletes(this.replicationStreamId);
|
|
122
|
+
const cursor = markers.find({ delete_after: { $lte: new Date() } });
|
|
123
|
+
let batch = [];
|
|
124
|
+
const deleteBatch = async () => {
|
|
125
|
+
if (batch.length === 0) {
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
const deleting = batch;
|
|
129
|
+
batch = [];
|
|
130
|
+
try {
|
|
131
|
+
await this.bucketData.delete(deleting.map((marker) => marker.path), { signal });
|
|
132
|
+
await markers.deleteMany({ _id: { $in: deleting.map((marker) => marker._id) } });
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
if (signal?.aborted) {
|
|
136
|
+
// Shutting down: stop instead of working through the remaining markers. They stay
|
|
137
|
+
// pending, and are cleaned up during the next compaction. Checked on the signal rather
|
|
138
|
+
// than the error, since a caller may abort with any reason.
|
|
139
|
+
throw error;
|
|
140
|
+
}
|
|
141
|
+
logger.warn('Failed to clean up object storage deletion markers; will retry during the next compaction', error);
|
|
142
|
+
}
|
|
143
|
+
};
|
|
144
|
+
for await (const marker of cursor) {
|
|
145
|
+
signal?.throwIfAborted();
|
|
146
|
+
batch.push(marker);
|
|
147
|
+
if (batch.length === 500) {
|
|
148
|
+
await deleteBatch();
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
await deleteBatch();
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
//# sourceMappingURL=ObjectStorageLifecycle.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ObjectStorageLifecycle.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAGvE,MAAM,CAAC,MAAM,8BAA8B,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC7D,MAAM,CAAC,MAAM,2CAA2C,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AACzE,MAAM,CAAC,MAAM,iCAAiC,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAChE;;;;GAIG;AACH,MAAM,iCAAiC,GAAG,GAAG,CAAC;AAQ9C;;;;;;;;;GASG;AACH,MAAM,OAAO,sBAAsB;IAId;IACA;IACA;IALV,UAAU,CAA0B;IAE7C,YACmB,EAA6B,EAC7B,mBAA2B,EAC3B,aAA4B;QAF5B,OAAE,GAAF,EAAE,CAA2B;QAC7B,wBAAmB,GAAnB,mBAAmB,CAAQ;QAC3B,kBAAa,GAAb,aAAa,CAAe;QAE7C,IAAI,CAAC,UAAU,GAAG,IAAI,uBAAuB,CAAC,aAAa,CAAC,CAAC;IAC/D,CAAC;IAED,YAAY;QACV,OAAO,eAAe,IAAI,CAAC,mBAAmB,GAAG,CAAC;IACpD,CAAC;IAED,gBAAgB,CAAC,YAAoB;QACnC,OAAO,GAAG,IAAI,CAAC,YAAY,EAAE,GAAG,YAAY,GAAG,CAAC;IAClD,CAAC;IAED,YAAY,CAAC,YAAoB,EAAE,MAAc,EAAE,KAAa,EAAE,KAAa;QAC7E,wEAAwE;QACxE,oEAAoE;QACpE,MAAM,aAAa,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;QACjD,MAAM,aAAa,GACjB,MAAM,CAAC,UAAU,CAAC,aAAa,EAAE,MAAM,CAAC,IAAI,iCAAiC;YAC3E,CAAC,CAAC,aAAa;YACf,CAAC,CAAC,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;QAC1E,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC,GAAG,aAAa,IAAI,KAAK,IAAI,KAAK,IAAI,UAAU,EAAE,OAAO,CAAC;IACzG,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,cAAc,CAAC,KAAe,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE;QACpD,MAAM,WAAW,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,8BAA8B,CAAC,CAAC;QAC7E,MAAM,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;QAC5F,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,UAAU,CAC5E,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACvB,GAAG,EAAE,MAAM,CAAC,QAAQ;gBACpB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,YAAY,EAAE,MAAM,CAAC,WAAW;aACjC,CAAC,CAAC,CACJ,CAAC;QACJ,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,UAAU,CAAC,MAAmC,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE;QAC9D,OAAO,GAAG,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,EAAE,GAAG,2CAA2C,CAAC;IACpG,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,cAAc,CAAC,OAAsC,EAAE,OAA4B;QACvF,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CAAC,gDAAgD,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YACjF,CAAC;QACH,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE;aACzB,2BAA2B,CAAC,IAAI,CAAC,mBAAmB,CAAC;aACrD,UAAU,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QACvF,IAAI,MAAM,CAAC,YAAY,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED;;;;;;;;;;OAUG;IACH,KAAK,CAAC,MAAM,CAAC,KAAuB,EAAE,OAA4B,EAAE,GAAG,GAAG,IAAI,IAAI,EAAE;QAClF,MAAM,OAAO,GAAkC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1E,GAAG,EAAE,IAAI,IAAI,CAAC,QAAQ,EAAE;YACxB,IAAI;YACJ,YAAY,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,iCAAiC,CAAC;SAC1E,CAAC,CAAC,CAAC;QACJ,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,MAAM,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QACvG,CAAC;IACH,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,OAAuC;QACxE,OAAO,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAAuC;QACnE,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC/B,MAAM,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,2BAA2B,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC9E,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,IAAI,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACpE,IAAI,KAAK,GAAkC,EAAE,CAAC;QAE9C,MAAM,WAAW,GAAG,KAAK,IAAI,EAAE;YAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO;YACT,CAAC;YACD,MAAM,QAAQ,GAAG,KAAK,CAAC;YACvB,KAAK,GAAG,EAAE,CAAC;YACX,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAC1B,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,EACrC,EAAE,MAAM,EAAE,CACX,CAAC;gBACF,MAAM,OAAO,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;YACnF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;oBACpB,kFAAkF;oBAClF,uFAAuF;oBACvF,4DAA4D;oBAC5D,MAAM,KAAK,CAAC;gBACd,CAAC;gBACD,MAAM,CAAC,IAAI,CAAC,2FAA2F,EAAE,KAAK,CAAC,CAAC;YAClH,CAAC;QACH,CAAC,CAAC;QAEF,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;YAClC,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACnB,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;gBACzB,MAAM,WAAW,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;QACD,MAAM,WAAW,EAAE,CAAC;IACtB,CAAC;CACF"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { BucketDefinitionId } from '@powersync/service-sync-rules';
|
|
3
|
+
import { BucketDataDocumentV3 } from '../models.js';
|
|
4
|
+
import { VersionedPowerSyncMongoV3 } from '../VersionedPowerSyncMongoV3.js';
|
|
5
|
+
export declare const OBJECT_STORAGE_USAGE_BASE_WRITER_ID = "__base__";
|
|
6
|
+
export declare const DEFAULT_OBJECT_STORAGE_USAGE_STALE_WRITER_MS: number;
|
|
7
|
+
export declare const DEFAULT_OBJECT_STORAGE_USAGE_FOLD_LIMIT = 100;
|
|
8
|
+
export interface ReplicationStreamObjectStorageDefinitionUsageResult {
|
|
9
|
+
replication_stream_id: number;
|
|
10
|
+
definition_id: BucketDefinitionId;
|
|
11
|
+
active_bytes: bigint;
|
|
12
|
+
}
|
|
13
|
+
export declare function createObjectStorageUsageWriterId(): string;
|
|
14
|
+
/**
|
|
15
|
+
* Tracks active S3 references. Values are signed writer-local deltas, so reference changes and
|
|
16
|
+
* accounting updates can be committed together without making replication and compaction share a
|
|
17
|
+
* hot counter document.
|
|
18
|
+
*/
|
|
19
|
+
export declare class ObjectStorageUsage {
|
|
20
|
+
private readonly db;
|
|
21
|
+
private readonly replicationStreamId;
|
|
22
|
+
readonly writerId?: string | undefined;
|
|
23
|
+
constructor(db: VersionedPowerSyncMongoV3, replicationStreamId: number, writerId?: string | undefined);
|
|
24
|
+
static bytes(document: Pick<BucketDataDocumentV3, 'storage_ref'>): bigint;
|
|
25
|
+
static readAllDefinitionUsage(db: VersionedPowerSyncMongoV3): Promise<ReplicationStreamObjectStorageDefinitionUsageResult[]>;
|
|
26
|
+
applyDelta(definitionId: BucketDefinitionId, delta: bigint, session: mongo.ClientSession): Promise<void>;
|
|
27
|
+
applyDeltas(deltas: ReadonlyMap<BucketDefinitionId, bigint>, session: mongo.ClientSession): Promise<void>;
|
|
28
|
+
removeDefinition(definitionId: BucketDefinitionId, session: mongo.ClientSession): Promise<void>;
|
|
29
|
+
removeStream(session: mongo.ClientSession): Promise<void>;
|
|
30
|
+
foldStaleWriterDeltas(options?: {
|
|
31
|
+
staleWriterMs?: number;
|
|
32
|
+
limit?: number;
|
|
33
|
+
}): Promise<void>;
|
|
34
|
+
private definitionPath;
|
|
35
|
+
private documentId;
|
|
36
|
+
private validateDefinitionId;
|
|
37
|
+
}
|
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
|
+
import { ReplicationAssertionError } from '@powersync/lib-services-framework';
|
|
3
|
+
import { randomUUID } from 'node:crypto';
|
|
4
|
+
export const OBJECT_STORAGE_USAGE_BASE_WRITER_ID = '__base__';
|
|
5
|
+
export const DEFAULT_OBJECT_STORAGE_USAGE_STALE_WRITER_MS = 30 * 60 * 1000;
|
|
6
|
+
export const DEFAULT_OBJECT_STORAGE_USAGE_FOLD_LIMIT = 100;
|
|
7
|
+
export function createObjectStorageUsageWriterId() {
|
|
8
|
+
return randomUUID();
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Tracks active S3 references. Values are signed writer-local deltas, so reference changes and
|
|
12
|
+
* accounting updates can be committed together without making replication and compaction share a
|
|
13
|
+
* hot counter document.
|
|
14
|
+
*/
|
|
15
|
+
export class ObjectStorageUsage {
|
|
16
|
+
db;
|
|
17
|
+
replicationStreamId;
|
|
18
|
+
writerId;
|
|
19
|
+
constructor(db, replicationStreamId, writerId) {
|
|
20
|
+
this.db = db;
|
|
21
|
+
this.replicationStreamId = replicationStreamId;
|
|
22
|
+
this.writerId = writerId;
|
|
23
|
+
if (writerId === OBJECT_STORAGE_USAGE_BASE_WRITER_ID) {
|
|
24
|
+
throw new ReplicationAssertionError('The reserved object-storage usage writer id cannot be used by a writer');
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
static bytes(document) {
|
|
28
|
+
return BigInt(document.storage_ref?.file_size ?? 0);
|
|
29
|
+
}
|
|
30
|
+
static async readAllDefinitionUsage(db) {
|
|
31
|
+
return db.client.withSession({ snapshot: true }, async (session) => {
|
|
32
|
+
// Deliberately read the whole usage collection. The number of replication streams is
|
|
33
|
+
// expected to stay low, so a collection scan is cheaper than issuing one _id range query
|
|
34
|
+
// per stream (and the usage collection is bounded by streams and writers, not buckets).
|
|
35
|
+
const entries = await db.objectStorageUsage
|
|
36
|
+
.aggregate([
|
|
37
|
+
{ $project: { replication_stream_id: '$_id.g', definitions: { $objectToArray: '$definitions' } } },
|
|
38
|
+
{ $unwind: '$definitions' },
|
|
39
|
+
{
|
|
40
|
+
$group: {
|
|
41
|
+
_id: {
|
|
42
|
+
replication_stream_id: '$replication_stream_id',
|
|
43
|
+
definition_id: '$definitions.k'
|
|
44
|
+
},
|
|
45
|
+
active_bytes: { $sum: '$definitions.v' }
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
], { session, readConcern: 'snapshot' })
|
|
49
|
+
.toArray()
|
|
50
|
+
.catch((error) => {
|
|
51
|
+
if (lib_mongo.isMongoNamespaceNotFoundError(error)) {
|
|
52
|
+
return [];
|
|
53
|
+
}
|
|
54
|
+
throw error;
|
|
55
|
+
});
|
|
56
|
+
return entries.map((entry) => {
|
|
57
|
+
const activeBytes = BigInt(entry.active_bytes ?? 0);
|
|
58
|
+
if (activeBytes < 0n) {
|
|
59
|
+
return {
|
|
60
|
+
replication_stream_id: entry._id.replication_stream_id,
|
|
61
|
+
definition_id: entry._id.definition_id,
|
|
62
|
+
active_bytes: 0n
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
return {
|
|
66
|
+
replication_stream_id: entry._id.replication_stream_id,
|
|
67
|
+
definition_id: entry._id.definition_id,
|
|
68
|
+
active_bytes: activeBytes
|
|
69
|
+
};
|
|
70
|
+
});
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
async applyDelta(definitionId, delta, session) {
|
|
74
|
+
await this.applyDeltas(new Map([[definitionId, delta]]), session);
|
|
75
|
+
}
|
|
76
|
+
async applyDeltas(deltas, session) {
|
|
77
|
+
if (this.writerId == null) {
|
|
78
|
+
throw new ReplicationAssertionError('A writer id is required to apply object-storage usage');
|
|
79
|
+
}
|
|
80
|
+
const increments = {};
|
|
81
|
+
for (const [definitionId, delta] of deltas) {
|
|
82
|
+
if (delta === 0n) {
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
increments[this.definitionPath(definitionId)] = delta;
|
|
86
|
+
}
|
|
87
|
+
if (Object.keys(increments).length === 0) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
await this.db.objectStorageUsage.updateOne({ _id: this.documentId() }, {
|
|
91
|
+
$inc: increments,
|
|
92
|
+
$currentDate: { updated_at: true }
|
|
93
|
+
}, { upsert: true, session });
|
|
94
|
+
}
|
|
95
|
+
async removeDefinition(definitionId, session) {
|
|
96
|
+
await this.db.objectStorageUsage.updateMany({ '_id.g': this.replicationStreamId }, { $unset: { [this.definitionPath(definitionId)]: 1 } }, { session });
|
|
97
|
+
}
|
|
98
|
+
async removeStream(session) {
|
|
99
|
+
await this.db.objectStorageUsage.deleteMany({ '_id.g': this.replicationStreamId }, { session });
|
|
100
|
+
}
|
|
101
|
+
async foldStaleWriterDeltas(options = {}) {
|
|
102
|
+
const staleWriterMs = options.staleWriterMs ?? DEFAULT_OBJECT_STORAGE_USAGE_STALE_WRITER_MS;
|
|
103
|
+
const limit = options.limit ?? DEFAULT_OBJECT_STORAGE_USAGE_FOLD_LIMIT;
|
|
104
|
+
if (limit <= 0) {
|
|
105
|
+
return;
|
|
106
|
+
}
|
|
107
|
+
await this.db.client.withSession((session) => session.withTransaction(async () => {
|
|
108
|
+
const staleDocuments = await this.db.objectStorageUsage
|
|
109
|
+
.find({
|
|
110
|
+
'_id.g': this.replicationStreamId,
|
|
111
|
+
'_id.w': { $ne: OBJECT_STORAGE_USAGE_BASE_WRITER_ID },
|
|
112
|
+
$expr: {
|
|
113
|
+
$lt: [
|
|
114
|
+
'$updated_at',
|
|
115
|
+
{ $dateSubtract: { startDate: '$$NOW', unit: 'millisecond', amount: staleWriterMs } }
|
|
116
|
+
]
|
|
117
|
+
}
|
|
118
|
+
}, { session, sort: { updated_at: 1 }, limit })
|
|
119
|
+
.toArray();
|
|
120
|
+
if (staleDocuments.length === 0) {
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
const deltas = new Map();
|
|
124
|
+
for (const document of staleDocuments) {
|
|
125
|
+
for (const [definitionId, delta] of Object.entries(document.definitions ?? {})) {
|
|
126
|
+
this.validateDefinitionId(definitionId);
|
|
127
|
+
deltas.set(definitionId, (deltas.get(definitionId) ?? 0n) + BigInt(delta));
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
const increments = {};
|
|
131
|
+
for (const [definitionId, delta] of deltas) {
|
|
132
|
+
if (delta !== 0n) {
|
|
133
|
+
increments[this.definitionPath(definitionId)] = delta;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
if (Object.keys(increments).length > 0) {
|
|
137
|
+
await this.db.objectStorageUsage.updateOne({ _id: this.documentId(OBJECT_STORAGE_USAGE_BASE_WRITER_ID) }, {
|
|
138
|
+
$inc: increments,
|
|
139
|
+
$currentDate: { updated_at: true }
|
|
140
|
+
}, { upsert: true, session });
|
|
141
|
+
}
|
|
142
|
+
await this.db.objectStorageUsage.deleteMany({
|
|
143
|
+
'_id.g': this.replicationStreamId,
|
|
144
|
+
_id: { $in: staleDocuments.map((document) => document._id) }
|
|
145
|
+
}, { session });
|
|
146
|
+
}, { readConcern: { level: 'snapshot' }, writeConcern: { w: 'majority' } }));
|
|
147
|
+
}
|
|
148
|
+
definitionPath(definitionId) {
|
|
149
|
+
this.validateDefinitionId(definitionId);
|
|
150
|
+
return `definitions.${definitionId}`;
|
|
151
|
+
}
|
|
152
|
+
documentId(writerId = this.writerId) {
|
|
153
|
+
if (writerId == null) {
|
|
154
|
+
throw new ReplicationAssertionError('A writer id is required to identify object-storage usage');
|
|
155
|
+
}
|
|
156
|
+
return {
|
|
157
|
+
g: this.replicationStreamId,
|
|
158
|
+
w: writerId
|
|
159
|
+
};
|
|
160
|
+
}
|
|
161
|
+
validateDefinitionId(definitionId) {
|
|
162
|
+
if (definitionId.length === 0 || definitionId.includes('.') || definitionId.includes('$')) {
|
|
163
|
+
throw new ReplicationAssertionError(`Invalid bucket definition id for object-storage usage: ${definitionId}`);
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
//# sourceMappingURL=ObjectStorageUsage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ObjectStorageUsage.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAE9E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIzC,MAAM,CAAC,MAAM,mCAAmC,GAAG,UAAU,CAAC;AAC9D,MAAM,CAAC,MAAM,4CAA4C,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC3E,MAAM,CAAC,MAAM,uCAAuC,GAAG,GAAG,CAAC;AAQ3D,MAAM,UAAU,gCAAgC;IAC9C,OAAO,UAAU,EAAE,CAAC;AACtB,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,kBAAkB;IAEV;IACA;IACR;IAHX,YACmB,EAA6B,EAC7B,mBAA2B,EACnC,QAAiB;QAFT,OAAE,GAAF,EAAE,CAA2B;QAC7B,wBAAmB,GAAnB,mBAAmB,CAAQ;QACnC,aAAQ,GAAR,QAAQ,CAAS;QAE1B,IAAI,QAAQ,KAAK,mCAAmC,EAAE,CAAC;YACrD,MAAM,IAAI,yBAAyB,CAAC,wEAAwE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,QAAmD;QAC9D,OAAO,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,sBAAsB,CACjC,EAA6B;QAE7B,OAAO,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACjE,qFAAqF;YACrF,yFAAyF;YACzF,wFAAwF;YACxF,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,kBAAkB;iBACxC,SAAS,CAIR;gBACE,EAAE,QAAQ,EAAE,EAAE,qBAAqB,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,cAAc,EAAE,EAAE,EAAE;gBAClG,EAAE,OAAO,EAAE,cAAc,EAAE;gBAC3B;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,qBAAqB,EAAE,wBAAwB;4BAC/C,aAAa,EAAE,gBAAgB;yBAChC;wBACD,YAAY,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;qBACzC;iBACF;aACF,EACD,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,CACrC;iBACA,OAAO,EAAE;iBACT,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACf,IAAI,SAAS,CAAC,6BAA6B,CAAC,KAAK,CAAC,EAAE,CAAC;oBACnD,OAAO,EAAE,CAAC;gBACZ,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;YAEL,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC3B,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;gBACpD,IAAI,WAAW,GAAG,EAAE,EAAE,CAAC;oBACrB,OAAO;wBACL,qBAAqB,EAAE,KAAK,CAAC,GAAG,CAAC,qBAAqB;wBACtD,aAAa,EAAE,KAAK,CAAC,GAAG,CAAC,aAAa;wBACtC,YAAY,EAAE,EAAE;qBACjB,CAAC;gBACJ,CAAC;gBACD,OAAO;oBACL,qBAAqB,EAAE,KAAK,CAAC,GAAG,CAAC,qBAAqB;oBACtD,aAAa,EAAE,KAAK,CAAC,GAAG,CAAC,aAAa;oBACtC,YAAY,EAAE,WAAW;iBAC1B,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,YAAgC,EAAE,KAAa,EAAE,OAA4B;QAC5F,MAAM,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,MAA+C,EAAE,OAA4B;QAC7F,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;YAC1B,MAAM,IAAI,yBAAyB,CAAC,uDAAuD,CAAC,CAAC;QAC/F,CAAC;QAED,MAAM,UAAU,GAA2B,EAAE,CAAC;QAC9C,KAAK,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YAC3C,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;gBACjB,SAAS;YACX,CAAC;YACD,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,GAAG,KAAK,CAAC;QACxD,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzC,OAAO;QACT,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,SAAS,CACxC,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,EAAE,EAC1B;YACE,IAAI,EAAE,UAAU;YAChB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;SACnC,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,YAAgC,EAAE,OAA4B;QACnF,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,UAAU,CACzC,EAAE,OAAO,EAAE,IAAI,CAAC,mBAAmB,EAAE,EACrC,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EACtD,EAAE,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC7C,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,mBAAmB,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAClG,CAAC;IAED,KAAK,CAAC,qBAAqB,CACzB,UAGI,EAAE;QAEN,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,4CAA4C,CAAC;QAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,uCAAuC,CAAC;QACvE,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,EAAE,CAC3C,OAAO,CAAC,eAAe,CACrB,KAAK,IAAI,EAAE;YACT,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB;iBACpD,IAAI,CACH;gBACE,OAAO,EAAE,IAAI,CAAC,mBAAmB;gBACjC,OAAO,EAAE,EAAE,GAAG,EAAE,mCAAmC,EAAE;gBACrD,KAAK,EAAE;oBACL,GAAG,EAAE;wBACH,aAAa;wBACb,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE;qBACtF;iBACF;aACF,EACD,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,CAC5C;iBACA,OAAO,EAAE,CAAC;YACb,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAChC,OAAO;YACT,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAA8B,CAAC;YACrD,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;gBACtC,KAAK,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,CAAC;oBAC/E,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;oBACxC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;YAED,MAAM,UAAU,GAA2B,EAAE,CAAC;YAC9C,KAAK,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;gBAC3C,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;oBACjB,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,GAAG,KAAK,CAAC;gBACxD,CAAC;YACH,CAAC;YACD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvC,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,SAAS,CACxC,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,mCAAmC,CAAC,EAAE,EAC7D;oBACE,IAAI,EAAE,UAAU;oBAChB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;iBACnC,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAC1B,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,UAAU,CACzC;gBACE,OAAO,EAAE,IAAI,CAAC,mBAAmB;gBACjC,GAAG,EAAE,EAAE,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;aAC7D,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;QACJ,CAAC,EACD,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,CACxE,CACF,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,YAAgC;QACrD,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;QACxC,OAAO,eAAe,YAAY,EAAE,CAAC;IACvC,CAAC;IAEO,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ;QACzC,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,IAAI,yBAAyB,CAAC,0DAA0D,CAAC,CAAC;QAClG,CAAC;QACD,OAAO;YACL,CAAC,EAAE,IAAI,CAAC,mBAAmB;YAC3B,CAAC,EAAE,QAAQ;SACZ,CAAC;IACJ,CAAC;IAEO,oBAAoB,CAAC,YAAgC;QAC3D,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1F,MAAM,IAAI,yBAAyB,CAAC,0DAA0D,YAAY,EAAE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;CACF"}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { S3Client } from '@aws-sdk/client-s3';
|
|
2
|
+
import { type DefaultsMode, type ResolvedDefaultsMode } from '@smithy/core/client';
|
|
3
|
+
import { type ObjectStorage, type ObjectStorageOperationOptions, type ObjectStoragePutMetadata } from './ObjectStorage.js';
|
|
4
|
+
export interface S3TimeoutProfile {
|
|
5
|
+
/**
|
|
6
|
+
* Per attempt: establishing the connection, including waiting for a socket from the pool.
|
|
7
|
+
*/
|
|
8
|
+
connectionTimeoutMs: number;
|
|
9
|
+
/**
|
|
10
|
+
* Per attempt: from starting the request until the response headers are received. For uploads
|
|
11
|
+
* this includes sending the body.
|
|
12
|
+
*/
|
|
13
|
+
requestTimeoutMs: number;
|
|
14
|
+
/**
|
|
15
|
+
* The complete operation, including SDK retries and streaming the response body.
|
|
16
|
+
*
|
|
17
|
+
* The SDK timeouts above stop applying once the response headers are received, so this is what
|
|
18
|
+
* bounds how long a single operation can occupy a concurrency slot.
|
|
19
|
+
*/
|
|
20
|
+
operationTimeoutMs: number;
|
|
21
|
+
/**
|
|
22
|
+
* Waiting for a concurrency slot, before the operation itself starts.
|
|
23
|
+
*/
|
|
24
|
+
queueTimeoutMs: number;
|
|
25
|
+
}
|
|
26
|
+
export declare function resolveTimeoutProfile(mode: ResolvedDefaultsMode): S3TimeoutProfile;
|
|
27
|
+
export interface S3ObjectStorageOptions {
|
|
28
|
+
bucket: string;
|
|
29
|
+
region?: string;
|
|
30
|
+
prefix?: string;
|
|
31
|
+
endpoint?: string;
|
|
32
|
+
forcePathStyle?: boolean;
|
|
33
|
+
accessKeyId?: string;
|
|
34
|
+
secretAccessKey?: string;
|
|
35
|
+
concurrencyLimit?: number;
|
|
36
|
+
/**
|
|
37
|
+
* AWS defaults mode, used as the baseline for the request timeouts.
|
|
38
|
+
*
|
|
39
|
+
* Defaults to the AWS_DEFAULTS_MODE environment variable. `auto` is treated as `standard`.
|
|
40
|
+
*/
|
|
41
|
+
defaultsMode?: DefaultsMode;
|
|
42
|
+
}
|
|
43
|
+
export declare class S3ObjectStorage implements ObjectStorage {
|
|
44
|
+
/**
|
|
45
|
+
* Public for tests only.
|
|
46
|
+
*/
|
|
47
|
+
readonly client: S3Client;
|
|
48
|
+
private bucket;
|
|
49
|
+
private prefix;
|
|
50
|
+
private readonly operationSemaphore;
|
|
51
|
+
private readonly timeouts;
|
|
52
|
+
constructor(options: S3ObjectStorageOptions);
|
|
53
|
+
put(path: string, data: Uint8Array, metadata: ObjectStoragePutMetadata, options?: ObjectStorageOperationOptions): Promise<void>;
|
|
54
|
+
get(path: string, options?: ObjectStorageOperationOptions): Promise<{
|
|
55
|
+
data: Uint8Array;
|
|
56
|
+
metadata: ObjectStoragePutMetadata;
|
|
57
|
+
}>;
|
|
58
|
+
list(prefix: string, options?: ObjectStorageOperationOptions): AsyncIterable<string>;
|
|
59
|
+
delete(paths: string[], options?: ObjectStorageOperationOptions): Promise<void>;
|
|
60
|
+
deletePrefix(prefix: string, options?: ObjectStorageOperationOptions): Promise<{
|
|
61
|
+
objectCount: number;
|
|
62
|
+
}>;
|
|
63
|
+
private listPage;
|
|
64
|
+
private deleteFullPaths;
|
|
65
|
+
private fullPath;
|
|
66
|
+
private withOperation;
|
|
67
|
+
}
|