@powersync/service-module-mongodb-storage 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/dist/storage/MongoBucketStorage.d.ts +5 -0
- package/dist/storage/MongoBucketStorage.js +7 -1
- package/dist/storage/MongoBucketStorage.js.map +1 -1
- package/dist/storage/implementation/MongoBucketBatch.d.ts +2 -1
- package/dist/storage/implementation/MongoBucketBatch.js +9 -8
- package/dist/storage/implementation/MongoBucketBatch.js.map +1 -1
- package/dist/storage/implementation/MongoStorageProvider.js +4 -2
- package/dist/storage/implementation/MongoStorageProvider.js.map +1 -1
- package/dist/storage/implementation/MongoWriteBatch.d.ts +49 -0
- package/dist/storage/implementation/MongoWriteBatch.js +149 -0
- package/dist/storage/implementation/MongoWriteBatch.js.map +1 -0
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js +2 -2
- package/dist/storage/implementation/MongoWriteCheckpointAPI.js.map +1 -1
- package/dist/storage/implementation/common/PersistedBatch.d.ts +5 -4
- package/dist/storage/implementation/common/PersistedBatch.js +6 -4
- package/dist/storage/implementation/common/PersistedBatch.js.map +1 -1
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.d.ts +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js +3 -0
- package/dist/storage/implementation/common/VersionedPowerSyncMongoBase.js.map +1 -1
- package/dist/storage/implementation/db.d.ts +5 -0
- package/dist/storage/implementation/db.js +13 -0
- package/dist/storage/implementation/db.js.map +1 -1
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js +13 -6
- package/dist/storage/implementation/v1/MongoBucketBatchV1.js.map +1 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.d.ts +2 -1
- package/dist/storage/implementation/v1/MongoCompactorV1.js +23 -17
- package/dist/storage/implementation/v1/MongoCompactorV1.js.map +1 -1
- package/dist/storage/implementation/v1/PersistedBatchV1.d.ts +7 -5
- package/dist/storage/implementation/v1/PersistedBatchV1.js +22 -28
- package/dist/storage/implementation/v1/PersistedBatchV1.js.map +1 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.d.ts +2 -1
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js +17 -14
- package/dist/storage/implementation/v3/MongoBucketBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.d.ts +6 -1
- package/dist/storage/implementation/v3/MongoCompactorV3.js +164 -69
- package/dist/storage/implementation/v3/MongoCompactorV3.js.map +1 -1
- package/dist/storage/implementation/v3/PersistedBatchV3.d.ts +12 -8
- package/dist/storage/implementation/v3/PersistedBatchV3.js +25 -32
- package/dist/storage/implementation/v3/PersistedBatchV3.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js +8 -1
- package/dist/storage/implementation/v3/object-storage/BucketDataObjectStorage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.d.ts +3 -3
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js +7 -9
- package/dist/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.d.ts +3 -2
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js +4 -7
- package/dist/storage/implementation/v3/object-storage/ObjectStorageUsage.js.map +1 -1
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js +21 -3
- package/dist/storage/implementation/v3/object-storage/S3ObjectStorage.js.map +1 -1
- package/dist/types/types.d.ts +5 -0
- package/dist/types/types.js +12 -0
- package/dist/types/types.js.map +1 -1
- package/dist/utils/test-utils.js +1 -0
- package/dist/utils/test-utils.js.map +1 -1
- package/package.json +6 -6
- package/src/storage/MongoBucketStorage.ts +12 -1
- package/src/storage/implementation/MongoBucketBatch.ts +13 -10
- package/src/storage/implementation/MongoStorageProvider.ts +12 -2
- package/src/storage/implementation/MongoWriteBatch.ts +172 -0
- package/src/storage/implementation/MongoWriteCheckpointAPI.ts +4 -2
- package/src/storage/implementation/common/PersistedBatch.ts +12 -8
- package/src/storage/implementation/common/VersionedPowerSyncMongoBase.ts +4 -0
- package/src/storage/implementation/db.ts +14 -0
- package/src/storage/implementation/v1/MongoBucketBatchV1.ts +17 -8
- package/src/storage/implementation/v1/MongoCompactorV1.ts +29 -26
- package/src/storage/implementation/v1/PersistedBatchV1.ts +25 -30
- package/src/storage/implementation/v3/MongoBucketBatchV3.ts +51 -49
- package/src/storage/implementation/v3/MongoCompactorV3.ts +276 -97
- package/src/storage/implementation/v3/PersistedBatchV3.ts +36 -36
- package/src/storage/implementation/v3/object-storage/BucketDataObjectStorage.ts +8 -1
- package/src/storage/implementation/v3/object-storage/ObjectStorageLifecycle.ts +11 -10
- package/src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts +8 -9
- package/src/storage/implementation/v3/object-storage/S3ObjectStorage.ts +23 -3
- package/src/types/types.ts +18 -0
- package/src/utils/test-utils.ts +1 -0
- package/test/src/chunk_compaction_config.test.ts +38 -0
- package/test/src/object_storage_usage.test.ts +16 -4
- package/test/src/storage_compacting.test.ts +257 -5
- package/test/src/storage_s3_reading.test.ts +47 -1
- package/tsconfig.tsbuildinfo +1 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ObjectStorageUsage.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAE9E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"ObjectStorageUsage.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/ObjectStorageUsage.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAE5D,OAAO,EAAE,yBAAyB,EAAE,MAAM,mCAAmC,CAAC;AAE9E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAKzC,MAAM,CAAC,MAAM,mCAAmC,GAAG,UAAU,CAAC;AAC9D,MAAM,CAAC,MAAM,4CAA4C,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;AAC3E,MAAM,CAAC,MAAM,uCAAuC,GAAG,GAAG,CAAC;AAQ3D,MAAM,UAAU,gCAAgC;IAC9C,OAAO,UAAU,EAAE,CAAC;AACtB,CAAC;AAED;;;;GAIG;AACH,MAAM,OAAO,kBAAkB;IAEV;IACA;IACR;IAHX,YACmB,EAA6B,EAC7B,mBAA2B,EACnC,QAAiB;QAFT,OAAE,GAAF,EAAE,CAA2B;QAC7B,wBAAmB,GAAnB,mBAAmB,CAAQ;QACnC,aAAQ,GAAR,QAAQ,CAAS;QAE1B,IAAI,QAAQ,KAAK,mCAAmC,EAAE,CAAC;YACrD,MAAM,IAAI,yBAAyB,CAAC,wEAAwE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,QAAmD;QAC9D,OAAO,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,sBAAsB,CACjC,EAA6B;QAE7B,OAAO,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE;YACjE,qFAAqF;YACrF,yFAAyF;YACzF,wFAAwF;YACxF,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,kBAAkB;iBACxC,SAAS,CAIR;gBACE,EAAE,QAAQ,EAAE,EAAE,qBAAqB,EAAE,QAAQ,EAAE,WAAW,EAAE,EAAE,cAAc,EAAE,cAAc,EAAE,EAAE,EAAE;gBAClG,EAAE,OAAO,EAAE,cAAc,EAAE;gBAC3B;oBACE,MAAM,EAAE;wBACN,GAAG,EAAE;4BACH,qBAAqB,EAAE,wBAAwB;4BAC/C,aAAa,EAAE,gBAAgB;yBAChC;wBACD,YAAY,EAAE,EAAE,IAAI,EAAE,gBAAgB,EAAE;qBACzC;iBACF;aACF,EACD,EAAE,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,CACrC;iBACA,OAAO,EAAE;iBACT,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACf,IAAI,SAAS,CAAC,6BAA6B,CAAC,KAAK,CAAC,EAAE,CAAC;oBACnD,OAAO,EAAE,CAAC;gBACZ,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;YAEL,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC3B,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,CAAC;gBACpD,IAAI,WAAW,GAAG,EAAE,EAAE,CAAC;oBACrB,OAAO;wBACL,qBAAqB,EAAE,KAAK,CAAC,GAAG,CAAC,qBAAqB;wBACtD,aAAa,EAAE,KAAK,CAAC,GAAG,CAAC,aAAa;wBACtC,YAAY,EAAE,EAAE;qBACjB,CAAC;gBACJ,CAAC;gBACD,OAAO;oBACL,qBAAqB,EAAE,KAAK,CAAC,GAAG,CAAC,qBAAqB;oBACtD,aAAa,EAAE,KAAK,CAAC,GAAG,CAAC,aAAa;oBACtC,YAAY,EAAE,WAAW;iBAC1B,CAAC;YACJ,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAED,UAAU,CAAC,YAAgC,EAAE,KAAa,EAAE,MAAuB;QACjF,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC7D,CAAC;IAED,WAAW,CAAC,MAA+C,EAAE,MAAuB;QAClF,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE,CAAC;YAC1B,MAAM,IAAI,yBAAyB,CAAC,uDAAuD,CAAC,CAAC;QAC/F,CAAC;QAED,MAAM,UAAU,GAA2B,EAAE,CAAC;QAC9C,KAAK,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;YAC3C,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;gBACjB,SAAS;YACX,CAAC;YACD,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,GAAG,KAAK,CAAC;QACxD,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzC,OAAO;QACT,CAAC;QAED,MAAM,CAAC,SAAS,CACd,IAAI,CAAC,EAAE,CAAC,kBAAkB,EAC1B,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,EAAE,EAC1B,EAAE,IAAI,EAAE,UAAU,EAAE,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,EAAE,EACxD,EAAE,MAAM,EAAE,IAAI,EAAE,CACjB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,YAAgC,EAAE,OAA4B;QACnF,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,UAAU,CACzC,EAAE,OAAO,EAAE,IAAI,CAAC,mBAAmB,EAAE,EACrC,EAAE,MAAM,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EACtD,EAAE,OAAO,EAAE,CACZ,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,OAA4B;QAC7C,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,UAAU,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,mBAAmB,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;IAClG,CAAC;IAED,KAAK,CAAC,qBAAqB,CACzB,UAGI,EAAE;QAEN,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,4CAA4C,CAAC;QAC5F,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,uCAAuC,CAAC;QACvE,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QAED,MAAM,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,OAAO,EAAE,EAAE,CAC3C,OAAO,CAAC,eAAe,CACrB,KAAK,IAAI,EAAE;YACT,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB;iBACpD,IAAI,CACH;gBACE,OAAO,EAAE,IAAI,CAAC,mBAAmB;gBACjC,OAAO,EAAE,EAAE,GAAG,EAAE,mCAAmC,EAAE;gBACrD,KAAK,EAAE;oBACL,GAAG,EAAE;wBACH,aAAa;wBACb,EAAE,aAAa,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE;qBACtF;iBACF;aACF,EACD,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,EAAE,EAAE,KAAK,EAAE,CAC5C;iBACA,OAAO,EAAE,CAAC;YACb,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAChC,OAAO;YACT,CAAC;YAED,MAAM,MAAM,GAAG,IAAI,GAAG,EAA8B,CAAC;YACrD,KAAK,MAAM,QAAQ,IAAI,cAAc,EAAE,CAAC;gBACtC,KAAK,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,WAAW,IAAI,EAAE,CAAC,EAAE,CAAC;oBAC/E,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;oBACxC,MAAM,CAAC,GAAG,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,EAAE,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;gBAC7E,CAAC;YACH,CAAC;YAED,MAAM,UAAU,GAA2B,EAAE,CAAC;YAC9C,KAAK,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;gBAC3C,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;oBACjB,UAAU,CAAC,IAAI,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC,GAAG,KAAK,CAAC;gBACxD,CAAC;YACH,CAAC;YACD,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvC,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,SAAS,CACxC,EAAE,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,mCAAmC,CAAC,EAAE,EAC7D;oBACE,IAAI,EAAE,UAAU;oBAChB,YAAY,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE;iBACnC,EACD,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAC1B,CAAC;YACJ,CAAC;YAED,MAAM,IAAI,CAAC,EAAE,CAAC,kBAAkB,CAAC,UAAU,CACzC;gBACE,OAAO,EAAE,IAAI,CAAC,mBAAmB;gBACjC,GAAG,EAAE,EAAE,GAAG,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE;aAC7D,EACD,EAAE,OAAO,EAAE,CACZ,CAAC;QACJ,CAAC,EACD,EAAE,WAAW,EAAE,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,YAAY,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE,EAAE,CACxE,CACF,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,YAAgC;QACrD,IAAI,CAAC,oBAAoB,CAAC,YAAY,CAAC,CAAC;QACxC,OAAO,eAAe,YAAY,EAAE,CAAC;IACvC,CAAC;IAEO,UAAU,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ;QACzC,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,IAAI,yBAAyB,CAAC,0DAA0D,CAAC,CAAC;QAClG,CAAC;QACD,OAAO;YACL,CAAC,EAAE,IAAI,CAAC,mBAAmB;YAC3B,CAAC,EAAE,QAAQ;SACZ,CAAC;IACJ,CAAC;IAEO,oBAAoB,CAAC,YAAgC;QAC3D,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1F,MAAM,IAAI,yBAAyB,CAAC,0DAA0D,YAAY,EAAE,CAAC,CAAC;QAChH,CAAC;IACH,CAAC;CACF"}
|
|
@@ -6,10 +6,12 @@ import { isThrottlingError, isTransientError } from '@smithy/core/retry';
|
|
|
6
6
|
import { NodeHttpHandler } from '@smithy/node-http-handler';
|
|
7
7
|
import { Semaphore } from 'async-mutex';
|
|
8
8
|
import { ObjectStorageError } from './ObjectStorage.js';
|
|
9
|
-
|
|
9
|
+
// Never statically import (or re-export) this module!
|
|
10
|
+
// We want to avoid loading the AWS SDK unless the user has configured S3 object storage, since it's a large dependency.
|
|
11
|
+
const DEFAULT_S3_OPERATION_CONCURRENCY = 64;
|
|
10
12
|
const S3_DELETE_PREFIX_BATCH_SIZE = 1000;
|
|
11
13
|
/**
|
|
12
|
-
*
|
|
14
|
+
* Keep prefix deletion below the default shared limit to leave capacity for other operations.
|
|
13
15
|
*/
|
|
14
16
|
const S3_DELETE_PREFIX_CONCURRENCY = 12;
|
|
15
17
|
const MAX_S3_PREFIX_BYTES = 256;
|
|
@@ -40,12 +42,19 @@ const SAFE_S3_KEY_BYTES = 896;
|
|
|
40
42
|
const BASELINE_CONNECTION_TIMEOUT_MS = 3_100;
|
|
41
43
|
const REQUEST_TIMEOUT_FACTOR = 5;
|
|
42
44
|
const OPERATION_TIMEOUT_FACTOR = 2;
|
|
45
|
+
/**
|
|
46
|
+
* Don't keep idle http sockets around for longer than this timeout.
|
|
47
|
+
*
|
|
48
|
+
* This covers the case where we have large bursts of activity during replication, followed by
|
|
49
|
+
* long periods of inactivity.
|
|
50
|
+
*/
|
|
51
|
+
const SOCKET_IDLE_TIMEOUT_MS = 60_000;
|
|
43
52
|
/**
|
|
44
53
|
* How long an operation may wait for a concurrency slot, as a multiple of the operation timeout.
|
|
45
54
|
*
|
|
46
55
|
* Every slot holder now releases within the operation timeout, so the queue always drains: this is
|
|
47
56
|
* a backpressure limit rather than a deadlock guard. Waiting this long means at least
|
|
48
|
-
* `factor * concurrencyLimit` operations ahead of us each took their full deadline -
|
|
57
|
+
* `factor * concurrencyLimit` operations ahead of us each took their full deadline - 256 at the
|
|
49
58
|
* default concurrency. Real operations complete in a fraction of the deadline, so the number of
|
|
50
59
|
* queued operations this actually tolerates is far higher, well above the ~2000 uploads that the
|
|
51
60
|
* largest possible replication flush (MAX_TRANSACTION_DOC_COUNT) can enqueue at once.
|
|
@@ -137,6 +146,15 @@ export class S3ObjectStorage {
|
|
|
137
146
|
forcePathStyle: options.forcePathStyle,
|
|
138
147
|
defaultsMode,
|
|
139
148
|
requestHandler: new NodeHttpHandler({
|
|
149
|
+
// Match the shared request limit instead of queuing again at the HTTP pool.
|
|
150
|
+
httpAgent: {
|
|
151
|
+
keepAlive: true,
|
|
152
|
+
maxSockets: concurrencyLimit,
|
|
153
|
+
// The timeout here is a little more general than an "idle timeout", but it does cover the
|
|
154
|
+
// case of closing idle sockets after a period of inactivity.
|
|
155
|
+
timeout: SOCKET_IDLE_TIMEOUT_MS
|
|
156
|
+
},
|
|
157
|
+
httpsAgent: { keepAlive: true, maxSockets: concurrencyLimit, timeout: SOCKET_IDLE_TIMEOUT_MS },
|
|
140
158
|
connectionTimeout: this.timeouts.connectionTimeoutMs,
|
|
141
159
|
requestTimeout: this.timeouts.requestTimeoutMs,
|
|
142
160
|
// Without this, exceeding requestTimeout only logs a warning.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"S3ObjectStorage.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/S3ObjectStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,oBAAoB,EAEpB,gBAAgB,EAChB,QAAQ,EACT,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EAAE,yBAAyB,EAAgD,MAAM,qBAAqB,CAAC;AAC9G,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAsB,MAAM,aAAa,CAAC;AAC5D,OAAO,EACL,kBAAkB,EAInB,MAAM,oBAAoB,CAAC;AAE5B,MAAM,gCAAgC,GAAG,EAAE,CAAC;AAC5C,MAAM,2BAA2B,GAAG,IAAI,CAAC;AACzC;;GAEG;AACH,MAAM,4BAA4B,GAAG,EAAE,CAAC;AACxC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,8BAA8B,GAAG,KAAK,CAAC;AAC7C,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACjC,MAAM,wBAAwB,GAAG,CAAC,CAAC;AACnC;;;;;;;;;GASG;AACH,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAyB/B,MAAM,UAAU,qBAAqB,CAAC,IAA0B;IAC9D,gGAAgG;IAChG,wEAAwE;IACxE,MAAM,EAAE,iBAAiB,EAAE,cAAc,EAAE,GAAG,yBAAyB,CAAC,IAAI,CAAC,CAAC;IAC9E,MAAM,mBAAmB,GAAG,iBAAiB,IAAI,8BAA8B,CAAC;IAChF,MAAM,gBAAgB,GAAG,cAAc,IAAI,mBAAmB,GAAG,sBAAsB,CAAC;IACxF,MAAM,kBAAkB,GAAG,gBAAgB,GAAG,wBAAwB,CAAC;IACvE,OAAO;QACL,mBAAmB;QACnB,gBAAgB;QAChB,kBAAkB;QAClB,cAAc,EAAE,kBAAkB,GAAG,oBAAoB;KAC1D,CAAC;AACJ,CAAC;AAED,MAAM,qBAAqB,GAAG,mBAAmB,CAAC;AAClD,MAAM,uBAAuB,GAA2B,CAAC,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;AAEtH;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAS,6BAA6B,CAAC,UAAyB;IAC9D,uEAAuE;IACvE,MAAM,IAAI,GAAG,CAAC,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1F,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACpB,MAAM,CAAC,IAAI,CACT,kFAAkF;YAChF,6EAA6E,CAChF,CAAC;QACF,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,MAAM,QAAQ,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC;IACjF,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CACb,6BAA6B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CACjH,CAAC;IACJ,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAoCD,MAAM,OAAO,eAAe;IAC1B;;OAEG;IACa,MAAM,CAAW;IACzB,MAAM,CAAS;IACf,MAAM,CAAS;IACN,kBAAkB,CAAqB;IACvC,QAAQ,CAAmB;IAE5C,YAAY,OAA+B;QACzC,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,gCAAgC,CAAC;QACtF,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,IAAI,gBAAgB,IAAI,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;QACnF,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,EAAE,CAAC;YACxE,MAAM,IAAI,KAAK,CAAC,+EAA+E,CAAC,CAAC;QACnG,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACpC,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,mBAAmB,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,4CAA4C,mBAAmB,cAAc,CAAC,CAAC;QACjG,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,kBAAkB,GAAG,IAAI,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAE1D,gGAAgG;QAChG,MAAM,YAAY,GAAG,6BAA6B,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACzE,IAAI,CAAC,QAAQ,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;QAEpD,IAAI,CAAC,MAAM,GAAG,IAAI,QAAQ,CAAC;YACzB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,YAAY;YACZ,cAAc,EAAE,IAAI,eAAe,CAAC;gBAClC,iBAAiB,EAAE,IAAI,CAAC,QAAQ,CAAC,mBAAmB;gBACpD,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,gBAAgB;gBAC9C,8DAA8D;gBAC9D,qBAAqB,EAAE,IAAI;aAC5B,CAAC;YACF,WAAW,EACT,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,eAAe;gBAC5C,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,eAAe,EAAE,OAAO,CAAC,eAAe,EAAE;gBAChF,CAAC,CAAC,SAAS;SAChB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,GAAG,CACP,IAAY,EACZ,IAAgB,EAChB,QAAkC,EAClC,OAAuC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,YAAY,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACtF,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CACpB,IAAI,gBAAgB,CAAC;gBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,QAAQ;gBACb,IAAI,EAAE,IAAI;gBACV,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACjC,eAAe,EAAE,QAAQ,CAAC,eAAe,IAAI,SAAS;aACvD,CAAC,EACF,EAAE,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,CAClC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CACP,IAAY,EACZ,OAAuC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,YAAY,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACxF,wFAAwF;QACxF,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CACrC,IAAI,gBAAgB,CAAC;gBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,QAAQ;aACd,CAAC,EACF,EAAE,WAAW,EAAE,MAAM,EAAE,CACxB,CAAC;YACF,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC;YAC7C,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,2BAA2B,CAAC,CAAC;YACnF,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBAC9D,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,8BAA8B,aAAa,EAAE,CAAC,CAAC;YACrG,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,aAAa,CAAC,CAAC;YAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAiC,CAAC;YAC1D,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,MAAM,CAAC,cAAc,EAAE,CAAC;gBACxB,MAAM,UAAU,GAAG,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC;gBAC7C,IAAI,UAAU,GAAG,aAAa,EAAE,CAAC;oBAC/B,MAAM,IAAI,KAAK,CACb,0CAA0C,QAAQ,cAAc,aAAa,6BAA6B,UAAU,EAAE,CACvH,CAAC;gBACJ,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBACxB,MAAM,GAAG,UAAU,CAAC;YACtB,CAAC;YACD,IAAI,MAAM,KAAK,aAAa,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,0CAA0C,QAAQ,cAAc,aAAa,oBAAoB,MAAM,EAAE,CAC1G,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,IAAI;gBACJ,QAAQ,EAAE;oBACR,WAAW,EAAE,QAAQ,CAAC,WAAW,IAAI,0BAA0B;oBAC/D,eAAe,EAAE,QAAQ,CAAC,eAAe,IAAI,IAAI;iBAClD;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACzD,MAAM,IAAI,kBAAkB,CAAC,wBAAwB,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YACrG,CAAC;YACD,MAAM,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,IAAI,CAAC,MAAc,EAAE,OAAuC;QACjE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,iBAAqC,CAAC;QAC1C,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAE/B,GAAG,CAAC;YACF,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;YAC5E,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;gBAC7C,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,SAAS;gBACX,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;YAC5E,CAAC;YACD,iBAAiB,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS,CAAC;YACtF,IAAI,QAAQ,CAAC,WAAW,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;gBACtD,MAAM,IAAI,KAAK,CAAC,kBAAkB,UAAU,6CAA6C,CAAC,CAAC;YAC7F,CAAC;QACH,CAAC,QAAQ,iBAAiB,IAAI,IAAI,EAAE;IACtC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAe,EAAE,OAAuC;QACnE,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,OAAuC;QACxE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC/B,IAAI,iBAAqC,CAAC;QAC1C,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAiB,CAAC;QAC1C,IAAI,WAA2C,CAAC;QAEhD,MAAM,cAAc,GAAG,CAAC,SAA4B,EAAE,EAAE;YACtD,IAAI,OAAsB,CAAC;YAC3B,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC;iBAC9C,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACf,WAAW,KAAK,EAAE,KAAK,EAAE,CAAC;YAC5B,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3C,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACxB,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,GAAG,EAAE;YAC5B,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM,WAAW,CAAC,KAAK,CAAC;YAC1B,CAAC;QACH,CAAC,CAAC;QAEF,IAAI,cAA8C,CAAC;QACnD,IAAI,CAAC;YACH,GAAG,CAAC;gBACF,MAAM,EAAE,cAAc,EAAE,CAAC;gBACzB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;gBAC5E,gBAAgB,EAAE,CAAC;gBACnB,MAAM,SAAS,GAAsB,EAAE,CAAC;gBACxC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;oBAC7C,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;wBACvB,SAAS,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;oBACtC,CAAC;gBACH,CAAC;gBACD,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC;gBAChC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC3B,OAAO,QAAQ,CAAC,IAAI,IAAI,4BAA4B,EAAE,CAAC;wBACrD,MAAM,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;wBAC7B,gBAAgB,EAAE,CAAC;oBACrB,CAAC;oBACD,cAAc,CAAC,SAAS,CAAC,CAAC;gBAC5B,CAAC;gBACD,iBAAiB,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS,CAAC;gBACtF,IAAI,QAAQ,CAAC,WAAW,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;oBACtD,MAAM,IAAI,KAAK,CAAC,kBAAkB,UAAU,6CAA6C,CAAC,CAAC;gBAC7F,CAAC;YACH,CAAC,QAAQ,iBAAiB,IAAI,IAAI,EAAE;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,cAAc,GAAG,EAAE,KAAK,EAAE,CAAC;QAC7B,CAAC;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5B,IAAI,cAAc,IAAI,IAAI,EAAE,CAAC;YAC3B,MAAM,cAAc,CAAC,KAAK,CAAC;QAC7B,CAAC;QACD,gBAAgB,EAAE,CAAC;QACnB,OAAO,EAAE,WAAW,EAAE,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,QAAQ,CACpB,UAAkB,EAClB,iBAAqC,EACrC,MAAoB;QAEpB,+FAA+F;QAC/F,sCAAsC;QACtC,YAAY,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;QAC7E,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAC3B,IAAI,oBAAoB,CAAC;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,UAAU;gBAClB,iBAAiB,EAAE,iBAAiB;gBACpC,OAAO,EAAE,2BAA2B;aACrC,CAAC,EACF,EAAE,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,CAClC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,SAA4B,EAAE,MAAoB;QAC9E,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACnC,YAAY,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,GAAG,SAAS,CAAC,MAAM,UAAU,EAAE,MAAM,CAAC,CAAC;QAClG,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAC/B,IAAI,oBAAoB,CAAC;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE;aAC5C,CAAC,EACF,EAAE,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,CAClC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxF,MAAM,IAAI,kBAAkB,CAAC,gCAAgC,MAAM,EAAE,EAAE;gBACrE,KAAK,EAAE,QAAQ,CAAC,MAAM;gBACtB,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aACrF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,IAAY;QAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACjD,IAAI,IAAI,GAAG,iBAAiB,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,uCAAuC,iBAAiB,QAAQ,CAAC,CAAC;QAC5G,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,SAAiB,EAAE,MAAc,EAAE,MAAoB;QACjF,MAAM,EAAE,cAAc,EAAE,CAAC;QACzB,MAAM,EAAE,kBAAkB,EAAE,cAAc,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC;QAE7D,4FAA4F;QAC5F,gCAAgC;QAChC,MAAM,aAAa,GAAG,aAAa,CAAC,cAAc,EAAE,GAAG,EAAE,CACvD,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,mBAAmB,cAAc,oCAAoC,CAAC,CACzG,CAAC;QACF,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,yBAAyB,CACxC,IAAI,CAAC,kBAAkB,EACvB,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAChF,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,aAAa,CAAC,MAAM,EAAE,CAAC;QACzB,CAAC;QACD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,4EAA4E;YAC5E,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,MAAM,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;QACpC,CAAC;QAED,MAAM,CAAC,EAAE,OAAO,CAAC,GAAG,QAAQ,CAAC;QAC7B,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACpB,OAAO,EAAE,CAAC;YACV,MAAM,CAAC,cAAc,EAAE,CAAC;QAC1B,CAAC;QACD,yFAAyF;QACzF,MAAM,QAAQ,GAAG,aAAa,CAAC,kBAAkB,EAAE,GAAG,EAAE,CACtD,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,2BAA2B,kBAAkB,KAAK,CAAC,CACtF,CAAC;QACF,MAAM,eAAe,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC9F,OAAO;YACL,MAAM,EAAE,eAAe;YACvB,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;gBACf,oFAAoF;gBACpF,qFAAqF;gBACrF,uFAAuF;gBACvF,wFAAwF;gBACxF,kEAAkE;gBAClE,IAAI,eAAe,CAAC,OAAO,EAAE,CAAC;oBAC5B,OAAO,eAAe,CAAC,MAAe,CAAC;gBACzC,CAAC;gBACD,OAAO,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACpD,CAAC;YACD,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,KAAK,IAAI,EAAE;gBAChC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,OAAO,EAAE,CAAC;YACZ,CAAC;SACF,CAAC;IACJ,CAAC;CACF;AAED;;;;;;GAMG;AACH,SAAS,aAAa,CAAC,SAAiB,EAAE,YAA2B;IACnE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;IAC5E,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;AAC1E,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,SAAiB,EAAE,MAAc,EAAE,OAAe;IACxE,OAAO,IAAI,kBAAkB,CAAC,MAAM,SAAS,QAAQ,MAAM,IAAI,OAAO,EAAE,EAAE;QACxE,KAAK,EAAE,IAAI,YAAY,CAAC,OAAO,EAAE,cAAc,CAAC;QAChD,SAAS,EAAE,IAAI;KAChB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAiB,EAAE,MAAc,EAAE,KAAc;IACzE,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,2EAA2E;IAC3E,kDAAkD;IAClD,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,KAAc,CAAC;IACxB,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,OAAO,IAAI,kBAAkB,CAAC,MAAM,SAAS,eAAe,MAAM,GAAG,MAAM,EAAE,EAAE;QAC7E,KAAK,EAAE,KAAK;QACZ,SAAS,EAAE,kBAAkB,CAAC,KAAK,CAAC;KACrC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,QAAQ,GAAG,KAA+C,CAAC;IACjE,uEAAuE;IACvE,4EAA4E;IAC5E,6EAA6E;IAC7E,qBAAqB;IACrB,8EAA8E;IAC9E,8CAA8C;IAC9C,OAAO,iBAAiB,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,CAAC;AACnE,CAAC"}
|
|
1
|
+
{"version":3,"file":"S3ObjectStorage.js","sourceRoot":"","sources":["../../../../../src/storage/implementation/v3/object-storage/S3ObjectStorage.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAChB,oBAAoB,EAEpB,gBAAgB,EAChB,QAAQ,EACT,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,MAAM,EAAE,MAAM,mCAAmC,CAAC;AAC3D,OAAO,EAAE,yBAAyB,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAClF,OAAO,EAAE,yBAAyB,EAAgD,MAAM,qBAAqB,CAAC;AAC9G,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACzE,OAAO,EAAE,eAAe,EAAE,MAAM,2BAA2B,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAsB,MAAM,aAAa,CAAC;AAC5D,OAAO,EACL,kBAAkB,EAInB,MAAM,oBAAoB,CAAC;AAE5B,sDAAsD;AACtD,wHAAwH;AAExH,MAAM,gCAAgC,GAAG,EAAE,CAAC;AAC5C,MAAM,2BAA2B,GAAG,IAAI,CAAC;AACzC;;GAEG;AACH,MAAM,4BAA4B,GAAG,EAAE,CAAC;AACxC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAChC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,8BAA8B,GAAG,KAAK,CAAC;AAC7C,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACjC,MAAM,wBAAwB,GAAG,CAAC,CAAC;AACnC;;;;;GAKG;AACH,MAAM,sBAAsB,GAAG,MAAM,CAAC;AAEtC;;;;;;;;;GASG;AACH,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAyB/B,MAAM,UAAU,qBAAqB,CAAC,IAA0B;IAC9D,gGAAgG;IAChG,wEAAwE;IACxE,MAAM,EAAE,iBAAiB,EAAE,cAAc,EAAE,GAAG,yBAAyB,CAAC,IAAI,CAAC,CAAC;IAC9E,MAAM,mBAAmB,GAAG,iBAAiB,IAAI,8BAA8B,CAAC;IAChF,MAAM,gBAAgB,GAAG,cAAc,IAAI,mBAAmB,GAAG,sBAAsB,CAAC;IACxF,MAAM,kBAAkB,GAAG,gBAAgB,GAAG,wBAAwB,CAAC;IACvE,OAAO;QACL,mBAAmB;QACnB,gBAAgB;QAChB,kBAAkB;QAClB,cAAc,EAAE,kBAAkB,GAAG,oBAAoB;KAC1D,CAAC;AACJ,CAAC;AAED,MAAM,qBAAqB,GAAG,mBAAmB,CAAC;AAClD,MAAM,uBAAuB,GAA2B,CAAC,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;AAEtH;;;;;;;;;;;;;;;;;;;GAmBG;AACH,SAAS,6BAA6B,CAAC,UAAyB;IAC9D,uEAAuE;IACvE,MAAM,IAAI,GAAG,CAAC,UAAU,IAAI,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;IAC1F,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACpB,MAAM,CAAC,IAAI,CACT,kFAAkF;YAChF,6EAA6E,CAChF,CAAC;QACF,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,MAAM,QAAQ,GAAG,uBAAuB,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC;IACjF,IAAI,QAAQ,IAAI,IAAI,EAAE,CAAC;QACrB,MAAM,IAAI,KAAK,CACb,6BAA6B,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,qBAAqB,uBAAuB,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CACjH,CAAC;IACJ,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAoCD,MAAM,OAAO,eAAe;IAC1B;;OAEG;IACa,MAAM,CAAW;IACzB,MAAM,CAAS;IACf,MAAM,CAAS;IACN,kBAAkB,CAAqB;IACvC,QAAQ,CAAmB;IAE5C,YAAY,OAA+B;QACzC,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,gCAAgC,CAAC;QACtF,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,gBAAgB,CAAC,IAAI,gBAAgB,IAAI,CAAC,EAAE,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;QACnF,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,eAAe,IAAI,IAAI,CAAC,EAAE,CAAC;YACxE,MAAM,IAAI,KAAK,CAAC,+EAA+E,CAAC,CAAC;QACnG,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAC;QACpC,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,mBAAmB,EAAE,CAAC;YAC5D,MAAM,IAAI,KAAK,CAAC,4CAA4C,mBAAmB,cAAc,CAAC,CAAC;QACjG,CAAC;QACD,IAAI,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACpE,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,kBAAkB,GAAG,IAAI,SAAS,CAAC,gBAAgB,CAAC,CAAC;QAE1D,gGAAgG;QAChG,MAAM,YAAY,GAAG,6BAA6B,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QACzE,IAAI,CAAC,QAAQ,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAAC;QAEpD,IAAI,CAAC,MAAM,GAAG,IAAI,QAAQ,CAAC;YACzB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,YAAY;YACZ,cAAc,EAAE,IAAI,eAAe,CAAC;gBAClC,4EAA4E;gBAC5E,SAAS,EAAE;oBACT,SAAS,EAAE,IAAI;oBACf,UAAU,EAAE,gBAAgB;oBAC5B,0FAA0F;oBAC1F,6DAA6D;oBAC7D,OAAO,EAAE,sBAAsB;iBAChC;gBACD,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,UAAU,EAAE,gBAAgB,EAAE,OAAO,EAAE,sBAAsB,EAAE;gBAC9F,iBAAiB,EAAE,IAAI,CAAC,QAAQ,CAAC,mBAAmB;gBACpD,cAAc,EAAE,IAAI,CAAC,QAAQ,CAAC,gBAAgB;gBAC9C,8DAA8D;gBAC9D,qBAAqB,EAAE,IAAI;aAC5B,CAAC;YACF,WAAW,EACT,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,eAAe;gBAC5C,CAAC,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,eAAe,EAAE,OAAO,CAAC,eAAe,EAAE;gBAChF,CAAC,CAAC,SAAS;SAChB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,GAAG,CACP,IAAY,EACZ,IAAgB,EAChB,QAAkC,EAClC,OAAuC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,YAAY,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACtF,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CACpB,IAAI,gBAAgB,CAAC;gBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,QAAQ;gBACb,IAAI,EAAE,IAAI;gBACV,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACjC,eAAe,EAAE,QAAQ,CAAC,eAAe,IAAI,SAAS;aACvD,CAAC,EACF,EAAE,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,CAClC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CACP,IAAY,EACZ,OAAuC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,YAAY,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;QACxF,wFAAwF;QACxF,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC;QAChC,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CACrC,IAAI,gBAAgB,CAAC;gBACnB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,GAAG,EAAE,QAAQ;aACd,CAAC,EACF,EAAE,WAAW,EAAE,MAAM,EAAE,CACxB,CAAC;YACF,MAAM,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC;YAC7C,IAAI,aAAa,IAAI,IAAI,EAAE,CAAC;gBAC1B,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,2BAA2B,CAAC,CAAC;YACnF,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;gBAC9D,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,8BAA8B,aAAa,EAAE,CAAC,CAAC;YACrG,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,aAAa,CAAC,CAAC;YAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAiC,CAAC;YAC1D,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBACjC,MAAM,CAAC,cAAc,EAAE,CAAC;gBACxB,MAAM,UAAU,GAAG,MAAM,GAAG,KAAK,CAAC,UAAU,CAAC;gBAC7C,IAAI,UAAU,GAAG,aAAa,EAAE,CAAC;oBAC/B,MAAM,IAAI,KAAK,CACb,0CAA0C,QAAQ,cAAc,aAAa,6BAA6B,UAAU,EAAE,CACvH,CAAC;gBACJ,CAAC;gBACD,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBACxB,MAAM,GAAG,UAAU,CAAC;YACtB,CAAC;YACD,IAAI,MAAM,KAAK,aAAa,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,0CAA0C,QAAQ,cAAc,aAAa,oBAAoB,MAAM,EAAE,CAC1G,CAAC;YACJ,CAAC;YAED,OAAO;gBACL,IAAI;gBACJ,QAAQ,EAAE;oBACR,WAAW,EAAE,QAAQ,CAAC,WAAW,IAAI,0BAA0B;oBAC/D,eAAe,EAAE,QAAQ,CAAC,eAAe,IAAI,IAAI;iBAClD;aACF,CAAC;QACJ,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACzD,MAAM,IAAI,kBAAkB,CAAC,wBAAwB,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;YACrG,CAAC;YACD,MAAM,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,KAAK,CAAC,CAAC,IAAI,CAAC,MAAc,EAAE,OAAuC;QACjE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,iBAAqC,CAAC;QAC1C,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAE/B,GAAG,CAAC;YACF,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;YAC5E,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;gBAC7C,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;oBACvB,SAAS;gBACX,CAAC;gBACD,MAAM,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;YAC5E,CAAC;YACD,iBAAiB,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS,CAAC;YACtF,IAAI,QAAQ,CAAC,WAAW,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;gBACtD,MAAM,IAAI,KAAK,CAAC,kBAAkB,UAAU,6CAA6C,CAAC,CAAC;YAC7F,CAAC;QACH,CAAC,QAAQ,iBAAiB,IAAI,IAAI,EAAE;IACtC,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAe,EAAE,OAAuC;QACnE,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACtE,MAAM,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,OAAuC;QACxE,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;QAC/B,IAAI,iBAAqC,CAAC;QAC1C,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAiB,CAAC;QAC1C,IAAI,WAA2C,CAAC;QAEhD,MAAM,cAAc,GAAG,CAAC,SAA4B,EAAE,EAAE;YACtD,IAAI,OAAsB,CAAC;YAC3B,OAAO,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,MAAM,CAAC;iBAC9C,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACf,WAAW,KAAK,EAAE,KAAK,EAAE,CAAC;YAC5B,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC;YAC3C,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACxB,CAAC,CAAC;QACF,MAAM,gBAAgB,GAAG,GAAG,EAAE;YAC5B,IAAI,WAAW,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM,WAAW,CAAC,KAAK,CAAC;YAC1B,CAAC;QACH,CAAC,CAAC;QAEF,IAAI,cAA8C,CAAC;QACnD,IAAI,CAAC;YACH,GAAG,CAAC;gBACF,MAAM,EAAE,cAAc,EAAE,CAAC;gBACzB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,iBAAiB,EAAE,MAAM,CAAC,CAAC;gBAC5E,gBAAgB,EAAE,CAAC;gBACnB,MAAM,SAAS,GAAsB,EAAE,CAAC;gBACxC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC;oBAC7C,IAAI,MAAM,CAAC,GAAG,IAAI,IAAI,EAAE,CAAC;wBACvB,SAAS,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC;oBACtC,CAAC;gBACH,CAAC;gBACD,WAAW,IAAI,SAAS,CAAC,MAAM,CAAC;gBAChC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBAC3B,OAAO,QAAQ,CAAC,IAAI,IAAI,4BAA4B,EAAE,CAAC;wBACrD,MAAM,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;wBAC7B,gBAAgB,EAAE,CAAC;oBACrB,CAAC;oBACD,cAAc,CAAC,SAAS,CAAC,CAAC;gBAC5B,CAAC;gBACD,iBAAiB,GAAG,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,qBAAqB,CAAC,CAAC,CAAC,SAAS,CAAC;gBACtF,IAAI,QAAQ,CAAC,WAAW,IAAI,iBAAiB,IAAI,IAAI,EAAE,CAAC;oBACtD,MAAM,IAAI,KAAK,CAAC,kBAAkB,UAAU,6CAA6C,CAAC,CAAC;gBAC7F,CAAC;YACH,CAAC,QAAQ,iBAAiB,IAAI,IAAI,EAAE;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,cAAc,GAAG,EAAE,KAAK,EAAE,CAAC;QAC7B,CAAC;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAC5B,IAAI,cAAc,IAAI,IAAI,EAAE,CAAC;YAC3B,MAAM,cAAc,CAAC,KAAK,CAAC;QAC7B,CAAC;QACD,gBAAgB,EAAE,CAAC;QACnB,OAAO,EAAE,WAAW,EAAE,CAAC;IACzB,CAAC;IAEO,KAAK,CAAC,QAAQ,CACpB,UAAkB,EAClB,iBAAqC,EACrC,MAAoB;QAEpB,+FAA+F;QAC/F,sCAAsC;QACtC,YAAY,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,CAAC,CAAC;QAC7E,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAC3B,IAAI,oBAAoB,CAAC;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,UAAU;gBAClB,iBAAiB,EAAE,iBAAiB;gBACpC,OAAO,EAAE,2BAA2B;aACrC,CAAC,EACF,EAAE,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,CAClC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,SAA4B,EAAE,MAAoB;QAC9E,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACnC,YAAY,SAAS,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,GAAG,SAAS,CAAC,MAAM,UAAU,EAAE,MAAM,CAAC,CAAC;QAClG,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAC/B,IAAI,oBAAoB,CAAC;gBACvB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,MAAM,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,IAAI,EAAE;aAC5C,CAAC,EACF,EAAE,WAAW,EAAE,SAAS,CAAC,MAAM,EAAE,CAClC,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,GAAG,KAAK,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxF,MAAM,IAAI,kBAAkB,CAAC,gCAAgC,MAAM,EAAE,EAAE;gBACrE,KAAK,EAAE,QAAQ,CAAC,MAAM;gBACtB,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,kBAAkB,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;aACrF,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,IAAY;QAC3B,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QAC/D,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACjD,IAAI,IAAI,GAAG,iBAAiB,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,oBAAoB,IAAI,uCAAuC,iBAAiB,QAAQ,CAAC,CAAC;QAC5G,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,SAAiB,EAAE,MAAc,EAAE,MAAoB;QACjF,MAAM,EAAE,cAAc,EAAE,CAAC;QACzB,MAAM,EAAE,kBAAkB,EAAE,cAAc,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC;QAE7D,4FAA4F;QAC5F,gCAAgC;QAChC,MAAM,aAAa,GAAG,aAAa,CAAC,cAAc,EAAE,GAAG,EAAE,CACvD,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,mBAAmB,cAAc,oCAAoC,CAAC,CACzG,CAAC;QACF,IAAI,QAAQ,CAAC;QACb,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,yBAAyB,CACxC,IAAI,CAAC,kBAAkB,EACvB,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAChF,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,aAAa,CAAC,MAAM,EAAE,CAAC;QACzB,CAAC;QACD,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,4EAA4E;YAC5E,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,MAAM,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC;QACpC,CAAC;QAED,MAAM,CAAC,EAAE,OAAO,CAAC,GAAG,QAAQ,CAAC;QAC7B,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;YACpB,OAAO,EAAE,CAAC;YACV,MAAM,CAAC,cAAc,EAAE,CAAC;QAC1B,CAAC;QACD,yFAAyF;QACzF,MAAM,QAAQ,GAAG,aAAa,CAAC,kBAAkB,EAAE,GAAG,EAAE,CACtD,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,2BAA2B,kBAAkB,KAAK,CAAC,CACtF,CAAC;QACF,MAAM,eAAe,GAAG,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC9F,OAAO;YACL,MAAM,EAAE,eAAe;YACvB,KAAK,EAAE,CAAC,KAAK,EAAE,EAAE;gBACf,oFAAoF;gBACpF,qFAAqF;gBACrF,uFAAuF;gBACvF,wFAAwF;gBACxF,kEAAkE;gBAClE,IAAI,eAAe,CAAC,OAAO,EAAE,CAAC;oBAC5B,OAAO,eAAe,CAAC,MAAe,CAAC;gBACzC,CAAC;gBACD,OAAO,gBAAgB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;YACpD,CAAC;YACD,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,KAAK,IAAI,EAAE;gBAChC,QAAQ,CAAC,MAAM,EAAE,CAAC;gBAClB,OAAO,EAAE,CAAC;YACZ,CAAC;SACF,CAAC;IACJ,CAAC;CACF;AAED;;;;;;GAMG;AACH,SAAS,aAAa,CAAC,SAAiB,EAAE,YAA2B;IACnE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;IAC5E,KAAK,CAAC,KAAK,EAAE,CAAC;IACd,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;AAC1E,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,SAAiB,EAAE,MAAc,EAAE,OAAe;IACxE,OAAO,IAAI,kBAAkB,CAAC,MAAM,SAAS,QAAQ,MAAM,IAAI,OAAO,EAAE,EAAE;QACxE,KAAK,EAAE,IAAI,YAAY,CAAC,OAAO,EAAE,cAAc,CAAC;QAChD,SAAS,EAAE,IAAI;KAChB,CAAC,CAAC;AACL,CAAC;AAED,SAAS,gBAAgB,CAAC,SAAiB,EAAE,MAAc,EAAE,KAAc;IACzE,IAAI,KAAK,YAAY,kBAAkB,EAAE,CAAC;QACxC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,2EAA2E;IAC3E,kDAAkD;IAClD,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,KAAc,CAAC;IACxB,CAAC;IACD,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAClE,OAAO,IAAI,kBAAkB,CAAC,MAAM,SAAS,eAAe,MAAM,GAAG,MAAM,EAAE,EAAE;QAC7E,KAAK,EAAE,KAAK;QACZ,SAAS,EAAE,kBAAkB,CAAC,KAAK,CAAC;KACrC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,OAAO,KAAK,IAAI,QAAQ,IAAI,KAAK,IAAI,IAAI,EAAE,CAAC;QAC9C,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,QAAQ,GAAG,KAA+C,CAAC;IACjE,uEAAuE;IACvE,4EAA4E;IAC5E,6EAA6E;IAC7E,qBAAqB;IACrB,8EAA8E;IAC9E,8CAA8C;IAC9C,OAAO,iBAAiB,CAAC,QAAQ,CAAC,IAAI,gBAAgB,CAAC,QAAQ,CAAC,CAAC;AACnE,CAAC"}
|
package/dist/types/types.d.ts
CHANGED
|
@@ -55,6 +55,8 @@ export declare const MongoStorageConfig: t.Intersection<t.Codec<{
|
|
|
55
55
|
* Defaults to 0.2.
|
|
56
56
|
*/
|
|
57
57
|
clear_batch_throttle_rate: t.OptionalCodec<t.Codec<number, number, string, t.CodecProps>>;
|
|
58
|
+
/** Maximum concurrent buckets across V3 chunk-compaction jobs. Range: 1–64. Default: 4 with object storage, otherwise 2. */
|
|
59
|
+
chunk_compaction_concurrency: t.OptionalCodec<t.Codec<number, number, string, t.CodecProps>>;
|
|
58
60
|
object_storage: t.OptionalCodec<t.Codec<{
|
|
59
61
|
type: "s3";
|
|
60
62
|
bucket: string;
|
|
@@ -84,5 +86,8 @@ export declare const MongoStorageConfig: t.Intersection<t.Codec<{
|
|
|
84
86
|
export type MongoStorageConfig = t.Encoded<typeof MongoStorageConfig>;
|
|
85
87
|
export type MongoStorageConfigDecoded = t.Decoded<typeof MongoStorageConfig>;
|
|
86
88
|
export declare const DEFAULT_CLEAR_BATCH_THROTTLE_RATE = 0.2;
|
|
89
|
+
export declare const DEFAULT_CHUNK_COMPACTION_CONCURRENCY = 2;
|
|
90
|
+
export declare const DEFAULT_OBJECT_STORAGE_CHUNK_COMPACTION_CONCURRENCY = 4;
|
|
91
|
+
export declare function normalizeChunkCompactionConcurrency(value: number | undefined, hasObjectStorage?: boolean): number;
|
|
87
92
|
export declare function normalizeClearBatchThrottleRate(value: number | undefined): number;
|
|
88
93
|
export declare function isMongoStorageConfig(config: service_types.configFile.GenericStorageConfig): config is MongoStorageConfig;
|
package/dist/types/types.js
CHANGED
|
@@ -49,9 +49,21 @@ export const MongoStorageConfig = service_types.configFile.BaseStorageConfig.and
|
|
|
49
49
|
* Defaults to 0.2.
|
|
50
50
|
*/
|
|
51
51
|
clear_batch_throttle_rate: t.number.optional(),
|
|
52
|
+
/** Maximum concurrent buckets across V3 chunk-compaction jobs. Range: 1–64. Default: 4 with object storage, otherwise 2. */
|
|
53
|
+
chunk_compaction_concurrency: t.number.optional(),
|
|
52
54
|
object_storage: S3ObjectStorageConfig.optional()
|
|
53
55
|
}));
|
|
54
56
|
export const DEFAULT_CLEAR_BATCH_THROTTLE_RATE = 0.2;
|
|
57
|
+
export const DEFAULT_CHUNK_COMPACTION_CONCURRENCY = 2;
|
|
58
|
+
export const DEFAULT_OBJECT_STORAGE_CHUNK_COMPACTION_CONCURRENCY = 4;
|
|
59
|
+
export function normalizeChunkCompactionConcurrency(value, hasObjectStorage = false) {
|
|
60
|
+
const concurrency = value ??
|
|
61
|
+
(hasObjectStorage ? DEFAULT_OBJECT_STORAGE_CHUNK_COMPACTION_CONCURRENCY : DEFAULT_CHUNK_COMPACTION_CONCURRENCY);
|
|
62
|
+
if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 64) {
|
|
63
|
+
throw new ServiceError(ErrorCode.PSYNC_S3201, 'storage.chunk_compaction_concurrency must be an integer between 1 and 64');
|
|
64
|
+
}
|
|
65
|
+
return concurrency;
|
|
66
|
+
}
|
|
55
67
|
export function normalizeClearBatchThrottleRate(value) {
|
|
56
68
|
const rate = value ?? DEFAULT_CLEAR_BATCH_THROTTLE_RATE;
|
|
57
69
|
if (!Number.isFinite(rate) || rate < 0 || rate > 20) {
|
package/dist/types/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,aAAa,CAAC,WAAW,CAAC,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;AAI3G,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IACrB,MAAM,EAAE,CAAC,CAAC,MAAM;IAChB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC3B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IACtC,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAClC,iBAAiB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACtC,iBAAiB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACtC,0DAA0D;IAC1D,aAAa,EAAE,cAAc,CAAC,QAAQ,EAAE;IACxC,oEAAoE;IACpE,yEAAyE;IACzE,sBAAsB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CAC5C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,aAAa,CAAC,WAAW,CACjE,SAAS,EACT,kBAAkB,EAClB,WAAW,EACX,oBAAoB,EACpB,SAAS,CACV,CAAC;AAIF,MAAM,CAAC,MAAM,kBAAkB,GAAG,aAAa,CAAC,UAAU,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,GAAG,CAC7G,CAAC,CAAC,MAAM,CAAC;IACP;;;;;;OAMG;IACH,oBAAoB,EAAE,0BAA0B,CAAC,QAAQ,EAAE;IAC3D;;;;;;;;OAQG;IACH,yBAAyB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAE9C,cAAc,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CACjD,CAAC,CACH,CAAC;AAKF,MAAM,CAAC,MAAM,iCAAiC,GAAG,GAAG,CAAC;
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/types/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,SAAS,MAAM,gCAAgC,CAAC;AAC5D,OAAO,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,mCAAmC,CAAC;AAC5E,OAAO,KAAK,aAAa,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,CAAC,MAAM,UAAU,CAAC;AAE9B;;;;;;;;;GASG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,aAAa,CAAC,WAAW,CAAC,UAAU,EAAE,WAAW,EAAE,cAAc,EAAE,QAAQ,CAAC,CAAC;AAI3G,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IACrB,MAAM,EAAE,CAAC,CAAC,MAAM;IAChB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC3B,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC3B,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAC7B,gBAAgB,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE;IACtC,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAClC,iBAAiB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACtC,iBAAiB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IACtC,0DAA0D;IAC1D,aAAa,EAAE,cAAc,CAAC,QAAQ,EAAE;IACxC,oEAAoE;IACpE,yEAAyE;IACzE,sBAAsB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;CAC5C,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,0BAA0B,GAAG,aAAa,CAAC,WAAW,CACjE,SAAS,EACT,kBAAkB,EAClB,WAAW,EACX,oBAAoB,EACpB,SAAS,CACV,CAAC;AAIF,MAAM,CAAC,MAAM,kBAAkB,GAAG,aAAa,CAAC,UAAU,CAAC,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,eAAe,CAAC,CAAC,GAAG,CAC7G,CAAC,CAAC,MAAM,CAAC;IACP;;;;;;OAMG;IACH,oBAAoB,EAAE,0BAA0B,CAAC,QAAQ,EAAE;IAC3D;;;;;;;;OAQG;IACH,yBAAyB,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAE9C,4HAA4H;IAC5H,4BAA4B,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,EAAE;IAEjD,cAAc,EAAE,qBAAqB,CAAC,QAAQ,EAAE;CACjD,CAAC,CACH,CAAC;AAKF,MAAM,CAAC,MAAM,iCAAiC,GAAG,GAAG,CAAC;AACrD,MAAM,CAAC,MAAM,oCAAoC,GAAG,CAAC,CAAC;AACtD,MAAM,CAAC,MAAM,mDAAmD,GAAG,CAAC,CAAC;AAErE,MAAM,UAAU,mCAAmC,CAAC,KAAyB,EAAE,gBAAgB,GAAG,KAAK;IACrG,MAAM,WAAW,GACf,KAAK;QACL,CAAC,gBAAgB,CAAC,CAAC,CAAC,mDAAmD,CAAC,CAAC,CAAC,oCAAoC,CAAC,CAAC;IAClH,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,IAAI,WAAW,GAAG,CAAC,IAAI,WAAW,GAAG,EAAE,EAAE,CAAC;QAC9E,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,0EAA0E,CAC3E,CAAC;IACJ,CAAC;IACD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,+BAA+B,CAAC,KAAyB;IACvE,MAAM,IAAI,GAAG,KAAK,IAAI,iCAAiC,CAAC;IACxD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,GAAG,EAAE,EAAE,CAAC;QACpD,MAAM,IAAI,YAAY,CACpB,SAAS,CAAC,WAAW,EACrB,4EAA4E,CAC7E,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,MAAqD;IAErD,OAAO,MAAM,CAAC,IAAI,IAAI,SAAS,CAAC,qBAAqB,CAAC;AACxD,CAAC"}
|
package/dist/utils/test-utils.js
CHANGED
|
@@ -20,6 +20,7 @@ export function mongoTestStorageFactoryGenerator(factoryOptions) {
|
|
|
20
20
|
checksumOptions: factoryOptions.checksumOptions,
|
|
21
21
|
supportsMultipleSyncConfigs: factoryOptions.supportsMultipleSyncConfigs,
|
|
22
22
|
objectStorage: factoryOptions.objectStorage,
|
|
23
|
+
chunkCompactionConcurrency: factoryOptions.chunkCompactionConcurrency,
|
|
23
24
|
inlineThresholdBytes: factoryOptions.inlineThresholdBytes
|
|
24
25
|
});
|
|
25
26
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAA6B,MAAM,kCAAkC,CAAC;AACjG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAOjE,MAAM,UAAU,gCAAgC,CAAC,cAAuC;IACtF,OAAO;QACL,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;YAEzE,8EAA8E;YAC9E,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBAC5F,MAAM,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;YAED,8EAA8E;YAC9E,MAAM,EAAE,CAAC,gCAAgC,EAAE,CAAC;YAE5C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;gBACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACnB,CAAC;YAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,EAAE;gBAChC,2BAA2B,EAAE,OAAO;gBACpC,eAAe,EAAE,cAAc,CAAC,eAAe;gBAC/C,2BAA2B,EAAE,cAAc,CAAC,2BAA2B;gBACvE,aAAa,EAAE,cAAc,CAAC,aAAa;gBAC3C,oBAAoB,EAAE,cAAc,CAAC,oBAAoB;aAC1D,CAAC,CAAC;QACL,CAAC;QACD,cAAc,EAAE,KAAK;KACtB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sCAAsC,CAAC,cAAuC;IAC5F,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;QAEzE,MAAM,EAAE,CAAC,mCAAmC,EAAE,CAAC;QAE/C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;YACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;QAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE;QACxC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"test-utils.js","sourceRoot":"","sources":["../../src/utils/test-utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gCAAgC,CAAC;AAEvD,OAAO,EAAE,kBAAkB,EAA6B,MAAM,kCAAkC,CAAC;AACjG,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,MAAM,iCAAiC,CAAC;AAOjE,MAAM,UAAU,gCAAgC,CAAC,cAAuC;IACtF,OAAO;QACL,OAAO,EAAE,KAAK,EAAE,OAA4B,EAAE,EAAE;YAC9C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;YAEzE,8EAA8E;YAC9E,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,eAAe,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC,iBAAiB,CAAC,cAAc,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;gBAC5F,MAAM,EAAE,CAAC,EAAE,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,CAAC;YACpD,CAAC;YAED,8EAA8E;YAC9E,MAAM,EAAE,CAAC,gCAAgC,EAAE,CAAC;YAE5C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;gBACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;YACnB,CAAC;YAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,EAAE;gBAChC,2BAA2B,EAAE,OAAO;gBACpC,eAAe,EAAE,cAAc,CAAC,eAAe;gBAC/C,2BAA2B,EAAE,cAAc,CAAC,2BAA2B;gBACvE,aAAa,EAAE,cAAc,CAAC,aAAa;gBAC3C,0BAA0B,EAAE,cAAc,CAAC,0BAA0B;gBACrE,oBAAoB,EAAE,cAAc,CAAC,oBAAoB;aAC1D,CAAC,CAAC;QACL,CAAC;QACD,cAAc,EAAE,KAAK;KACtB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,sCAAsC,CAAC,cAAuC;IAC5F,OAAO,KAAK,EAAE,OAA4B,EAAE,EAAE;QAC5C,MAAM,EAAE,GAAG,oBAAoB,CAAC,cAAc,CAAC,GAAG,EAAE,cAAc,CAAC,IAAI,CAAC,CAAC;QAEzE,MAAM,EAAE,CAAC,mCAAmC,EAAE,CAAC;QAE/C,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,CAAC;YACzB,MAAM,EAAE,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;QAED,OAAO,IAAI,kBAAkB,CAAC,EAAE,CAAC,CAAC;IACpC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAG,CAAC,GAAW,EAAE,IAAa,EAAE,EAAE;IACjE,0EAA0E;IAC1E,oEAAoE;IACpE,MAAM,MAAM,GAAG,IAAI,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE;QACxC,gBAAgB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACvC,eAAe,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;QACtC,wBAAwB,EAAE,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;KAChD,CAAC,CAAC;IACH,OAAO,IAAI,cAAc,CAAC,MAAM,CAAC,CAAC;AACpC,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@powersync/service-module-mongodb-storage",
|
|
3
3
|
"repository": "https://github.com/powersync-ja/powersync-service",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.22.0",
|
|
5
5
|
"license": "FSL-1.1-ALv2",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"publishConfig": {
|
|
@@ -21,15 +21,15 @@
|
|
|
21
21
|
"lru-cache": "^10.2.2",
|
|
22
22
|
"ts-codec": "^1.3.0",
|
|
23
23
|
"uuid": "^14.0.0",
|
|
24
|
-
"@powersync/lib-service-mongodb": "0.6.
|
|
25
|
-
"@powersync/lib-services-framework": "0.10.
|
|
26
|
-
"@powersync/service-core": "1.26.
|
|
24
|
+
"@powersync/lib-service-mongodb": "0.6.32",
|
|
25
|
+
"@powersync/lib-services-framework": "0.10.2",
|
|
26
|
+
"@powersync/service-core": "1.26.1",
|
|
27
27
|
"@powersync/service-jsonbig": "0.17.13",
|
|
28
|
-
"@powersync/service-sync-rules": "0.
|
|
28
|
+
"@powersync/service-sync-rules": "0.42.0",
|
|
29
29
|
"@powersync/service-types": "0.18.0"
|
|
30
30
|
},
|
|
31
31
|
"devDependencies": {
|
|
32
|
-
"@powersync/service-core-tests": "0.18.
|
|
32
|
+
"@powersync/service-core-tests": "0.18.2"
|
|
33
33
|
},
|
|
34
34
|
"scripts": {
|
|
35
35
|
"build": "tsc -b",
|
|
@@ -22,8 +22,9 @@ import {
|
|
|
22
22
|
formatIncrementalSyncConfigUpdateLog,
|
|
23
23
|
isCompatible
|
|
24
24
|
} from '@powersync/service-core';
|
|
25
|
+
import { Semaphore } from 'async-mutex';
|
|
25
26
|
import { ObjectId } from 'bson';
|
|
26
|
-
import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE } from '../types/types.js';
|
|
27
|
+
import { DEFAULT_CLEAR_BATCH_THROTTLE_RATE, normalizeChunkCompactionConcurrency } from '../types/types.js';
|
|
27
28
|
import { generateReplicationStreamName } from '../utils/util.js';
|
|
28
29
|
import type { MongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
|
|
29
30
|
import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBucketStorage.js';
|
|
@@ -42,6 +43,8 @@ export interface MongoBucketStorageOptions {
|
|
|
42
43
|
checksumOptions?: Omit<MongoChecksumOptions, 'storageConfig'>;
|
|
43
44
|
objectStorage?: ObjectStorage;
|
|
44
45
|
inlineThresholdBytes?: number;
|
|
46
|
+
/** Shared across chunk-compaction jobs. Default: 4 with object storage, otherwise 2. */
|
|
47
|
+
chunkCompactionConcurrency?: number;
|
|
45
48
|
/**
|
|
46
49
|
* Prefix for replication stream name and Postgres logical replication slot name.
|
|
47
50
|
*/
|
|
@@ -69,12 +72,20 @@ export class MongoBucketStorage extends storage.BucketStorageFactory {
|
|
|
69
72
|
private activeStorageCache: MongoSyncBucketStorage | undefined;
|
|
70
73
|
|
|
71
74
|
public readonly db: PowerSyncMongo;
|
|
75
|
+
public readonly chunkCompactionConcurrency: number;
|
|
76
|
+
public readonly chunkCompactionSlots: Semaphore;
|
|
72
77
|
|
|
73
78
|
constructor(
|
|
74
79
|
db: PowerSyncMongo,
|
|
75
80
|
private options: MongoBucketStorageOptions
|
|
76
81
|
) {
|
|
77
82
|
super();
|
|
83
|
+
this.chunkCompactionConcurrency = normalizeChunkCompactionConcurrency(
|
|
84
|
+
options.chunkCompactionConcurrency,
|
|
85
|
+
options.objectStorage != null
|
|
86
|
+
);
|
|
87
|
+
// All replication streams created by this factory share the configured limit.
|
|
88
|
+
this.chunkCompactionSlots = new Semaphore(this.chunkCompactionConcurrency);
|
|
78
89
|
this.client = db.client;
|
|
79
90
|
this.db = db;
|
|
80
91
|
this.replicationStreamNamePrefix = options.replicationStreamNamePrefix;
|
|
@@ -36,6 +36,7 @@ import type { VersionedPowerSyncMongo } from './db.js';
|
|
|
36
36
|
import { MAX_ROW_SIZE } from './MongoBucketBatchShared.js';
|
|
37
37
|
import { MongoIdSequence } from './MongoIdSequence.js';
|
|
38
38
|
import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js';
|
|
39
|
+
import { MongoWriteBatch } from './MongoWriteBatch.js';
|
|
39
40
|
import { OperationBatch, RecordOperation } from './OperationBatch.js';
|
|
40
41
|
import { ObjectStorage } from './v3/object-storage/ObjectStorage.js';
|
|
41
42
|
import { createObjectStorageUsageWriterId } from './v3/object-storage/ObjectStorageUsage.js';
|
|
@@ -239,12 +240,14 @@ export abstract class MongoBucketBatch
|
|
|
239
240
|
if (this.write_checkpoint_batch.length > 0) {
|
|
240
241
|
this.logger.info(`Writing ${this.write_checkpoint_batch.length} custom write checkpoints`);
|
|
241
242
|
await this.batchCreateCustomWriteCheckpoints(session, opSeq.next());
|
|
242
|
-
this.write_checkpoint_batch = [];
|
|
243
243
|
}
|
|
244
244
|
|
|
245
245
|
last_op = opSeq.last();
|
|
246
246
|
});
|
|
247
247
|
|
|
248
|
+
// Keep checkpoints available if the transaction retries after writing them.
|
|
249
|
+
this.write_checkpoint_batch = [];
|
|
250
|
+
|
|
248
251
|
if (clearedError) {
|
|
249
252
|
this.clearedError = true;
|
|
250
253
|
}
|
|
@@ -706,7 +709,9 @@ export abstract class MongoBucketBatch
|
|
|
706
709
|
|
|
707
710
|
await callback(session, opSeq);
|
|
708
711
|
|
|
709
|
-
|
|
712
|
+
const writes = this.db.createWriteBatch(session, { ordered: false });
|
|
713
|
+
writes.updateOne(
|
|
714
|
+
this.db.op_id_sequence,
|
|
710
715
|
{
|
|
711
716
|
_id: 'main'
|
|
712
717
|
},
|
|
@@ -714,13 +719,11 @@ export abstract class MongoBucketBatch
|
|
|
714
719
|
$set: {
|
|
715
720
|
op_id: opSeq.last()
|
|
716
721
|
}
|
|
717
|
-
},
|
|
718
|
-
{
|
|
719
|
-
session
|
|
720
722
|
}
|
|
721
723
|
);
|
|
722
724
|
|
|
723
|
-
|
|
725
|
+
writes.updateOne(
|
|
726
|
+
this.db.sync_rules,
|
|
724
727
|
{
|
|
725
728
|
_id: this.replicationStreamId
|
|
726
729
|
},
|
|
@@ -728,13 +731,13 @@ export abstract class MongoBucketBatch
|
|
|
728
731
|
$set: {
|
|
729
732
|
last_keepalive_ts: new Date()
|
|
730
733
|
}
|
|
731
|
-
}
|
|
732
|
-
{ session }
|
|
734
|
+
}
|
|
733
735
|
);
|
|
734
736
|
|
|
735
737
|
// Allow subclasses to persist additional flush-time state in the same transaction
|
|
736
738
|
// (e.g. v3 advances the stream-level last_persisted_op).
|
|
737
|
-
|
|
739
|
+
this.onReplicationTransactionFlush(writes, opSeq.last());
|
|
740
|
+
await writes.execute();
|
|
738
741
|
// We don't notify checkpoint here - we don't make any checkpoint updates directly
|
|
739
742
|
});
|
|
740
743
|
}
|
|
@@ -745,7 +748,7 @@ export abstract class MongoBucketBatch
|
|
|
745
748
|
* The base implementation does nothing; v3 storage overrides this to `$max` the stream-level
|
|
746
749
|
* `last_persisted_op` durably within the same transaction.
|
|
747
750
|
*/
|
|
748
|
-
protected
|
|
751
|
+
protected onReplicationTransactionFlush(_writes: MongoWriteBatch, _lastOp: InternalOpId): void {
|
|
749
752
|
// No-op by default (v1 behaviour unchanged).
|
|
750
753
|
}
|
|
751
754
|
|
|
@@ -1,7 +1,11 @@
|
|
|
1
1
|
import * as lib_mongo from '@powersync/lib-service-mongodb';
|
|
2
2
|
import { ErrorCode, logger, ServiceAssertionError, ServiceError } from '@powersync/lib-services-framework';
|
|
3
3
|
import { POWERSYNC_VERSION, storage } from '@powersync/service-core';
|
|
4
|
-
import {
|
|
4
|
+
import {
|
|
5
|
+
MongoStorageConfig,
|
|
6
|
+
normalizeChunkCompactionConcurrency,
|
|
7
|
+
normalizeClearBatchThrottleRate
|
|
8
|
+
} from '../../types/types.js';
|
|
5
9
|
import { MongoBucketStorage } from '../MongoBucketStorage.js';
|
|
6
10
|
import { MongoReportStorage } from '../MongoReportStorage.js';
|
|
7
11
|
import { PowerSyncMongo } from './db.js';
|
|
@@ -24,6 +28,10 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
24
28
|
}
|
|
25
29
|
|
|
26
30
|
const decodedConfig = MongoStorageConfig.decode(storage as any);
|
|
31
|
+
const chunkCompactionConcurrency = normalizeChunkCompactionConcurrency(
|
|
32
|
+
decodedConfig.chunk_compaction_concurrency,
|
|
33
|
+
decodedConfig.object_storage != null
|
|
34
|
+
);
|
|
27
35
|
|
|
28
36
|
let objectStorage: ObjectStorage | undefined;
|
|
29
37
|
if (decodedConfig.object_storage?.type === 's3') {
|
|
@@ -47,6 +55,8 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
47
55
|
maxPoolSize: resolvedConfig.storage.max_pool_size ?? 8
|
|
48
56
|
});
|
|
49
57
|
|
|
58
|
+
const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
|
|
59
|
+
|
|
50
60
|
let shuttingDown = false;
|
|
51
61
|
|
|
52
62
|
// Explicitly connect on startup.
|
|
@@ -56,7 +66,6 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
56
66
|
// Errors here will cause the process to exit.
|
|
57
67
|
await client.connect();
|
|
58
68
|
|
|
59
|
-
const database = new PowerSyncMongo(client, { database: resolvedConfig.storage.database });
|
|
60
69
|
const readPreference =
|
|
61
70
|
decodedConfig.bulk_read_preference == null
|
|
62
71
|
? undefined
|
|
@@ -66,6 +75,7 @@ export class MongoStorageProvider implements storage.StorageProvider {
|
|
|
66
75
|
maxStalenessSeconds: decodedConfig.bulk_read_preference == 'primary' ? undefined : 90
|
|
67
76
|
});
|
|
68
77
|
const syncStorageFactory = new MongoBucketStorage(database, {
|
|
78
|
+
chunkCompactionConcurrency,
|
|
69
79
|
replicationStreamNamePrefix: resolvedConfig.slot_name_prefix,
|
|
70
80
|
readPreference,
|
|
71
81
|
clearBatchThrottleRate: normalizeClearBatchThrottleRate(decodedConfig.clear_batch_throttle_rate),
|
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
import { mongo } from '@powersync/lib-service-mongodb';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Explicitly groups writes with no intervening reads or result dependencies.
|
|
5
|
+
* Callers retain responsibility for transaction and ordering boundaries.
|
|
6
|
+
*
|
|
7
|
+
* On MongoDB 8.0+, this uses the client-level bulkWrite API.
|
|
8
|
+
*
|
|
9
|
+
* On MongoDB earlier versions, this uses the the collection-level bulkWrite API, or individual operations.
|
|
10
|
+
* This is less efficient, but produces the same results.
|
|
11
|
+
*/
|
|
12
|
+
export class MongoWriteBatch {
|
|
13
|
+
private readonly operations: mongo.AnyClientBulkWriteModel<mongo.Document>[] = [];
|
|
14
|
+
private readonly deleteChecks: { index: number; check: (count: number) => void }[] = [];
|
|
15
|
+
private readonly fallback: (() => Promise<unknown>)[] = [];
|
|
16
|
+
|
|
17
|
+
private readonly ordered: boolean;
|
|
18
|
+
|
|
19
|
+
constructor(
|
|
20
|
+
private readonly client: mongo.MongoClient,
|
|
21
|
+
private readonly supportsClientBulkWrite: boolean,
|
|
22
|
+
private readonly session: mongo.ClientSession | undefined,
|
|
23
|
+
options: { ordered: boolean }
|
|
24
|
+
) {
|
|
25
|
+
this.ordered = options.ordered;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Perform a collection-level bulkWrite.
|
|
30
|
+
*
|
|
31
|
+
* On MongoDB 8.0+, these operations join the client-level bulkWrite with the batch's ordering.
|
|
32
|
+
* The driver may split commands to respect server batch limits.
|
|
33
|
+
*
|
|
34
|
+
* On earlier MongoDB versions, this always uses an unordered collection-level bulkWrite.
|
|
35
|
+
* Operations within this collection bulk must therefore be independent of their order.
|
|
36
|
+
*/
|
|
37
|
+
bulkWriteUnordered<T extends mongo.Document>(
|
|
38
|
+
collection: mongo.Collection<T>,
|
|
39
|
+
operations: mongo.AnyBulkWriteOperation<T>[]
|
|
40
|
+
): void {
|
|
41
|
+
if (operations.length === 0) return;
|
|
42
|
+
if (!this.supportsClientBulkWrite) {
|
|
43
|
+
this.fallback.push(() => collection.bulkWrite(operations, { ordered: false, session: this.session }));
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
for (const operation of operations) {
|
|
47
|
+
// Collection and client models contain the same operation fields, with a
|
|
48
|
+
// different envelope. Keep the schema checked at the collection boundary.
|
|
49
|
+
const [name, model] = Object.entries(operation)[0];
|
|
50
|
+
this.operations.push({ ...model, name, namespace: collection.namespace });
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
insertOne<T extends mongo.Document>(collection: mongo.Collection<T>, document: mongo.OptionalUnlessRequiredId<T>) {
|
|
55
|
+
if (!this.supportsClientBulkWrite) {
|
|
56
|
+
this.fallback.push(() => collection.insertOne(document, { session: this.session }));
|
|
57
|
+
return;
|
|
58
|
+
}
|
|
59
|
+
this.addModel<T>({ name: 'insertOne', namespace: collection.namespace, document: document as mongo.OptionalId<T> });
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
insertMany<T extends mongo.Document>(
|
|
63
|
+
collection: mongo.Collection<T>,
|
|
64
|
+
documents: mongo.OptionalUnlessRequiredId<T>[]
|
|
65
|
+
) {
|
|
66
|
+
if (documents.length === 0) return;
|
|
67
|
+
if (!this.supportsClientBulkWrite) {
|
|
68
|
+
this.fallback.push(() => collection.insertMany(documents, { session: this.session }));
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
for (const document of documents) {
|
|
72
|
+
this.addModel<T>({
|
|
73
|
+
name: 'insertOne',
|
|
74
|
+
namespace: collection.namespace,
|
|
75
|
+
document: document as mongo.OptionalId<T>
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
deleteMany<T extends mongo.Document>(
|
|
81
|
+
collection: mongo.Collection<T>,
|
|
82
|
+
filter: mongo.Filter<T>,
|
|
83
|
+
checkDeletedCount?: (count: number) => void
|
|
84
|
+
) {
|
|
85
|
+
if (checkDeletedCount) {
|
|
86
|
+
// Client-bulk results are checked after all writes execute. A failed
|
|
87
|
+
// invariant must roll back the entire batch, not leave later writes visible.
|
|
88
|
+
if (!this.session?.inTransaction()) throw new Error('Checked deletes require a transaction');
|
|
89
|
+
}
|
|
90
|
+
if (!this.supportsClientBulkWrite) {
|
|
91
|
+
this.fallback.push(async () => {
|
|
92
|
+
const result = await collection.deleteMany(filter, { session: this.session });
|
|
93
|
+
checkDeletedCount?.(result.deletedCount);
|
|
94
|
+
});
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
if (checkDeletedCount) {
|
|
98
|
+
this.deleteChecks.push({ index: this.operations.length, check: checkDeletedCount });
|
|
99
|
+
}
|
|
100
|
+
this.addModel<T>({ name: 'deleteMany', namespace: collection.namespace, filter });
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
updateOne<T extends mongo.Document>(
|
|
104
|
+
collection: mongo.Collection<T>,
|
|
105
|
+
filter: mongo.Filter<T>,
|
|
106
|
+
update: mongo.UpdateFilter<T> | mongo.Document[],
|
|
107
|
+
options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
|
|
108
|
+
) {
|
|
109
|
+
if (!this.supportsClientBulkWrite) {
|
|
110
|
+
this.fallback.push(() => collection.updateOne(filter, update, { ...options, session: this.session }));
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
this.addModel({ name: 'updateOne', namespace: collection.namespace, filter, update, ...options });
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
updateMany<T extends mongo.Document>(
|
|
117
|
+
collection: mongo.Collection<T>,
|
|
118
|
+
filter: mongo.Filter<T>,
|
|
119
|
+
update: mongo.UpdateFilter<T> | mongo.Document[],
|
|
120
|
+
options: Pick<mongo.UpdateOptions, 'upsert' | 'arrayFilters'> = {}
|
|
121
|
+
) {
|
|
122
|
+
if (!this.supportsClientBulkWrite) {
|
|
123
|
+
this.fallback.push(() => collection.updateMany(filter, update, { ...options, session: this.session }));
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
this.addModel({ name: 'updateMany', namespace: collection.namespace, filter, update, ...options });
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
private addModel<T extends mongo.Document>(operation: mongo.AnyClientBulkWriteModel<T>) {
|
|
130
|
+
// The collection-specific methods validate the schema before erasing it here.
|
|
131
|
+
this.operations.push(operation as unknown as mongo.AnyClientBulkWriteModel<mongo.Document>);
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Execute the queued writes.
|
|
136
|
+
*
|
|
137
|
+
* On MongoDB 8.0+, this uses the client-level bulkWrite API, typically resulting in a single command.
|
|
138
|
+
* The command may be split if there are a large number of operations.
|
|
139
|
+
*
|
|
140
|
+
* On earlier MongoDB versions, this executes each queued operation individually.
|
|
141
|
+
*
|
|
142
|
+
* If no operations were queued, this is a no-op.
|
|
143
|
+
*/
|
|
144
|
+
async execute(): Promise<void> {
|
|
145
|
+
if (!this.supportsClientBulkWrite) {
|
|
146
|
+
// Sessions cannot execute concurrent operations within a transaction.
|
|
147
|
+
for (const write of this.fallback) await write();
|
|
148
|
+
return;
|
|
149
|
+
}
|
|
150
|
+
if (this.operations.length === 0) return;
|
|
151
|
+
|
|
152
|
+
try {
|
|
153
|
+
const result = await this.client.bulkWrite(this.operations, {
|
|
154
|
+
session: this.session,
|
|
155
|
+
ordered: this.ordered,
|
|
156
|
+
verboseResults: this.deleteChecks.length > 0
|
|
157
|
+
});
|
|
158
|
+
for (const { index, check } of this.deleteChecks) {
|
|
159
|
+
const deleted = result.deleteResults?.get(index);
|
|
160
|
+
if (deleted == null) throw new Error('Missing client bulk delete result');
|
|
161
|
+
check(deleted.deletedCount);
|
|
162
|
+
}
|
|
163
|
+
} catch (error) {
|
|
164
|
+
// The driver wraps command failures without copying their error labels.
|
|
165
|
+
// Preserve withTransaction's retry handling for e.g. write conflicts.
|
|
166
|
+
if (error instanceof mongo.MongoClientBulkWriteError && error.cause instanceof mongo.MongoError) {
|
|
167
|
+
throw error.cause;
|
|
168
|
+
}
|
|
169
|
+
throw error;
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
}
|
|
@@ -95,7 +95,8 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
95
95
|
},
|
|
96
96
|
upsert: true
|
|
97
97
|
}
|
|
98
|
-
}))
|
|
98
|
+
})),
|
|
99
|
+
{ ordered: false }
|
|
99
100
|
);
|
|
100
101
|
|
|
101
102
|
const userIds = generatedCheckpoints.map((checkpoint) => checkpoint.user_id);
|
|
@@ -180,7 +181,8 @@ export class MongoWriteCheckpointAPI implements storage.WriteCheckpointAPI {
|
|
|
180
181
|
upsert: true
|
|
181
182
|
}
|
|
182
183
|
};
|
|
183
|
-
})
|
|
184
|
+
}),
|
|
185
|
+
{ ordered: false }
|
|
184
186
|
);
|
|
185
187
|
|
|
186
188
|
// Fetch the final ids separately so stale requests can still return the
|