opendal 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +89 -56
- data/Cargo.toml +1 -1
- data/DEPENDENCIES.rust.tsv +55 -41
- data/LICENSE +201 -0
- data/LICENSE-MPL-2.0.txt +373 -0
- data/NOTICE +5 -0
- data/core/CHANGELOG.md +175 -0
- data/core/Cargo.lock +225 -298
- data/core/Cargo.toml +100 -94
- data/core/DEPENDENCIES.rust.tsv +44 -73
- data/core/benches/metadata/README.md +21 -0
- data/core/benches/metadata/main.rs +541 -0
- data/core/benches/vs_fs/Cargo.toml +1 -1
- data/core/benches/vs_s3/Cargo.toml +1 -1
- data/core/core/Cargo.toml +7 -2
- data/core/core/src/blocking/delete.rs +5 -1
- data/core/core/src/blocking/operator.rs +12 -0
- data/core/core/src/blocking/read/std_reader.rs +25 -0
- data/core/core/src/docs/concepts.rs +70 -94
- data/core/core/src/docs/internals/accessor.rs +62 -293
- data/core/core/src/docs/internals/layer.rs +43 -59
- data/core/core/src/docs/internals/mod.rs +21 -23
- data/core/core/src/docs/mod.rs +10 -6
- data/core/core/src/docs/performance/concurrent_write.md +92 -55
- data/core/core/src/docs/performance/http_optimization.md +120 -104
- data/core/core/src/docs/performance/mod.rs +8 -6
- data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
- data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
- data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
- data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
- data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
- data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
- data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
- data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
- data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
- data/core/core/src/docs/rfcs/README.md +14 -3
- data/core/core/src/docs/rfcs/mod.rs +46 -22
- data/core/core/src/docs/specs/README.md +34 -0
- data/core/core/src/docs/specs/conditional-operations.md +156 -0
- data/core/core/src/docs/specs/metadata.md +63 -0
- data/core/core/src/docs/specs/mod.rs +26 -0
- data/core/core/src/docs/upgrade.md +114 -1
- data/core/core/src/layers/capability_override.rs +15 -2
- data/core/core/src/layers/complete.rs +347 -43
- data/core/core/src/layers/correctness_check.rs +718 -51
- data/core/core/src/layers/error_context.rs +76 -2
- data/core/core/src/layers/simulate.rs +91 -28
- data/core/core/src/lib.rs +1 -0
- data/core/core/src/raw/accessor.rs +91 -15
- data/core/core/src/raw/enum_utils.rs +7 -0
- data/core/core/src/raw/http_util/header.rs +22 -17
- data/core/core/src/raw/oio/compose/api.rs +92 -0
- data/core/core/src/raw/oio/compose/mod.rs +19 -0
- data/core/core/src/raw/oio/copy/api.rs +3 -3
- data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
- data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
- data/core/core/src/raw/oio/entry.rs +48 -22
- data/core/core/src/raw/oio/list/flat_list.rs +1 -1
- data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
- data/core/core/src/raw/oio/mod.rs +3 -0
- data/core/core/src/raw/oio/read/position_read.rs +1 -1
- data/core/core/src/raw/oio/write/api.rs +40 -0
- data/core/core/src/raw/oio/write/append_write.rs +4 -4
- data/core/core/src/raw/oio/write/block_write.rs +2 -2
- data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
- data/core/core/src/raw/oio/write/position_write.rs +1 -1
- data/core/core/src/raw/operation.rs +6 -0
- data/core/core/src/raw/ops.rs +1191 -456
- data/core/core/src/raw/rps.rs +11 -0
- data/core/core/src/raw/time.rs +14 -3
- data/core/core/src/services/memory/backend.rs +5 -10
- data/core/core/src/services/memory/lister.rs +4 -4
- data/core/core/src/services/memory/writer.rs +7 -6
- data/core/core/src/types/buffer.rs +102 -12
- data/core/core/src/types/bytes_range.rs +20 -1
- data/core/core/src/types/capability.rs +65 -0
- data/core/core/src/types/compact.rs +329 -0
- data/core/core/src/types/compose/composer.rs +183 -0
- data/core/core/src/types/compose/input.rs +57 -0
- data/core/core/src/types/compose/mod.rs +23 -0
- data/core/core/src/types/context/read.rs +16 -11
- data/core/core/src/types/context/write.rs +150 -2
- data/core/core/src/types/copy.rs +5 -3
- data/core/core/src/types/delete/deleter.rs +20 -12
- data/core/core/src/types/delete/input.rs +22 -64
- data/core/core/src/types/delete/mod.rs +0 -1
- data/core/core/src/types/entry.rs +8 -14
- data/core/core/src/types/error.rs +28 -11
- data/core/core/src/types/metadata.rs +709 -322
- data/core/core/src/types/mod.rs +9 -1
- data/core/core/src/types/operator/operator.rs +278 -29
- data/core/core/src/types/operator/operator_futures.rs +466 -105
- data/core/core/src/types/options.rs +631 -158
- data/core/core/src/types/read/futures_async_reader.rs +663 -50
- data/core/core/src/types/read/reader.rs +23 -5
- data/core/core/src/types/write/futures_async_writer.rs +27 -0
- data/core/core/src/types/write/writer.rs +47 -5
- data/core/http-transports/reqwest/Cargo.toml +1 -1
- data/core/layers/async-backtrace/Cargo.toml +2 -2
- data/core/layers/async-backtrace/src/lib.rs +21 -2
- data/core/layers/await-tree/Cargo.toml +2 -2
- data/core/layers/await-tree/src/lib.rs +25 -2
- data/core/layers/capability-check/Cargo.toml +2 -2
- data/core/layers/capability-check/src/lib.rs +98 -10
- data/core/layers/chaos/Cargo.toml +2 -2
- data/core/layers/chaos/src/lib.rs +18 -3
- data/core/layers/concurrent-limit/Cargo.toml +3 -3
- data/core/layers/concurrent-limit/src/lib.rs +53 -13
- data/core/layers/dtrace/Cargo.toml +2 -2
- data/core/layers/dtrace/src/lib.rs +29 -2
- data/core/layers/fastmetrics/Cargo.toml +3 -3
- data/core/layers/fastrace/Cargo.toml +2 -2
- data/core/layers/fastrace/src/lib.rs +28 -2
- data/core/layers/foyer/Cargo.toml +2 -2
- data/core/layers/foyer/src/full.rs +16 -19
- data/core/layers/foyer/src/lib.rs +67 -14
- data/core/layers/hotpath/Cargo.toml +2 -2
- data/core/layers/hotpath/src/lib.rs +20 -2
- data/core/layers/immutable-index/Cargo.toml +2 -2
- data/core/layers/immutable-index/src/lib.rs +48 -24
- data/core/layers/logging/Cargo.toml +2 -2
- data/core/layers/logging/src/lib.rs +46 -2
- data/core/layers/metrics/Cargo.toml +3 -3
- data/core/layers/mime-guess/Cargo.toml +2 -2
- data/core/layers/mime-guess/src/lib.rs +31 -4
- data/core/layers/observe-metrics-common/Cargo.toml +2 -2
- data/core/layers/observe-metrics-common/src/lib.rs +55 -2
- data/core/layers/otelmetrics/Cargo.toml +3 -3
- data/core/layers/oteltrace/Cargo.toml +2 -2
- data/core/layers/oteltrace/src/lib.rs +27 -2
- data/core/layers/prometheus/Cargo.toml +3 -3
- data/core/layers/prometheus-client/Cargo.toml +3 -3
- data/core/layers/retry/Cargo.toml +4 -4
- data/core/layers/retry/src/lib.rs +76 -19
- data/core/layers/route/Cargo.toml +3 -3
- data/core/layers/route/src/lib.rs +34 -7
- data/core/layers/tail-cut/Cargo.toml +2 -2
- data/core/layers/tail-cut/src/lib.rs +28 -2
- data/core/layers/throttle/Cargo.toml +2 -2
- data/core/layers/throttle/src/lib.rs +19 -2
- data/core/layers/timeout/Cargo.toml +3 -3
- data/core/layers/timeout/src/lib.rs +38 -13
- data/core/layers/tracing/Cargo.toml +2 -2
- data/core/layers/tracing/src/lib.rs +24 -3
- data/core/services/aliyun-drive/Cargo.toml +2 -2
- data/core/services/aliyun-drive/src/backend.rs +34 -24
- data/core/services/aliyun-drive/src/core.rs +143 -54
- data/core/services/aliyun-drive/src/lister.rs +19 -18
- data/core/services/aliyun-drive/src/reader.rs +4 -1
- data/core/services/aliyun-drive/src/writer.rs +3 -3
- data/core/services/alluxio/Cargo.toml +1 -1
- data/core/services/alluxio/src/backend.rs +1 -1
- data/core/services/alluxio/src/core.rs +113 -94
- data/core/services/alluxio/src/reader.rs +7 -3
- data/core/services/alluxio/src/writer.rs +2 -2
- data/core/services/azblob/Cargo.toml +2 -2
- data/core/services/azblob/src/backend.rs +17 -9
- data/core/services/azblob/src/copier.rs +41 -15
- data/core/services/azblob/src/core.rs +372 -135
- data/core/services/azblob/src/deleter.rs +41 -18
- data/core/services/azblob/src/lister.rs +16 -10
- data/core/services/azblob/src/reader.rs +4 -1
- data/core/services/azblob/src/writer.rs +48 -15
- data/core/services/azdls/Cargo.toml +3 -3
- data/core/services/azdls/src/backend.rs +45 -15
- data/core/services/azdls/src/config.rs +4 -1
- data/core/services/azdls/src/core.rs +276 -87
- data/core/services/azdls/src/deleter.rs +13 -4
- data/core/services/azdls/src/lister.rs +18 -12
- data/core/services/azdls/src/reader.rs +6 -2
- data/core/services/azdls/src/writer.rs +44 -23
- data/core/services/azfile/Cargo.toml +2 -2
- data/core/services/azfile/src/backend.rs +24 -8
- data/core/services/azfile/src/core.rs +195 -77
- data/core/services/azfile/src/deleter.rs +8 -1
- data/core/services/azfile/src/lister.rs +20 -14
- data/core/services/azfile/src/reader.rs +5 -2
- data/core/services/azfile/src/writer.rs +27 -15
- data/core/services/azure-common/Cargo.toml +1 -1
- data/core/services/b2/Cargo.toml +2 -2
- data/core/services/b2/src/backend.rs +30 -11
- data/core/services/b2/src/core.rs +134 -120
- data/core/services/b2/src/deleter.rs +1 -1
- data/core/services/b2/src/lister.rs +5 -2
- data/core/services/b2/src/reader.rs +5 -2
- data/core/services/b2/src/writer.rs +29 -12
- data/core/services/cacache/Cargo.toml +1 -1
- data/core/services/cacache/src/backend.rs +4 -5
- data/core/services/cacache/src/reader.rs +4 -1
- data/core/services/cacache/src/writer.rs +6 -2
- data/core/services/cloudflare-kv/Cargo.toml +1 -1
- data/core/services/cloudflare-kv/src/backend.rs +26 -15
- data/core/services/cloudflare-kv/src/config.rs +13 -2
- data/core/services/cloudflare-kv/src/core.rs +62 -56
- data/core/services/cloudflare-kv/src/deleter.rs +12 -3
- data/core/services/cloudflare-kv/src/lister.rs +66 -17
- data/core/services/cloudflare-kv/src/reader.rs +14 -9
- data/core/services/cloudflare-kv/src/writer.rs +10 -7
- data/core/services/compfs/Cargo.toml +1 -1
- data/core/services/compfs/src/backend.rs +11 -7
- data/core/services/compfs/src/lister.rs +7 -7
- data/core/services/compfs/src/writer.rs +1 -1
- data/core/services/cos/Cargo.toml +1 -1
- data/core/services/cos/src/backend.rs +129 -16
- data/core/services/cos/src/core.rs +198 -80
- data/core/services/cos/src/deleter.rs +4 -1
- data/core/services/cos/src/lister.rs +42 -27
- data/core/services/cos/src/reader.rs +6 -2
- data/core/services/cos/src/writer.rs +45 -21
- data/core/services/d1/Cargo.toml +1 -1
- data/core/services/d1/src/backend.rs +6 -5
- data/core/services/d1/src/core.rs +79 -63
- data/core/services/d1/src/reader.rs +4 -1
- data/core/services/d1/src/writer.rs +6 -2
- data/core/services/dashmap/Cargo.toml +1 -1
- data/core/services/dashmap/src/backend.rs +2 -2
- data/core/services/dashmap/src/reader.rs +4 -1
- data/core/services/dashmap/src/writer.rs +11 -8
- data/core/services/dbfs/Cargo.toml +1 -1
- data/core/services/dbfs/src/backend.rs +21 -15
- data/core/services/dbfs/src/core.rs +65 -60
- data/core/services/dbfs/src/deleter.rs +4 -1
- data/core/services/dbfs/src/lister.rs +10 -10
- data/core/services/dbfs/src/writer.rs +6 -3
- data/core/services/dropbox/Cargo.toml +2 -2
- data/core/services/dropbox/src/backend.rs +30 -23
- data/core/services/dropbox/src/core.rs +65 -63
- data/core/services/dropbox/src/deleter.rs +1 -1
- data/core/services/dropbox/src/lister.rs +17 -8
- data/core/services/dropbox/src/reader.rs +4 -1
- data/core/services/dropbox/src/writer.rs +13 -10
- data/core/services/etcd/Cargo.toml +1 -1
- data/core/services/etcd/src/backend.rs +8 -5
- data/core/services/etcd/src/core.rs +5 -11
- data/core/services/etcd/src/lister.rs +6 -1
- data/core/services/etcd/src/reader.rs +4 -1
- data/core/services/etcd/src/writer.rs +6 -3
- data/core/services/foundationdb/Cargo.toml +1 -1
- data/core/services/foundationdb/src/backend.rs +6 -5
- data/core/services/foundationdb/src/reader.rs +4 -1
- data/core/services/foundationdb/src/writer.rs +6 -2
- data/core/services/foyer/Cargo.toml +3 -3
- data/core/services/foyer/src/backend.rs +6 -5
- data/core/services/foyer/src/reader.rs +4 -1
- data/core/services/foyer/src/writer.rs +6 -2
- data/core/services/fs/Cargo.toml +1 -1
- data/core/services/fs/src/backend.rs +4 -3
- data/core/services/fs/src/core.rs +44 -26
- data/core/services/fs/src/lister.rs +9 -6
- data/core/services/fs/src/writer.rs +20 -13
- data/core/services/ftp/Cargo.toml +4 -6
- data/core/services/ftp/src/backend.rs +9 -5
- data/core/services/ftp/src/core.rs +9 -7
- data/core/services/ftp/src/lister.rs +7 -8
- data/core/services/ftp/src/reader.rs +3 -3
- data/core/services/ftp/src/writer.rs +3 -3
- data/core/services/gcs/Cargo.toml +4 -2
- data/core/services/gcs/src/backend.rs +53 -7
- data/core/services/gcs/src/composer.rs +705 -0
- data/core/services/gcs/src/copier.rs +14 -4
- data/core/services/gcs/src/core.rs +828 -124
- data/core/services/gcs/src/deleter.rs +33 -18
- data/core/services/gcs/src/lib.rs +1 -0
- data/core/services/gcs/src/lister.rs +21 -12
- data/core/services/gcs/src/reader.rs +12 -5
- data/core/services/gcs/src/writer.rs +632 -10
- data/core/services/gcs-grpc/Cargo.toml +58 -0
- data/core/services/gcs-grpc/README.md +19 -0
- data/core/services/gcs-grpc/proto/README.md +23 -0
- data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
- data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
- data/core/services/gcs-grpc/src/backend.rs +449 -0
- data/core/services/gcs-grpc/src/config.rs +116 -0
- data/core/services/gcs-grpc/src/copier.rs +98 -0
- data/core/services/gcs-grpc/src/core.rs +298 -0
- data/core/services/gcs-grpc/src/deleter.rs +65 -0
- data/core/services/gcs-grpc/src/docs.md +19 -0
- data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
- data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
- data/core/services/gcs-grpc/src/lib.rs +43 -0
- data/core/services/gcs-grpc/src/lister.rs +125 -0
- data/core/services/gcs-grpc/src/reader.rs +140 -0
- data/core/services/gcs-grpc/src/writer.rs +780 -0
- data/core/services/gdrive/Cargo.toml +2 -2
- data/core/services/gdrive/src/backend.rs +123 -69
- data/core/services/gdrive/src/core.rs +87 -69
- data/core/services/gdrive/src/deleter.rs +8 -2
- data/core/services/gdrive/src/lister.rs +38 -26
- data/core/services/gdrive/src/path_index.rs +2 -2
- data/core/services/gdrive/src/reader.rs +9 -3
- data/core/services/gdrive/src/writer.rs +13 -7
- data/core/services/ghac/Cargo.toml +2 -2
- data/core/services/ghac/src/backend.rs +5 -2
- data/core/services/ghac/src/core.rs +59 -37
- data/core/services/ghac/src/reader.rs +5 -2
- data/core/services/ghac/src/writer.rs +16 -6
- data/core/services/github/Cargo.toml +1 -1
- data/core/services/github/src/backend.rs +16 -8
- data/core/services/github/src/core.rs +88 -76
- data/core/services/github/src/lister.rs +17 -15
- data/core/services/github/src/reader.rs +5 -2
- data/core/services/github/src/writer.rs +12 -6
- data/core/services/goosefs/Cargo.toml +2 -2
- data/core/services/goosefs/src/backend.rs +2 -2
- data/core/services/goosefs/src/core.rs +291 -208
- data/core/services/goosefs/src/docs.md +3 -0
- data/core/services/goosefs/src/lister.rs +2 -3
- data/core/services/goosefs/src/reader.rs +7 -3
- data/core/services/goosefs/src/writer.rs +21 -10
- data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
- data/core/services/gridfs/Cargo.toml +2 -2
- data/core/services/gridfs/src/backend.rs +12 -8
- data/core/services/gridfs/src/core.rs +1 -1
- data/core/services/gridfs/src/reader.rs +4 -1
- data/core/services/gridfs/src/writer.rs +6 -2
- data/core/services/hdfs/Cargo.toml +1 -1
- data/core/services/hdfs/src/backend.rs +1 -1
- data/core/services/hdfs/src/core.rs +7 -4
- data/core/services/hdfs/src/lister.rs +7 -9
- data/core/services/hdfs/src/writer.rs +5 -1
- data/core/services/hdfs-native/Cargo.toml +1 -1
- data/core/services/hdfs-native/src/backend.rs +1 -1
- data/core/services/hdfs-native/src/core.rs +44 -50
- data/core/services/hdfs-native/src/lister.rs +8 -10
- data/core/services/hdfs-native/src/writer.rs +5 -1
- data/core/services/hf/Cargo.toml +4 -3
- data/core/services/hf/src/backend.rs +4 -3
- data/core/services/hf/src/core.rs +907 -153
- data/core/services/hf/src/lister.rs +16 -6
- data/core/services/hf/src/reader.rs +377 -57
- data/core/services/hf/src/writer.rs +6 -2
- data/core/services/http/Cargo.toml +3 -3
- data/core/services/http/src/backend.rs +8 -6
- data/core/services/http/src/core.rs +57 -35
- data/core/services/http/src/reader.rs +5 -2
- data/core/services/ipfs/Cargo.toml +1 -1
- data/core/services/ipfs/src/backend.rs +20 -9
- data/core/services/ipfs/src/core.rs +43 -34
- data/core/services/ipfs/src/reader.rs +5 -2
- data/core/services/ipmfs/Cargo.toml +1 -1
- data/core/services/ipmfs/src/backend.rs +17 -8
- data/core/services/ipmfs/src/core.rs +66 -60
- data/core/services/ipmfs/src/deleter.rs +5 -2
- data/core/services/ipmfs/src/lister.rs +15 -12
- data/core/services/ipmfs/src/reader.rs +5 -2
- data/core/services/ipmfs/src/writer.rs +6 -3
- data/core/services/koofr/Cargo.toml +2 -2
- data/core/services/koofr/src/backend.rs +46 -17
- data/core/services/koofr/src/core.rs +64 -48
- data/core/services/koofr/src/deleter.rs +4 -1
- data/core/services/koofr/src/lister.rs +11 -10
- data/core/services/koofr/src/reader.rs +5 -2
- data/core/services/koofr/src/writer.rs +13 -7
- data/core/services/lakefs/Cargo.toml +1 -1
- data/core/services/lakefs/src/backend.rs +25 -8
- data/core/services/lakefs/src/core.rs +79 -73
- data/core/services/lakefs/src/deleter.rs +4 -1
- data/core/services/lakefs/src/lister.rs +17 -14
- data/core/services/lakefs/src/reader.rs +5 -2
- data/core/services/lakefs/src/writer.rs +11 -10
- data/core/services/memcached/Cargo.toml +1 -1
- data/core/services/memcached/src/backend.rs +57 -7
- data/core/services/memcached/src/config.rs +13 -2
- data/core/services/memcached/src/core.rs +11 -1
- data/core/services/memcached/src/reader.rs +4 -1
- data/core/services/memcached/src/writer.rs +6 -2
- data/core/services/mini_moka/Cargo.toml +1 -1
- data/core/services/mini_moka/src/backend.rs +8 -8
- data/core/services/mini_moka/src/reader.rs +4 -1
- data/core/services/mini_moka/src/writer.rs +11 -7
- data/core/services/moka/Cargo.toml +1 -1
- data/core/services/moka/src/backend.rs +7 -7
- data/core/services/moka/src/reader.rs +4 -1
- data/core/services/moka/src/writer.rs +10 -6
- data/core/services/mongodb/Cargo.toml +2 -2
- data/core/services/mongodb/src/backend.rs +7 -6
- data/core/services/mongodb/src/core.rs +1 -1
- data/core/services/mongodb/src/reader.rs +4 -1
- data/core/services/mongodb/src/writer.rs +6 -2
- data/core/services/monoiofs/Cargo.toml +2 -2
- data/core/services/monoiofs/src/backend.rs +16 -10
- data/core/services/monoiofs/src/core.rs +8 -9
- data/core/services/monoiofs/src/reader.rs +8 -10
- data/core/services/monoiofs/src/writer.rs +19 -24
- data/core/services/mysql/Cargo.toml +2 -2
- data/core/services/mysql/src/backend.rs +7 -6
- data/core/services/mysql/src/core.rs +1 -1
- data/core/services/mysql/src/lister.rs +7 -3
- data/core/services/mysql/src/reader.rs +4 -1
- data/core/services/mysql/src/writer.rs +6 -2
- data/core/services/obs/Cargo.toml +1 -1
- data/core/services/obs/src/backend.rs +29 -11
- data/core/services/obs/src/core.rs +81 -82
- data/core/services/obs/src/deleter.rs +4 -1
- data/core/services/obs/src/lister.rs +12 -5
- data/core/services/obs/src/reader.rs +5 -2
- data/core/services/obs/src/writer.rs +40 -19
- data/core/services/onedrive/Cargo.toml +2 -2
- data/core/services/onedrive/src/backend.rs +30 -21
- data/core/services/onedrive/src/config.rs +2 -2
- data/core/services/onedrive/src/core.rs +191 -90
- data/core/services/onedrive/src/deleter.rs +5 -2
- data/core/services/onedrive/src/graph_model.rs +10 -51
- data/core/services/onedrive/src/lister.rs +60 -24
- data/core/services/onedrive/src/reader.rs +17 -2
- data/core/services/onedrive/src/writer.rs +26 -15
- data/core/services/opfs/Cargo.toml +1 -1
- data/core/services/opfs/src/backend.rs +6 -13
- data/core/services/opfs/src/core.rs +33 -37
- data/core/services/opfs/src/lister.rs +4 -4
- data/core/services/opfs/src/writer.rs +1 -3
- data/core/services/oss/Cargo.toml +1 -1
- data/core/services/oss/src/backend.rs +40 -8
- data/core/services/oss/src/core.rs +192 -92
- data/core/services/oss/src/deleter.rs +16 -6
- data/core/services/oss/src/lister.rs +43 -27
- data/core/services/oss/src/reader.rs +6 -2
- data/core/services/oss/src/writer.rs +40 -22
- data/core/services/pcloud/Cargo.toml +1 -1
- data/core/services/pcloud/src/backend.rs +40 -6
- data/core/services/pcloud/src/core.rs +80 -66
- data/core/services/pcloud/src/deleter.rs +8 -1
- data/core/services/pcloud/src/lister.rs +4 -1
- data/core/services/pcloud/src/reader.rs +5 -2
- data/core/services/pcloud/src/writer.rs +6 -3
- data/core/services/persy/Cargo.toml +1 -1
- data/core/services/persy/src/backend.rs +6 -5
- data/core/services/persy/src/core.rs +60 -15
- data/core/services/persy/src/reader.rs +4 -1
- data/core/services/persy/src/writer.rs +6 -2
- data/core/services/postgresql/Cargo.toml +2 -2
- data/core/services/postgresql/src/backend.rs +7 -6
- data/core/services/postgresql/src/core.rs +1 -1
- data/core/services/postgresql/src/reader.rs +4 -1
- data/core/services/postgresql/src/writer.rs +6 -2
- data/core/services/redb/Cargo.toml +1 -1
- data/core/services/redb/src/backend.rs +6 -5
- data/core/services/redb/src/reader.rs +4 -1
- data/core/services/redb/src/writer.rs +6 -2
- data/core/services/redis/Cargo.toml +1 -1
- data/core/services/redis/src/backend.rs +18 -8
- data/core/services/redis/src/config.rs +13 -2
- data/core/services/redis/src/reader.rs +8 -4
- data/core/services/redis/src/writer.rs +6 -2
- data/core/services/rocksdb/Cargo.toml +1 -1
- data/core/services/rocksdb/src/backend.rs +6 -5
- data/core/services/rocksdb/src/lister.rs +6 -5
- data/core/services/rocksdb/src/reader.rs +4 -1
- data/core/services/rocksdb/src/writer.rs +6 -2
- data/core/services/s3/Cargo.toml +5 -4
- data/core/services/s3/src/backend.rs +995 -39
- data/core/services/s3/src/compatible_services.md +57 -0
- data/core/services/s3/src/config.rs +16 -0
- data/core/services/s3/src/copier.rs +190 -55
- data/core/services/s3/src/core.rs +299 -190
- data/core/services/s3/src/deleter.rs +26 -7
- data/core/services/s3/src/docs.md +57 -0
- data/core/services/s3/src/lister.rs +55 -36
- data/core/services/s3/src/reader.rs +22 -2
- data/core/services/s3/src/writer.rs +154 -32
- data/core/services/seafile/Cargo.toml +2 -2
- data/core/services/seafile/src/backend.rs +3 -3
- data/core/services/seafile/src/core.rs +105 -79
- data/core/services/seafile/src/lister.rs +11 -8
- data/core/services/seafile/src/reader.rs +5 -2
- data/core/services/seafile/src/writer.rs +6 -3
- data/core/services/sftp/Cargo.toml +1 -1
- data/core/services/sftp/src/backend.rs +19 -4
- data/core/services/sftp/src/core.rs +51 -46
- data/core/services/sftp/src/lister.rs +4 -4
- data/core/services/sftp/src/writer.rs +1 -1
- data/core/services/sled/Cargo.toml +1 -1
- data/core/services/sled/src/backend.rs +6 -5
- data/core/services/sled/src/core.rs +28 -6
- data/core/services/sled/src/lister.rs +6 -5
- data/core/services/sled/src/reader.rs +4 -1
- data/core/services/sled/src/writer.rs +6 -2
- data/core/services/sqlite/Cargo.toml +2 -2
- data/core/services/sqlite/src/backend.rs +12 -7
- data/core/services/sqlite/src/core.rs +3 -3
- data/core/services/sqlite/src/reader.rs +4 -1
- data/core/services/sqlite/src/writer.rs +7 -3
- data/core/services/surrealdb/Cargo.toml +2 -2
- data/core/services/surrealdb/src/backend.rs +7 -6
- data/core/services/surrealdb/src/core.rs +1 -1
- data/core/services/surrealdb/src/reader.rs +4 -1
- data/core/services/surrealdb/src/writer.rs +6 -2
- data/core/services/swift/Cargo.toml +1 -1
- data/core/services/swift/src/backend.rs +38 -8
- data/core/services/swift/src/core.rs +63 -67
- data/core/services/swift/src/deleter.rs +8 -2
- data/core/services/swift/src/lister.rs +11 -8
- data/core/services/swift/src/reader.rs +5 -2
- data/core/services/swift/src/writer.rs +17 -8
- data/core/services/tikv/Cargo.toml +2 -2
- data/core/services/tikv/src/backend.rs +7 -6
- data/core/services/tikv/src/core.rs +31 -14
- data/core/services/tikv/src/reader.rs +4 -1
- data/core/services/tikv/src/writer.rs +6 -2
- data/core/services/tos/Cargo.toml +2 -2
- data/core/services/tos/src/backend.rs +45 -21
- data/core/services/tos/src/copier.rs +44 -17
- data/core/services/tos/src/core.rs +147 -117
- data/core/services/tos/src/deleter.rs +9 -3
- data/core/services/tos/src/docs.md +1 -1
- data/core/services/tos/src/lister.rs +43 -28
- data/core/services/tos/src/reader.rs +5 -2
- data/core/services/tos/src/writer.rs +37 -16
- data/core/services/upyun/Cargo.toml +1 -1
- data/core/services/upyun/src/backend.rs +30 -7
- data/core/services/upyun/src/core.rs +122 -87
- data/core/services/upyun/src/deleter.rs +4 -1
- data/core/services/upyun/src/lister.rs +10 -10
- data/core/services/upyun/src/reader.rs +5 -2
- data/core/services/upyun/src/writer.rs +19 -7
- data/core/services/vercel-artifacts/Cargo.toml +1 -1
- data/core/services/vercel-artifacts/src/backend.rs +6 -3
- data/core/services/vercel-artifacts/src/core.rs +32 -27
- data/core/services/vercel-artifacts/src/reader.rs +5 -2
- data/core/services/vercel-artifacts/src/writer.rs +6 -3
- data/core/services/vercel-blob/Cargo.toml +1 -1
- data/core/services/vercel-blob/src/backend.rs +23 -6
- data/core/services/vercel-blob/src/core.rs +92 -81
- data/core/services/vercel-blob/src/lister.rs +12 -3
- data/core/services/vercel-blob/src/reader.rs +5 -2
- data/core/services/vercel-blob/src/writer.rs +19 -7
- data/core/services/webdav/Cargo.toml +2 -2
- data/core/services/webdav/src/backend.rs +25 -5
- data/core/services/webdav/src/core.rs +116 -59
- data/core/services/webdav/src/deleter.rs +4 -1
- data/core/services/webdav/src/lister.rs +4 -1
- data/core/services/webdav/src/reader.rs +6 -2
- data/core/services/webdav/src/writer.rs +17 -7
- data/core/services/webhdfs/Cargo.toml +2 -2
- data/core/services/webhdfs/src/backend.rs +24 -11
- data/core/services/webhdfs/src/core.rs +99 -98
- data/core/services/webhdfs/src/deleter.rs +5 -2
- data/core/services/webhdfs/src/lister.rs +21 -9
- data/core/services/webhdfs/src/reader.rs +5 -2
- data/core/services/webhdfs/src/writer.rs +49 -14
- data/core/services/yandex-disk/Cargo.toml +1 -1
- data/core/services/yandex-disk/src/backend.rs +28 -6
- data/core/services/yandex-disk/src/core.rs +97 -79
- data/core/services/yandex-disk/src/deleter.rs +4 -1
- data/core/services/yandex-disk/src/lister.rs +25 -9
- data/core/services/yandex-disk/src/reader.rs +5 -2
- data/core/services/yandex-disk/src/writer.rs +6 -3
- data/core/src/lib.rs +5 -0
- data/core/testkit/Cargo.toml +4 -4
- data/core/testkit/src/utils.rs +53 -8
- data/core/tests/behavior/async_compose.rs +297 -0
- data/core/tests/behavior/async_copy.rs +162 -1
- data/core/tests/behavior/async_delete.rs +364 -3
- data/core/tests/behavior/async_read.rs +86 -0
- data/core/tests/behavior/async_restore.rs +162 -0
- data/core/tests/behavior/async_stat.rs +90 -0
- data/core/tests/behavior/async_write.rs +330 -2
- data/core/tests/behavior/main.rs +4 -0
- data/core/upgrade.md +114 -1
- data/src/capability.rs +0 -1
- data/src/io.rs +0 -1
- data/src/lister.rs +0 -1
- data/src/metadata.rs +0 -1
- data/src/operator_info.rs +0 -1
- metadata +44 -2
|
@@ -1,138 +1,154 @@
|
|
|
1
1
|
# HTTP Optimization
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
3
|
+
Most OpenDAL services use HTTP, so transport configuration can dominate
|
|
4
|
+
throughput and tail latency. This guide uses reqwest because the `opendal`
|
|
5
|
+
facade installs it as the default transport when an HTTP transport feature is
|
|
6
|
+
enabled.
|
|
7
|
+
|
|
8
|
+
The default `auto-register-services` feature calls `opendal::install_default()`
|
|
9
|
+
before `main`. Applications that need a different process-wide default must
|
|
10
|
+
disable `auto-register-services`, initialize the service registry when needed,
|
|
11
|
+
and install their transport before another default is installed.
|
|
12
|
+
|
|
13
|
+
A simpler option is to replace the transport for one operator. Add
|
|
14
|
+
`opendal-http-transport-reqwest` and `reqwest` as direct dependencies, build one
|
|
15
|
+
client, and reuse the resulting operator:
|
|
16
|
+
|
|
17
|
+
```rust,ignore
|
|
18
|
+
use opendal::{HttpTransporter, Operator};
|
|
19
|
+
use opendal_http_transport_reqwest::ReqwestTransport;
|
|
20
|
+
|
|
21
|
+
fn with_reqwest_client(op: Operator, client: reqwest::Client) -> Operator {
|
|
22
|
+
let transport = HttpTransporter::new(ReqwestTransport::new(client));
|
|
23
|
+
let ctx = op.base_context().with_http_transport(transport);
|
|
24
|
+
op.with_context(ctx)
|
|
25
|
+
}
|
|
26
|
+
```
|
|
20
27
|
|
|
21
|
-
|
|
28
|
+
Creating a client or operator per request discards DNS and connection-pool
|
|
29
|
+
state and repeatedly pays connection setup costs.
|
|
22
30
|
|
|
23
|
-
|
|
31
|
+
## Prefer HTTP/1.1 for large transfers
|
|
24
32
|
|
|
25
|
-
|
|
33
|
+
For high-throughput uploads and downloads of large objects, benchmark HTTP/1.1
|
|
34
|
+
first. OpenDAL users often see higher aggregate throughput with HTTP/1.1 because
|
|
35
|
+
reqwest can open several pooled connections, while HTTP/2 normally multiplexes
|
|
36
|
+
requests over one connection.
|
|
26
37
|
|
|
27
|
-
|
|
38
|
+
One HTTP/2 connection can become the bottleneck when it reaches its stream
|
|
39
|
+
limit or the bandwidth available to one TCP connection. The hyper client does
|
|
40
|
+
not currently open another pooled HTTP/2 connection when the existing one
|
|
41
|
+
reaches its maximum concurrent streams; see [hyper issue #3623]. This matters
|
|
42
|
+
most for long-lived streams and highly concurrent large transfers.
|
|
28
43
|
|
|
29
|
-
|
|
30
|
-
let client = reqwest::ClientBuilder::new()
|
|
31
|
-
// Disable http2 for better performance.
|
|
32
|
-
.http1_only()
|
|
33
|
-
.build()
|
|
34
|
-
.expect("http client must be created");
|
|
44
|
+
Force HTTP/1.1 with `ClientBuilder::http1_only`:
|
|
35
45
|
|
|
36
|
-
|
|
37
|
-
let
|
|
38
|
-
|
|
46
|
+
```rust,ignore
|
|
47
|
+
let client = reqwest::Client::builder()
|
|
48
|
+
.http1_only()
|
|
49
|
+
.build()?;
|
|
50
|
+
let op = with_reqwest_client(op, client);
|
|
39
51
|
```
|
|
40
52
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
53
|
+
HTTP/1.1 can open many TCP connections when no idle connection is available.
|
|
54
|
+
Bound OpenDAL request concurrency so the client cannot exhaust host resources
|
|
55
|
+
or overload the service. HTTP/2 can still be better for many small requests or
|
|
56
|
+
when connection count matters, so measure both protocols against the real
|
|
57
|
+
endpoint and object-size distribution.
|
|
44
58
|
|
|
45
|
-
|
|
59
|
+
## Enable DNS caching
|
|
46
60
|
|
|
47
|
-
|
|
61
|
+
Without the optional Hickory resolver, reqwest uses a `getaddrinfo`-based
|
|
62
|
+
resolver and does not maintain a reqwest-level DNS cache. Repeated resolution
|
|
63
|
+
can become visible at high request rates and can overload a constrained local
|
|
64
|
+
resolver.
|
|
48
65
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
// Enable hickory dns for dns caching and async dns resolve.
|
|
52
|
-
.hickory_dns(true)
|
|
53
|
-
.build()
|
|
54
|
-
.expect("http client must be created");
|
|
66
|
+
Enable reqwest's `hickory-dns` feature in the application and select it on the
|
|
67
|
+
client:
|
|
55
68
|
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
69
|
+
```toml
|
|
70
|
+
[dependencies]
|
|
71
|
+
reqwest = { version = "0.13", features = ["hickory-dns"] }
|
|
59
72
|
```
|
|
60
73
|
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
let mut opts = ResolverOpts::default();
|
|
67
|
-
// Only query for the ipv4 address.
|
|
68
|
-
opts.ip_strategy = LookupIpStrategy::Ipv4Only;
|
|
69
|
-
// Use larger cache size for better performance.
|
|
70
|
-
opts.cache_size = 1024;
|
|
71
|
-
// Positive TTL is set to 5 minutes.
|
|
72
|
-
opts.positive_min_ttl = Some(Duration::from_secs(300));
|
|
73
|
-
// Negative TTL is set to 1 minute.
|
|
74
|
-
opts.negative_min_ttl = Some(Duration::from_secs(60));
|
|
75
|
-
|
|
76
|
-
Arc::new(
|
|
77
|
-
HickoryResolver::default()
|
|
78
|
-
// Always shuffle the DNS results for better performance.
|
|
79
|
-
.with_shuffle(true)
|
|
80
|
-
.with_options(opts),
|
|
81
|
-
)
|
|
82
|
-
});
|
|
83
|
-
|
|
84
|
-
let client = reqwest::ClientBuilder::new()
|
|
85
|
-
// Use our global hickory resolver instead.
|
|
86
|
-
.dns_resolver(GLOBAL_HICKORY_RESOLVER.clone())
|
|
87
|
-
.build()
|
|
88
|
-
.expect("http client must be created");
|
|
89
|
-
|
|
90
|
-
// Replace the operator's base HTTP transport.
|
|
91
|
-
let transport = HttpTransporter::new(ReqwestTransport::new(client));
|
|
92
|
-
let op = op.with_context(OperationContext::new().with_http_transport(transport));
|
|
74
|
+
```rust,ignore
|
|
75
|
+
let client = reqwest::Client::builder()
|
|
76
|
+
.hickory_dns(true)
|
|
77
|
+
.build()?;
|
|
78
|
+
let op = with_reqwest_client(op, client);
|
|
93
79
|
```
|
|
94
80
|
|
|
95
|
-
The
|
|
81
|
+
The built-in Hickory settings are a good starting point. Use
|
|
82
|
+
`ClientBuilder::dns_resolver` when the application needs to share a custom
|
|
83
|
+
resolver or control cache size, positive and negative TTLs, IP selection, or
|
|
84
|
+
address shuffling. Longer TTLs reduce resolver traffic but retain stale records
|
|
85
|
+
longer, so test failover behavior before increasing them.
|
|
96
86
|
|
|
97
|
-
|
|
87
|
+
## Set timeouts deliberately
|
|
98
88
|
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
89
|
+
Reqwest 0.13 has no request, read, or connect timeout by default. At minimum,
|
|
90
|
+
set a connect timeout so an unreachable endpoint cannot leave connection setup
|
|
91
|
+
unbounded:
|
|
102
92
|
|
|
103
|
-
|
|
93
|
+
```rust,ignore
|
|
94
|
+
use std::time::Duration;
|
|
104
95
|
|
|
105
|
-
|
|
96
|
+
let client = reqwest::Client::builder()
|
|
97
|
+
.connect_timeout(Duration::from_secs(5))
|
|
98
|
+
.read_timeout(Duration::from_secs(30))
|
|
99
|
+
.build()?;
|
|
100
|
+
let op = with_reqwest_client(op, client);
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
`read_timeout` limits inactivity between successful reads. A total request
|
|
104
|
+
timeout created with `ClientBuilder::timeout` covers the complete transfer, so
|
|
105
|
+
do not set it below the expected duration of a healthy large upload or
|
|
106
|
+
download.
|
|
106
107
|
|
|
107
|
-
|
|
108
|
+
The facade's optional `layers-timeout` feature also provides `TimeoutLayer` for
|
|
109
|
+
OpenDAL operation and I/O timeouts:
|
|
108
110
|
|
|
109
|
-
|
|
111
|
+
```rust,ignore
|
|
112
|
+
use std::time::Duration;
|
|
110
113
|
|
|
111
|
-
|
|
112
|
-
let client = reqwest::ClientBuilder::new()
|
|
113
|
-
// Set a connect timeout of 5 seconds.
|
|
114
|
-
.connect_timeout(Duration::from_secs(5))
|
|
115
|
-
.build()
|
|
116
|
-
.expect("http client must be created");
|
|
114
|
+
use opendal::layers::TimeoutLayer;
|
|
117
115
|
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
116
|
+
let op = op.layer(
|
|
117
|
+
TimeoutLayer::new()
|
|
118
|
+
.with_timeout(Duration::from_secs(30))
|
|
119
|
+
.with_io_timeout(Duration::from_secs(60)),
|
|
120
|
+
);
|
|
121
121
|
```
|
|
122
122
|
|
|
123
|
-
|
|
123
|
+
Transport timeouts bound HTTP phases. `TimeoutLayer` applies at OpenDAL's
|
|
124
|
+
operation and I/O boundaries. Configure both when the application needs bounds
|
|
125
|
+
at both levels.
|
|
124
126
|
|
|
125
|
-
|
|
126
|
-
let op = op.layer(TimeoutLayer::new());
|
|
127
|
-
```
|
|
127
|
+
## Size the connection pool
|
|
128
128
|
|
|
129
|
-
|
|
129
|
+
Reqwest reuses idle connections. In reqwest 0.13, the default idle timeout is
|
|
130
|
+
90 seconds and the default maximum number of idle connections per host is
|
|
131
|
+
unlimited. Tune both values for the traffic pattern; neither setting limits
|
|
132
|
+
active connections:
|
|
130
133
|
|
|
131
|
-
|
|
134
|
+
```rust,ignore
|
|
135
|
+
use std::time::Duration;
|
|
136
|
+
|
|
137
|
+
let client = reqwest::Client::builder()
|
|
138
|
+
.pool_idle_timeout(Duration::from_secs(60))
|
|
139
|
+
.pool_max_idle_per_host(32)
|
|
140
|
+
.build()?;
|
|
141
|
+
let op = with_reqwest_client(op, client);
|
|
142
|
+
```
|
|
132
143
|
|
|
133
|
-
|
|
144
|
+
- Keep enough idle connections for expected HTTP/1.1 concurrency so bursts do
|
|
145
|
+
not repeat DNS, TCP, and TLS setup.
|
|
146
|
+
- Set a finite idle per-host limit for applications that contact many
|
|
147
|
+
endpoints. Bound active connections with OpenDAL request concurrency.
|
|
148
|
+
- Shorten the idle timeout when traffic is sparse and connection reuse is rare.
|
|
134
149
|
|
|
135
|
-
|
|
150
|
+
Measure connection reuse, open sockets, handshake rate, request concurrency,
|
|
151
|
+
throughput, and tail latency together. A pool setting that improves a steady
|
|
152
|
+
single-endpoint workload can waste resources in a bursty multi-endpoint one.
|
|
136
153
|
|
|
137
|
-
|
|
138
|
-
- [pool_max_idle_per_host](https://docs.rs/reqwest/0.13.2/reqwest/struct.ClientBuilder.html#method.pool_max_idle_per_host): Sets the maximum idle connection per host allowed in the pool.
|
|
154
|
+
[hyper issue #3623]: https://github.com/hyperium/hyper/issues/3623
|
|
@@ -17,13 +17,15 @@
|
|
|
17
17
|
|
|
18
18
|
//! OpenDAL Performance Guide
|
|
19
19
|
//!
|
|
20
|
-
//! OpenDAL
|
|
21
|
-
//!
|
|
22
|
-
//!
|
|
23
|
-
//!
|
|
20
|
+
//! OpenDAL keeps the storage abstraction lightweight, but applications still
|
|
21
|
+
//! need to configure buffering, concurrency, and HTTP transport for their
|
|
22
|
+
//! workload. This guide gives concrete Rust configuration examples and explains
|
|
23
|
+
//! the trade-offs behind them.
|
|
24
24
|
//!
|
|
25
|
-
//!
|
|
26
|
-
//!
|
|
25
|
+
//! - [Concurrent writes][concurrent_write] explains how OpenDAL schedules write
|
|
26
|
+
//! parts and how to tune `concurrent` and `chunk`.
|
|
27
|
+
//! - [HTTP optimization][http_optimization] explains when to try HTTP/1.1 and
|
|
28
|
+
//! how to configure DNS, timeouts, and connection pools.
|
|
27
29
|
|
|
28
30
|
#[doc = include_str!("concurrent_write.md")]
|
|
29
31
|
pub mod concurrent_write {}
|
|
@@ -72,6 +72,10 @@ let reader = op.reader_with("path/to/file")
|
|
|
72
72
|
.await?;
|
|
73
73
|
```
|
|
74
74
|
|
|
75
|
+
All conditional reads resolve the current live target before evaluating the
|
|
76
|
+
condition. If no live target exists, the operation returns
|
|
77
|
+
`ErrorKind::NotFound`.
|
|
78
|
+
|
|
75
79
|
|
|
76
80
|
# Reference-level explanation
|
|
77
81
|
|