opendal 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +89 -56
- data/Cargo.toml +1 -1
- data/DEPENDENCIES.rust.tsv +55 -41
- data/LICENSE +201 -0
- data/LICENSE-MPL-2.0.txt +373 -0
- data/NOTICE +5 -0
- data/core/CHANGELOG.md +175 -0
- data/core/Cargo.lock +225 -298
- data/core/Cargo.toml +100 -94
- data/core/DEPENDENCIES.rust.tsv +44 -73
- data/core/benches/metadata/README.md +21 -0
- data/core/benches/metadata/main.rs +541 -0
- data/core/benches/vs_fs/Cargo.toml +1 -1
- data/core/benches/vs_s3/Cargo.toml +1 -1
- data/core/core/Cargo.toml +7 -2
- data/core/core/src/blocking/delete.rs +5 -1
- data/core/core/src/blocking/operator.rs +12 -0
- data/core/core/src/blocking/read/std_reader.rs +25 -0
- data/core/core/src/docs/concepts.rs +70 -94
- data/core/core/src/docs/internals/accessor.rs +62 -293
- data/core/core/src/docs/internals/layer.rs +43 -59
- data/core/core/src/docs/internals/mod.rs +21 -23
- data/core/core/src/docs/mod.rs +10 -6
- data/core/core/src/docs/performance/concurrent_write.md +92 -55
- data/core/core/src/docs/performance/http_optimization.md +120 -104
- data/core/core/src/docs/performance/mod.rs +8 -6
- data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
- data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
- data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
- data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
- data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
- data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
- data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
- data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
- data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
- data/core/core/src/docs/rfcs/README.md +14 -3
- data/core/core/src/docs/rfcs/mod.rs +46 -22
- data/core/core/src/docs/specs/README.md +34 -0
- data/core/core/src/docs/specs/conditional-operations.md +156 -0
- data/core/core/src/docs/specs/metadata.md +63 -0
- data/core/core/src/docs/specs/mod.rs +26 -0
- data/core/core/src/docs/upgrade.md +114 -1
- data/core/core/src/layers/capability_override.rs +15 -2
- data/core/core/src/layers/complete.rs +347 -43
- data/core/core/src/layers/correctness_check.rs +718 -51
- data/core/core/src/layers/error_context.rs +76 -2
- data/core/core/src/layers/simulate.rs +91 -28
- data/core/core/src/lib.rs +1 -0
- data/core/core/src/raw/accessor.rs +91 -15
- data/core/core/src/raw/enum_utils.rs +7 -0
- data/core/core/src/raw/http_util/header.rs +22 -17
- data/core/core/src/raw/oio/compose/api.rs +92 -0
- data/core/core/src/raw/oio/compose/mod.rs +19 -0
- data/core/core/src/raw/oio/copy/api.rs +3 -3
- data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
- data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
- data/core/core/src/raw/oio/entry.rs +48 -22
- data/core/core/src/raw/oio/list/flat_list.rs +1 -1
- data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
- data/core/core/src/raw/oio/mod.rs +3 -0
- data/core/core/src/raw/oio/read/position_read.rs +1 -1
- data/core/core/src/raw/oio/write/api.rs +40 -0
- data/core/core/src/raw/oio/write/append_write.rs +4 -4
- data/core/core/src/raw/oio/write/block_write.rs +2 -2
- data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
- data/core/core/src/raw/oio/write/position_write.rs +1 -1
- data/core/core/src/raw/operation.rs +6 -0
- data/core/core/src/raw/ops.rs +1191 -456
- data/core/core/src/raw/rps.rs +11 -0
- data/core/core/src/raw/time.rs +14 -3
- data/core/core/src/services/memory/backend.rs +5 -10
- data/core/core/src/services/memory/lister.rs +4 -4
- data/core/core/src/services/memory/writer.rs +7 -6
- data/core/core/src/types/buffer.rs +102 -12
- data/core/core/src/types/bytes_range.rs +20 -1
- data/core/core/src/types/capability.rs +65 -0
- data/core/core/src/types/compact.rs +329 -0
- data/core/core/src/types/compose/composer.rs +183 -0
- data/core/core/src/types/compose/input.rs +57 -0
- data/core/core/src/types/compose/mod.rs +23 -0
- data/core/core/src/types/context/read.rs +16 -11
- data/core/core/src/types/context/write.rs +150 -2
- data/core/core/src/types/copy.rs +5 -3
- data/core/core/src/types/delete/deleter.rs +20 -12
- data/core/core/src/types/delete/input.rs +22 -64
- data/core/core/src/types/delete/mod.rs +0 -1
- data/core/core/src/types/entry.rs +8 -14
- data/core/core/src/types/error.rs +28 -11
- data/core/core/src/types/metadata.rs +709 -322
- data/core/core/src/types/mod.rs +9 -1
- data/core/core/src/types/operator/operator.rs +278 -29
- data/core/core/src/types/operator/operator_futures.rs +466 -105
- data/core/core/src/types/options.rs +631 -158
- data/core/core/src/types/read/futures_async_reader.rs +663 -50
- data/core/core/src/types/read/reader.rs +23 -5
- data/core/core/src/types/write/futures_async_writer.rs +27 -0
- data/core/core/src/types/write/writer.rs +47 -5
- data/core/http-transports/reqwest/Cargo.toml +1 -1
- data/core/layers/async-backtrace/Cargo.toml +2 -2
- data/core/layers/async-backtrace/src/lib.rs +21 -2
- data/core/layers/await-tree/Cargo.toml +2 -2
- data/core/layers/await-tree/src/lib.rs +25 -2
- data/core/layers/capability-check/Cargo.toml +2 -2
- data/core/layers/capability-check/src/lib.rs +98 -10
- data/core/layers/chaos/Cargo.toml +2 -2
- data/core/layers/chaos/src/lib.rs +18 -3
- data/core/layers/concurrent-limit/Cargo.toml +3 -3
- data/core/layers/concurrent-limit/src/lib.rs +53 -13
- data/core/layers/dtrace/Cargo.toml +2 -2
- data/core/layers/dtrace/src/lib.rs +29 -2
- data/core/layers/fastmetrics/Cargo.toml +3 -3
- data/core/layers/fastrace/Cargo.toml +2 -2
- data/core/layers/fastrace/src/lib.rs +28 -2
- data/core/layers/foyer/Cargo.toml +2 -2
- data/core/layers/foyer/src/full.rs +16 -19
- data/core/layers/foyer/src/lib.rs +67 -14
- data/core/layers/hotpath/Cargo.toml +2 -2
- data/core/layers/hotpath/src/lib.rs +20 -2
- data/core/layers/immutable-index/Cargo.toml +2 -2
- data/core/layers/immutable-index/src/lib.rs +48 -24
- data/core/layers/logging/Cargo.toml +2 -2
- data/core/layers/logging/src/lib.rs +46 -2
- data/core/layers/metrics/Cargo.toml +3 -3
- data/core/layers/mime-guess/Cargo.toml +2 -2
- data/core/layers/mime-guess/src/lib.rs +31 -4
- data/core/layers/observe-metrics-common/Cargo.toml +2 -2
- data/core/layers/observe-metrics-common/src/lib.rs +55 -2
- data/core/layers/otelmetrics/Cargo.toml +3 -3
- data/core/layers/oteltrace/Cargo.toml +2 -2
- data/core/layers/oteltrace/src/lib.rs +27 -2
- data/core/layers/prometheus/Cargo.toml +3 -3
- data/core/layers/prometheus-client/Cargo.toml +3 -3
- data/core/layers/retry/Cargo.toml +4 -4
- data/core/layers/retry/src/lib.rs +76 -19
- data/core/layers/route/Cargo.toml +3 -3
- data/core/layers/route/src/lib.rs +34 -7
- data/core/layers/tail-cut/Cargo.toml +2 -2
- data/core/layers/tail-cut/src/lib.rs +28 -2
- data/core/layers/throttle/Cargo.toml +2 -2
- data/core/layers/throttle/src/lib.rs +19 -2
- data/core/layers/timeout/Cargo.toml +3 -3
- data/core/layers/timeout/src/lib.rs +38 -13
- data/core/layers/tracing/Cargo.toml +2 -2
- data/core/layers/tracing/src/lib.rs +24 -3
- data/core/services/aliyun-drive/Cargo.toml +2 -2
- data/core/services/aliyun-drive/src/backend.rs +34 -24
- data/core/services/aliyun-drive/src/core.rs +143 -54
- data/core/services/aliyun-drive/src/lister.rs +19 -18
- data/core/services/aliyun-drive/src/reader.rs +4 -1
- data/core/services/aliyun-drive/src/writer.rs +3 -3
- data/core/services/alluxio/Cargo.toml +1 -1
- data/core/services/alluxio/src/backend.rs +1 -1
- data/core/services/alluxio/src/core.rs +113 -94
- data/core/services/alluxio/src/reader.rs +7 -3
- data/core/services/alluxio/src/writer.rs +2 -2
- data/core/services/azblob/Cargo.toml +2 -2
- data/core/services/azblob/src/backend.rs +17 -9
- data/core/services/azblob/src/copier.rs +41 -15
- data/core/services/azblob/src/core.rs +372 -135
- data/core/services/azblob/src/deleter.rs +41 -18
- data/core/services/azblob/src/lister.rs +16 -10
- data/core/services/azblob/src/reader.rs +4 -1
- data/core/services/azblob/src/writer.rs +48 -15
- data/core/services/azdls/Cargo.toml +3 -3
- data/core/services/azdls/src/backend.rs +45 -15
- data/core/services/azdls/src/config.rs +4 -1
- data/core/services/azdls/src/core.rs +276 -87
- data/core/services/azdls/src/deleter.rs +13 -4
- data/core/services/azdls/src/lister.rs +18 -12
- data/core/services/azdls/src/reader.rs +6 -2
- data/core/services/azdls/src/writer.rs +44 -23
- data/core/services/azfile/Cargo.toml +2 -2
- data/core/services/azfile/src/backend.rs +24 -8
- data/core/services/azfile/src/core.rs +195 -77
- data/core/services/azfile/src/deleter.rs +8 -1
- data/core/services/azfile/src/lister.rs +20 -14
- data/core/services/azfile/src/reader.rs +5 -2
- data/core/services/azfile/src/writer.rs +27 -15
- data/core/services/azure-common/Cargo.toml +1 -1
- data/core/services/b2/Cargo.toml +2 -2
- data/core/services/b2/src/backend.rs +30 -11
- data/core/services/b2/src/core.rs +134 -120
- data/core/services/b2/src/deleter.rs +1 -1
- data/core/services/b2/src/lister.rs +5 -2
- data/core/services/b2/src/reader.rs +5 -2
- data/core/services/b2/src/writer.rs +29 -12
- data/core/services/cacache/Cargo.toml +1 -1
- data/core/services/cacache/src/backend.rs +4 -5
- data/core/services/cacache/src/reader.rs +4 -1
- data/core/services/cacache/src/writer.rs +6 -2
- data/core/services/cloudflare-kv/Cargo.toml +1 -1
- data/core/services/cloudflare-kv/src/backend.rs +26 -15
- data/core/services/cloudflare-kv/src/config.rs +13 -2
- data/core/services/cloudflare-kv/src/core.rs +62 -56
- data/core/services/cloudflare-kv/src/deleter.rs +12 -3
- data/core/services/cloudflare-kv/src/lister.rs +66 -17
- data/core/services/cloudflare-kv/src/reader.rs +14 -9
- data/core/services/cloudflare-kv/src/writer.rs +10 -7
- data/core/services/compfs/Cargo.toml +1 -1
- data/core/services/compfs/src/backend.rs +11 -7
- data/core/services/compfs/src/lister.rs +7 -7
- data/core/services/compfs/src/writer.rs +1 -1
- data/core/services/cos/Cargo.toml +1 -1
- data/core/services/cos/src/backend.rs +129 -16
- data/core/services/cos/src/core.rs +198 -80
- data/core/services/cos/src/deleter.rs +4 -1
- data/core/services/cos/src/lister.rs +42 -27
- data/core/services/cos/src/reader.rs +6 -2
- data/core/services/cos/src/writer.rs +45 -21
- data/core/services/d1/Cargo.toml +1 -1
- data/core/services/d1/src/backend.rs +6 -5
- data/core/services/d1/src/core.rs +79 -63
- data/core/services/d1/src/reader.rs +4 -1
- data/core/services/d1/src/writer.rs +6 -2
- data/core/services/dashmap/Cargo.toml +1 -1
- data/core/services/dashmap/src/backend.rs +2 -2
- data/core/services/dashmap/src/reader.rs +4 -1
- data/core/services/dashmap/src/writer.rs +11 -8
- data/core/services/dbfs/Cargo.toml +1 -1
- data/core/services/dbfs/src/backend.rs +21 -15
- data/core/services/dbfs/src/core.rs +65 -60
- data/core/services/dbfs/src/deleter.rs +4 -1
- data/core/services/dbfs/src/lister.rs +10 -10
- data/core/services/dbfs/src/writer.rs +6 -3
- data/core/services/dropbox/Cargo.toml +2 -2
- data/core/services/dropbox/src/backend.rs +30 -23
- data/core/services/dropbox/src/core.rs +65 -63
- data/core/services/dropbox/src/deleter.rs +1 -1
- data/core/services/dropbox/src/lister.rs +17 -8
- data/core/services/dropbox/src/reader.rs +4 -1
- data/core/services/dropbox/src/writer.rs +13 -10
- data/core/services/etcd/Cargo.toml +1 -1
- data/core/services/etcd/src/backend.rs +8 -5
- data/core/services/etcd/src/core.rs +5 -11
- data/core/services/etcd/src/lister.rs +6 -1
- data/core/services/etcd/src/reader.rs +4 -1
- data/core/services/etcd/src/writer.rs +6 -3
- data/core/services/foundationdb/Cargo.toml +1 -1
- data/core/services/foundationdb/src/backend.rs +6 -5
- data/core/services/foundationdb/src/reader.rs +4 -1
- data/core/services/foundationdb/src/writer.rs +6 -2
- data/core/services/foyer/Cargo.toml +3 -3
- data/core/services/foyer/src/backend.rs +6 -5
- data/core/services/foyer/src/reader.rs +4 -1
- data/core/services/foyer/src/writer.rs +6 -2
- data/core/services/fs/Cargo.toml +1 -1
- data/core/services/fs/src/backend.rs +4 -3
- data/core/services/fs/src/core.rs +44 -26
- data/core/services/fs/src/lister.rs +9 -6
- data/core/services/fs/src/writer.rs +20 -13
- data/core/services/ftp/Cargo.toml +4 -6
- data/core/services/ftp/src/backend.rs +9 -5
- data/core/services/ftp/src/core.rs +9 -7
- data/core/services/ftp/src/lister.rs +7 -8
- data/core/services/ftp/src/reader.rs +3 -3
- data/core/services/ftp/src/writer.rs +3 -3
- data/core/services/gcs/Cargo.toml +4 -2
- data/core/services/gcs/src/backend.rs +53 -7
- data/core/services/gcs/src/composer.rs +705 -0
- data/core/services/gcs/src/copier.rs +14 -4
- data/core/services/gcs/src/core.rs +828 -124
- data/core/services/gcs/src/deleter.rs +33 -18
- data/core/services/gcs/src/lib.rs +1 -0
- data/core/services/gcs/src/lister.rs +21 -12
- data/core/services/gcs/src/reader.rs +12 -5
- data/core/services/gcs/src/writer.rs +632 -10
- data/core/services/gcs-grpc/Cargo.toml +58 -0
- data/core/services/gcs-grpc/README.md +19 -0
- data/core/services/gcs-grpc/proto/README.md +23 -0
- data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
- data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
- data/core/services/gcs-grpc/src/backend.rs +449 -0
- data/core/services/gcs-grpc/src/config.rs +116 -0
- data/core/services/gcs-grpc/src/copier.rs +98 -0
- data/core/services/gcs-grpc/src/core.rs +298 -0
- data/core/services/gcs-grpc/src/deleter.rs +65 -0
- data/core/services/gcs-grpc/src/docs.md +19 -0
- data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
- data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
- data/core/services/gcs-grpc/src/lib.rs +43 -0
- data/core/services/gcs-grpc/src/lister.rs +125 -0
- data/core/services/gcs-grpc/src/reader.rs +140 -0
- data/core/services/gcs-grpc/src/writer.rs +780 -0
- data/core/services/gdrive/Cargo.toml +2 -2
- data/core/services/gdrive/src/backend.rs +123 -69
- data/core/services/gdrive/src/core.rs +87 -69
- data/core/services/gdrive/src/deleter.rs +8 -2
- data/core/services/gdrive/src/lister.rs +38 -26
- data/core/services/gdrive/src/path_index.rs +2 -2
- data/core/services/gdrive/src/reader.rs +9 -3
- data/core/services/gdrive/src/writer.rs +13 -7
- data/core/services/ghac/Cargo.toml +2 -2
- data/core/services/ghac/src/backend.rs +5 -2
- data/core/services/ghac/src/core.rs +59 -37
- data/core/services/ghac/src/reader.rs +5 -2
- data/core/services/ghac/src/writer.rs +16 -6
- data/core/services/github/Cargo.toml +1 -1
- data/core/services/github/src/backend.rs +16 -8
- data/core/services/github/src/core.rs +88 -76
- data/core/services/github/src/lister.rs +17 -15
- data/core/services/github/src/reader.rs +5 -2
- data/core/services/github/src/writer.rs +12 -6
- data/core/services/goosefs/Cargo.toml +2 -2
- data/core/services/goosefs/src/backend.rs +2 -2
- data/core/services/goosefs/src/core.rs +291 -208
- data/core/services/goosefs/src/docs.md +3 -0
- data/core/services/goosefs/src/lister.rs +2 -3
- data/core/services/goosefs/src/reader.rs +7 -3
- data/core/services/goosefs/src/writer.rs +21 -10
- data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
- data/core/services/gridfs/Cargo.toml +2 -2
- data/core/services/gridfs/src/backend.rs +12 -8
- data/core/services/gridfs/src/core.rs +1 -1
- data/core/services/gridfs/src/reader.rs +4 -1
- data/core/services/gridfs/src/writer.rs +6 -2
- data/core/services/hdfs/Cargo.toml +1 -1
- data/core/services/hdfs/src/backend.rs +1 -1
- data/core/services/hdfs/src/core.rs +7 -4
- data/core/services/hdfs/src/lister.rs +7 -9
- data/core/services/hdfs/src/writer.rs +5 -1
- data/core/services/hdfs-native/Cargo.toml +1 -1
- data/core/services/hdfs-native/src/backend.rs +1 -1
- data/core/services/hdfs-native/src/core.rs +44 -50
- data/core/services/hdfs-native/src/lister.rs +8 -10
- data/core/services/hdfs-native/src/writer.rs +5 -1
- data/core/services/hf/Cargo.toml +4 -3
- data/core/services/hf/src/backend.rs +4 -3
- data/core/services/hf/src/core.rs +907 -153
- data/core/services/hf/src/lister.rs +16 -6
- data/core/services/hf/src/reader.rs +377 -57
- data/core/services/hf/src/writer.rs +6 -2
- data/core/services/http/Cargo.toml +3 -3
- data/core/services/http/src/backend.rs +8 -6
- data/core/services/http/src/core.rs +57 -35
- data/core/services/http/src/reader.rs +5 -2
- data/core/services/ipfs/Cargo.toml +1 -1
- data/core/services/ipfs/src/backend.rs +20 -9
- data/core/services/ipfs/src/core.rs +43 -34
- data/core/services/ipfs/src/reader.rs +5 -2
- data/core/services/ipmfs/Cargo.toml +1 -1
- data/core/services/ipmfs/src/backend.rs +17 -8
- data/core/services/ipmfs/src/core.rs +66 -60
- data/core/services/ipmfs/src/deleter.rs +5 -2
- data/core/services/ipmfs/src/lister.rs +15 -12
- data/core/services/ipmfs/src/reader.rs +5 -2
- data/core/services/ipmfs/src/writer.rs +6 -3
- data/core/services/koofr/Cargo.toml +2 -2
- data/core/services/koofr/src/backend.rs +46 -17
- data/core/services/koofr/src/core.rs +64 -48
- data/core/services/koofr/src/deleter.rs +4 -1
- data/core/services/koofr/src/lister.rs +11 -10
- data/core/services/koofr/src/reader.rs +5 -2
- data/core/services/koofr/src/writer.rs +13 -7
- data/core/services/lakefs/Cargo.toml +1 -1
- data/core/services/lakefs/src/backend.rs +25 -8
- data/core/services/lakefs/src/core.rs +79 -73
- data/core/services/lakefs/src/deleter.rs +4 -1
- data/core/services/lakefs/src/lister.rs +17 -14
- data/core/services/lakefs/src/reader.rs +5 -2
- data/core/services/lakefs/src/writer.rs +11 -10
- data/core/services/memcached/Cargo.toml +1 -1
- data/core/services/memcached/src/backend.rs +57 -7
- data/core/services/memcached/src/config.rs +13 -2
- data/core/services/memcached/src/core.rs +11 -1
- data/core/services/memcached/src/reader.rs +4 -1
- data/core/services/memcached/src/writer.rs +6 -2
- data/core/services/mini_moka/Cargo.toml +1 -1
- data/core/services/mini_moka/src/backend.rs +8 -8
- data/core/services/mini_moka/src/reader.rs +4 -1
- data/core/services/mini_moka/src/writer.rs +11 -7
- data/core/services/moka/Cargo.toml +1 -1
- data/core/services/moka/src/backend.rs +7 -7
- data/core/services/moka/src/reader.rs +4 -1
- data/core/services/moka/src/writer.rs +10 -6
- data/core/services/mongodb/Cargo.toml +2 -2
- data/core/services/mongodb/src/backend.rs +7 -6
- data/core/services/mongodb/src/core.rs +1 -1
- data/core/services/mongodb/src/reader.rs +4 -1
- data/core/services/mongodb/src/writer.rs +6 -2
- data/core/services/monoiofs/Cargo.toml +2 -2
- data/core/services/monoiofs/src/backend.rs +16 -10
- data/core/services/monoiofs/src/core.rs +8 -9
- data/core/services/monoiofs/src/reader.rs +8 -10
- data/core/services/monoiofs/src/writer.rs +19 -24
- data/core/services/mysql/Cargo.toml +2 -2
- data/core/services/mysql/src/backend.rs +7 -6
- data/core/services/mysql/src/core.rs +1 -1
- data/core/services/mysql/src/lister.rs +7 -3
- data/core/services/mysql/src/reader.rs +4 -1
- data/core/services/mysql/src/writer.rs +6 -2
- data/core/services/obs/Cargo.toml +1 -1
- data/core/services/obs/src/backend.rs +29 -11
- data/core/services/obs/src/core.rs +81 -82
- data/core/services/obs/src/deleter.rs +4 -1
- data/core/services/obs/src/lister.rs +12 -5
- data/core/services/obs/src/reader.rs +5 -2
- data/core/services/obs/src/writer.rs +40 -19
- data/core/services/onedrive/Cargo.toml +2 -2
- data/core/services/onedrive/src/backend.rs +30 -21
- data/core/services/onedrive/src/config.rs +2 -2
- data/core/services/onedrive/src/core.rs +191 -90
- data/core/services/onedrive/src/deleter.rs +5 -2
- data/core/services/onedrive/src/graph_model.rs +10 -51
- data/core/services/onedrive/src/lister.rs +60 -24
- data/core/services/onedrive/src/reader.rs +17 -2
- data/core/services/onedrive/src/writer.rs +26 -15
- data/core/services/opfs/Cargo.toml +1 -1
- data/core/services/opfs/src/backend.rs +6 -13
- data/core/services/opfs/src/core.rs +33 -37
- data/core/services/opfs/src/lister.rs +4 -4
- data/core/services/opfs/src/writer.rs +1 -3
- data/core/services/oss/Cargo.toml +1 -1
- data/core/services/oss/src/backend.rs +40 -8
- data/core/services/oss/src/core.rs +192 -92
- data/core/services/oss/src/deleter.rs +16 -6
- data/core/services/oss/src/lister.rs +43 -27
- data/core/services/oss/src/reader.rs +6 -2
- data/core/services/oss/src/writer.rs +40 -22
- data/core/services/pcloud/Cargo.toml +1 -1
- data/core/services/pcloud/src/backend.rs +40 -6
- data/core/services/pcloud/src/core.rs +80 -66
- data/core/services/pcloud/src/deleter.rs +8 -1
- data/core/services/pcloud/src/lister.rs +4 -1
- data/core/services/pcloud/src/reader.rs +5 -2
- data/core/services/pcloud/src/writer.rs +6 -3
- data/core/services/persy/Cargo.toml +1 -1
- data/core/services/persy/src/backend.rs +6 -5
- data/core/services/persy/src/core.rs +60 -15
- data/core/services/persy/src/reader.rs +4 -1
- data/core/services/persy/src/writer.rs +6 -2
- data/core/services/postgresql/Cargo.toml +2 -2
- data/core/services/postgresql/src/backend.rs +7 -6
- data/core/services/postgresql/src/core.rs +1 -1
- data/core/services/postgresql/src/reader.rs +4 -1
- data/core/services/postgresql/src/writer.rs +6 -2
- data/core/services/redb/Cargo.toml +1 -1
- data/core/services/redb/src/backend.rs +6 -5
- data/core/services/redb/src/reader.rs +4 -1
- data/core/services/redb/src/writer.rs +6 -2
- data/core/services/redis/Cargo.toml +1 -1
- data/core/services/redis/src/backend.rs +18 -8
- data/core/services/redis/src/config.rs +13 -2
- data/core/services/redis/src/reader.rs +8 -4
- data/core/services/redis/src/writer.rs +6 -2
- data/core/services/rocksdb/Cargo.toml +1 -1
- data/core/services/rocksdb/src/backend.rs +6 -5
- data/core/services/rocksdb/src/lister.rs +6 -5
- data/core/services/rocksdb/src/reader.rs +4 -1
- data/core/services/rocksdb/src/writer.rs +6 -2
- data/core/services/s3/Cargo.toml +5 -4
- data/core/services/s3/src/backend.rs +995 -39
- data/core/services/s3/src/compatible_services.md +57 -0
- data/core/services/s3/src/config.rs +16 -0
- data/core/services/s3/src/copier.rs +190 -55
- data/core/services/s3/src/core.rs +299 -190
- data/core/services/s3/src/deleter.rs +26 -7
- data/core/services/s3/src/docs.md +57 -0
- data/core/services/s3/src/lister.rs +55 -36
- data/core/services/s3/src/reader.rs +22 -2
- data/core/services/s3/src/writer.rs +154 -32
- data/core/services/seafile/Cargo.toml +2 -2
- data/core/services/seafile/src/backend.rs +3 -3
- data/core/services/seafile/src/core.rs +105 -79
- data/core/services/seafile/src/lister.rs +11 -8
- data/core/services/seafile/src/reader.rs +5 -2
- data/core/services/seafile/src/writer.rs +6 -3
- data/core/services/sftp/Cargo.toml +1 -1
- data/core/services/sftp/src/backend.rs +19 -4
- data/core/services/sftp/src/core.rs +51 -46
- data/core/services/sftp/src/lister.rs +4 -4
- data/core/services/sftp/src/writer.rs +1 -1
- data/core/services/sled/Cargo.toml +1 -1
- data/core/services/sled/src/backend.rs +6 -5
- data/core/services/sled/src/core.rs +28 -6
- data/core/services/sled/src/lister.rs +6 -5
- data/core/services/sled/src/reader.rs +4 -1
- data/core/services/sled/src/writer.rs +6 -2
- data/core/services/sqlite/Cargo.toml +2 -2
- data/core/services/sqlite/src/backend.rs +12 -7
- data/core/services/sqlite/src/core.rs +3 -3
- data/core/services/sqlite/src/reader.rs +4 -1
- data/core/services/sqlite/src/writer.rs +7 -3
- data/core/services/surrealdb/Cargo.toml +2 -2
- data/core/services/surrealdb/src/backend.rs +7 -6
- data/core/services/surrealdb/src/core.rs +1 -1
- data/core/services/surrealdb/src/reader.rs +4 -1
- data/core/services/surrealdb/src/writer.rs +6 -2
- data/core/services/swift/Cargo.toml +1 -1
- data/core/services/swift/src/backend.rs +38 -8
- data/core/services/swift/src/core.rs +63 -67
- data/core/services/swift/src/deleter.rs +8 -2
- data/core/services/swift/src/lister.rs +11 -8
- data/core/services/swift/src/reader.rs +5 -2
- data/core/services/swift/src/writer.rs +17 -8
- data/core/services/tikv/Cargo.toml +2 -2
- data/core/services/tikv/src/backend.rs +7 -6
- data/core/services/tikv/src/core.rs +31 -14
- data/core/services/tikv/src/reader.rs +4 -1
- data/core/services/tikv/src/writer.rs +6 -2
- data/core/services/tos/Cargo.toml +2 -2
- data/core/services/tos/src/backend.rs +45 -21
- data/core/services/tos/src/copier.rs +44 -17
- data/core/services/tos/src/core.rs +147 -117
- data/core/services/tos/src/deleter.rs +9 -3
- data/core/services/tos/src/docs.md +1 -1
- data/core/services/tos/src/lister.rs +43 -28
- data/core/services/tos/src/reader.rs +5 -2
- data/core/services/tos/src/writer.rs +37 -16
- data/core/services/upyun/Cargo.toml +1 -1
- data/core/services/upyun/src/backend.rs +30 -7
- data/core/services/upyun/src/core.rs +122 -87
- data/core/services/upyun/src/deleter.rs +4 -1
- data/core/services/upyun/src/lister.rs +10 -10
- data/core/services/upyun/src/reader.rs +5 -2
- data/core/services/upyun/src/writer.rs +19 -7
- data/core/services/vercel-artifacts/Cargo.toml +1 -1
- data/core/services/vercel-artifacts/src/backend.rs +6 -3
- data/core/services/vercel-artifacts/src/core.rs +32 -27
- data/core/services/vercel-artifacts/src/reader.rs +5 -2
- data/core/services/vercel-artifacts/src/writer.rs +6 -3
- data/core/services/vercel-blob/Cargo.toml +1 -1
- data/core/services/vercel-blob/src/backend.rs +23 -6
- data/core/services/vercel-blob/src/core.rs +92 -81
- data/core/services/vercel-blob/src/lister.rs +12 -3
- data/core/services/vercel-blob/src/reader.rs +5 -2
- data/core/services/vercel-blob/src/writer.rs +19 -7
- data/core/services/webdav/Cargo.toml +2 -2
- data/core/services/webdav/src/backend.rs +25 -5
- data/core/services/webdav/src/core.rs +116 -59
- data/core/services/webdav/src/deleter.rs +4 -1
- data/core/services/webdav/src/lister.rs +4 -1
- data/core/services/webdav/src/reader.rs +6 -2
- data/core/services/webdav/src/writer.rs +17 -7
- data/core/services/webhdfs/Cargo.toml +2 -2
- data/core/services/webhdfs/src/backend.rs +24 -11
- data/core/services/webhdfs/src/core.rs +99 -98
- data/core/services/webhdfs/src/deleter.rs +5 -2
- data/core/services/webhdfs/src/lister.rs +21 -9
- data/core/services/webhdfs/src/reader.rs +5 -2
- data/core/services/webhdfs/src/writer.rs +49 -14
- data/core/services/yandex-disk/Cargo.toml +1 -1
- data/core/services/yandex-disk/src/backend.rs +28 -6
- data/core/services/yandex-disk/src/core.rs +97 -79
- data/core/services/yandex-disk/src/deleter.rs +4 -1
- data/core/services/yandex-disk/src/lister.rs +25 -9
- data/core/services/yandex-disk/src/reader.rs +5 -2
- data/core/services/yandex-disk/src/writer.rs +6 -3
- data/core/src/lib.rs +5 -0
- data/core/testkit/Cargo.toml +4 -4
- data/core/testkit/src/utils.rs +53 -8
- data/core/tests/behavior/async_compose.rs +297 -0
- data/core/tests/behavior/async_copy.rs +162 -1
- data/core/tests/behavior/async_delete.rs +364 -3
- data/core/tests/behavior/async_read.rs +86 -0
- data/core/tests/behavior/async_restore.rs +162 -0
- data/core/tests/behavior/async_stat.rs +90 -0
- data/core/tests/behavior/async_write.rs +330 -2
- data/core/tests/behavior/main.rs +4 -0
- data/core/upgrade.md +114 -1
- data/src/capability.rs +0 -1
- data/src/io.rs +0 -1
- data/src/lister.rs +0 -1
- data/src/metadata.rs +0 -1
- data/src/operator_info.rs +0 -1
- metadata +44 -2
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
// under the License.
|
|
17
17
|
|
|
18
18
|
use std::fmt::Debug;
|
|
19
|
+
use std::time::Duration;
|
|
19
20
|
|
|
20
21
|
use http::Request;
|
|
21
22
|
use http::Response;
|
|
@@ -99,6 +100,27 @@ impl TosCore {
|
|
|
99
100
|
)
|
|
100
101
|
}
|
|
101
102
|
|
|
103
|
+
pub async fn sign_query<T>(
|
|
104
|
+
&self,
|
|
105
|
+
ctx: &OperationContext,
|
|
106
|
+
req: Request<T>,
|
|
107
|
+
duration: Duration,
|
|
108
|
+
) -> Result<Request<T>> {
|
|
109
|
+
if self.skip_signature {
|
|
110
|
+
return Ok(req);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
let (mut parts, body) = req.into_parts();
|
|
114
|
+
self.signer(ctx)
|
|
115
|
+
.sign(&mut parts, Some(duration))
|
|
116
|
+
.await
|
|
117
|
+
.map_err(|e| new_request_sign_error(e.into()))?;
|
|
118
|
+
|
|
119
|
+
parts.headers.remove(HOST);
|
|
120
|
+
|
|
121
|
+
Ok(Request::from_parts(parts, body))
|
|
122
|
+
}
|
|
123
|
+
|
|
102
124
|
pub async fn send(
|
|
103
125
|
&self,
|
|
104
126
|
ctx: &OperationContext,
|
|
@@ -413,12 +435,11 @@ impl TosCore {
|
|
|
413
435
|
self.send(ctx, req).await
|
|
414
436
|
}
|
|
415
437
|
|
|
416
|
-
pub
|
|
438
|
+
pub fn tos_delete_object_request(
|
|
417
439
|
&self,
|
|
418
|
-
ctx: &OperationContext,
|
|
419
440
|
path: &str,
|
|
420
441
|
args: &OpDelete,
|
|
421
|
-
) -> Result<
|
|
442
|
+
) -> Result<Request<Buffer>> {
|
|
422
443
|
let p = build_abs_path(&self.root, path);
|
|
423
444
|
|
|
424
445
|
let mut url = format!(
|
|
@@ -444,11 +465,19 @@ impl TosCore {
|
|
|
444
465
|
|
|
445
466
|
let req = Request::delete(&url);
|
|
446
467
|
|
|
447
|
-
|
|
448
|
-
.extension(Operation::Delete)
|
|
468
|
+
req.extension(Operation::Delete)
|
|
449
469
|
.extension(ServiceOperation("DeleteObject"))
|
|
450
470
|
.body(Buffer::new())
|
|
451
|
-
.map_err(new_request_build_error)
|
|
471
|
+
.map_err(new_request_build_error)
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
pub async fn tos_delete_object(
|
|
475
|
+
&self,
|
|
476
|
+
ctx: &OperationContext,
|
|
477
|
+
path: &str,
|
|
478
|
+
args: &OpDelete,
|
|
479
|
+
) -> Result<Response<Buffer>> {
|
|
480
|
+
let req = self.tos_delete_object_request(path, args)?;
|
|
452
481
|
|
|
453
482
|
self.send(ctx, req).await
|
|
454
483
|
}
|
|
@@ -1052,99 +1081,103 @@ pub struct CopyObjectOutput {
|
|
|
1052
1081
|
pub etag: String,
|
|
1053
1082
|
}
|
|
1054
1083
|
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
use quick_xml::de;
|
|
1059
|
-
use serde::Deserialize;
|
|
1060
|
-
|
|
1061
|
-
use opendal_core::raw::*;
|
|
1062
|
-
use opendal_core::*;
|
|
1063
|
-
|
|
1064
|
-
/// TosError is the error returned by TOS service.
|
|
1065
|
-
#[derive(Default, Debug, Deserialize, PartialEq, Eq)]
|
|
1066
|
-
#[serde(default, rename_all = "PascalCase")]
|
|
1067
|
-
pub(crate) struct TosError {
|
|
1068
|
-
pub code: String,
|
|
1069
|
-
pub message: String,
|
|
1070
|
-
pub resource: String,
|
|
1071
|
-
pub request_id: String,
|
|
1072
|
-
}
|
|
1084
|
+
use bytes::Buf;
|
|
1085
|
+
use http::StatusCode;
|
|
1086
|
+
use quick_xml::de;
|
|
1073
1087
|
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1088
|
+
/// TosError is the error returned by TOS service.
|
|
1089
|
+
#[derive(Default, Debug, Deserialize, PartialEq, Eq)]
|
|
1090
|
+
#[serde(default, rename_all = "PascalCase")]
|
|
1091
|
+
pub(crate) struct TosError {
|
|
1092
|
+
pub code: String,
|
|
1093
|
+
pub message: String,
|
|
1094
|
+
pub resource: String,
|
|
1095
|
+
pub request_id: String,
|
|
1096
|
+
}
|
|
1078
1097
|
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
}
|
|
1085
|
-
StatusCode::CONFLICT => (ErrorKind::ConditionNotMatch, true),
|
|
1086
|
-
StatusCode::TOO_MANY_REQUESTS => (ErrorKind::RateLimited, true),
|
|
1087
|
-
code if code.is_server_error() => (ErrorKind::Unexpected, true),
|
|
1088
|
-
_ => (ErrorKind::Unexpected, false),
|
|
1089
|
-
};
|
|
1090
|
-
|
|
1091
|
-
let (message, tos_err) = parse_tos_error(&bs)
|
|
1092
|
-
.map(|tos_err| (format!("{tos_err:?}"), Some(tos_err)))
|
|
1093
|
-
.unwrap_or_else(|| (String::from_utf8_lossy(&bs).into_owned(), None));
|
|
1094
|
-
|
|
1095
|
-
if let Some(tos_err) = tos_err {
|
|
1096
|
-
(kind, retryable) =
|
|
1097
|
-
parse_tos_error_code(tos_err.code.as_str()).unwrap_or((kind, retryable));
|
|
1098
|
-
}
|
|
1098
|
+
/// Context needed to classify an error from this service.
|
|
1099
|
+
#[derive(Clone, Copy, Debug)]
|
|
1100
|
+
pub(crate) struct ErrorContext {
|
|
1101
|
+
service_operation: ServiceOperation,
|
|
1102
|
+
}
|
|
1099
1103
|
|
|
1100
|
-
|
|
1104
|
+
impl ErrorContext {
|
|
1105
|
+
pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
|
|
1106
|
+
Self { service_operation }
|
|
1107
|
+
}
|
|
1108
|
+
}
|
|
1101
1109
|
|
|
1102
|
-
|
|
1110
|
+
/// Parse an error response using its service request context.
|
|
1111
|
+
pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
|
|
1112
|
+
let (parts, body) = resp.into_parts();
|
|
1113
|
+
let bs = body.to_bytes();
|
|
1103
1114
|
|
|
1104
|
-
|
|
1105
|
-
|
|
1115
|
+
let (mut kind, mut retryable) = match parts.status {
|
|
1116
|
+
StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
|
|
1117
|
+
StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
|
|
1118
|
+
StatusCode::NOT_MODIFIED | StatusCode::PRECONDITION_FAILED => {
|
|
1119
|
+
(ErrorKind::ConditionNotMatch, false)
|
|
1106
1120
|
}
|
|
1107
|
-
|
|
1108
|
-
|
|
1121
|
+
StatusCode::CONFLICT => (ErrorKind::ConditionNotMatch, true),
|
|
1122
|
+
StatusCode::TOO_MANY_REQUESTS => (ErrorKind::RateLimited, true),
|
|
1123
|
+
code if code.is_server_error() => (ErrorKind::Unexpected, true),
|
|
1124
|
+
_ => (ErrorKind::Unexpected, false),
|
|
1125
|
+
};
|
|
1126
|
+
|
|
1127
|
+
let (message, tos_err) = parse_tos_error(&bs)
|
|
1128
|
+
.map(|tos_err| (format!("{tos_err:?}"), Some(tos_err)))
|
|
1129
|
+
.unwrap_or_else(|| (String::from_utf8_lossy(&bs).into_owned(), None));
|
|
1130
|
+
|
|
1131
|
+
if let Some(tos_err) = tos_err {
|
|
1132
|
+
(kind, retryable) =
|
|
1133
|
+
parse_tos_error_code(tos_err.code.as_str()).unwrap_or((kind, retryable));
|
|
1109
1134
|
}
|
|
1110
1135
|
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1136
|
+
let mut err = Error::new(kind, message);
|
|
1137
|
+
|
|
1138
|
+
err = err.with_context("service_operation", ctx.service_operation.0);
|
|
1139
|
+
err = with_error_response_context(err, parts);
|
|
1140
|
+
|
|
1141
|
+
if retryable {
|
|
1142
|
+
err = err.set_temporary();
|
|
1115
1143
|
}
|
|
1116
1144
|
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1145
|
+
err
|
|
1146
|
+
}
|
|
1147
|
+
|
|
1148
|
+
fn parse_tos_error(bs: &bytes::Bytes) -> Option<TosError> {
|
|
1149
|
+
de::from_reader::<_, TosError>(bs.chunk().reader())
|
|
1150
|
+
.or_else(|_| serde_json::from_slice::<TosError>(bs))
|
|
1151
|
+
.ok()
|
|
1152
|
+
}
|
|
1153
|
+
|
|
1154
|
+
/// Returns the `Error kind` of this code and whether the error is retryable.
|
|
1155
|
+
/// TOS error codes: https://www.volcengine.com/docs/6349/74874
|
|
1156
|
+
pub fn parse_tos_error_code(code: &str) -> Option<(ErrorKind, bool)> {
|
|
1157
|
+
match code {
|
|
1158
|
+
"NoSuchBucket" => Some((ErrorKind::ConfigInvalid, false)),
|
|
1159
|
+
"NoSuchKey" => Some((ErrorKind::NotFound, false)),
|
|
1160
|
+
"DuplicateObject" => Some((ErrorKind::ConditionNotMatch, false)),
|
|
1161
|
+
"RequestTimeout" => Some((ErrorKind::Unexpected, true)),
|
|
1162
|
+
"InternalError" => Some((ErrorKind::Unexpected, true)),
|
|
1163
|
+
"OperationAborted" => Some((ErrorKind::Unexpected, true)),
|
|
1164
|
+
"SlowDown" => Some((ErrorKind::RateLimited, true)),
|
|
1165
|
+
"ServiceUnavailable" => Some((ErrorKind::Unexpected, true)),
|
|
1166
|
+
"TooManyRequests" => Some((ErrorKind::RateLimited, true)),
|
|
1167
|
+
"ExceedAccountQPSLimit" | "ExceedAccountRateLimit" => Some((ErrorKind::RateLimited, true)),
|
|
1168
|
+
"ExceedBucketQPSLimit" | "ExceedBucketRateLimit" => Some((ErrorKind::RateLimited, true)),
|
|
1169
|
+
_ => None,
|
|
1138
1170
|
}
|
|
1171
|
+
}
|
|
1139
1172
|
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1173
|
+
#[cfg(test)]
|
|
1174
|
+
mod tests {
|
|
1175
|
+
use super::*;
|
|
1143
1176
|
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1177
|
+
#[test]
|
|
1178
|
+
fn test_parse_error() {
|
|
1179
|
+
let bs = bytes::Bytes::from(
|
|
1180
|
+
r#"
|
|
1148
1181
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
1149
1182
|
<Error>
|
|
1150
1183
|
<Code>NoSuchKey</Code>
|
|
@@ -1153,34 +1186,34 @@ mod error {
|
|
|
1153
1186
|
<RequestId>4442587FB7D0A2F9</RequestId>
|
|
1154
1187
|
</Error>
|
|
1155
1188
|
"#,
|
|
1156
|
-
|
|
1189
|
+
);
|
|
1157
1190
|
|
|
1158
|
-
|
|
1159
|
-
|
|
1191
|
+
let out: TosError = de::from_reader(bs.reader()).expect("must success");
|
|
1192
|
+
println!("{out:?}");
|
|
1160
1193
|
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1194
|
+
assert_eq!(out.code, "NoSuchKey");
|
|
1195
|
+
assert_eq!(out.message, "The resource you requested does not exist");
|
|
1196
|
+
assert_eq!(out.resource, "/mybucket/myfoto.jpg");
|
|
1197
|
+
assert_eq!(out.request_id, "4442587FB7D0A2F9");
|
|
1198
|
+
}
|
|
1166
1199
|
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1200
|
+
#[test]
|
|
1201
|
+
fn test_parse_json_error() {
|
|
1202
|
+
let bs = bytes::Bytes::from(
|
|
1203
|
+
r#"{"Code":"DuplicateObject","RequestId":"request-id","Message":"The object already exists."}"#,
|
|
1204
|
+
);
|
|
1172
1205
|
|
|
1173
|
-
|
|
1206
|
+
let out = parse_tos_error(&bs).expect("must success");
|
|
1174
1207
|
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1208
|
+
assert_eq!(out.code, "DuplicateObject");
|
|
1209
|
+
assert_eq!(out.message, "The object already exists.");
|
|
1210
|
+
assert_eq!(out.request_id, "request-id");
|
|
1211
|
+
}
|
|
1179
1212
|
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1213
|
+
#[test]
|
|
1214
|
+
fn test_parse_error_from_unrelated_input() {
|
|
1215
|
+
let bs = bytes::Bytes::from(
|
|
1216
|
+
r#"
|
|
1184
1217
|
<?xml version="1.0" encoding="UTF-8"?>
|
|
1185
1218
|
<CompleteMultipartUploadResult xmlns="http://tos.apache.org/doc/2024-01-01/">
|
|
1186
1219
|
<Bucket>example-bucket</Bucket>
|
|
@@ -1188,16 +1221,13 @@ mod error {
|
|
|
1188
1221
|
<UploadId>VXBsb2FkIElEIGZvciA2aWWpbmcncyBteS1tb3ZpZS5tMnRzIHVwbG9hZA</UploadId>
|
|
1189
1222
|
</CompleteMultipartUploadResult>
|
|
1190
1223
|
"#,
|
|
1191
|
-
|
|
1224
|
+
);
|
|
1192
1225
|
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
}
|
|
1226
|
+
let out: TosError = de::from_reader(bs.reader()).expect("must success");
|
|
1227
|
+
assert_eq!(out, TosError::default());
|
|
1196
1228
|
}
|
|
1197
1229
|
}
|
|
1198
1230
|
|
|
1199
|
-
pub(super) use error::*;
|
|
1200
|
-
|
|
1201
1231
|
mod utils {
|
|
1202
1232
|
use http::HeaderMap;
|
|
1203
1233
|
use http::header::ETAG;
|
|
@@ -1216,13 +1246,13 @@ mod utils {
|
|
|
1216
1246
|
path: &str,
|
|
1217
1247
|
headers: &HeaderMap,
|
|
1218
1248
|
) -> opendal_core::Result<Metadata> {
|
|
1219
|
-
let mut metadata = parse_into_metadata(path, headers)
|
|
1249
|
+
let mut metadata = parse_into_metadata(path, headers)?.into_builder();
|
|
1220
1250
|
|
|
1221
1251
|
if let Some(etag) = tos_parse_etag(headers)? {
|
|
1222
|
-
metadata.
|
|
1252
|
+
metadata.etag(etag);
|
|
1223
1253
|
}
|
|
1224
1254
|
|
|
1225
|
-
Ok(metadata)
|
|
1255
|
+
Ok(metadata.build())
|
|
1226
1256
|
}
|
|
1227
1257
|
}
|
|
1228
1258
|
|
|
@@ -50,7 +50,10 @@ impl oio::BatchDelete for TosDeleter {
|
|
|
50
50
|
match status {
|
|
51
51
|
StatusCode::NO_CONTENT => Ok(()),
|
|
52
52
|
StatusCode::NOT_FOUND => Ok(()),
|
|
53
|
-
_ => Err(crate::core::parse_error(
|
|
53
|
+
_ => Err(crate::core::parse_error(
|
|
54
|
+
crate::core::ErrorContext::new(ServiceOperation("DeleteObject")),
|
|
55
|
+
resp,
|
|
56
|
+
)),
|
|
54
57
|
}
|
|
55
58
|
}
|
|
56
59
|
|
|
@@ -59,7 +62,10 @@ impl oio::BatchDelete for TosDeleter {
|
|
|
59
62
|
|
|
60
63
|
let status = resp.status();
|
|
61
64
|
if status != StatusCode::OK {
|
|
62
|
-
return Err(crate::core::parse_error(
|
|
65
|
+
return Err(crate::core::parse_error(
|
|
66
|
+
crate::core::ErrorContext::new(ServiceOperation("DeleteMultipleObjects")),
|
|
67
|
+
resp,
|
|
68
|
+
));
|
|
63
69
|
}
|
|
64
70
|
|
|
65
71
|
let bs = resp.into_body();
|
|
@@ -90,7 +96,7 @@ impl oio::BatchDelete for TosDeleter {
|
|
|
90
96
|
let deleted = deleted.swap_remove(idx);
|
|
91
97
|
let op = if op.version().is_some() {
|
|
92
98
|
if let Some(version) = &deleted.version_id {
|
|
93
|
-
op.
|
|
99
|
+
op.into_version(version)
|
|
94
100
|
} else {
|
|
95
101
|
op
|
|
96
102
|
}
|
|
@@ -10,7 +10,7 @@ Depending on its configuration and the backing system, this service can expose:
|
|
|
10
10
|
- [x] list
|
|
11
11
|
- [x] copy
|
|
12
12
|
- [ ] rename
|
|
13
|
-
- [
|
|
13
|
+
- [x] presign
|
|
14
14
|
|
|
15
15
|
Inspect the effective capability set with [`opendal_core::Operator::info`] and
|
|
16
16
|
[`opendal_core::OperatorInfo::capability`] after building an operator.
|
|
@@ -21,8 +21,7 @@ use bytes::Buf;
|
|
|
21
21
|
|
|
22
22
|
use crate::core::parse_error;
|
|
23
23
|
use crate::core::*;
|
|
24
|
-
use opendal_core::
|
|
25
|
-
use opendal_core::Metadata;
|
|
24
|
+
use opendal_core::MetadataBuilder;
|
|
26
25
|
use opendal_core::OperationContext;
|
|
27
26
|
use opendal_core::Result;
|
|
28
27
|
use opendal_core::raw::*;
|
|
@@ -77,7 +76,10 @@ impl oio::PageList for TosLister {
|
|
|
77
76
|
.await?;
|
|
78
77
|
|
|
79
78
|
if resp.status() != http::StatusCode::OK {
|
|
80
|
-
return Err(parse_error(
|
|
79
|
+
return Err(parse_error(
|
|
80
|
+
ErrorContext::new(ServiceOperation("ListObjectsV2")),
|
|
81
|
+
resp,
|
|
82
|
+
));
|
|
81
83
|
}
|
|
82
84
|
|
|
83
85
|
let bs = resp.into_body();
|
|
@@ -90,7 +92,7 @@ impl oio::PageList for TosLister {
|
|
|
90
92
|
for prefix in output.common_prefixes {
|
|
91
93
|
let de = oio::Entry::new(
|
|
92
94
|
&build_rel_path(&self.core.root, &prefix.prefix),
|
|
93
|
-
|
|
95
|
+
MetadataBuilder::dir().build(),
|
|
94
96
|
);
|
|
95
97
|
|
|
96
98
|
ctx.entries.push_back(de);
|
|
@@ -102,16 +104,19 @@ impl oio::PageList for TosLister {
|
|
|
102
104
|
path = "/".to_string();
|
|
103
105
|
}
|
|
104
106
|
|
|
105
|
-
let mut meta =
|
|
106
|
-
|
|
107
|
+
let mut meta = if path.ends_with('/') {
|
|
108
|
+
MetadataBuilder::dir()
|
|
109
|
+
} else {
|
|
110
|
+
MetadataBuilder::file(object.size)
|
|
111
|
+
};
|
|
112
|
+
meta.is_current(Some(true));
|
|
107
113
|
if let Some(etag) = &object.etag {
|
|
108
|
-
meta.
|
|
109
|
-
meta.
|
|
114
|
+
meta.etag(etag);
|
|
115
|
+
meta.content_md5(etag.trim_matches('"'));
|
|
110
116
|
}
|
|
111
|
-
meta.
|
|
112
|
-
meta.set_last_modified(object.last_modified.parse::<Timestamp>()?);
|
|
117
|
+
meta.last_modified(object.last_modified.parse::<Timestamp>()?);
|
|
113
118
|
|
|
114
|
-
let de = oio::Entry::with(path, meta);
|
|
119
|
+
let de = oio::Entry::with(path, meta.build());
|
|
115
120
|
ctx.entries.push_back(de);
|
|
116
121
|
}
|
|
117
122
|
|
|
@@ -172,7 +177,10 @@ impl oio::PageList for TosObjectVersionsLister {
|
|
|
172
177
|
.await?;
|
|
173
178
|
|
|
174
179
|
if resp.status() != http::StatusCode::OK {
|
|
175
|
-
return Err(parse_error(
|
|
180
|
+
return Err(parse_error(
|
|
181
|
+
ErrorContext::new(ServiceOperation("ListObjectVersions")),
|
|
182
|
+
resp,
|
|
183
|
+
));
|
|
176
184
|
}
|
|
177
185
|
|
|
178
186
|
let bs = resp.into_body();
|
|
@@ -193,7 +201,7 @@ impl oio::PageList for TosObjectVersionsLister {
|
|
|
193
201
|
for prefix in output.common_prefixes {
|
|
194
202
|
let de = oio::Entry::new(
|
|
195
203
|
&build_rel_path(&self.core.root, &prefix.prefix),
|
|
196
|
-
|
|
204
|
+
MetadataBuilder::dir().build(),
|
|
197
205
|
);
|
|
198
206
|
|
|
199
207
|
ctx.entries.push_back(de);
|
|
@@ -209,18 +217,21 @@ impl oio::PageList for TosObjectVersionsLister {
|
|
|
209
217
|
path = "/".to_string();
|
|
210
218
|
}
|
|
211
219
|
|
|
212
|
-
let mut meta =
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
220
|
+
let mut meta = if path.ends_with('/') {
|
|
221
|
+
MetadataBuilder::dir()
|
|
222
|
+
} else {
|
|
223
|
+
MetadataBuilder::file(version_object.size)
|
|
224
|
+
};
|
|
225
|
+
meta.version(&version_object.version_id);
|
|
226
|
+
meta.is_current(Some(version_object.is_latest));
|
|
227
|
+
meta.last_modified(version_object.last_modified.parse::<Timestamp>()?);
|
|
217
228
|
|
|
218
229
|
if let Some(etag) = version_object.etag {
|
|
219
|
-
meta.
|
|
220
|
-
meta.
|
|
230
|
+
meta.etag(&etag);
|
|
231
|
+
meta.content_md5(etag.trim_matches('"'));
|
|
221
232
|
}
|
|
222
233
|
|
|
223
|
-
let de = oio::Entry::with(path, meta);
|
|
234
|
+
let de = oio::Entry::with(path, meta.build());
|
|
224
235
|
ctx.entries.push_back(de);
|
|
225
236
|
}
|
|
226
237
|
|
|
@@ -231,13 +242,17 @@ impl oio::PageList for TosObjectVersionsLister {
|
|
|
231
242
|
path = "/".to_string();
|
|
232
243
|
}
|
|
233
244
|
|
|
234
|
-
let mut meta =
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
245
|
+
let mut meta = if path.ends_with('/') {
|
|
246
|
+
MetadataBuilder::dir()
|
|
247
|
+
} else {
|
|
248
|
+
MetadataBuilder::file(0)
|
|
249
|
+
};
|
|
250
|
+
meta.version(&delete_marker.version_id);
|
|
251
|
+
meta.is_deleted(true);
|
|
252
|
+
meta.is_current(Some(delete_marker.is_latest));
|
|
253
|
+
meta.last_modified(delete_marker.last_modified.parse::<Timestamp>()?);
|
|
254
|
+
|
|
255
|
+
let de = oio::Entry::with(path, meta.build());
|
|
241
256
|
ctx.entries.push_back(de);
|
|
242
257
|
}
|
|
243
258
|
}
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
// under the License.
|
|
17
17
|
|
|
18
18
|
use super::backend::*;
|
|
19
|
-
use crate::core::parse_error;
|
|
19
|
+
use crate::core::{ErrorContext, parse_error};
|
|
20
20
|
use http::Response;
|
|
21
21
|
use http::StatusCode;
|
|
22
22
|
use opendal_core::BytesRange;
|
|
@@ -67,7 +67,10 @@ impl oio::StreamRead for TosReader {
|
|
|
67
67
|
_ => {
|
|
68
68
|
let (part, mut body) = resp.into_parts();
|
|
69
69
|
let buf = body.to_buffer().await?;
|
|
70
|
-
return Err(parse_error(
|
|
70
|
+
return Err(parse_error(
|
|
71
|
+
ErrorContext::new(ServiceOperation("GetObject")),
|
|
72
|
+
Response::from_parts(part, buf),
|
|
73
|
+
));
|
|
71
74
|
}
|
|
72
75
|
};
|
|
73
76
|
|
|
@@ -47,19 +47,24 @@ impl TosWriter {
|
|
|
47
47
|
}
|
|
48
48
|
|
|
49
49
|
fn parse_header_into_meta(path: &str, headers: &http::HeaderMap) -> Result<Metadata> {
|
|
50
|
-
let mut meta =
|
|
50
|
+
let mut meta = if path.ends_with('/') {
|
|
51
|
+
MetadataBuilder::dir()
|
|
52
|
+
} else {
|
|
53
|
+
MetadataBuilder::unknown()
|
|
54
|
+
};
|
|
51
55
|
if let Some(etag) = tos_parse_etag(headers)? {
|
|
52
|
-
meta.
|
|
56
|
+
meta.etag(etag);
|
|
53
57
|
}
|
|
54
58
|
if let Some(version) = parse_header_to_str(headers, X_TOS_VERSION_ID)? {
|
|
55
|
-
meta.
|
|
59
|
+
meta.version(version);
|
|
56
60
|
}
|
|
57
|
-
if
|
|
58
|
-
|
|
61
|
+
if !path.ends_with('/')
|
|
62
|
+
&& let Some(value) =
|
|
63
|
+
parse_header_to_str(headers, X_TOS_OBJECT_SIZE)?.and_then(|size| size.parse().ok())
|
|
59
64
|
{
|
|
60
|
-
meta.
|
|
65
|
+
meta.set_file(value);
|
|
61
66
|
}
|
|
62
|
-
Ok(meta)
|
|
67
|
+
Ok(meta.build())
|
|
63
68
|
}
|
|
64
69
|
}
|
|
65
70
|
|
|
@@ -77,7 +82,10 @@ impl oio::MultipartWrite for TosWriter {
|
|
|
77
82
|
|
|
78
83
|
match status {
|
|
79
84
|
StatusCode::CREATED | StatusCode::OK => Ok(meta),
|
|
80
|
-
_ => Err(parse_error(
|
|
85
|
+
_ => Err(parse_error(
|
|
86
|
+
ErrorContext::new(ServiceOperation("PutObject")),
|
|
87
|
+
resp,
|
|
88
|
+
)),
|
|
81
89
|
}
|
|
82
90
|
}
|
|
83
91
|
|
|
@@ -97,7 +105,10 @@ impl oio::MultipartWrite for TosWriter {
|
|
|
97
105
|
|
|
98
106
|
Ok(result.upload_id)
|
|
99
107
|
}
|
|
100
|
-
_ => Err(parse_error(
|
|
108
|
+
_ => Err(parse_error(
|
|
109
|
+
ErrorContext::new(ServiceOperation("CreateMultipartUpload")),
|
|
110
|
+
resp,
|
|
111
|
+
)),
|
|
101
112
|
}
|
|
102
113
|
}
|
|
103
114
|
|
|
@@ -135,7 +146,10 @@ impl oio::MultipartWrite for TosWriter {
|
|
|
135
146
|
size: None,
|
|
136
147
|
})
|
|
137
148
|
}
|
|
138
|
-
_ => Err(parse_error(
|
|
149
|
+
_ => Err(parse_error(
|
|
150
|
+
ErrorContext::new(ServiceOperation("UploadPart")),
|
|
151
|
+
resp,
|
|
152
|
+
)),
|
|
139
153
|
}
|
|
140
154
|
}
|
|
141
155
|
|
|
@@ -158,7 +172,8 @@ impl oio::MultipartWrite for TosWriter {
|
|
|
158
172
|
.await?;
|
|
159
173
|
|
|
160
174
|
let status = resp.status();
|
|
161
|
-
let mut meta =
|
|
175
|
+
let mut meta =
|
|
176
|
+
TosWriter::parse_header_into_meta(&self.path, resp.headers())?.into_builder();
|
|
162
177
|
|
|
163
178
|
match status {
|
|
164
179
|
StatusCode::OK => {
|
|
@@ -171,15 +186,18 @@ impl oio::MultipartWrite for TosWriter {
|
|
|
171
186
|
if !ret.etag.is_empty() {
|
|
172
187
|
// CompleteMultipartUpload response wraps ETag in quotes:
|
|
173
188
|
// https://www.volcengine.com/docs/6349/74868
|
|
174
|
-
meta.
|
|
189
|
+
meta.etag(ret.etag.trim_matches('"'));
|
|
175
190
|
}
|
|
176
191
|
if !ret.version_id.is_empty() {
|
|
177
|
-
meta.
|
|
192
|
+
meta.version(&ret.version_id);
|
|
178
193
|
}
|
|
179
194
|
|
|
180
|
-
Ok(meta)
|
|
195
|
+
Ok(meta.build())
|
|
181
196
|
}
|
|
182
|
-
_ => Err(parse_error(
|
|
197
|
+
_ => Err(parse_error(
|
|
198
|
+
ErrorContext::new(ServiceOperation("CompleteMultipartUpload")),
|
|
199
|
+
resp,
|
|
200
|
+
)),
|
|
183
201
|
}
|
|
184
202
|
}
|
|
185
203
|
|
|
@@ -191,7 +209,10 @@ impl oio::MultipartWrite for TosWriter {
|
|
|
191
209
|
|
|
192
210
|
match resp.status() {
|
|
193
211
|
StatusCode::NO_CONTENT => Ok(()),
|
|
194
|
-
_ => Err(parse_error(
|
|
212
|
+
_ => Err(parse_error(
|
|
213
|
+
ErrorContext::new(ServiceOperation("AbortMultipartUpload")),
|
|
214
|
+
resp,
|
|
215
|
+
)),
|
|
195
216
|
}
|
|
196
217
|
}
|
|
197
218
|
}
|
|
@@ -38,7 +38,7 @@ hmac = "0.13.0"
|
|
|
38
38
|
http = { workspace = true }
|
|
39
39
|
log = { workspace = true }
|
|
40
40
|
md-5 = "0.11.0"
|
|
41
|
-
opendal-core = { path = "../../core", version = "0.
|
|
41
|
+
opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
|
|
42
42
|
quick-xml = { workspace = true, features = ["serialize", "overlapped-lists"] }
|
|
43
43
|
serde = { workspace = true, features = ["derive"] }
|
|
44
44
|
serde_json = { workspace = true }
|