opendal 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +89 -56
- data/Cargo.toml +1 -1
- data/DEPENDENCIES.rust.tsv +55 -41
- data/LICENSE +201 -0
- data/LICENSE-MPL-2.0.txt +373 -0
- data/NOTICE +5 -0
- data/core/CHANGELOG.md +175 -0
- data/core/Cargo.lock +225 -298
- data/core/Cargo.toml +100 -94
- data/core/DEPENDENCIES.rust.tsv +44 -73
- data/core/benches/metadata/README.md +21 -0
- data/core/benches/metadata/main.rs +541 -0
- data/core/benches/vs_fs/Cargo.toml +1 -1
- data/core/benches/vs_s3/Cargo.toml +1 -1
- data/core/core/Cargo.toml +7 -2
- data/core/core/src/blocking/delete.rs +5 -1
- data/core/core/src/blocking/operator.rs +12 -0
- data/core/core/src/blocking/read/std_reader.rs +25 -0
- data/core/core/src/docs/concepts.rs +70 -94
- data/core/core/src/docs/internals/accessor.rs +62 -293
- data/core/core/src/docs/internals/layer.rs +43 -59
- data/core/core/src/docs/internals/mod.rs +21 -23
- data/core/core/src/docs/mod.rs +10 -6
- data/core/core/src/docs/performance/concurrent_write.md +92 -55
- data/core/core/src/docs/performance/http_optimization.md +120 -104
- data/core/core/src/docs/performance/mod.rs +8 -6
- data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
- data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
- data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
- data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
- data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
- data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
- data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
- data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
- data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
- data/core/core/src/docs/rfcs/README.md +14 -3
- data/core/core/src/docs/rfcs/mod.rs +46 -22
- data/core/core/src/docs/specs/README.md +34 -0
- data/core/core/src/docs/specs/conditional-operations.md +156 -0
- data/core/core/src/docs/specs/metadata.md +63 -0
- data/core/core/src/docs/specs/mod.rs +26 -0
- data/core/core/src/docs/upgrade.md +114 -1
- data/core/core/src/layers/capability_override.rs +15 -2
- data/core/core/src/layers/complete.rs +347 -43
- data/core/core/src/layers/correctness_check.rs +718 -51
- data/core/core/src/layers/error_context.rs +76 -2
- data/core/core/src/layers/simulate.rs +91 -28
- data/core/core/src/lib.rs +1 -0
- data/core/core/src/raw/accessor.rs +91 -15
- data/core/core/src/raw/enum_utils.rs +7 -0
- data/core/core/src/raw/http_util/header.rs +22 -17
- data/core/core/src/raw/oio/compose/api.rs +92 -0
- data/core/core/src/raw/oio/compose/mod.rs +19 -0
- data/core/core/src/raw/oio/copy/api.rs +3 -3
- data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
- data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
- data/core/core/src/raw/oio/entry.rs +48 -22
- data/core/core/src/raw/oio/list/flat_list.rs +1 -1
- data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
- data/core/core/src/raw/oio/mod.rs +3 -0
- data/core/core/src/raw/oio/read/position_read.rs +1 -1
- data/core/core/src/raw/oio/write/api.rs +40 -0
- data/core/core/src/raw/oio/write/append_write.rs +4 -4
- data/core/core/src/raw/oio/write/block_write.rs +2 -2
- data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
- data/core/core/src/raw/oio/write/position_write.rs +1 -1
- data/core/core/src/raw/operation.rs +6 -0
- data/core/core/src/raw/ops.rs +1191 -456
- data/core/core/src/raw/rps.rs +11 -0
- data/core/core/src/raw/time.rs +14 -3
- data/core/core/src/services/memory/backend.rs +5 -10
- data/core/core/src/services/memory/lister.rs +4 -4
- data/core/core/src/services/memory/writer.rs +7 -6
- data/core/core/src/types/buffer.rs +102 -12
- data/core/core/src/types/bytes_range.rs +20 -1
- data/core/core/src/types/capability.rs +65 -0
- data/core/core/src/types/compact.rs +329 -0
- data/core/core/src/types/compose/composer.rs +183 -0
- data/core/core/src/types/compose/input.rs +57 -0
- data/core/core/src/types/compose/mod.rs +23 -0
- data/core/core/src/types/context/read.rs +16 -11
- data/core/core/src/types/context/write.rs +150 -2
- data/core/core/src/types/copy.rs +5 -3
- data/core/core/src/types/delete/deleter.rs +20 -12
- data/core/core/src/types/delete/input.rs +22 -64
- data/core/core/src/types/delete/mod.rs +0 -1
- data/core/core/src/types/entry.rs +8 -14
- data/core/core/src/types/error.rs +28 -11
- data/core/core/src/types/metadata.rs +709 -322
- data/core/core/src/types/mod.rs +9 -1
- data/core/core/src/types/operator/operator.rs +278 -29
- data/core/core/src/types/operator/operator_futures.rs +466 -105
- data/core/core/src/types/options.rs +631 -158
- data/core/core/src/types/read/futures_async_reader.rs +663 -50
- data/core/core/src/types/read/reader.rs +23 -5
- data/core/core/src/types/write/futures_async_writer.rs +27 -0
- data/core/core/src/types/write/writer.rs +47 -5
- data/core/http-transports/reqwest/Cargo.toml +1 -1
- data/core/layers/async-backtrace/Cargo.toml +2 -2
- data/core/layers/async-backtrace/src/lib.rs +21 -2
- data/core/layers/await-tree/Cargo.toml +2 -2
- data/core/layers/await-tree/src/lib.rs +25 -2
- data/core/layers/capability-check/Cargo.toml +2 -2
- data/core/layers/capability-check/src/lib.rs +98 -10
- data/core/layers/chaos/Cargo.toml +2 -2
- data/core/layers/chaos/src/lib.rs +18 -3
- data/core/layers/concurrent-limit/Cargo.toml +3 -3
- data/core/layers/concurrent-limit/src/lib.rs +53 -13
- data/core/layers/dtrace/Cargo.toml +2 -2
- data/core/layers/dtrace/src/lib.rs +29 -2
- data/core/layers/fastmetrics/Cargo.toml +3 -3
- data/core/layers/fastrace/Cargo.toml +2 -2
- data/core/layers/fastrace/src/lib.rs +28 -2
- data/core/layers/foyer/Cargo.toml +2 -2
- data/core/layers/foyer/src/full.rs +16 -19
- data/core/layers/foyer/src/lib.rs +67 -14
- data/core/layers/hotpath/Cargo.toml +2 -2
- data/core/layers/hotpath/src/lib.rs +20 -2
- data/core/layers/immutable-index/Cargo.toml +2 -2
- data/core/layers/immutable-index/src/lib.rs +48 -24
- data/core/layers/logging/Cargo.toml +2 -2
- data/core/layers/logging/src/lib.rs +46 -2
- data/core/layers/metrics/Cargo.toml +3 -3
- data/core/layers/mime-guess/Cargo.toml +2 -2
- data/core/layers/mime-guess/src/lib.rs +31 -4
- data/core/layers/observe-metrics-common/Cargo.toml +2 -2
- data/core/layers/observe-metrics-common/src/lib.rs +55 -2
- data/core/layers/otelmetrics/Cargo.toml +3 -3
- data/core/layers/oteltrace/Cargo.toml +2 -2
- data/core/layers/oteltrace/src/lib.rs +27 -2
- data/core/layers/prometheus/Cargo.toml +3 -3
- data/core/layers/prometheus-client/Cargo.toml +3 -3
- data/core/layers/retry/Cargo.toml +4 -4
- data/core/layers/retry/src/lib.rs +76 -19
- data/core/layers/route/Cargo.toml +3 -3
- data/core/layers/route/src/lib.rs +34 -7
- data/core/layers/tail-cut/Cargo.toml +2 -2
- data/core/layers/tail-cut/src/lib.rs +28 -2
- data/core/layers/throttle/Cargo.toml +2 -2
- data/core/layers/throttle/src/lib.rs +19 -2
- data/core/layers/timeout/Cargo.toml +3 -3
- data/core/layers/timeout/src/lib.rs +38 -13
- data/core/layers/tracing/Cargo.toml +2 -2
- data/core/layers/tracing/src/lib.rs +24 -3
- data/core/services/aliyun-drive/Cargo.toml +2 -2
- data/core/services/aliyun-drive/src/backend.rs +34 -24
- data/core/services/aliyun-drive/src/core.rs +143 -54
- data/core/services/aliyun-drive/src/lister.rs +19 -18
- data/core/services/aliyun-drive/src/reader.rs +4 -1
- data/core/services/aliyun-drive/src/writer.rs +3 -3
- data/core/services/alluxio/Cargo.toml +1 -1
- data/core/services/alluxio/src/backend.rs +1 -1
- data/core/services/alluxio/src/core.rs +113 -94
- data/core/services/alluxio/src/reader.rs +7 -3
- data/core/services/alluxio/src/writer.rs +2 -2
- data/core/services/azblob/Cargo.toml +2 -2
- data/core/services/azblob/src/backend.rs +17 -9
- data/core/services/azblob/src/copier.rs +41 -15
- data/core/services/azblob/src/core.rs +372 -135
- data/core/services/azblob/src/deleter.rs +41 -18
- data/core/services/azblob/src/lister.rs +16 -10
- data/core/services/azblob/src/reader.rs +4 -1
- data/core/services/azblob/src/writer.rs +48 -15
- data/core/services/azdls/Cargo.toml +3 -3
- data/core/services/azdls/src/backend.rs +45 -15
- data/core/services/azdls/src/config.rs +4 -1
- data/core/services/azdls/src/core.rs +276 -87
- data/core/services/azdls/src/deleter.rs +13 -4
- data/core/services/azdls/src/lister.rs +18 -12
- data/core/services/azdls/src/reader.rs +6 -2
- data/core/services/azdls/src/writer.rs +44 -23
- data/core/services/azfile/Cargo.toml +2 -2
- data/core/services/azfile/src/backend.rs +24 -8
- data/core/services/azfile/src/core.rs +195 -77
- data/core/services/azfile/src/deleter.rs +8 -1
- data/core/services/azfile/src/lister.rs +20 -14
- data/core/services/azfile/src/reader.rs +5 -2
- data/core/services/azfile/src/writer.rs +27 -15
- data/core/services/azure-common/Cargo.toml +1 -1
- data/core/services/b2/Cargo.toml +2 -2
- data/core/services/b2/src/backend.rs +30 -11
- data/core/services/b2/src/core.rs +134 -120
- data/core/services/b2/src/deleter.rs +1 -1
- data/core/services/b2/src/lister.rs +5 -2
- data/core/services/b2/src/reader.rs +5 -2
- data/core/services/b2/src/writer.rs +29 -12
- data/core/services/cacache/Cargo.toml +1 -1
- data/core/services/cacache/src/backend.rs +4 -5
- data/core/services/cacache/src/reader.rs +4 -1
- data/core/services/cacache/src/writer.rs +6 -2
- data/core/services/cloudflare-kv/Cargo.toml +1 -1
- data/core/services/cloudflare-kv/src/backend.rs +26 -15
- data/core/services/cloudflare-kv/src/config.rs +13 -2
- data/core/services/cloudflare-kv/src/core.rs +62 -56
- data/core/services/cloudflare-kv/src/deleter.rs +12 -3
- data/core/services/cloudflare-kv/src/lister.rs +66 -17
- data/core/services/cloudflare-kv/src/reader.rs +14 -9
- data/core/services/cloudflare-kv/src/writer.rs +10 -7
- data/core/services/compfs/Cargo.toml +1 -1
- data/core/services/compfs/src/backend.rs +11 -7
- data/core/services/compfs/src/lister.rs +7 -7
- data/core/services/compfs/src/writer.rs +1 -1
- data/core/services/cos/Cargo.toml +1 -1
- data/core/services/cos/src/backend.rs +129 -16
- data/core/services/cos/src/core.rs +198 -80
- data/core/services/cos/src/deleter.rs +4 -1
- data/core/services/cos/src/lister.rs +42 -27
- data/core/services/cos/src/reader.rs +6 -2
- data/core/services/cos/src/writer.rs +45 -21
- data/core/services/d1/Cargo.toml +1 -1
- data/core/services/d1/src/backend.rs +6 -5
- data/core/services/d1/src/core.rs +79 -63
- data/core/services/d1/src/reader.rs +4 -1
- data/core/services/d1/src/writer.rs +6 -2
- data/core/services/dashmap/Cargo.toml +1 -1
- data/core/services/dashmap/src/backend.rs +2 -2
- data/core/services/dashmap/src/reader.rs +4 -1
- data/core/services/dashmap/src/writer.rs +11 -8
- data/core/services/dbfs/Cargo.toml +1 -1
- data/core/services/dbfs/src/backend.rs +21 -15
- data/core/services/dbfs/src/core.rs +65 -60
- data/core/services/dbfs/src/deleter.rs +4 -1
- data/core/services/dbfs/src/lister.rs +10 -10
- data/core/services/dbfs/src/writer.rs +6 -3
- data/core/services/dropbox/Cargo.toml +2 -2
- data/core/services/dropbox/src/backend.rs +30 -23
- data/core/services/dropbox/src/core.rs +65 -63
- data/core/services/dropbox/src/deleter.rs +1 -1
- data/core/services/dropbox/src/lister.rs +17 -8
- data/core/services/dropbox/src/reader.rs +4 -1
- data/core/services/dropbox/src/writer.rs +13 -10
- data/core/services/etcd/Cargo.toml +1 -1
- data/core/services/etcd/src/backend.rs +8 -5
- data/core/services/etcd/src/core.rs +5 -11
- data/core/services/etcd/src/lister.rs +6 -1
- data/core/services/etcd/src/reader.rs +4 -1
- data/core/services/etcd/src/writer.rs +6 -3
- data/core/services/foundationdb/Cargo.toml +1 -1
- data/core/services/foundationdb/src/backend.rs +6 -5
- data/core/services/foundationdb/src/reader.rs +4 -1
- data/core/services/foundationdb/src/writer.rs +6 -2
- data/core/services/foyer/Cargo.toml +3 -3
- data/core/services/foyer/src/backend.rs +6 -5
- data/core/services/foyer/src/reader.rs +4 -1
- data/core/services/foyer/src/writer.rs +6 -2
- data/core/services/fs/Cargo.toml +1 -1
- data/core/services/fs/src/backend.rs +4 -3
- data/core/services/fs/src/core.rs +44 -26
- data/core/services/fs/src/lister.rs +9 -6
- data/core/services/fs/src/writer.rs +20 -13
- data/core/services/ftp/Cargo.toml +4 -6
- data/core/services/ftp/src/backend.rs +9 -5
- data/core/services/ftp/src/core.rs +9 -7
- data/core/services/ftp/src/lister.rs +7 -8
- data/core/services/ftp/src/reader.rs +3 -3
- data/core/services/ftp/src/writer.rs +3 -3
- data/core/services/gcs/Cargo.toml +4 -2
- data/core/services/gcs/src/backend.rs +53 -7
- data/core/services/gcs/src/composer.rs +705 -0
- data/core/services/gcs/src/copier.rs +14 -4
- data/core/services/gcs/src/core.rs +828 -124
- data/core/services/gcs/src/deleter.rs +33 -18
- data/core/services/gcs/src/lib.rs +1 -0
- data/core/services/gcs/src/lister.rs +21 -12
- data/core/services/gcs/src/reader.rs +12 -5
- data/core/services/gcs/src/writer.rs +632 -10
- data/core/services/gcs-grpc/Cargo.toml +58 -0
- data/core/services/gcs-grpc/README.md +19 -0
- data/core/services/gcs-grpc/proto/README.md +23 -0
- data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
- data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
- data/core/services/gcs-grpc/src/backend.rs +449 -0
- data/core/services/gcs-grpc/src/config.rs +116 -0
- data/core/services/gcs-grpc/src/copier.rs +98 -0
- data/core/services/gcs-grpc/src/core.rs +298 -0
- data/core/services/gcs-grpc/src/deleter.rs +65 -0
- data/core/services/gcs-grpc/src/docs.md +19 -0
- data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
- data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
- data/core/services/gcs-grpc/src/lib.rs +43 -0
- data/core/services/gcs-grpc/src/lister.rs +125 -0
- data/core/services/gcs-grpc/src/reader.rs +140 -0
- data/core/services/gcs-grpc/src/writer.rs +780 -0
- data/core/services/gdrive/Cargo.toml +2 -2
- data/core/services/gdrive/src/backend.rs +123 -69
- data/core/services/gdrive/src/core.rs +87 -69
- data/core/services/gdrive/src/deleter.rs +8 -2
- data/core/services/gdrive/src/lister.rs +38 -26
- data/core/services/gdrive/src/path_index.rs +2 -2
- data/core/services/gdrive/src/reader.rs +9 -3
- data/core/services/gdrive/src/writer.rs +13 -7
- data/core/services/ghac/Cargo.toml +2 -2
- data/core/services/ghac/src/backend.rs +5 -2
- data/core/services/ghac/src/core.rs +59 -37
- data/core/services/ghac/src/reader.rs +5 -2
- data/core/services/ghac/src/writer.rs +16 -6
- data/core/services/github/Cargo.toml +1 -1
- data/core/services/github/src/backend.rs +16 -8
- data/core/services/github/src/core.rs +88 -76
- data/core/services/github/src/lister.rs +17 -15
- data/core/services/github/src/reader.rs +5 -2
- data/core/services/github/src/writer.rs +12 -6
- data/core/services/goosefs/Cargo.toml +2 -2
- data/core/services/goosefs/src/backend.rs +2 -2
- data/core/services/goosefs/src/core.rs +291 -208
- data/core/services/goosefs/src/docs.md +3 -0
- data/core/services/goosefs/src/lister.rs +2 -3
- data/core/services/goosefs/src/reader.rs +7 -3
- data/core/services/goosefs/src/writer.rs +21 -10
- data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
- data/core/services/gridfs/Cargo.toml +2 -2
- data/core/services/gridfs/src/backend.rs +12 -8
- data/core/services/gridfs/src/core.rs +1 -1
- data/core/services/gridfs/src/reader.rs +4 -1
- data/core/services/gridfs/src/writer.rs +6 -2
- data/core/services/hdfs/Cargo.toml +1 -1
- data/core/services/hdfs/src/backend.rs +1 -1
- data/core/services/hdfs/src/core.rs +7 -4
- data/core/services/hdfs/src/lister.rs +7 -9
- data/core/services/hdfs/src/writer.rs +5 -1
- data/core/services/hdfs-native/Cargo.toml +1 -1
- data/core/services/hdfs-native/src/backend.rs +1 -1
- data/core/services/hdfs-native/src/core.rs +44 -50
- data/core/services/hdfs-native/src/lister.rs +8 -10
- data/core/services/hdfs-native/src/writer.rs +5 -1
- data/core/services/hf/Cargo.toml +4 -3
- data/core/services/hf/src/backend.rs +4 -3
- data/core/services/hf/src/core.rs +907 -153
- data/core/services/hf/src/lister.rs +16 -6
- data/core/services/hf/src/reader.rs +377 -57
- data/core/services/hf/src/writer.rs +6 -2
- data/core/services/http/Cargo.toml +3 -3
- data/core/services/http/src/backend.rs +8 -6
- data/core/services/http/src/core.rs +57 -35
- data/core/services/http/src/reader.rs +5 -2
- data/core/services/ipfs/Cargo.toml +1 -1
- data/core/services/ipfs/src/backend.rs +20 -9
- data/core/services/ipfs/src/core.rs +43 -34
- data/core/services/ipfs/src/reader.rs +5 -2
- data/core/services/ipmfs/Cargo.toml +1 -1
- data/core/services/ipmfs/src/backend.rs +17 -8
- data/core/services/ipmfs/src/core.rs +66 -60
- data/core/services/ipmfs/src/deleter.rs +5 -2
- data/core/services/ipmfs/src/lister.rs +15 -12
- data/core/services/ipmfs/src/reader.rs +5 -2
- data/core/services/ipmfs/src/writer.rs +6 -3
- data/core/services/koofr/Cargo.toml +2 -2
- data/core/services/koofr/src/backend.rs +46 -17
- data/core/services/koofr/src/core.rs +64 -48
- data/core/services/koofr/src/deleter.rs +4 -1
- data/core/services/koofr/src/lister.rs +11 -10
- data/core/services/koofr/src/reader.rs +5 -2
- data/core/services/koofr/src/writer.rs +13 -7
- data/core/services/lakefs/Cargo.toml +1 -1
- data/core/services/lakefs/src/backend.rs +25 -8
- data/core/services/lakefs/src/core.rs +79 -73
- data/core/services/lakefs/src/deleter.rs +4 -1
- data/core/services/lakefs/src/lister.rs +17 -14
- data/core/services/lakefs/src/reader.rs +5 -2
- data/core/services/lakefs/src/writer.rs +11 -10
- data/core/services/memcached/Cargo.toml +1 -1
- data/core/services/memcached/src/backend.rs +57 -7
- data/core/services/memcached/src/config.rs +13 -2
- data/core/services/memcached/src/core.rs +11 -1
- data/core/services/memcached/src/reader.rs +4 -1
- data/core/services/memcached/src/writer.rs +6 -2
- data/core/services/mini_moka/Cargo.toml +1 -1
- data/core/services/mini_moka/src/backend.rs +8 -8
- data/core/services/mini_moka/src/reader.rs +4 -1
- data/core/services/mini_moka/src/writer.rs +11 -7
- data/core/services/moka/Cargo.toml +1 -1
- data/core/services/moka/src/backend.rs +7 -7
- data/core/services/moka/src/reader.rs +4 -1
- data/core/services/moka/src/writer.rs +10 -6
- data/core/services/mongodb/Cargo.toml +2 -2
- data/core/services/mongodb/src/backend.rs +7 -6
- data/core/services/mongodb/src/core.rs +1 -1
- data/core/services/mongodb/src/reader.rs +4 -1
- data/core/services/mongodb/src/writer.rs +6 -2
- data/core/services/monoiofs/Cargo.toml +2 -2
- data/core/services/monoiofs/src/backend.rs +16 -10
- data/core/services/monoiofs/src/core.rs +8 -9
- data/core/services/monoiofs/src/reader.rs +8 -10
- data/core/services/monoiofs/src/writer.rs +19 -24
- data/core/services/mysql/Cargo.toml +2 -2
- data/core/services/mysql/src/backend.rs +7 -6
- data/core/services/mysql/src/core.rs +1 -1
- data/core/services/mysql/src/lister.rs +7 -3
- data/core/services/mysql/src/reader.rs +4 -1
- data/core/services/mysql/src/writer.rs +6 -2
- data/core/services/obs/Cargo.toml +1 -1
- data/core/services/obs/src/backend.rs +29 -11
- data/core/services/obs/src/core.rs +81 -82
- data/core/services/obs/src/deleter.rs +4 -1
- data/core/services/obs/src/lister.rs +12 -5
- data/core/services/obs/src/reader.rs +5 -2
- data/core/services/obs/src/writer.rs +40 -19
- data/core/services/onedrive/Cargo.toml +2 -2
- data/core/services/onedrive/src/backend.rs +30 -21
- data/core/services/onedrive/src/config.rs +2 -2
- data/core/services/onedrive/src/core.rs +191 -90
- data/core/services/onedrive/src/deleter.rs +5 -2
- data/core/services/onedrive/src/graph_model.rs +10 -51
- data/core/services/onedrive/src/lister.rs +60 -24
- data/core/services/onedrive/src/reader.rs +17 -2
- data/core/services/onedrive/src/writer.rs +26 -15
- data/core/services/opfs/Cargo.toml +1 -1
- data/core/services/opfs/src/backend.rs +6 -13
- data/core/services/opfs/src/core.rs +33 -37
- data/core/services/opfs/src/lister.rs +4 -4
- data/core/services/opfs/src/writer.rs +1 -3
- data/core/services/oss/Cargo.toml +1 -1
- data/core/services/oss/src/backend.rs +40 -8
- data/core/services/oss/src/core.rs +192 -92
- data/core/services/oss/src/deleter.rs +16 -6
- data/core/services/oss/src/lister.rs +43 -27
- data/core/services/oss/src/reader.rs +6 -2
- data/core/services/oss/src/writer.rs +40 -22
- data/core/services/pcloud/Cargo.toml +1 -1
- data/core/services/pcloud/src/backend.rs +40 -6
- data/core/services/pcloud/src/core.rs +80 -66
- data/core/services/pcloud/src/deleter.rs +8 -1
- data/core/services/pcloud/src/lister.rs +4 -1
- data/core/services/pcloud/src/reader.rs +5 -2
- data/core/services/pcloud/src/writer.rs +6 -3
- data/core/services/persy/Cargo.toml +1 -1
- data/core/services/persy/src/backend.rs +6 -5
- data/core/services/persy/src/core.rs +60 -15
- data/core/services/persy/src/reader.rs +4 -1
- data/core/services/persy/src/writer.rs +6 -2
- data/core/services/postgresql/Cargo.toml +2 -2
- data/core/services/postgresql/src/backend.rs +7 -6
- data/core/services/postgresql/src/core.rs +1 -1
- data/core/services/postgresql/src/reader.rs +4 -1
- data/core/services/postgresql/src/writer.rs +6 -2
- data/core/services/redb/Cargo.toml +1 -1
- data/core/services/redb/src/backend.rs +6 -5
- data/core/services/redb/src/reader.rs +4 -1
- data/core/services/redb/src/writer.rs +6 -2
- data/core/services/redis/Cargo.toml +1 -1
- data/core/services/redis/src/backend.rs +18 -8
- data/core/services/redis/src/config.rs +13 -2
- data/core/services/redis/src/reader.rs +8 -4
- data/core/services/redis/src/writer.rs +6 -2
- data/core/services/rocksdb/Cargo.toml +1 -1
- data/core/services/rocksdb/src/backend.rs +6 -5
- data/core/services/rocksdb/src/lister.rs +6 -5
- data/core/services/rocksdb/src/reader.rs +4 -1
- data/core/services/rocksdb/src/writer.rs +6 -2
- data/core/services/s3/Cargo.toml +5 -4
- data/core/services/s3/src/backend.rs +995 -39
- data/core/services/s3/src/compatible_services.md +57 -0
- data/core/services/s3/src/config.rs +16 -0
- data/core/services/s3/src/copier.rs +190 -55
- data/core/services/s3/src/core.rs +299 -190
- data/core/services/s3/src/deleter.rs +26 -7
- data/core/services/s3/src/docs.md +57 -0
- data/core/services/s3/src/lister.rs +55 -36
- data/core/services/s3/src/reader.rs +22 -2
- data/core/services/s3/src/writer.rs +154 -32
- data/core/services/seafile/Cargo.toml +2 -2
- data/core/services/seafile/src/backend.rs +3 -3
- data/core/services/seafile/src/core.rs +105 -79
- data/core/services/seafile/src/lister.rs +11 -8
- data/core/services/seafile/src/reader.rs +5 -2
- data/core/services/seafile/src/writer.rs +6 -3
- data/core/services/sftp/Cargo.toml +1 -1
- data/core/services/sftp/src/backend.rs +19 -4
- data/core/services/sftp/src/core.rs +51 -46
- data/core/services/sftp/src/lister.rs +4 -4
- data/core/services/sftp/src/writer.rs +1 -1
- data/core/services/sled/Cargo.toml +1 -1
- data/core/services/sled/src/backend.rs +6 -5
- data/core/services/sled/src/core.rs +28 -6
- data/core/services/sled/src/lister.rs +6 -5
- data/core/services/sled/src/reader.rs +4 -1
- data/core/services/sled/src/writer.rs +6 -2
- data/core/services/sqlite/Cargo.toml +2 -2
- data/core/services/sqlite/src/backend.rs +12 -7
- data/core/services/sqlite/src/core.rs +3 -3
- data/core/services/sqlite/src/reader.rs +4 -1
- data/core/services/sqlite/src/writer.rs +7 -3
- data/core/services/surrealdb/Cargo.toml +2 -2
- data/core/services/surrealdb/src/backend.rs +7 -6
- data/core/services/surrealdb/src/core.rs +1 -1
- data/core/services/surrealdb/src/reader.rs +4 -1
- data/core/services/surrealdb/src/writer.rs +6 -2
- data/core/services/swift/Cargo.toml +1 -1
- data/core/services/swift/src/backend.rs +38 -8
- data/core/services/swift/src/core.rs +63 -67
- data/core/services/swift/src/deleter.rs +8 -2
- data/core/services/swift/src/lister.rs +11 -8
- data/core/services/swift/src/reader.rs +5 -2
- data/core/services/swift/src/writer.rs +17 -8
- data/core/services/tikv/Cargo.toml +2 -2
- data/core/services/tikv/src/backend.rs +7 -6
- data/core/services/tikv/src/core.rs +31 -14
- data/core/services/tikv/src/reader.rs +4 -1
- data/core/services/tikv/src/writer.rs +6 -2
- data/core/services/tos/Cargo.toml +2 -2
- data/core/services/tos/src/backend.rs +45 -21
- data/core/services/tos/src/copier.rs +44 -17
- data/core/services/tos/src/core.rs +147 -117
- data/core/services/tos/src/deleter.rs +9 -3
- data/core/services/tos/src/docs.md +1 -1
- data/core/services/tos/src/lister.rs +43 -28
- data/core/services/tos/src/reader.rs +5 -2
- data/core/services/tos/src/writer.rs +37 -16
- data/core/services/upyun/Cargo.toml +1 -1
- data/core/services/upyun/src/backend.rs +30 -7
- data/core/services/upyun/src/core.rs +122 -87
- data/core/services/upyun/src/deleter.rs +4 -1
- data/core/services/upyun/src/lister.rs +10 -10
- data/core/services/upyun/src/reader.rs +5 -2
- data/core/services/upyun/src/writer.rs +19 -7
- data/core/services/vercel-artifacts/Cargo.toml +1 -1
- data/core/services/vercel-artifacts/src/backend.rs +6 -3
- data/core/services/vercel-artifacts/src/core.rs +32 -27
- data/core/services/vercel-artifacts/src/reader.rs +5 -2
- data/core/services/vercel-artifacts/src/writer.rs +6 -3
- data/core/services/vercel-blob/Cargo.toml +1 -1
- data/core/services/vercel-blob/src/backend.rs +23 -6
- data/core/services/vercel-blob/src/core.rs +92 -81
- data/core/services/vercel-blob/src/lister.rs +12 -3
- data/core/services/vercel-blob/src/reader.rs +5 -2
- data/core/services/vercel-blob/src/writer.rs +19 -7
- data/core/services/webdav/Cargo.toml +2 -2
- data/core/services/webdav/src/backend.rs +25 -5
- data/core/services/webdav/src/core.rs +116 -59
- data/core/services/webdav/src/deleter.rs +4 -1
- data/core/services/webdav/src/lister.rs +4 -1
- data/core/services/webdav/src/reader.rs +6 -2
- data/core/services/webdav/src/writer.rs +17 -7
- data/core/services/webhdfs/Cargo.toml +2 -2
- data/core/services/webhdfs/src/backend.rs +24 -11
- data/core/services/webhdfs/src/core.rs +99 -98
- data/core/services/webhdfs/src/deleter.rs +5 -2
- data/core/services/webhdfs/src/lister.rs +21 -9
- data/core/services/webhdfs/src/reader.rs +5 -2
- data/core/services/webhdfs/src/writer.rs +49 -14
- data/core/services/yandex-disk/Cargo.toml +1 -1
- data/core/services/yandex-disk/src/backend.rs +28 -6
- data/core/services/yandex-disk/src/core.rs +97 -79
- data/core/services/yandex-disk/src/deleter.rs +4 -1
- data/core/services/yandex-disk/src/lister.rs +25 -9
- data/core/services/yandex-disk/src/reader.rs +5 -2
- data/core/services/yandex-disk/src/writer.rs +6 -3
- data/core/src/lib.rs +5 -0
- data/core/testkit/Cargo.toml +4 -4
- data/core/testkit/src/utils.rs +53 -8
- data/core/tests/behavior/async_compose.rs +297 -0
- data/core/tests/behavior/async_copy.rs +162 -1
- data/core/tests/behavior/async_delete.rs +364 -3
- data/core/tests/behavior/async_read.rs +86 -0
- data/core/tests/behavior/async_restore.rs +162 -0
- data/core/tests/behavior/async_stat.rs +90 -0
- data/core/tests/behavior/async_write.rs +330 -2
- data/core/tests/behavior/main.rs +4 -0
- data/core/upgrade.md +114 -1
- data/src/capability.rs +0 -1
- data/src/io.rs +0 -1
- data/src/lister.rs +0 -1
- data/src/metadata.rs +0 -1
- data/src/operator_info.rs +0 -1
- metadata +44 -2
|
@@ -222,6 +222,7 @@ impl Service for D1Backend {
|
|
|
222
222
|
type Lister = ();
|
|
223
223
|
type Deleter = oio::OneShotDeleter<D1Deleter>;
|
|
224
224
|
type Copier = ();
|
|
225
|
+
type Composer = ();
|
|
225
226
|
|
|
226
227
|
fn info(&self) -> ServiceInfo {
|
|
227
228
|
self.info.clone()
|
|
@@ -247,12 +248,13 @@ impl Service for D1Backend {
|
|
|
247
248
|
let p = build_abs_path(&self.root, path);
|
|
248
249
|
|
|
249
250
|
if p == build_abs_path(&self.root, "") {
|
|
250
|
-
Ok(RpStat::new(
|
|
251
|
+
Ok(RpStat::new(MetadataBuilder::dir().build()))
|
|
251
252
|
} else {
|
|
252
253
|
match self.core.get_length(ctx, &p).await? {
|
|
253
|
-
Some(length) => Ok(RpStat::new(
|
|
254
|
-
|
|
255
|
-
|
|
254
|
+
Some(length) => Ok(RpStat::new({
|
|
255
|
+
let metadata = MetadataBuilder::file(length as u64);
|
|
256
|
+
metadata.build()
|
|
257
|
+
})),
|
|
256
258
|
None => Err(Error::new(ErrorKind::NotFound, "kv not found in d1")),
|
|
257
259
|
}
|
|
258
260
|
}
|
|
@@ -304,7 +306,6 @@ impl Service for D1Backend {
|
|
|
304
306
|
_from: &str,
|
|
305
307
|
_to: &str,
|
|
306
308
|
_args: OpCopy,
|
|
307
|
-
_opts: OpCopier,
|
|
308
309
|
) -> Result<Self::Copier> {
|
|
309
310
|
Err(Error::new(
|
|
310
311
|
ErrorKind::Unsupported,
|
|
@@ -103,7 +103,10 @@ impl D1Core {
|
|
|
103
103
|
let d1_response = D1Response::parse(&bs)?;
|
|
104
104
|
Ok(d1_response.get_result(&self.value_field))
|
|
105
105
|
}
|
|
106
|
-
_ => Err(parse_error(
|
|
106
|
+
_ => Err(parse_error(
|
|
107
|
+
ErrorContext::new(ServiceOperation("Query")),
|
|
108
|
+
resp,
|
|
109
|
+
)),
|
|
107
110
|
}
|
|
108
111
|
}
|
|
109
112
|
|
|
@@ -124,7 +127,10 @@ impl D1Core {
|
|
|
124
127
|
let d1_response = D1Response::parse(&bs)?;
|
|
125
128
|
d1_response.get_usize_result("content_length")
|
|
126
129
|
}
|
|
127
|
-
_ => Err(parse_error(
|
|
130
|
+
_ => Err(parse_error(
|
|
131
|
+
ErrorContext::new(ServiceOperation("Query")),
|
|
132
|
+
resp,
|
|
133
|
+
)),
|
|
128
134
|
}
|
|
129
135
|
}
|
|
130
136
|
|
|
@@ -133,9 +139,9 @@ impl D1Core {
|
|
|
133
139
|
let key_field = &self.key_field;
|
|
134
140
|
let value_field = &self.value_field;
|
|
135
141
|
let query = format!(
|
|
136
|
-
r#"INSERT INTO "{table}" ("{key_field}", "{value_field}")
|
|
137
|
-
VALUES (
|
|
138
|
-
ON CONFLICT ("{key_field}")
|
|
142
|
+
r#"INSERT INTO "{table}" ("{key_field}", "{value_field}")
|
|
143
|
+
VALUES (?, ?)
|
|
144
|
+
ON CONFLICT ("{key_field}")
|
|
139
145
|
DO UPDATE SET "{value_field}" = EXCLUDED."{value_field}""#,
|
|
140
146
|
);
|
|
141
147
|
|
|
@@ -146,7 +152,10 @@ impl D1Core {
|
|
|
146
152
|
let status = resp.status();
|
|
147
153
|
match status {
|
|
148
154
|
StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(()),
|
|
149
|
-
_ => Err(parse_error(
|
|
155
|
+
_ => Err(parse_error(
|
|
156
|
+
ErrorContext::new(ServiceOperation("Query")),
|
|
157
|
+
resp,
|
|
158
|
+
)),
|
|
150
159
|
}
|
|
151
160
|
}
|
|
152
161
|
|
|
@@ -162,73 +171,80 @@ impl D1Core {
|
|
|
162
171
|
let status = resp.status();
|
|
163
172
|
match status {
|
|
164
173
|
StatusCode::OK | StatusCode::PARTIAL_CONTENT => Ok(()),
|
|
165
|
-
_ => Err(parse_error(
|
|
174
|
+
_ => Err(parse_error(
|
|
175
|
+
ErrorContext::new(ServiceOperation("Query")),
|
|
176
|
+
resp,
|
|
177
|
+
)),
|
|
166
178
|
}
|
|
167
179
|
}
|
|
168
180
|
}
|
|
169
181
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
use http::StatusCode;
|
|
174
|
-
use serde_json::de;
|
|
175
|
-
|
|
176
|
-
use crate::model::*;
|
|
177
|
-
use opendal_core::raw::*;
|
|
178
|
-
use opendal_core::*;
|
|
179
|
-
|
|
180
|
-
/// Parse error response into Error.
|
|
181
|
-
pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
|
|
182
|
-
let (parts, body) = resp.into_parts();
|
|
183
|
-
let bs = body.to_bytes();
|
|
184
|
-
|
|
185
|
-
let (mut kind, mut retryable) = match parts.status {
|
|
186
|
-
StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
|
|
187
|
-
// Some services (like owncloud) return 403 while file locked.
|
|
188
|
-
StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, true),
|
|
189
|
-
// Allowing retry for resource locked.
|
|
190
|
-
StatusCode::LOCKED => (ErrorKind::Unexpected, true),
|
|
191
|
-
StatusCode::INTERNAL_SERVER_ERROR
|
|
192
|
-
| StatusCode::BAD_GATEWAY
|
|
193
|
-
| StatusCode::SERVICE_UNAVAILABLE
|
|
194
|
-
| StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
|
|
195
|
-
_ => (ErrorKind::Unexpected, false),
|
|
196
|
-
};
|
|
197
|
-
|
|
198
|
-
let (message, d1_err) = de::from_reader::<_, D1Response>(bs.clone().reader())
|
|
199
|
-
.map(|d1_err| (format!("{d1_err:?}"), Some(d1_err)))
|
|
200
|
-
.unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
|
|
201
|
-
|
|
202
|
-
if let Some(d1_err) = d1_err {
|
|
203
|
-
(kind, retryable) = parse_d1_error_code(d1_err.errors).unwrap_or((kind, retryable));
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
let mut err = Error::new(kind, message);
|
|
182
|
+
use bytes::Buf;
|
|
183
|
+
use http::Response;
|
|
184
|
+
use serde_json::de;
|
|
207
185
|
|
|
208
|
-
|
|
186
|
+
/// Context needed to classify an error from this service.
|
|
187
|
+
#[derive(Clone, Copy, Debug)]
|
|
188
|
+
pub(crate) struct ErrorContext {
|
|
189
|
+
service_operation: ServiceOperation,
|
|
190
|
+
}
|
|
209
191
|
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
}
|
|
192
|
+
impl ErrorContext {
|
|
193
|
+
pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
|
|
194
|
+
Self { service_operation }
|
|
195
|
+
}
|
|
196
|
+
}
|
|
213
197
|
|
|
214
|
-
|
|
198
|
+
/// Parse an error response using its service request context.
|
|
199
|
+
pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
|
|
200
|
+
let (parts, body) = resp.into_parts();
|
|
201
|
+
let bs = body.to_bytes();
|
|
202
|
+
|
|
203
|
+
let (mut kind, mut retryable) = match parts.status {
|
|
204
|
+
StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
|
|
205
|
+
// Some services (like owncloud) return 403 while file locked.
|
|
206
|
+
StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, true),
|
|
207
|
+
// Allowing retry for resource locked.
|
|
208
|
+
StatusCode::LOCKED => (ErrorKind::Unexpected, true),
|
|
209
|
+
StatusCode::INTERNAL_SERVER_ERROR
|
|
210
|
+
| StatusCode::BAD_GATEWAY
|
|
211
|
+
| StatusCode::SERVICE_UNAVAILABLE
|
|
212
|
+
| StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
|
|
213
|
+
_ => (ErrorKind::Unexpected, false),
|
|
214
|
+
};
|
|
215
|
+
|
|
216
|
+
let (message, d1_err) = de::from_reader::<_, D1Response>(bs.clone().reader())
|
|
217
|
+
.map(|d1_err| (format!("{d1_err:?}"), Some(d1_err)))
|
|
218
|
+
.unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
|
|
219
|
+
|
|
220
|
+
if let Some(d1_err) = d1_err {
|
|
221
|
+
(kind, retryable) = parse_d1_error_code(d1_err.errors).unwrap_or((kind, retryable));
|
|
215
222
|
}
|
|
216
223
|
|
|
217
|
-
|
|
218
|
-
if errors.is_empty() {
|
|
219
|
-
return None;
|
|
220
|
-
}
|
|
224
|
+
let mut err = Error::new(kind, message);
|
|
221
225
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
// Authentication error.
|
|
228
|
-
10000 => Some((ErrorKind::PermissionDenied, false)),
|
|
229
|
-
_ => None,
|
|
230
|
-
}
|
|
226
|
+
err = err.with_context("service_operation", ctx.service_operation.0);
|
|
227
|
+
err = with_error_response_context(err, parts);
|
|
228
|
+
|
|
229
|
+
if retryable {
|
|
230
|
+
err = err.set_temporary();
|
|
231
231
|
}
|
|
232
|
+
|
|
233
|
+
err
|
|
232
234
|
}
|
|
233
235
|
|
|
234
|
-
pub(
|
|
236
|
+
pub fn parse_d1_error_code(errors: Vec<D1Error>) -> Option<(ErrorKind, bool)> {
|
|
237
|
+
if errors.is_empty() {
|
|
238
|
+
return None;
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
match errors[0].code {
|
|
242
|
+
// The request is malformed: failed to decode id.
|
|
243
|
+
7400 => Some((ErrorKind::Unexpected, false)),
|
|
244
|
+
// no such column: Xxxx.
|
|
245
|
+
7500 => Some((ErrorKind::NotFound, false)),
|
|
246
|
+
// Authentication error.
|
|
247
|
+
10000 => Some((ErrorKind::PermissionDenied, false)),
|
|
248
|
+
_ => None,
|
|
249
|
+
}
|
|
250
|
+
}
|
|
@@ -48,7 +48,10 @@ impl oio::StreamRead for D1Reader {
|
|
|
48
48
|
}
|
|
49
49
|
};
|
|
50
50
|
let content = bs.slice(range.to_content_range(bs.len())?);
|
|
51
|
-
let metadata =
|
|
51
|
+
let metadata = {
|
|
52
|
+
let metadata = MetadataBuilder::file(bs.len() as u64);
|
|
53
|
+
metadata.build()
|
|
54
|
+
};
|
|
52
55
|
Ok((
|
|
53
56
|
RpRead::new(metadata),
|
|
54
57
|
Box::new(content) as Box<dyn oio::ReadStreamDyn>,
|
|
@@ -50,8 +50,12 @@ impl oio::Write for D1Writer {
|
|
|
50
50
|
let length = buf.len() as u64;
|
|
51
51
|
self.core.set(&self.ctx, &self.path, buf).await?;
|
|
52
52
|
|
|
53
|
-
let
|
|
54
|
-
|
|
53
|
+
let metadata = if self.path.ends_with('/') {
|
|
54
|
+
MetadataBuilder::dir()
|
|
55
|
+
} else {
|
|
56
|
+
MetadataBuilder::file(length)
|
|
57
|
+
};
|
|
58
|
+
Ok(metadata.build())
|
|
55
59
|
}
|
|
56
60
|
|
|
57
61
|
async fn abort(&mut self) -> Result<()> {
|
|
@@ -34,7 +34,7 @@ all-features = true
|
|
|
34
34
|
[dependencies]
|
|
35
35
|
dashmap = "6"
|
|
36
36
|
log = { workspace = true }
|
|
37
|
-
opendal-core = { path = "../../core", version = "0.
|
|
37
|
+
opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
|
|
38
38
|
serde = { workspace = true, features = ["derive"] }
|
|
39
39
|
|
|
40
40
|
[dev-dependencies]
|
|
@@ -116,6 +116,7 @@ impl Service for DashmapBackend {
|
|
|
116
116
|
type Lister = oio::HierarchyLister<DashmapLister>;
|
|
117
117
|
type Deleter = oio::OneShotDeleter<DashmapDeleter>;
|
|
118
118
|
type Copier = ();
|
|
119
|
+
type Composer = ();
|
|
119
120
|
|
|
120
121
|
fn info(&self) -> ServiceInfo {
|
|
121
122
|
self.info.clone()
|
|
@@ -149,7 +150,7 @@ impl Service for DashmapBackend {
|
|
|
149
150
|
if p.ends_with('/') {
|
|
150
151
|
let has_children = self.core.cache.iter().any(|kv| kv.key().starts_with(&p));
|
|
151
152
|
if has_children {
|
|
152
|
-
return Ok(RpStat::new(
|
|
153
|
+
return Ok(RpStat::new(MetadataBuilder::dir().build()));
|
|
153
154
|
}
|
|
154
155
|
}
|
|
155
156
|
Err(Error::new(ErrorKind::NotFound, "key not found in dashmap"))
|
|
@@ -204,7 +205,6 @@ impl Service for DashmapBackend {
|
|
|
204
205
|
_from: &str,
|
|
205
206
|
_to: &str,
|
|
206
207
|
_args: OpCopy,
|
|
207
|
-
_opts: OpCopier,
|
|
208
208
|
) -> Result<Self::Copier> {
|
|
209
209
|
Err(Error::new(
|
|
210
210
|
ErrorKind::Unsupported,
|
|
@@ -46,7 +46,10 @@ impl oio::StreamRead for DashmapReader {
|
|
|
46
46
|
let buffer = value
|
|
47
47
|
.content
|
|
48
48
|
.slice(range.to_content_range(value.content.len())?);
|
|
49
|
-
let metadata =
|
|
49
|
+
let metadata = {
|
|
50
|
+
let metadata = MetadataBuilder::file(total_size);
|
|
51
|
+
metadata.build()
|
|
52
|
+
};
|
|
50
53
|
Ok((
|
|
51
54
|
RpRead::new(metadata),
|
|
52
55
|
Box::new(buffer) as Box<dyn oio::ReadStreamDyn>,
|
|
@@ -50,24 +50,27 @@ impl oio::Write for DashmapWriter {
|
|
|
50
50
|
async fn close(&mut self) -> Result<Metadata> {
|
|
51
51
|
let content = self.buf.clone().collect();
|
|
52
52
|
|
|
53
|
-
let
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
53
|
+
let mut meta = if self.path.ends_with('/') {
|
|
54
|
+
MetadataBuilder::dir()
|
|
55
|
+
} else {
|
|
56
|
+
MetadataBuilder::file(content.len() as u64)
|
|
57
|
+
};
|
|
58
|
+
meta.last_modified(Timestamp::now());
|
|
57
59
|
|
|
58
60
|
if let Some(v) = self.op.content_type() {
|
|
59
|
-
meta.
|
|
61
|
+
meta.content_type(v);
|
|
60
62
|
}
|
|
61
63
|
if let Some(v) = self.op.content_disposition() {
|
|
62
|
-
meta.
|
|
64
|
+
meta.content_disposition(v);
|
|
63
65
|
}
|
|
64
66
|
if let Some(v) = self.op.cache_control() {
|
|
65
|
-
meta.
|
|
67
|
+
meta.cache_control(v);
|
|
66
68
|
}
|
|
67
69
|
if let Some(v) = self.op.content_encoding() {
|
|
68
|
-
meta.
|
|
70
|
+
meta.content_encoding(v);
|
|
69
71
|
}
|
|
70
72
|
|
|
73
|
+
let meta = meta.build();
|
|
71
74
|
self.core.set(
|
|
72
75
|
&self.path,
|
|
73
76
|
DashmapValue {
|
|
@@ -36,7 +36,7 @@ base64 = { workspace = true }
|
|
|
36
36
|
bytes = { workspace = true }
|
|
37
37
|
http = { workspace = true }
|
|
38
38
|
log = { workspace = true }
|
|
39
|
-
opendal-core = { path = "../../core", version = "0.
|
|
39
|
+
opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
|
|
40
40
|
serde = { workspace = true, features = ["derive"] }
|
|
41
41
|
serde_json = { workspace = true }
|
|
42
42
|
|
|
@@ -24,8 +24,8 @@ use serde::Deserialize;
|
|
|
24
24
|
|
|
25
25
|
use super::DBFS_SCHEME;
|
|
26
26
|
use super::config::DbfsConfig;
|
|
27
|
-
use super::core::DbfsCore;
|
|
28
27
|
use super::core::parse_error;
|
|
28
|
+
use super::core::{DbfsCore, ErrorContext};
|
|
29
29
|
use super::deleter::DbfsDeleter;
|
|
30
30
|
use super::lister::DbfsLister;
|
|
31
31
|
use super::writer::DbfsWriter;
|
|
@@ -144,6 +144,7 @@ impl Service for DbfsBackend {
|
|
|
144
144
|
type Lister = oio::PageLister<DbfsLister>;
|
|
145
145
|
type Deleter = oio::OneShotDeleter<DbfsDeleter>;
|
|
146
146
|
type Copier = ();
|
|
147
|
+
type Composer = ();
|
|
147
148
|
|
|
148
149
|
fn info(&self) -> ServiceInfo {
|
|
149
150
|
ServiceInfo::new(DBFS_SCHEME, &self.core.root, "")
|
|
@@ -165,14 +166,17 @@ impl Service for DbfsBackend {
|
|
|
165
166
|
|
|
166
167
|
match status {
|
|
167
168
|
StatusCode::CREATED | StatusCode::OK => Ok(RpCreateDir::default()),
|
|
168
|
-
_ => Err(parse_error(
|
|
169
|
+
_ => Err(parse_error(
|
|
170
|
+
ErrorContext::new(ServiceOperation("Mkdirs")),
|
|
171
|
+
resp,
|
|
172
|
+
)),
|
|
169
173
|
}
|
|
170
174
|
}
|
|
171
175
|
|
|
172
176
|
async fn stat(&self, ctx: &OperationContext, path: &str, _: OpStat) -> Result<RpStat> {
|
|
173
177
|
// Stat root always returns a DIR.
|
|
174
178
|
if path == "/" {
|
|
175
|
-
return Ok(RpStat::new(
|
|
179
|
+
return Ok(RpStat::new(MetadataBuilder::dir().build()));
|
|
176
180
|
}
|
|
177
181
|
|
|
178
182
|
let resp = self.core.dbfs_get_status(ctx, path).await?;
|
|
@@ -181,26 +185,26 @@ impl Service for DbfsBackend {
|
|
|
181
185
|
|
|
182
186
|
match status {
|
|
183
187
|
StatusCode::OK => {
|
|
184
|
-
let mut meta =
|
|
188
|
+
let mut meta = MetadataBuilder::unknown();
|
|
185
189
|
let bs = resp.into_body();
|
|
186
190
|
let decoded_response: DbfsStatus =
|
|
187
191
|
serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
|
|
188
|
-
meta.
|
|
192
|
+
meta.last_modified(Timestamp::from_millisecond(
|
|
189
193
|
decoded_response.modification_time,
|
|
190
194
|
)?);
|
|
191
195
|
match decoded_response.is_dir {
|
|
192
|
-
true => meta.
|
|
193
|
-
false =>
|
|
194
|
-
meta.set_mode(EntryMode::FILE);
|
|
195
|
-
meta.set_content_length(decoded_response.file_size as u64)
|
|
196
|
-
}
|
|
196
|
+
true => meta.set_dir(),
|
|
197
|
+
false => meta.set_file(decoded_response.file_size as u64),
|
|
197
198
|
};
|
|
198
|
-
Ok(RpStat::new(meta))
|
|
199
|
+
Ok(RpStat::new(meta.build()))
|
|
199
200
|
}
|
|
200
201
|
StatusCode::NOT_FOUND if path.ends_with('/') => {
|
|
201
|
-
Ok(RpStat::new(
|
|
202
|
+
Ok(RpStat::new(MetadataBuilder::dir().build()))
|
|
202
203
|
}
|
|
203
|
-
_ => Err(parse_error(
|
|
204
|
+
_ => Err(parse_error(
|
|
205
|
+
ErrorContext::new(ServiceOperation("GetStatus")),
|
|
206
|
+
resp,
|
|
207
|
+
)),
|
|
204
208
|
}
|
|
205
209
|
}
|
|
206
210
|
|
|
@@ -260,7 +264,10 @@ impl Service for DbfsBackend {
|
|
|
260
264
|
|
|
261
265
|
match status {
|
|
262
266
|
StatusCode::OK => Ok(RpRename::default()),
|
|
263
|
-
_ => Err(parse_error(
|
|
267
|
+
_ => Err(parse_error(
|
|
268
|
+
ErrorContext::new(ServiceOperation("Move")),
|
|
269
|
+
resp,
|
|
270
|
+
)),
|
|
264
271
|
}
|
|
265
272
|
}
|
|
266
273
|
|
|
@@ -270,7 +277,6 @@ impl Service for DbfsBackend {
|
|
|
270
277
|
_from: &str,
|
|
271
278
|
_to: &str,
|
|
272
279
|
_args: OpCopy,
|
|
273
|
-
_opts: OpCopier,
|
|
274
280
|
) -> Result<Self::Copier> {
|
|
275
281
|
Err(Error::new(
|
|
276
282
|
ErrorKind::Unsupported,
|
|
@@ -24,6 +24,7 @@ use http::Request;
|
|
|
24
24
|
use http::Response;
|
|
25
25
|
use http::StatusCode;
|
|
26
26
|
use http::header;
|
|
27
|
+
use serde::Deserialize;
|
|
27
28
|
use serde_json::json;
|
|
28
29
|
|
|
29
30
|
use opendal_core::raw::*;
|
|
@@ -62,7 +63,7 @@ impl DbfsCore {
|
|
|
62
63
|
.to_string();
|
|
63
64
|
|
|
64
65
|
let req_body = &json!({
|
|
65
|
-
"path":
|
|
66
|
+
"path": p,
|
|
66
67
|
});
|
|
67
68
|
let body = Buffer::from(Bytes::from(req_body.to_string()));
|
|
68
69
|
|
|
@@ -91,7 +92,7 @@ impl DbfsCore {
|
|
|
91
92
|
.to_string();
|
|
92
93
|
|
|
93
94
|
let request_body = &json!({
|
|
94
|
-
"path":
|
|
95
|
+
"path": p,
|
|
95
96
|
// TODO: support recursive toggle, should we add a new field in OpDelete?
|
|
96
97
|
"recursive": true,
|
|
97
98
|
});
|
|
@@ -123,8 +124,8 @@ impl DbfsCore {
|
|
|
123
124
|
req = req.header(header::AUTHORIZATION, auth_header_content);
|
|
124
125
|
|
|
125
126
|
let req_body = &json!({
|
|
126
|
-
"source_path":
|
|
127
|
-
"destination_path":
|
|
127
|
+
"source_path": source,
|
|
128
|
+
"destination_path": target,
|
|
128
129
|
});
|
|
129
130
|
|
|
130
131
|
let body = Buffer::from(Bytes::from(req_body.to_string()));
|
|
@@ -172,7 +173,7 @@ impl DbfsCore {
|
|
|
172
173
|
req = req.header(header::AUTHORIZATION, auth_header_content);
|
|
173
174
|
|
|
174
175
|
let req_body = &json!({
|
|
175
|
-
"path": path,
|
|
176
|
+
"path": build_rooted_abs_path(&self.root, path),
|
|
176
177
|
"contents": contents,
|
|
177
178
|
"overwrite": true,
|
|
178
179
|
});
|
|
@@ -222,71 +223,75 @@ impl DbfsCore {
|
|
|
222
223
|
StatusCode::NOT_FOUND => {
|
|
223
224
|
self.dbfs_create_dir(ctx, path).await?;
|
|
224
225
|
}
|
|
225
|
-
_ =>
|
|
226
|
+
_ => {
|
|
227
|
+
return Err(parse_error(
|
|
228
|
+
ErrorContext::new(ServiceOperation("GetStatus")),
|
|
229
|
+
resp,
|
|
230
|
+
));
|
|
231
|
+
}
|
|
226
232
|
}
|
|
227
233
|
Ok(())
|
|
228
234
|
}
|
|
229
235
|
}
|
|
230
236
|
|
|
231
|
-
|
|
232
|
-
|
|
237
|
+
/// DbfsError is the error returned by DBFS service.
|
|
238
|
+
#[derive(Default, Deserialize)]
|
|
239
|
+
struct DbfsError {
|
|
240
|
+
error_code: String,
|
|
241
|
+
message: String,
|
|
242
|
+
}
|
|
233
243
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
244
|
+
impl Debug for DbfsError {
|
|
245
|
+
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
246
|
+
f.debug_struct("DbfsError")
|
|
247
|
+
.field("error_code", &self.error_code)
|
|
248
|
+
// replace `\n` to ` ` for better reading.
|
|
249
|
+
.field("message", &self.message.replace('\n', " "))
|
|
250
|
+
.finish()
|
|
251
|
+
}
|
|
252
|
+
}
|
|
237
253
|
|
|
238
|
-
|
|
239
|
-
|
|
254
|
+
/// Context needed to classify an error from this service.
|
|
255
|
+
#[derive(Clone, Copy, Debug)]
|
|
256
|
+
pub(crate) struct ErrorContext {
|
|
257
|
+
service_operation: ServiceOperation,
|
|
258
|
+
}
|
|
240
259
|
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
error_code: String,
|
|
245
|
-
message: String,
|
|
260
|
+
impl ErrorContext {
|
|
261
|
+
pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
|
|
262
|
+
Self { service_operation }
|
|
246
263
|
}
|
|
264
|
+
}
|
|
247
265
|
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
266
|
+
/// Parse an error response using its service request context.
|
|
267
|
+
pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
|
|
268
|
+
let (parts, body) = resp.into_parts();
|
|
269
|
+
let bs = body.to_bytes();
|
|
270
|
+
|
|
271
|
+
let (kind, retryable) = match parts.status {
|
|
272
|
+
StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
|
|
273
|
+
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, false),
|
|
274
|
+
StatusCode::PRECONDITION_FAILED => (ErrorKind::ConditionNotMatch, false),
|
|
275
|
+
StatusCode::INTERNAL_SERVER_ERROR
|
|
276
|
+
| StatusCode::BAD_GATEWAY
|
|
277
|
+
| StatusCode::SERVICE_UNAVAILABLE
|
|
278
|
+
| StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
|
|
279
|
+
_ => (ErrorKind::Unexpected, false),
|
|
280
|
+
};
|
|
281
|
+
|
|
282
|
+
let message = match serde_json::from_slice::<DbfsError>(&bs) {
|
|
283
|
+
Ok(dbfs_error) => format!("{:?}", dbfs_error.message),
|
|
284
|
+
Err(_) => String::from_utf8_lossy(&bs).into_owned(),
|
|
285
|
+
};
|
|
286
|
+
|
|
287
|
+
let mut err = Error::new(kind, message);
|
|
288
|
+
|
|
289
|
+
err = err.with_context("service_operation", ctx.service_operation.0);
|
|
290
|
+
err = with_error_response_context(err, parts);
|
|
291
|
+
|
|
292
|
+
if retryable {
|
|
293
|
+
err = err.set_temporary();
|
|
256
294
|
}
|
|
257
295
|
|
|
258
|
-
|
|
259
|
-
let (parts, body) = resp.into_parts();
|
|
260
|
-
let bs = body.to_bytes();
|
|
261
|
-
|
|
262
|
-
let (kind, retryable) = match parts.status {
|
|
263
|
-
StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
|
|
264
|
-
StatusCode::UNAUTHORIZED | StatusCode::FORBIDDEN => {
|
|
265
|
-
(ErrorKind::PermissionDenied, false)
|
|
266
|
-
}
|
|
267
|
-
StatusCode::PRECONDITION_FAILED => (ErrorKind::ConditionNotMatch, false),
|
|
268
|
-
StatusCode::INTERNAL_SERVER_ERROR
|
|
269
|
-
| StatusCode::BAD_GATEWAY
|
|
270
|
-
| StatusCode::SERVICE_UNAVAILABLE
|
|
271
|
-
| StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
|
|
272
|
-
_ => (ErrorKind::Unexpected, false),
|
|
273
|
-
};
|
|
274
|
-
|
|
275
|
-
let message = match serde_json::from_slice::<DbfsError>(&bs) {
|
|
276
|
-
Ok(dbfs_error) => format!("{:?}", dbfs_error.message),
|
|
277
|
-
Err(_) => String::from_utf8_lossy(&bs).into_owned(),
|
|
278
|
-
};
|
|
279
|
-
|
|
280
|
-
let mut err = Error::new(kind, message);
|
|
281
|
-
|
|
282
|
-
err = with_error_response_context(err, parts);
|
|
283
|
-
|
|
284
|
-
if retryable {
|
|
285
|
-
err = err.set_temporary();
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
err
|
|
289
|
-
}
|
|
296
|
+
err
|
|
290
297
|
}
|
|
291
|
-
|
|
292
|
-
pub(super) use error::*;
|
|
@@ -44,7 +44,10 @@ impl oio::OneShotDelete for DbfsDeleter {
|
|
|
44
44
|
match status {
|
|
45
45
|
// NOTE: Server will return 200 even if the path doesn't exist.
|
|
46
46
|
StatusCode::OK => Ok(()),
|
|
47
|
-
_ => Err(parse_error(
|
|
47
|
+
_ => Err(parse_error(
|
|
48
|
+
ErrorContext::new(ServiceOperation("Delete")),
|
|
49
|
+
resp,
|
|
50
|
+
)),
|
|
48
51
|
}
|
|
49
52
|
}
|
|
50
53
|
}
|
|
@@ -21,8 +21,8 @@ use bytes::Buf;
|
|
|
21
21
|
use http::StatusCode;
|
|
22
22
|
use serde::Deserialize;
|
|
23
23
|
|
|
24
|
-
use super::core::DbfsCore;
|
|
25
24
|
use super::core::parse_error;
|
|
25
|
+
use super::core::{DbfsCore, ErrorContext};
|
|
26
26
|
use opendal_core::raw::*;
|
|
27
27
|
use opendal_core::*;
|
|
28
28
|
|
|
@@ -48,7 +48,7 @@ impl oio::PageList for DbfsLister {
|
|
|
48
48
|
ctx.done = true;
|
|
49
49
|
return Ok(());
|
|
50
50
|
}
|
|
51
|
-
let error = parse_error(response);
|
|
51
|
+
let error = parse_error(ErrorContext::new(ServiceOperation("List")), response);
|
|
52
52
|
return Err(error);
|
|
53
53
|
}
|
|
54
54
|
|
|
@@ -59,18 +59,18 @@ impl oio::PageList for DbfsLister {
|
|
|
59
59
|
ctx.done = true;
|
|
60
60
|
|
|
61
61
|
for status in decoded_response.files {
|
|
62
|
+
let path = build_rel_path(&self.core.root, &status.path);
|
|
62
63
|
let entry: oio::Entry = match status.is_dir {
|
|
63
64
|
true => {
|
|
64
|
-
let normalized_path = format!("{}/"
|
|
65
|
-
let mut meta =
|
|
66
|
-
meta.
|
|
67
|
-
oio::Entry::new(&normalized_path, meta)
|
|
65
|
+
let normalized_path = format!("{path}/");
|
|
66
|
+
let mut meta = MetadataBuilder::dir();
|
|
67
|
+
meta.last_modified(Timestamp::from_millisecond(status.modification_time)?);
|
|
68
|
+
oio::Entry::new(&normalized_path, meta.build())
|
|
68
69
|
}
|
|
69
70
|
false => {
|
|
70
|
-
let mut meta =
|
|
71
|
-
meta.
|
|
72
|
-
meta.
|
|
73
|
-
oio::Entry::new(&status.path, meta)
|
|
71
|
+
let mut meta = MetadataBuilder::file(status.file_size as u64);
|
|
72
|
+
meta.last_modified(Timestamp::from_millisecond(status.modification_time)?);
|
|
73
|
+
oio::Entry::new(&path, meta.build())
|
|
74
74
|
}
|
|
75
75
|
};
|
|
76
76
|
ctx.entries.push_back(entry);
|