opendal 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +89 -56
- data/Cargo.toml +1 -1
- data/DEPENDENCIES.rust.tsv +55 -41
- data/LICENSE +201 -0
- data/LICENSE-MPL-2.0.txt +373 -0
- data/NOTICE +5 -0
- data/core/CHANGELOG.md +175 -0
- data/core/Cargo.lock +225 -298
- data/core/Cargo.toml +100 -94
- data/core/DEPENDENCIES.rust.tsv +44 -73
- data/core/benches/metadata/README.md +21 -0
- data/core/benches/metadata/main.rs +541 -0
- data/core/benches/vs_fs/Cargo.toml +1 -1
- data/core/benches/vs_s3/Cargo.toml +1 -1
- data/core/core/Cargo.toml +7 -2
- data/core/core/src/blocking/delete.rs +5 -1
- data/core/core/src/blocking/operator.rs +12 -0
- data/core/core/src/blocking/read/std_reader.rs +25 -0
- data/core/core/src/docs/concepts.rs +70 -94
- data/core/core/src/docs/internals/accessor.rs +62 -293
- data/core/core/src/docs/internals/layer.rs +43 -59
- data/core/core/src/docs/internals/mod.rs +21 -23
- data/core/core/src/docs/mod.rs +10 -6
- data/core/core/src/docs/performance/concurrent_write.md +92 -55
- data/core/core/src/docs/performance/http_optimization.md +120 -104
- data/core/core/src/docs/performance/mod.rs +8 -6
- data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
- data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
- data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
- data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
- data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
- data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
- data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
- data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
- data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
- data/core/core/src/docs/rfcs/README.md +14 -3
- data/core/core/src/docs/rfcs/mod.rs +46 -22
- data/core/core/src/docs/specs/README.md +34 -0
- data/core/core/src/docs/specs/conditional-operations.md +156 -0
- data/core/core/src/docs/specs/metadata.md +63 -0
- data/core/core/src/docs/specs/mod.rs +26 -0
- data/core/core/src/docs/upgrade.md +114 -1
- data/core/core/src/layers/capability_override.rs +15 -2
- data/core/core/src/layers/complete.rs +347 -43
- data/core/core/src/layers/correctness_check.rs +718 -51
- data/core/core/src/layers/error_context.rs +76 -2
- data/core/core/src/layers/simulate.rs +91 -28
- data/core/core/src/lib.rs +1 -0
- data/core/core/src/raw/accessor.rs +91 -15
- data/core/core/src/raw/enum_utils.rs +7 -0
- data/core/core/src/raw/http_util/header.rs +22 -17
- data/core/core/src/raw/oio/compose/api.rs +92 -0
- data/core/core/src/raw/oio/compose/mod.rs +19 -0
- data/core/core/src/raw/oio/copy/api.rs +3 -3
- data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
- data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
- data/core/core/src/raw/oio/entry.rs +48 -22
- data/core/core/src/raw/oio/list/flat_list.rs +1 -1
- data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
- data/core/core/src/raw/oio/mod.rs +3 -0
- data/core/core/src/raw/oio/read/position_read.rs +1 -1
- data/core/core/src/raw/oio/write/api.rs +40 -0
- data/core/core/src/raw/oio/write/append_write.rs +4 -4
- data/core/core/src/raw/oio/write/block_write.rs +2 -2
- data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
- data/core/core/src/raw/oio/write/position_write.rs +1 -1
- data/core/core/src/raw/operation.rs +6 -0
- data/core/core/src/raw/ops.rs +1191 -456
- data/core/core/src/raw/rps.rs +11 -0
- data/core/core/src/raw/time.rs +14 -3
- data/core/core/src/services/memory/backend.rs +5 -10
- data/core/core/src/services/memory/lister.rs +4 -4
- data/core/core/src/services/memory/writer.rs +7 -6
- data/core/core/src/types/buffer.rs +102 -12
- data/core/core/src/types/bytes_range.rs +20 -1
- data/core/core/src/types/capability.rs +65 -0
- data/core/core/src/types/compact.rs +329 -0
- data/core/core/src/types/compose/composer.rs +183 -0
- data/core/core/src/types/compose/input.rs +57 -0
- data/core/core/src/types/compose/mod.rs +23 -0
- data/core/core/src/types/context/read.rs +16 -11
- data/core/core/src/types/context/write.rs +150 -2
- data/core/core/src/types/copy.rs +5 -3
- data/core/core/src/types/delete/deleter.rs +20 -12
- data/core/core/src/types/delete/input.rs +22 -64
- data/core/core/src/types/delete/mod.rs +0 -1
- data/core/core/src/types/entry.rs +8 -14
- data/core/core/src/types/error.rs +28 -11
- data/core/core/src/types/metadata.rs +709 -322
- data/core/core/src/types/mod.rs +9 -1
- data/core/core/src/types/operator/operator.rs +278 -29
- data/core/core/src/types/operator/operator_futures.rs +466 -105
- data/core/core/src/types/options.rs +631 -158
- data/core/core/src/types/read/futures_async_reader.rs +663 -50
- data/core/core/src/types/read/reader.rs +23 -5
- data/core/core/src/types/write/futures_async_writer.rs +27 -0
- data/core/core/src/types/write/writer.rs +47 -5
- data/core/http-transports/reqwest/Cargo.toml +1 -1
- data/core/layers/async-backtrace/Cargo.toml +2 -2
- data/core/layers/async-backtrace/src/lib.rs +21 -2
- data/core/layers/await-tree/Cargo.toml +2 -2
- data/core/layers/await-tree/src/lib.rs +25 -2
- data/core/layers/capability-check/Cargo.toml +2 -2
- data/core/layers/capability-check/src/lib.rs +98 -10
- data/core/layers/chaos/Cargo.toml +2 -2
- data/core/layers/chaos/src/lib.rs +18 -3
- data/core/layers/concurrent-limit/Cargo.toml +3 -3
- data/core/layers/concurrent-limit/src/lib.rs +53 -13
- data/core/layers/dtrace/Cargo.toml +2 -2
- data/core/layers/dtrace/src/lib.rs +29 -2
- data/core/layers/fastmetrics/Cargo.toml +3 -3
- data/core/layers/fastrace/Cargo.toml +2 -2
- data/core/layers/fastrace/src/lib.rs +28 -2
- data/core/layers/foyer/Cargo.toml +2 -2
- data/core/layers/foyer/src/full.rs +16 -19
- data/core/layers/foyer/src/lib.rs +67 -14
- data/core/layers/hotpath/Cargo.toml +2 -2
- data/core/layers/hotpath/src/lib.rs +20 -2
- data/core/layers/immutable-index/Cargo.toml +2 -2
- data/core/layers/immutable-index/src/lib.rs +48 -24
- data/core/layers/logging/Cargo.toml +2 -2
- data/core/layers/logging/src/lib.rs +46 -2
- data/core/layers/metrics/Cargo.toml +3 -3
- data/core/layers/mime-guess/Cargo.toml +2 -2
- data/core/layers/mime-guess/src/lib.rs +31 -4
- data/core/layers/observe-metrics-common/Cargo.toml +2 -2
- data/core/layers/observe-metrics-common/src/lib.rs +55 -2
- data/core/layers/otelmetrics/Cargo.toml +3 -3
- data/core/layers/oteltrace/Cargo.toml +2 -2
- data/core/layers/oteltrace/src/lib.rs +27 -2
- data/core/layers/prometheus/Cargo.toml +3 -3
- data/core/layers/prometheus-client/Cargo.toml +3 -3
- data/core/layers/retry/Cargo.toml +4 -4
- data/core/layers/retry/src/lib.rs +76 -19
- data/core/layers/route/Cargo.toml +3 -3
- data/core/layers/route/src/lib.rs +34 -7
- data/core/layers/tail-cut/Cargo.toml +2 -2
- data/core/layers/tail-cut/src/lib.rs +28 -2
- data/core/layers/throttle/Cargo.toml +2 -2
- data/core/layers/throttle/src/lib.rs +19 -2
- data/core/layers/timeout/Cargo.toml +3 -3
- data/core/layers/timeout/src/lib.rs +38 -13
- data/core/layers/tracing/Cargo.toml +2 -2
- data/core/layers/tracing/src/lib.rs +24 -3
- data/core/services/aliyun-drive/Cargo.toml +2 -2
- data/core/services/aliyun-drive/src/backend.rs +34 -24
- data/core/services/aliyun-drive/src/core.rs +143 -54
- data/core/services/aliyun-drive/src/lister.rs +19 -18
- data/core/services/aliyun-drive/src/reader.rs +4 -1
- data/core/services/aliyun-drive/src/writer.rs +3 -3
- data/core/services/alluxio/Cargo.toml +1 -1
- data/core/services/alluxio/src/backend.rs +1 -1
- data/core/services/alluxio/src/core.rs +113 -94
- data/core/services/alluxio/src/reader.rs +7 -3
- data/core/services/alluxio/src/writer.rs +2 -2
- data/core/services/azblob/Cargo.toml +2 -2
- data/core/services/azblob/src/backend.rs +17 -9
- data/core/services/azblob/src/copier.rs +41 -15
- data/core/services/azblob/src/core.rs +372 -135
- data/core/services/azblob/src/deleter.rs +41 -18
- data/core/services/azblob/src/lister.rs +16 -10
- data/core/services/azblob/src/reader.rs +4 -1
- data/core/services/azblob/src/writer.rs +48 -15
- data/core/services/azdls/Cargo.toml +3 -3
- data/core/services/azdls/src/backend.rs +45 -15
- data/core/services/azdls/src/config.rs +4 -1
- data/core/services/azdls/src/core.rs +276 -87
- data/core/services/azdls/src/deleter.rs +13 -4
- data/core/services/azdls/src/lister.rs +18 -12
- data/core/services/azdls/src/reader.rs +6 -2
- data/core/services/azdls/src/writer.rs +44 -23
- data/core/services/azfile/Cargo.toml +2 -2
- data/core/services/azfile/src/backend.rs +24 -8
- data/core/services/azfile/src/core.rs +195 -77
- data/core/services/azfile/src/deleter.rs +8 -1
- data/core/services/azfile/src/lister.rs +20 -14
- data/core/services/azfile/src/reader.rs +5 -2
- data/core/services/azfile/src/writer.rs +27 -15
- data/core/services/azure-common/Cargo.toml +1 -1
- data/core/services/b2/Cargo.toml +2 -2
- data/core/services/b2/src/backend.rs +30 -11
- data/core/services/b2/src/core.rs +134 -120
- data/core/services/b2/src/deleter.rs +1 -1
- data/core/services/b2/src/lister.rs +5 -2
- data/core/services/b2/src/reader.rs +5 -2
- data/core/services/b2/src/writer.rs +29 -12
- data/core/services/cacache/Cargo.toml +1 -1
- data/core/services/cacache/src/backend.rs +4 -5
- data/core/services/cacache/src/reader.rs +4 -1
- data/core/services/cacache/src/writer.rs +6 -2
- data/core/services/cloudflare-kv/Cargo.toml +1 -1
- data/core/services/cloudflare-kv/src/backend.rs +26 -15
- data/core/services/cloudflare-kv/src/config.rs +13 -2
- data/core/services/cloudflare-kv/src/core.rs +62 -56
- data/core/services/cloudflare-kv/src/deleter.rs +12 -3
- data/core/services/cloudflare-kv/src/lister.rs +66 -17
- data/core/services/cloudflare-kv/src/reader.rs +14 -9
- data/core/services/cloudflare-kv/src/writer.rs +10 -7
- data/core/services/compfs/Cargo.toml +1 -1
- data/core/services/compfs/src/backend.rs +11 -7
- data/core/services/compfs/src/lister.rs +7 -7
- data/core/services/compfs/src/writer.rs +1 -1
- data/core/services/cos/Cargo.toml +1 -1
- data/core/services/cos/src/backend.rs +129 -16
- data/core/services/cos/src/core.rs +198 -80
- data/core/services/cos/src/deleter.rs +4 -1
- data/core/services/cos/src/lister.rs +42 -27
- data/core/services/cos/src/reader.rs +6 -2
- data/core/services/cos/src/writer.rs +45 -21
- data/core/services/d1/Cargo.toml +1 -1
- data/core/services/d1/src/backend.rs +6 -5
- data/core/services/d1/src/core.rs +79 -63
- data/core/services/d1/src/reader.rs +4 -1
- data/core/services/d1/src/writer.rs +6 -2
- data/core/services/dashmap/Cargo.toml +1 -1
- data/core/services/dashmap/src/backend.rs +2 -2
- data/core/services/dashmap/src/reader.rs +4 -1
- data/core/services/dashmap/src/writer.rs +11 -8
- data/core/services/dbfs/Cargo.toml +1 -1
- data/core/services/dbfs/src/backend.rs +21 -15
- data/core/services/dbfs/src/core.rs +65 -60
- data/core/services/dbfs/src/deleter.rs +4 -1
- data/core/services/dbfs/src/lister.rs +10 -10
- data/core/services/dbfs/src/writer.rs +6 -3
- data/core/services/dropbox/Cargo.toml +2 -2
- data/core/services/dropbox/src/backend.rs +30 -23
- data/core/services/dropbox/src/core.rs +65 -63
- data/core/services/dropbox/src/deleter.rs +1 -1
- data/core/services/dropbox/src/lister.rs +17 -8
- data/core/services/dropbox/src/reader.rs +4 -1
- data/core/services/dropbox/src/writer.rs +13 -10
- data/core/services/etcd/Cargo.toml +1 -1
- data/core/services/etcd/src/backend.rs +8 -5
- data/core/services/etcd/src/core.rs +5 -11
- data/core/services/etcd/src/lister.rs +6 -1
- data/core/services/etcd/src/reader.rs +4 -1
- data/core/services/etcd/src/writer.rs +6 -3
- data/core/services/foundationdb/Cargo.toml +1 -1
- data/core/services/foundationdb/src/backend.rs +6 -5
- data/core/services/foundationdb/src/reader.rs +4 -1
- data/core/services/foundationdb/src/writer.rs +6 -2
- data/core/services/foyer/Cargo.toml +3 -3
- data/core/services/foyer/src/backend.rs +6 -5
- data/core/services/foyer/src/reader.rs +4 -1
- data/core/services/foyer/src/writer.rs +6 -2
- data/core/services/fs/Cargo.toml +1 -1
- data/core/services/fs/src/backend.rs +4 -3
- data/core/services/fs/src/core.rs +44 -26
- data/core/services/fs/src/lister.rs +9 -6
- data/core/services/fs/src/writer.rs +20 -13
- data/core/services/ftp/Cargo.toml +4 -6
- data/core/services/ftp/src/backend.rs +9 -5
- data/core/services/ftp/src/core.rs +9 -7
- data/core/services/ftp/src/lister.rs +7 -8
- data/core/services/ftp/src/reader.rs +3 -3
- data/core/services/ftp/src/writer.rs +3 -3
- data/core/services/gcs/Cargo.toml +4 -2
- data/core/services/gcs/src/backend.rs +53 -7
- data/core/services/gcs/src/composer.rs +705 -0
- data/core/services/gcs/src/copier.rs +14 -4
- data/core/services/gcs/src/core.rs +828 -124
- data/core/services/gcs/src/deleter.rs +33 -18
- data/core/services/gcs/src/lib.rs +1 -0
- data/core/services/gcs/src/lister.rs +21 -12
- data/core/services/gcs/src/reader.rs +12 -5
- data/core/services/gcs/src/writer.rs +632 -10
- data/core/services/gcs-grpc/Cargo.toml +58 -0
- data/core/services/gcs-grpc/README.md +19 -0
- data/core/services/gcs-grpc/proto/README.md +23 -0
- data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
- data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
- data/core/services/gcs-grpc/src/backend.rs +449 -0
- data/core/services/gcs-grpc/src/config.rs +116 -0
- data/core/services/gcs-grpc/src/copier.rs +98 -0
- data/core/services/gcs-grpc/src/core.rs +298 -0
- data/core/services/gcs-grpc/src/deleter.rs +65 -0
- data/core/services/gcs-grpc/src/docs.md +19 -0
- data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
- data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
- data/core/services/gcs-grpc/src/lib.rs +43 -0
- data/core/services/gcs-grpc/src/lister.rs +125 -0
- data/core/services/gcs-grpc/src/reader.rs +140 -0
- data/core/services/gcs-grpc/src/writer.rs +780 -0
- data/core/services/gdrive/Cargo.toml +2 -2
- data/core/services/gdrive/src/backend.rs +123 -69
- data/core/services/gdrive/src/core.rs +87 -69
- data/core/services/gdrive/src/deleter.rs +8 -2
- data/core/services/gdrive/src/lister.rs +38 -26
- data/core/services/gdrive/src/path_index.rs +2 -2
- data/core/services/gdrive/src/reader.rs +9 -3
- data/core/services/gdrive/src/writer.rs +13 -7
- data/core/services/ghac/Cargo.toml +2 -2
- data/core/services/ghac/src/backend.rs +5 -2
- data/core/services/ghac/src/core.rs +59 -37
- data/core/services/ghac/src/reader.rs +5 -2
- data/core/services/ghac/src/writer.rs +16 -6
- data/core/services/github/Cargo.toml +1 -1
- data/core/services/github/src/backend.rs +16 -8
- data/core/services/github/src/core.rs +88 -76
- data/core/services/github/src/lister.rs +17 -15
- data/core/services/github/src/reader.rs +5 -2
- data/core/services/github/src/writer.rs +12 -6
- data/core/services/goosefs/Cargo.toml +2 -2
- data/core/services/goosefs/src/backend.rs +2 -2
- data/core/services/goosefs/src/core.rs +291 -208
- data/core/services/goosefs/src/docs.md +3 -0
- data/core/services/goosefs/src/lister.rs +2 -3
- data/core/services/goosefs/src/reader.rs +7 -3
- data/core/services/goosefs/src/writer.rs +21 -10
- data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
- data/core/services/gridfs/Cargo.toml +2 -2
- data/core/services/gridfs/src/backend.rs +12 -8
- data/core/services/gridfs/src/core.rs +1 -1
- data/core/services/gridfs/src/reader.rs +4 -1
- data/core/services/gridfs/src/writer.rs +6 -2
- data/core/services/hdfs/Cargo.toml +1 -1
- data/core/services/hdfs/src/backend.rs +1 -1
- data/core/services/hdfs/src/core.rs +7 -4
- data/core/services/hdfs/src/lister.rs +7 -9
- data/core/services/hdfs/src/writer.rs +5 -1
- data/core/services/hdfs-native/Cargo.toml +1 -1
- data/core/services/hdfs-native/src/backend.rs +1 -1
- data/core/services/hdfs-native/src/core.rs +44 -50
- data/core/services/hdfs-native/src/lister.rs +8 -10
- data/core/services/hdfs-native/src/writer.rs +5 -1
- data/core/services/hf/Cargo.toml +4 -3
- data/core/services/hf/src/backend.rs +4 -3
- data/core/services/hf/src/core.rs +907 -153
- data/core/services/hf/src/lister.rs +16 -6
- data/core/services/hf/src/reader.rs +377 -57
- data/core/services/hf/src/writer.rs +6 -2
- data/core/services/http/Cargo.toml +3 -3
- data/core/services/http/src/backend.rs +8 -6
- data/core/services/http/src/core.rs +57 -35
- data/core/services/http/src/reader.rs +5 -2
- data/core/services/ipfs/Cargo.toml +1 -1
- data/core/services/ipfs/src/backend.rs +20 -9
- data/core/services/ipfs/src/core.rs +43 -34
- data/core/services/ipfs/src/reader.rs +5 -2
- data/core/services/ipmfs/Cargo.toml +1 -1
- data/core/services/ipmfs/src/backend.rs +17 -8
- data/core/services/ipmfs/src/core.rs +66 -60
- data/core/services/ipmfs/src/deleter.rs +5 -2
- data/core/services/ipmfs/src/lister.rs +15 -12
- data/core/services/ipmfs/src/reader.rs +5 -2
- data/core/services/ipmfs/src/writer.rs +6 -3
- data/core/services/koofr/Cargo.toml +2 -2
- data/core/services/koofr/src/backend.rs +46 -17
- data/core/services/koofr/src/core.rs +64 -48
- data/core/services/koofr/src/deleter.rs +4 -1
- data/core/services/koofr/src/lister.rs +11 -10
- data/core/services/koofr/src/reader.rs +5 -2
- data/core/services/koofr/src/writer.rs +13 -7
- data/core/services/lakefs/Cargo.toml +1 -1
- data/core/services/lakefs/src/backend.rs +25 -8
- data/core/services/lakefs/src/core.rs +79 -73
- data/core/services/lakefs/src/deleter.rs +4 -1
- data/core/services/lakefs/src/lister.rs +17 -14
- data/core/services/lakefs/src/reader.rs +5 -2
- data/core/services/lakefs/src/writer.rs +11 -10
- data/core/services/memcached/Cargo.toml +1 -1
- data/core/services/memcached/src/backend.rs +57 -7
- data/core/services/memcached/src/config.rs +13 -2
- data/core/services/memcached/src/core.rs +11 -1
- data/core/services/memcached/src/reader.rs +4 -1
- data/core/services/memcached/src/writer.rs +6 -2
- data/core/services/mini_moka/Cargo.toml +1 -1
- data/core/services/mini_moka/src/backend.rs +8 -8
- data/core/services/mini_moka/src/reader.rs +4 -1
- data/core/services/mini_moka/src/writer.rs +11 -7
- data/core/services/moka/Cargo.toml +1 -1
- data/core/services/moka/src/backend.rs +7 -7
- data/core/services/moka/src/reader.rs +4 -1
- data/core/services/moka/src/writer.rs +10 -6
- data/core/services/mongodb/Cargo.toml +2 -2
- data/core/services/mongodb/src/backend.rs +7 -6
- data/core/services/mongodb/src/core.rs +1 -1
- data/core/services/mongodb/src/reader.rs +4 -1
- data/core/services/mongodb/src/writer.rs +6 -2
- data/core/services/monoiofs/Cargo.toml +2 -2
- data/core/services/monoiofs/src/backend.rs +16 -10
- data/core/services/monoiofs/src/core.rs +8 -9
- data/core/services/monoiofs/src/reader.rs +8 -10
- data/core/services/monoiofs/src/writer.rs +19 -24
- data/core/services/mysql/Cargo.toml +2 -2
- data/core/services/mysql/src/backend.rs +7 -6
- data/core/services/mysql/src/core.rs +1 -1
- data/core/services/mysql/src/lister.rs +7 -3
- data/core/services/mysql/src/reader.rs +4 -1
- data/core/services/mysql/src/writer.rs +6 -2
- data/core/services/obs/Cargo.toml +1 -1
- data/core/services/obs/src/backend.rs +29 -11
- data/core/services/obs/src/core.rs +81 -82
- data/core/services/obs/src/deleter.rs +4 -1
- data/core/services/obs/src/lister.rs +12 -5
- data/core/services/obs/src/reader.rs +5 -2
- data/core/services/obs/src/writer.rs +40 -19
- data/core/services/onedrive/Cargo.toml +2 -2
- data/core/services/onedrive/src/backend.rs +30 -21
- data/core/services/onedrive/src/config.rs +2 -2
- data/core/services/onedrive/src/core.rs +191 -90
- data/core/services/onedrive/src/deleter.rs +5 -2
- data/core/services/onedrive/src/graph_model.rs +10 -51
- data/core/services/onedrive/src/lister.rs +60 -24
- data/core/services/onedrive/src/reader.rs +17 -2
- data/core/services/onedrive/src/writer.rs +26 -15
- data/core/services/opfs/Cargo.toml +1 -1
- data/core/services/opfs/src/backend.rs +6 -13
- data/core/services/opfs/src/core.rs +33 -37
- data/core/services/opfs/src/lister.rs +4 -4
- data/core/services/opfs/src/writer.rs +1 -3
- data/core/services/oss/Cargo.toml +1 -1
- data/core/services/oss/src/backend.rs +40 -8
- data/core/services/oss/src/core.rs +192 -92
- data/core/services/oss/src/deleter.rs +16 -6
- data/core/services/oss/src/lister.rs +43 -27
- data/core/services/oss/src/reader.rs +6 -2
- data/core/services/oss/src/writer.rs +40 -22
- data/core/services/pcloud/Cargo.toml +1 -1
- data/core/services/pcloud/src/backend.rs +40 -6
- data/core/services/pcloud/src/core.rs +80 -66
- data/core/services/pcloud/src/deleter.rs +8 -1
- data/core/services/pcloud/src/lister.rs +4 -1
- data/core/services/pcloud/src/reader.rs +5 -2
- data/core/services/pcloud/src/writer.rs +6 -3
- data/core/services/persy/Cargo.toml +1 -1
- data/core/services/persy/src/backend.rs +6 -5
- data/core/services/persy/src/core.rs +60 -15
- data/core/services/persy/src/reader.rs +4 -1
- data/core/services/persy/src/writer.rs +6 -2
- data/core/services/postgresql/Cargo.toml +2 -2
- data/core/services/postgresql/src/backend.rs +7 -6
- data/core/services/postgresql/src/core.rs +1 -1
- data/core/services/postgresql/src/reader.rs +4 -1
- data/core/services/postgresql/src/writer.rs +6 -2
- data/core/services/redb/Cargo.toml +1 -1
- data/core/services/redb/src/backend.rs +6 -5
- data/core/services/redb/src/reader.rs +4 -1
- data/core/services/redb/src/writer.rs +6 -2
- data/core/services/redis/Cargo.toml +1 -1
- data/core/services/redis/src/backend.rs +18 -8
- data/core/services/redis/src/config.rs +13 -2
- data/core/services/redis/src/reader.rs +8 -4
- data/core/services/redis/src/writer.rs +6 -2
- data/core/services/rocksdb/Cargo.toml +1 -1
- data/core/services/rocksdb/src/backend.rs +6 -5
- data/core/services/rocksdb/src/lister.rs +6 -5
- data/core/services/rocksdb/src/reader.rs +4 -1
- data/core/services/rocksdb/src/writer.rs +6 -2
- data/core/services/s3/Cargo.toml +5 -4
- data/core/services/s3/src/backend.rs +995 -39
- data/core/services/s3/src/compatible_services.md +57 -0
- data/core/services/s3/src/config.rs +16 -0
- data/core/services/s3/src/copier.rs +190 -55
- data/core/services/s3/src/core.rs +299 -190
- data/core/services/s3/src/deleter.rs +26 -7
- data/core/services/s3/src/docs.md +57 -0
- data/core/services/s3/src/lister.rs +55 -36
- data/core/services/s3/src/reader.rs +22 -2
- data/core/services/s3/src/writer.rs +154 -32
- data/core/services/seafile/Cargo.toml +2 -2
- data/core/services/seafile/src/backend.rs +3 -3
- data/core/services/seafile/src/core.rs +105 -79
- data/core/services/seafile/src/lister.rs +11 -8
- data/core/services/seafile/src/reader.rs +5 -2
- data/core/services/seafile/src/writer.rs +6 -3
- data/core/services/sftp/Cargo.toml +1 -1
- data/core/services/sftp/src/backend.rs +19 -4
- data/core/services/sftp/src/core.rs +51 -46
- data/core/services/sftp/src/lister.rs +4 -4
- data/core/services/sftp/src/writer.rs +1 -1
- data/core/services/sled/Cargo.toml +1 -1
- data/core/services/sled/src/backend.rs +6 -5
- data/core/services/sled/src/core.rs +28 -6
- data/core/services/sled/src/lister.rs +6 -5
- data/core/services/sled/src/reader.rs +4 -1
- data/core/services/sled/src/writer.rs +6 -2
- data/core/services/sqlite/Cargo.toml +2 -2
- data/core/services/sqlite/src/backend.rs +12 -7
- data/core/services/sqlite/src/core.rs +3 -3
- data/core/services/sqlite/src/reader.rs +4 -1
- data/core/services/sqlite/src/writer.rs +7 -3
- data/core/services/surrealdb/Cargo.toml +2 -2
- data/core/services/surrealdb/src/backend.rs +7 -6
- data/core/services/surrealdb/src/core.rs +1 -1
- data/core/services/surrealdb/src/reader.rs +4 -1
- data/core/services/surrealdb/src/writer.rs +6 -2
- data/core/services/swift/Cargo.toml +1 -1
- data/core/services/swift/src/backend.rs +38 -8
- data/core/services/swift/src/core.rs +63 -67
- data/core/services/swift/src/deleter.rs +8 -2
- data/core/services/swift/src/lister.rs +11 -8
- data/core/services/swift/src/reader.rs +5 -2
- data/core/services/swift/src/writer.rs +17 -8
- data/core/services/tikv/Cargo.toml +2 -2
- data/core/services/tikv/src/backend.rs +7 -6
- data/core/services/tikv/src/core.rs +31 -14
- data/core/services/tikv/src/reader.rs +4 -1
- data/core/services/tikv/src/writer.rs +6 -2
- data/core/services/tos/Cargo.toml +2 -2
- data/core/services/tos/src/backend.rs +45 -21
- data/core/services/tos/src/copier.rs +44 -17
- data/core/services/tos/src/core.rs +147 -117
- data/core/services/tos/src/deleter.rs +9 -3
- data/core/services/tos/src/docs.md +1 -1
- data/core/services/tos/src/lister.rs +43 -28
- data/core/services/tos/src/reader.rs +5 -2
- data/core/services/tos/src/writer.rs +37 -16
- data/core/services/upyun/Cargo.toml +1 -1
- data/core/services/upyun/src/backend.rs +30 -7
- data/core/services/upyun/src/core.rs +122 -87
- data/core/services/upyun/src/deleter.rs +4 -1
- data/core/services/upyun/src/lister.rs +10 -10
- data/core/services/upyun/src/reader.rs +5 -2
- data/core/services/upyun/src/writer.rs +19 -7
- data/core/services/vercel-artifacts/Cargo.toml +1 -1
- data/core/services/vercel-artifacts/src/backend.rs +6 -3
- data/core/services/vercel-artifacts/src/core.rs +32 -27
- data/core/services/vercel-artifacts/src/reader.rs +5 -2
- data/core/services/vercel-artifacts/src/writer.rs +6 -3
- data/core/services/vercel-blob/Cargo.toml +1 -1
- data/core/services/vercel-blob/src/backend.rs +23 -6
- data/core/services/vercel-blob/src/core.rs +92 -81
- data/core/services/vercel-blob/src/lister.rs +12 -3
- data/core/services/vercel-blob/src/reader.rs +5 -2
- data/core/services/vercel-blob/src/writer.rs +19 -7
- data/core/services/webdav/Cargo.toml +2 -2
- data/core/services/webdav/src/backend.rs +25 -5
- data/core/services/webdav/src/core.rs +116 -59
- data/core/services/webdav/src/deleter.rs +4 -1
- data/core/services/webdav/src/lister.rs +4 -1
- data/core/services/webdav/src/reader.rs +6 -2
- data/core/services/webdav/src/writer.rs +17 -7
- data/core/services/webhdfs/Cargo.toml +2 -2
- data/core/services/webhdfs/src/backend.rs +24 -11
- data/core/services/webhdfs/src/core.rs +99 -98
- data/core/services/webhdfs/src/deleter.rs +5 -2
- data/core/services/webhdfs/src/lister.rs +21 -9
- data/core/services/webhdfs/src/reader.rs +5 -2
- data/core/services/webhdfs/src/writer.rs +49 -14
- data/core/services/yandex-disk/Cargo.toml +1 -1
- data/core/services/yandex-disk/src/backend.rs +28 -6
- data/core/services/yandex-disk/src/core.rs +97 -79
- data/core/services/yandex-disk/src/deleter.rs +4 -1
- data/core/services/yandex-disk/src/lister.rs +25 -9
- data/core/services/yandex-disk/src/reader.rs +5 -2
- data/core/services/yandex-disk/src/writer.rs +6 -3
- data/core/src/lib.rs +5 -0
- data/core/testkit/Cargo.toml +4 -4
- data/core/testkit/src/utils.rs +53 -8
- data/core/tests/behavior/async_compose.rs +297 -0
- data/core/tests/behavior/async_copy.rs +162 -1
- data/core/tests/behavior/async_delete.rs +364 -3
- data/core/tests/behavior/async_read.rs +86 -0
- data/core/tests/behavior/async_restore.rs +162 -0
- data/core/tests/behavior/async_stat.rs +90 -0
- data/core/tests/behavior/async_write.rs +330 -2
- data/core/tests/behavior/main.rs +4 -0
- data/core/upgrade.md +114 -1
- data/src/capability.rs +0 -1
- data/src/io.rs +0 -1
- data/src/lister.rs +0 -1
- data/src/metadata.rs +0 -1
- data/src/operator_info.rs +0 -1
- metadata +44 -2
|
@@ -20,11 +20,11 @@ use std::sync::Arc;
|
|
|
20
20
|
use bytes::Buf;
|
|
21
21
|
use http::StatusCode;
|
|
22
22
|
|
|
23
|
-
use super::core::B2Core;
|
|
24
23
|
use super::core::StartLargeFileResponse;
|
|
25
24
|
use super::core::UploadPartResponse;
|
|
26
25
|
use super::core::UploadResponse;
|
|
27
26
|
use super::core::parse_error;
|
|
27
|
+
use super::core::{B2Core, ErrorContext};
|
|
28
28
|
use opendal_core::raw::*;
|
|
29
29
|
use opendal_core::*;
|
|
30
30
|
|
|
@@ -49,19 +49,21 @@ impl B2Writer {
|
|
|
49
49
|
}
|
|
50
50
|
|
|
51
51
|
pub fn parse_body_into_meta(path: &str, resp: UploadResponse) -> Metadata {
|
|
52
|
-
let mut meta =
|
|
52
|
+
let mut meta = if path.ends_with('/') {
|
|
53
|
+
MetadataBuilder::dir()
|
|
54
|
+
} else {
|
|
55
|
+
MetadataBuilder::file(resp.content_length)
|
|
56
|
+
};
|
|
53
57
|
|
|
54
58
|
if let Some(md5) = resp.content_md5 {
|
|
55
|
-
meta.
|
|
59
|
+
meta.content_md5(&md5);
|
|
56
60
|
}
|
|
57
61
|
|
|
58
62
|
if let Some(content_type) = resp.content_type {
|
|
59
|
-
meta.
|
|
63
|
+
meta.content_type(&content_type);
|
|
60
64
|
}
|
|
61
65
|
|
|
62
|
-
meta.
|
|
63
|
-
|
|
64
|
-
meta
|
|
66
|
+
meta.build()
|
|
65
67
|
}
|
|
66
68
|
}
|
|
67
69
|
|
|
@@ -85,7 +87,10 @@ impl oio::MultipartWrite for B2Writer {
|
|
|
85
87
|
|
|
86
88
|
Ok(meta)
|
|
87
89
|
}
|
|
88
|
-
_ => Err(parse_error(
|
|
90
|
+
_ => Err(parse_error(
|
|
91
|
+
ErrorContext::new(ServiceOperation("UploadFile")),
|
|
92
|
+
resp,
|
|
93
|
+
)),
|
|
89
94
|
}
|
|
90
95
|
}
|
|
91
96
|
|
|
@@ -106,7 +111,10 @@ impl oio::MultipartWrite for B2Writer {
|
|
|
106
111
|
|
|
107
112
|
Ok(result.file_id)
|
|
108
113
|
}
|
|
109
|
-
_ => Err(parse_error(
|
|
114
|
+
_ => Err(parse_error(
|
|
115
|
+
ErrorContext::new(ServiceOperation("StartLargeFile")),
|
|
116
|
+
resp,
|
|
117
|
+
)),
|
|
110
118
|
}
|
|
111
119
|
}
|
|
112
120
|
|
|
@@ -141,7 +149,10 @@ impl oio::MultipartWrite for B2Writer {
|
|
|
141
149
|
size: None,
|
|
142
150
|
})
|
|
143
151
|
}
|
|
144
|
-
_ => Err(parse_error(
|
|
152
|
+
_ => Err(parse_error(
|
|
153
|
+
ErrorContext::new(ServiceOperation("UploadPart")),
|
|
154
|
+
resp,
|
|
155
|
+
)),
|
|
145
156
|
}
|
|
146
157
|
}
|
|
147
158
|
|
|
@@ -180,7 +191,10 @@ impl oio::MultipartWrite for B2Writer {
|
|
|
180
191
|
|
|
181
192
|
Ok(meta)
|
|
182
193
|
}
|
|
183
|
-
_ => Err(parse_error(
|
|
194
|
+
_ => Err(parse_error(
|
|
195
|
+
ErrorContext::new(ServiceOperation("FinishLargeFile")),
|
|
196
|
+
resp,
|
|
197
|
+
)),
|
|
184
198
|
}
|
|
185
199
|
}
|
|
186
200
|
|
|
@@ -189,7 +203,10 @@ impl oio::MultipartWrite for B2Writer {
|
|
|
189
203
|
match resp.status() {
|
|
190
204
|
// b2 returns code 200 if abort succeeds.
|
|
191
205
|
StatusCode::OK => Ok(()),
|
|
192
|
-
_ => Err(parse_error(
|
|
206
|
+
_ => Err(parse_error(
|
|
207
|
+
ErrorContext::new(ServiceOperation("CancelLargeFile")),
|
|
208
|
+
resp,
|
|
209
|
+
)),
|
|
193
210
|
}
|
|
194
211
|
}
|
|
195
212
|
}
|
|
@@ -37,7 +37,7 @@ cacache = { version = "13.0", default-features = false, features = [
|
|
|
37
37
|
"tokio-runtime",
|
|
38
38
|
"mmap",
|
|
39
39
|
] }
|
|
40
|
-
opendal-core = { path = "../../core", version = "0.
|
|
40
|
+
opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
|
|
41
41
|
serde = { workspace = true, features = ["derive"] }
|
|
42
42
|
|
|
43
43
|
[dev-dependencies]
|
|
@@ -87,6 +87,7 @@ impl Service for CacacheBackend {
|
|
|
87
87
|
type Lister = ();
|
|
88
88
|
type Deleter = oio::OneShotDeleter<CacacheDeleter>;
|
|
89
89
|
type Copier = ();
|
|
90
|
+
type Composer = ();
|
|
90
91
|
|
|
91
92
|
fn info(&self) -> ServiceInfo {
|
|
92
93
|
self.info.clone()
|
|
@@ -113,14 +114,13 @@ impl Service for CacacheBackend {
|
|
|
113
114
|
|
|
114
115
|
match metadata {
|
|
115
116
|
Some(meta) => {
|
|
116
|
-
let mut md =
|
|
117
|
-
md.set_content_length(meta.size as u64);
|
|
117
|
+
let mut md = MetadataBuilder::file(meta.size as u64);
|
|
118
118
|
// Convert u128 milliseconds to Timestamp
|
|
119
119
|
let millis = meta.time as i64;
|
|
120
120
|
if let Ok(dt) = Timestamp::from_millisecond(millis) {
|
|
121
|
-
md.
|
|
121
|
+
md.last_modified(dt);
|
|
122
122
|
}
|
|
123
|
-
Ok(RpStat::new(md))
|
|
123
|
+
Ok(RpStat::new(md.build()))
|
|
124
124
|
}
|
|
125
125
|
None => Err(Error::new(ErrorKind::NotFound, "entry not found")),
|
|
126
126
|
}
|
|
@@ -167,7 +167,6 @@ impl Service for CacacheBackend {
|
|
|
167
167
|
_from: &str,
|
|
168
168
|
_to: &str,
|
|
169
169
|
_args: OpCopy,
|
|
170
|
-
_opts: OpCopier,
|
|
171
170
|
) -> Result<Self::Copier> {
|
|
172
171
|
Err(Error::new(
|
|
173
172
|
ErrorKind::Unsupported,
|
|
@@ -44,7 +44,10 @@ impl oio::StreamRead for CacacheReader {
|
|
|
44
44
|
Some(bytes) => {
|
|
45
45
|
let content_length = bytes.len() as u64;
|
|
46
46
|
let buffer = Buffer::from(bytes.slice(range.to_content_range(bytes.len())?));
|
|
47
|
-
let metadata =
|
|
47
|
+
let metadata = {
|
|
48
|
+
let metadata = MetadataBuilder::file(content_length);
|
|
49
|
+
metadata.build()
|
|
50
|
+
};
|
|
48
51
|
Ok((
|
|
49
52
|
RpRead::new(metadata),
|
|
50
53
|
Box::new(buffer) as Box<dyn oio::ReadStreamDyn>,
|
|
@@ -50,8 +50,12 @@ impl oio::Write for CacacheWriter {
|
|
|
50
50
|
|
|
51
51
|
self.core.set(&self.path, buf.to_bytes()).await?;
|
|
52
52
|
|
|
53
|
-
let
|
|
54
|
-
|
|
53
|
+
let metadata = if self.path.ends_with('/') {
|
|
54
|
+
MetadataBuilder::dir()
|
|
55
|
+
} else {
|
|
56
|
+
MetadataBuilder::file(length)
|
|
57
|
+
};
|
|
58
|
+
Ok(metadata.build())
|
|
55
59
|
}
|
|
56
60
|
|
|
57
61
|
async fn abort(&mut self) -> Result<()> {
|
|
@@ -34,6 +34,6 @@ all-features = true
|
|
|
34
34
|
[dependencies]
|
|
35
35
|
bytes = { workspace = true }
|
|
36
36
|
http = { workspace = true }
|
|
37
|
-
opendal-core = { path = "../../core", version = "0.
|
|
37
|
+
opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
|
|
38
38
|
serde = { workspace = true, features = ["derive"] }
|
|
39
39
|
serde_json = { workspace = true }
|
|
@@ -25,8 +25,8 @@ use opendal_core::*;
|
|
|
25
25
|
|
|
26
26
|
use super::CLOUDFLARE_KV_SCHEME;
|
|
27
27
|
use super::config::CloudflareKvConfig;
|
|
28
|
-
use super::core::CloudflareKvCore;
|
|
29
28
|
use super::core::parse_error;
|
|
29
|
+
use super::core::{CloudflareKvCore, ErrorContext};
|
|
30
30
|
use super::deleter::CloudflareKvDeleter;
|
|
31
31
|
use super::lister::CloudflareKvLister;
|
|
32
32
|
use super::model::*;
|
|
@@ -37,6 +37,7 @@ use super::writer::CloudflareWriter;
|
|
|
37
37
|
#[derive(Default)]
|
|
38
38
|
pub struct CloudflareKvBuilder {
|
|
39
39
|
pub(super) config: CloudflareKvConfig,
|
|
40
|
+
pub(super) default_ttl: Option<Duration>,
|
|
40
41
|
}
|
|
41
42
|
|
|
42
43
|
impl Debug for CloudflareKvBuilder {
|
|
@@ -76,7 +77,7 @@ impl CloudflareKvBuilder {
|
|
|
76
77
|
///
|
|
77
78
|
/// If set, we will specify `EX` for write operations.
|
|
78
79
|
pub fn default_ttl(mut self, ttl: Duration) -> Self {
|
|
79
|
-
self.
|
|
80
|
+
self.default_ttl = Some(ttl);
|
|
80
81
|
self
|
|
81
82
|
}
|
|
82
83
|
|
|
@@ -96,6 +97,14 @@ impl Builder for CloudflareKvBuilder {
|
|
|
96
97
|
type Config = CloudflareKvConfig;
|
|
97
98
|
|
|
98
99
|
fn build(self) -> Result<impl Service> {
|
|
100
|
+
let default_ttl = match self.default_ttl {
|
|
101
|
+
Some(ttl) => Some(ttl),
|
|
102
|
+
None => self
|
|
103
|
+
.config
|
|
104
|
+
.default_ttl
|
|
105
|
+
.map(signed_duration_to_duration)
|
|
106
|
+
.transpose()?,
|
|
107
|
+
};
|
|
99
108
|
let api_token = match &self.config.api_token {
|
|
100
109
|
Some(api_token) => format_authorization_by_bearer(api_token)?,
|
|
101
110
|
None => {
|
|
@@ -121,7 +130,7 @@ impl Builder for CloudflareKvBuilder {
|
|
|
121
130
|
};
|
|
122
131
|
|
|
123
132
|
// Validate default TTL is at least 60 seconds if specified
|
|
124
|
-
if let Some(ttl) =
|
|
133
|
+
if let Some(ttl) = default_ttl
|
|
125
134
|
&& ttl < Duration::from_secs(60)
|
|
126
135
|
{
|
|
127
136
|
return Err(Error::new(
|
|
@@ -143,7 +152,7 @@ impl Builder for CloudflareKvBuilder {
|
|
|
143
152
|
api_token,
|
|
144
153
|
account_id,
|
|
145
154
|
namespace_id,
|
|
146
|
-
expiration_ttl:
|
|
155
|
+
expiration_ttl: default_ttl,
|
|
147
156
|
info: ServiceInfo::new(CLOUDFLARE_KV_SCHEME, &root, ""),
|
|
148
157
|
capability: Capability {
|
|
149
158
|
create_dir: true,
|
|
@@ -189,6 +198,7 @@ impl Service for CloudflareKvBackend {
|
|
|
189
198
|
type Lister = oio::PageLister<CloudflareKvLister>;
|
|
190
199
|
type Deleter = oio::BatchDeleter<CloudflareKvDeleter>;
|
|
191
200
|
type Copier = ();
|
|
201
|
+
type Composer = ();
|
|
192
202
|
|
|
193
203
|
fn info(&self) -> ServiceInfo {
|
|
194
204
|
self.core.info.clone()
|
|
@@ -266,7 +276,7 @@ impl Service for CloudflareKvBackend {
|
|
|
266
276
|
if let Some(entries) = list_result.result
|
|
267
277
|
&& !entries.is_empty()
|
|
268
278
|
{
|
|
269
|
-
return Ok(RpStat::new(
|
|
279
|
+
return Ok(RpStat::new(MetadataBuilder::dir().build()));
|
|
270
280
|
}
|
|
271
281
|
|
|
272
282
|
// Empty or no results means not found
|
|
@@ -278,7 +288,10 @@ impl Service for CloudflareKvBackend {
|
|
|
278
288
|
}
|
|
279
289
|
|
|
280
290
|
// For all other error cases, parse the error response
|
|
281
|
-
return Err(parse_error(
|
|
291
|
+
return Err(parse_error(
|
|
292
|
+
ErrorContext::new(ServiceOperation("GetMetadata")),
|
|
293
|
+
resp,
|
|
294
|
+
));
|
|
282
295
|
}
|
|
283
296
|
|
|
284
297
|
let resp_body = resp.into_body();
|
|
@@ -354,16 +367,15 @@ impl Service for CloudflareKvBackend {
|
|
|
354
367
|
));
|
|
355
368
|
}
|
|
356
369
|
|
|
357
|
-
let meta =
|
|
358
|
-
|
|
370
|
+
let mut meta = if metadata.is_dir {
|
|
371
|
+
MetadataBuilder::dir()
|
|
359
372
|
} else {
|
|
360
|
-
|
|
361
|
-
}
|
|
362
|
-
.
|
|
363
|
-
|
|
364
|
-
.with_last_modified(metadata.last_modified.parse::<Timestamp>()?);
|
|
373
|
+
MetadataBuilder::file(metadata.content_length as u64)
|
|
374
|
+
};
|
|
375
|
+
meta.etag(metadata.etag)
|
|
376
|
+
.last_modified(metadata.last_modified.parse::<Timestamp>()?);
|
|
365
377
|
|
|
366
|
-
Ok(RpStat::new(meta))
|
|
378
|
+
Ok(RpStat::new(meta.build()))
|
|
367
379
|
}
|
|
368
380
|
fn read(&self, ctx: &OperationContext, path: &str, args: OpRead) -> Result<Self::Reader> {
|
|
369
381
|
let output: oio::StreamReader<CloudflareKvReader> = {
|
|
@@ -438,7 +450,6 @@ impl Service for CloudflareKvBackend {
|
|
|
438
450
|
_from: &str,
|
|
439
451
|
_to: &str,
|
|
440
452
|
_args: OpCopy,
|
|
441
|
-
_opts: OpCopier,
|
|
442
453
|
) -> Result<Self::Copier> {
|
|
443
454
|
Err(Error::new(
|
|
444
455
|
ErrorKind::Unsupported,
|
|
@@ -34,7 +34,7 @@ pub struct CloudflareKvConfig {
|
|
|
34
34
|
/// The namespace ID. Used as URI path parameter.
|
|
35
35
|
pub namespace_id: Option<String>,
|
|
36
36
|
/// The default ttl for write operations.
|
|
37
|
-
pub default_ttl: Option<
|
|
37
|
+
pub default_ttl: Option<SignedDuration>,
|
|
38
38
|
|
|
39
39
|
/// Root within this backend.
|
|
40
40
|
pub root: Option<String>,
|
|
@@ -94,7 +94,10 @@ impl opendal_core::Configurator for CloudflareKvConfig {
|
|
|
94
94
|
}
|
|
95
95
|
|
|
96
96
|
fn into_builder(self) -> Self::Builder {
|
|
97
|
-
CloudflareKvBuilder {
|
|
97
|
+
CloudflareKvBuilder {
|
|
98
|
+
config: self,
|
|
99
|
+
default_ttl: None,
|
|
100
|
+
}
|
|
98
101
|
}
|
|
99
102
|
}
|
|
100
103
|
|
|
@@ -125,4 +128,12 @@ mod tests {
|
|
|
125
128
|
|
|
126
129
|
assert!(CloudflareKvConfig::from_uri(&uri).is_err());
|
|
127
130
|
}
|
|
131
|
+
|
|
132
|
+
#[test]
|
|
133
|
+
fn from_iter_parses_default_ttl() {
|
|
134
|
+
let cfg = CloudflareKvConfig::from_iter([("default_ttl".to_string(), "PT1M".to_string())])
|
|
135
|
+
.unwrap();
|
|
136
|
+
|
|
137
|
+
assert_eq!(cfg.default_ttl, Some(SignedDuration::from_mins(1)));
|
|
138
|
+
}
|
|
128
139
|
}
|
|
@@ -174,69 +174,75 @@ impl CloudflareKvCore {
|
|
|
174
174
|
}
|
|
175
175
|
}
|
|
176
176
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
use http::StatusCode;
|
|
181
|
-
use opendal_core::raw::*;
|
|
182
|
-
use opendal_core::*;
|
|
183
|
-
use serde_json::de;
|
|
184
|
-
|
|
185
|
-
use crate::model::CfKvError;
|
|
186
|
-
use crate::model::CfKvResponse;
|
|
187
|
-
|
|
188
|
-
/// Parse error response into Error.
|
|
189
|
-
pub(crate) fn parse_error(resp: Response<Buffer>) -> Error {
|
|
190
|
-
let (parts, body) = resp.into_parts();
|
|
191
|
-
let bs = body.to_bytes();
|
|
192
|
-
|
|
193
|
-
let (mut kind, mut retryable) = match parts.status {
|
|
194
|
-
StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
|
|
195
|
-
// Some services (like owncloud) return 403 while file locked.
|
|
196
|
-
StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, true),
|
|
197
|
-
// Allowing retry for resource locked.
|
|
198
|
-
StatusCode::LOCKED => (ErrorKind::Unexpected, true),
|
|
199
|
-
StatusCode::INTERNAL_SERVER_ERROR
|
|
200
|
-
| StatusCode::BAD_GATEWAY
|
|
201
|
-
| StatusCode::SERVICE_UNAVAILABLE
|
|
202
|
-
| StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
|
|
203
|
-
_ => (ErrorKind::Unexpected, false),
|
|
204
|
-
};
|
|
205
|
-
|
|
206
|
-
let (message, err) = de::from_reader::<_, CfKvResponse>(bs.clone().reader())
|
|
207
|
-
.map(|err| (format!("{err:?}"), Some(err)))
|
|
208
|
-
.unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
|
|
209
|
-
|
|
210
|
-
if let Some(err) = err {
|
|
211
|
-
(kind, retryable) = parse_cfkv_error_code(err.errors).unwrap_or((kind, retryable));
|
|
212
|
-
}
|
|
177
|
+
use bytes::Buf;
|
|
178
|
+
use http::StatusCode;
|
|
179
|
+
use serde_json::de;
|
|
213
180
|
|
|
214
|
-
|
|
181
|
+
use crate::model::CfKvError;
|
|
182
|
+
use crate::model::CfKvResponse;
|
|
215
183
|
|
|
216
|
-
|
|
184
|
+
/// Context needed to classify an error from this service.
|
|
185
|
+
#[derive(Clone, Copy, Debug)]
|
|
186
|
+
pub(crate) struct ErrorContext {
|
|
187
|
+
service_operation: ServiceOperation,
|
|
188
|
+
}
|
|
217
189
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
}
|
|
190
|
+
impl ErrorContext {
|
|
191
|
+
pub(crate) const fn new(service_operation: ServiceOperation) -> Self {
|
|
192
|
+
Self { service_operation }
|
|
193
|
+
}
|
|
194
|
+
}
|
|
221
195
|
|
|
222
|
-
|
|
196
|
+
/// Parse an error response using its service request context.
|
|
197
|
+
pub(crate) fn parse_error(ctx: ErrorContext, resp: Response<Buffer>) -> Error {
|
|
198
|
+
let (parts, body) = resp.into_parts();
|
|
199
|
+
let bs = body.to_bytes();
|
|
200
|
+
|
|
201
|
+
let (mut kind, mut retryable) = match parts.status {
|
|
202
|
+
StatusCode::NOT_FOUND => (ErrorKind::NotFound, false),
|
|
203
|
+
// Some services (like owncloud) return 403 while file locked.
|
|
204
|
+
StatusCode::FORBIDDEN => (ErrorKind::PermissionDenied, true),
|
|
205
|
+
// Allowing retry for resource locked.
|
|
206
|
+
StatusCode::LOCKED => (ErrorKind::Unexpected, true),
|
|
207
|
+
StatusCode::INTERNAL_SERVER_ERROR
|
|
208
|
+
| StatusCode::BAD_GATEWAY
|
|
209
|
+
| StatusCode::SERVICE_UNAVAILABLE
|
|
210
|
+
| StatusCode::GATEWAY_TIMEOUT => (ErrorKind::Unexpected, true),
|
|
211
|
+
_ => (ErrorKind::Unexpected, false),
|
|
212
|
+
};
|
|
213
|
+
|
|
214
|
+
let (message, err) = de::from_reader::<_, CfKvResponse>(bs.clone().reader())
|
|
215
|
+
.map(|err| (format!("{err:?}"), Some(err)))
|
|
216
|
+
.unwrap_or_else(|_| (String::from_utf8_lossy(&bs).into_owned(), None));
|
|
217
|
+
|
|
218
|
+
if let Some(err) = err {
|
|
219
|
+
(kind, retryable) = parse_cfkv_error_code(err.errors).unwrap_or((kind, retryable));
|
|
223
220
|
}
|
|
224
221
|
|
|
225
|
-
|
|
226
|
-
if errors.is_empty() {
|
|
227
|
-
return None;
|
|
228
|
-
}
|
|
222
|
+
let mut err = Error::new(kind, message);
|
|
229
223
|
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
// Authentication error.
|
|
236
|
-
10000 => Some((ErrorKind::PermissionDenied, false)),
|
|
237
|
-
_ => None,
|
|
238
|
-
}
|
|
224
|
+
err = err.with_context("service_operation", ctx.service_operation.0);
|
|
225
|
+
err = with_error_response_context(err, parts);
|
|
226
|
+
|
|
227
|
+
if retryable {
|
|
228
|
+
err = err.set_temporary();
|
|
239
229
|
}
|
|
230
|
+
|
|
231
|
+
err
|
|
240
232
|
}
|
|
241
233
|
|
|
242
|
-
pub(
|
|
234
|
+
pub(crate) fn parse_cfkv_error_code(errors: Vec<CfKvError>) -> Option<(ErrorKind, bool)> {
|
|
235
|
+
if errors.is_empty() {
|
|
236
|
+
return None;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
match errors[0].code {
|
|
240
|
+
// The request is malformed: failed to decode id.
|
|
241
|
+
7400 => Some((ErrorKind::Unexpected, false)),
|
|
242
|
+
// no such column: Xxxx.
|
|
243
|
+
7500 => Some((ErrorKind::NotFound, false)),
|
|
244
|
+
// Authentication error.
|
|
245
|
+
10000 => Some((ErrorKind::PermissionDenied, false)),
|
|
246
|
+
_ => None,
|
|
247
|
+
}
|
|
248
|
+
}
|
|
@@ -49,7 +49,10 @@ impl oio::BatchDelete for CloudflareKvDeleter {
|
|
|
49
49
|
let status = resp.status();
|
|
50
50
|
|
|
51
51
|
if status != StatusCode::OK {
|
|
52
|
-
return Err(parse_error(
|
|
52
|
+
return Err(parse_error(
|
|
53
|
+
ErrorContext::new(ServiceOperation("BulkDelete")),
|
|
54
|
+
resp.clone(),
|
|
55
|
+
));
|
|
53
56
|
}
|
|
54
57
|
|
|
55
58
|
let bs = resp.clone().into_body();
|
|
@@ -57,7 +60,10 @@ impl oio::BatchDelete for CloudflareKvDeleter {
|
|
|
57
60
|
serde_json::from_reader(bs.reader()).map_err(new_json_deserialize_error)?;
|
|
58
61
|
|
|
59
62
|
if !res.success {
|
|
60
|
-
return Err(parse_error(
|
|
63
|
+
return Err(parse_error(
|
|
64
|
+
ErrorContext::new(ServiceOperation("BulkDelete")),
|
|
65
|
+
resp.clone(),
|
|
66
|
+
));
|
|
61
67
|
}
|
|
62
68
|
|
|
63
69
|
Ok(())
|
|
@@ -77,7 +83,10 @@ impl oio::BatchDelete for CloudflareKvDeleter {
|
|
|
77
83
|
let status = resp.status();
|
|
78
84
|
|
|
79
85
|
if status != StatusCode::OK {
|
|
80
|
-
return Err(parse_error(
|
|
86
|
+
return Err(parse_error(
|
|
87
|
+
ErrorContext::new(ServiceOperation("BulkDelete")),
|
|
88
|
+
resp,
|
|
89
|
+
));
|
|
81
90
|
}
|
|
82
91
|
|
|
83
92
|
let bs = resp.into_body();
|
|
@@ -21,8 +21,8 @@ use bytes::Buf;
|
|
|
21
21
|
use opendal_core::raw::*;
|
|
22
22
|
use opendal_core::*;
|
|
23
23
|
|
|
24
|
-
use super::core::CloudflareKvCore;
|
|
25
24
|
use super::core::parse_error;
|
|
25
|
+
use super::core::{CloudflareKvCore, ErrorContext};
|
|
26
26
|
use super::model::{CfKvListKey, CfKvListResponse};
|
|
27
27
|
|
|
28
28
|
pub struct CloudflareKvLister {
|
|
@@ -34,6 +34,22 @@ pub struct CloudflareKvLister {
|
|
|
34
34
|
recursive: bool,
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
/// Strip the service root from a root-prefixed key.
|
|
38
|
+
///
|
|
39
|
+
/// The keys returned by the KV API carry the service root as a prefix, so only that prefix may be
|
|
40
|
+
/// removed. `str::replace` removes *every* occurrence anywhere in the key, which silently mangles
|
|
41
|
+
/// any key that repeats the root as an inner path segment.
|
|
42
|
+
///
|
|
43
|
+
/// The sibling object-store listers reach for `build_rel_path` here, but that helper
|
|
44
|
+
/// `debug_assert!`s that the path really does start with the root -- a guarantee this service does
|
|
45
|
+
/// not enforce on the values the API hands back -- so this stays total and leaves a
|
|
46
|
+
/// non-root-prefixed key untouched.
|
|
47
|
+
fn relative_to_root(root: &str, name: &str) -> String {
|
|
48
|
+
name.strip_prefix(root.trim_start_matches('/'))
|
|
49
|
+
.unwrap_or(name)
|
|
50
|
+
.to_string()
|
|
51
|
+
}
|
|
52
|
+
|
|
37
53
|
impl CloudflareKvLister {
|
|
38
54
|
pub fn new(
|
|
39
55
|
core: Arc<CloudflareKvCore>,
|
|
@@ -60,7 +76,7 @@ impl CloudflareKvLister {
|
|
|
60
76
|
name += "/";
|
|
61
77
|
}
|
|
62
78
|
|
|
63
|
-
let mut name =
|
|
79
|
+
let mut name = relative_to_root(root, &name);
|
|
64
80
|
|
|
65
81
|
// If it is the root directory, it needs to be processed as /
|
|
66
82
|
if name.is_empty() {
|
|
@@ -68,14 +84,15 @@ impl CloudflareKvLister {
|
|
|
68
84
|
}
|
|
69
85
|
|
|
70
86
|
let entry_metadata = if name.ends_with('/') {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
87
|
+
let mut metadata = MetadataBuilder::dir();
|
|
88
|
+
metadata.etag(build_tmp_path_of(&name));
|
|
89
|
+
metadata.build()
|
|
74
90
|
} else {
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
.
|
|
78
|
-
.
|
|
91
|
+
let mut result = MetadataBuilder::file(metadata.content_length as u64);
|
|
92
|
+
result
|
|
93
|
+
.etag(&metadata.etag)
|
|
94
|
+
.last_modified(metadata.last_modified.parse::<Timestamp>()?);
|
|
95
|
+
result.build()
|
|
79
96
|
};
|
|
80
97
|
|
|
81
98
|
Ok(oio::Entry::new(&name, entry_metadata))
|
|
@@ -91,13 +108,12 @@ impl CloudflareKvLister {
|
|
|
91
108
|
let entry = self.build_entry_for_item(item, root)?;
|
|
92
109
|
ctx.entries.push_back(entry);
|
|
93
110
|
} else if !result.is_empty() {
|
|
94
|
-
let path_name =
|
|
95
|
-
let entry = oio::Entry::new(
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
);
|
|
111
|
+
let path_name = relative_to_root(root, &self.path);
|
|
112
|
+
let entry = oio::Entry::new(&format!("{path_name}/"), {
|
|
113
|
+
let mut metadata = MetadataBuilder::dir();
|
|
114
|
+
metadata.etag(build_tmp_path_of(&path_name));
|
|
115
|
+
metadata.build()
|
|
116
|
+
});
|
|
101
117
|
ctx.entries.push_back(entry);
|
|
102
118
|
}
|
|
103
119
|
ctx.done = true;
|
|
@@ -114,7 +130,10 @@ impl oio::PageList for CloudflareKvLister {
|
|
|
114
130
|
.await?;
|
|
115
131
|
|
|
116
132
|
if resp.status() != http::StatusCode::OK {
|
|
117
|
-
return Err(parse_error(
|
|
133
|
+
return Err(parse_error(
|
|
134
|
+
ErrorContext::new(ServiceOperation("ListKeys")),
|
|
135
|
+
resp,
|
|
136
|
+
));
|
|
118
137
|
}
|
|
119
138
|
|
|
120
139
|
let bs = resp.into_body();
|
|
@@ -171,3 +190,33 @@ impl oio::PageList for CloudflareKvLister {
|
|
|
171
190
|
Ok(())
|
|
172
191
|
}
|
|
173
192
|
}
|
|
193
|
+
|
|
194
|
+
#[cfg(test)]
|
|
195
|
+
mod tests {
|
|
196
|
+
use super::*;
|
|
197
|
+
|
|
198
|
+
#[test]
|
|
199
|
+
fn relative_to_root_strips_only_the_prefix() {
|
|
200
|
+
// The root repeated as an inner segment must survive; only the leading copy goes.
|
|
201
|
+
assert_eq!(
|
|
202
|
+
relative_to_root("/data/", "data/backup/data/file.txt"),
|
|
203
|
+
"backup/data/file.txt"
|
|
204
|
+
);
|
|
205
|
+
assert_eq!(relative_to_root("/data/", "data/file.txt"), "file.txt");
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
#[test]
|
|
209
|
+
fn relative_to_root_does_not_match_mid_key() {
|
|
210
|
+
// "replace" would turn this into "xfile.txt" by deleting a substring that is not a prefix.
|
|
211
|
+
assert_eq!(
|
|
212
|
+
relative_to_root("/data/", "xdata/file.txt"),
|
|
213
|
+
"xdata/file.txt"
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
#[test]
|
|
218
|
+
fn relative_to_root_handles_the_root_itself_and_a_bare_root() {
|
|
219
|
+
assert_eq!(relative_to_root("/data/", "data/"), "");
|
|
220
|
+
assert_eq!(relative_to_root("/", "file.txt"), "file.txt");
|
|
221
|
+
}
|
|
222
|
+
}
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
// under the License.
|
|
17
17
|
|
|
18
18
|
use super::backend::*;
|
|
19
|
-
use super::core::parse_error;
|
|
19
|
+
use super::core::{ErrorContext, parse_error};
|
|
20
20
|
use super::model::*;
|
|
21
21
|
use bytes::Buf;
|
|
22
22
|
use http::StatusCode;
|
|
@@ -58,20 +58,22 @@ impl oio::StreamRead for CloudflareKvReader {
|
|
|
58
58
|
let status = resp.status();
|
|
59
59
|
|
|
60
60
|
if status != StatusCode::OK {
|
|
61
|
-
return Err(parse_error(
|
|
61
|
+
return Err(parse_error(
|
|
62
|
+
ErrorContext::new(ServiceOperation("GetValue")),
|
|
63
|
+
resp,
|
|
64
|
+
));
|
|
62
65
|
}
|
|
63
66
|
|
|
64
67
|
let resp_body = resp.into_body();
|
|
65
68
|
|
|
66
|
-
if args.
|
|
67
|
-
|| args.if_none_match().is_some()
|
|
68
|
-
|| args.if_modified_since().is_some()
|
|
69
|
-
|| args.if_unmodified_since().is_some()
|
|
70
|
-
{
|
|
69
|
+
if args.is_conditional() {
|
|
71
70
|
let meta_resp = backend.core.metadata(&self.ctx, &path).await?;
|
|
72
71
|
|
|
73
72
|
if meta_resp.status() != StatusCode::OK {
|
|
74
|
-
return Err(parse_error(
|
|
73
|
+
return Err(parse_error(
|
|
74
|
+
ErrorContext::new(ServiceOperation("GetMetadata")),
|
|
75
|
+
meta_resp,
|
|
76
|
+
));
|
|
75
77
|
}
|
|
76
78
|
|
|
77
79
|
let cf_response: CfKvStatResponse =
|
|
@@ -133,7 +135,10 @@ impl oio::StreamRead for CloudflareKvReader {
|
|
|
133
135
|
|
|
134
136
|
let total_size = resp_body.len() as u64;
|
|
135
137
|
let buffer = resp_body.slice(range.to_content_range(resp_body.len())?);
|
|
136
|
-
let metadata =
|
|
138
|
+
let metadata = {
|
|
139
|
+
let metadata = MetadataBuilder::file(total_size);
|
|
140
|
+
metadata.build()
|
|
141
|
+
};
|
|
137
142
|
Ok((
|
|
138
143
|
RpRead::new(metadata),
|
|
139
144
|
Box::new(buffer) as Box<dyn oio::ReadStreamDyn>,
|