opendal 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +89 -56
- data/Cargo.toml +1 -1
- data/DEPENDENCIES.rust.tsv +55 -41
- data/LICENSE +201 -0
- data/LICENSE-MPL-2.0.txt +373 -0
- data/NOTICE +5 -0
- data/core/CHANGELOG.md +175 -0
- data/core/Cargo.lock +225 -298
- data/core/Cargo.toml +100 -94
- data/core/DEPENDENCIES.rust.tsv +44 -73
- data/core/benches/metadata/README.md +21 -0
- data/core/benches/metadata/main.rs +541 -0
- data/core/benches/vs_fs/Cargo.toml +1 -1
- data/core/benches/vs_s3/Cargo.toml +1 -1
- data/core/core/Cargo.toml +7 -2
- data/core/core/src/blocking/delete.rs +5 -1
- data/core/core/src/blocking/operator.rs +12 -0
- data/core/core/src/blocking/read/std_reader.rs +25 -0
- data/core/core/src/docs/concepts.rs +70 -94
- data/core/core/src/docs/internals/accessor.rs +62 -293
- data/core/core/src/docs/internals/layer.rs +43 -59
- data/core/core/src/docs/internals/mod.rs +21 -23
- data/core/core/src/docs/mod.rs +10 -6
- data/core/core/src/docs/performance/concurrent_write.md +92 -55
- data/core/core/src/docs/performance/http_optimization.md +120 -104
- data/core/core/src/docs/performance/mod.rs +8 -6
- data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
- data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
- data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
- data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
- data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
- data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
- data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
- data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
- data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
- data/core/core/src/docs/rfcs/README.md +14 -3
- data/core/core/src/docs/rfcs/mod.rs +46 -22
- data/core/core/src/docs/specs/README.md +34 -0
- data/core/core/src/docs/specs/conditional-operations.md +156 -0
- data/core/core/src/docs/specs/metadata.md +63 -0
- data/core/core/src/docs/specs/mod.rs +26 -0
- data/core/core/src/docs/upgrade.md +114 -1
- data/core/core/src/layers/capability_override.rs +15 -2
- data/core/core/src/layers/complete.rs +347 -43
- data/core/core/src/layers/correctness_check.rs +718 -51
- data/core/core/src/layers/error_context.rs +76 -2
- data/core/core/src/layers/simulate.rs +91 -28
- data/core/core/src/lib.rs +1 -0
- data/core/core/src/raw/accessor.rs +91 -15
- data/core/core/src/raw/enum_utils.rs +7 -0
- data/core/core/src/raw/http_util/header.rs +22 -17
- data/core/core/src/raw/oio/compose/api.rs +92 -0
- data/core/core/src/raw/oio/compose/mod.rs +19 -0
- data/core/core/src/raw/oio/copy/api.rs +3 -3
- data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
- data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
- data/core/core/src/raw/oio/entry.rs +48 -22
- data/core/core/src/raw/oio/list/flat_list.rs +1 -1
- data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
- data/core/core/src/raw/oio/mod.rs +3 -0
- data/core/core/src/raw/oio/read/position_read.rs +1 -1
- data/core/core/src/raw/oio/write/api.rs +40 -0
- data/core/core/src/raw/oio/write/append_write.rs +4 -4
- data/core/core/src/raw/oio/write/block_write.rs +2 -2
- data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
- data/core/core/src/raw/oio/write/position_write.rs +1 -1
- data/core/core/src/raw/operation.rs +6 -0
- data/core/core/src/raw/ops.rs +1191 -456
- data/core/core/src/raw/rps.rs +11 -0
- data/core/core/src/raw/time.rs +14 -3
- data/core/core/src/services/memory/backend.rs +5 -10
- data/core/core/src/services/memory/lister.rs +4 -4
- data/core/core/src/services/memory/writer.rs +7 -6
- data/core/core/src/types/buffer.rs +102 -12
- data/core/core/src/types/bytes_range.rs +20 -1
- data/core/core/src/types/capability.rs +65 -0
- data/core/core/src/types/compact.rs +329 -0
- data/core/core/src/types/compose/composer.rs +183 -0
- data/core/core/src/types/compose/input.rs +57 -0
- data/core/core/src/types/compose/mod.rs +23 -0
- data/core/core/src/types/context/read.rs +16 -11
- data/core/core/src/types/context/write.rs +150 -2
- data/core/core/src/types/copy.rs +5 -3
- data/core/core/src/types/delete/deleter.rs +20 -12
- data/core/core/src/types/delete/input.rs +22 -64
- data/core/core/src/types/delete/mod.rs +0 -1
- data/core/core/src/types/entry.rs +8 -14
- data/core/core/src/types/error.rs +28 -11
- data/core/core/src/types/metadata.rs +709 -322
- data/core/core/src/types/mod.rs +9 -1
- data/core/core/src/types/operator/operator.rs +278 -29
- data/core/core/src/types/operator/operator_futures.rs +466 -105
- data/core/core/src/types/options.rs +631 -158
- data/core/core/src/types/read/futures_async_reader.rs +663 -50
- data/core/core/src/types/read/reader.rs +23 -5
- data/core/core/src/types/write/futures_async_writer.rs +27 -0
- data/core/core/src/types/write/writer.rs +47 -5
- data/core/http-transports/reqwest/Cargo.toml +1 -1
- data/core/layers/async-backtrace/Cargo.toml +2 -2
- data/core/layers/async-backtrace/src/lib.rs +21 -2
- data/core/layers/await-tree/Cargo.toml +2 -2
- data/core/layers/await-tree/src/lib.rs +25 -2
- data/core/layers/capability-check/Cargo.toml +2 -2
- data/core/layers/capability-check/src/lib.rs +98 -10
- data/core/layers/chaos/Cargo.toml +2 -2
- data/core/layers/chaos/src/lib.rs +18 -3
- data/core/layers/concurrent-limit/Cargo.toml +3 -3
- data/core/layers/concurrent-limit/src/lib.rs +53 -13
- data/core/layers/dtrace/Cargo.toml +2 -2
- data/core/layers/dtrace/src/lib.rs +29 -2
- data/core/layers/fastmetrics/Cargo.toml +3 -3
- data/core/layers/fastrace/Cargo.toml +2 -2
- data/core/layers/fastrace/src/lib.rs +28 -2
- data/core/layers/foyer/Cargo.toml +2 -2
- data/core/layers/foyer/src/full.rs +16 -19
- data/core/layers/foyer/src/lib.rs +67 -14
- data/core/layers/hotpath/Cargo.toml +2 -2
- data/core/layers/hotpath/src/lib.rs +20 -2
- data/core/layers/immutable-index/Cargo.toml +2 -2
- data/core/layers/immutable-index/src/lib.rs +48 -24
- data/core/layers/logging/Cargo.toml +2 -2
- data/core/layers/logging/src/lib.rs +46 -2
- data/core/layers/metrics/Cargo.toml +3 -3
- data/core/layers/mime-guess/Cargo.toml +2 -2
- data/core/layers/mime-guess/src/lib.rs +31 -4
- data/core/layers/observe-metrics-common/Cargo.toml +2 -2
- data/core/layers/observe-metrics-common/src/lib.rs +55 -2
- data/core/layers/otelmetrics/Cargo.toml +3 -3
- data/core/layers/oteltrace/Cargo.toml +2 -2
- data/core/layers/oteltrace/src/lib.rs +27 -2
- data/core/layers/prometheus/Cargo.toml +3 -3
- data/core/layers/prometheus-client/Cargo.toml +3 -3
- data/core/layers/retry/Cargo.toml +4 -4
- data/core/layers/retry/src/lib.rs +76 -19
- data/core/layers/route/Cargo.toml +3 -3
- data/core/layers/route/src/lib.rs +34 -7
- data/core/layers/tail-cut/Cargo.toml +2 -2
- data/core/layers/tail-cut/src/lib.rs +28 -2
- data/core/layers/throttle/Cargo.toml +2 -2
- data/core/layers/throttle/src/lib.rs +19 -2
- data/core/layers/timeout/Cargo.toml +3 -3
- data/core/layers/timeout/src/lib.rs +38 -13
- data/core/layers/tracing/Cargo.toml +2 -2
- data/core/layers/tracing/src/lib.rs +24 -3
- data/core/services/aliyun-drive/Cargo.toml +2 -2
- data/core/services/aliyun-drive/src/backend.rs +34 -24
- data/core/services/aliyun-drive/src/core.rs +143 -54
- data/core/services/aliyun-drive/src/lister.rs +19 -18
- data/core/services/aliyun-drive/src/reader.rs +4 -1
- data/core/services/aliyun-drive/src/writer.rs +3 -3
- data/core/services/alluxio/Cargo.toml +1 -1
- data/core/services/alluxio/src/backend.rs +1 -1
- data/core/services/alluxio/src/core.rs +113 -94
- data/core/services/alluxio/src/reader.rs +7 -3
- data/core/services/alluxio/src/writer.rs +2 -2
- data/core/services/azblob/Cargo.toml +2 -2
- data/core/services/azblob/src/backend.rs +17 -9
- data/core/services/azblob/src/copier.rs +41 -15
- data/core/services/azblob/src/core.rs +372 -135
- data/core/services/azblob/src/deleter.rs +41 -18
- data/core/services/azblob/src/lister.rs +16 -10
- data/core/services/azblob/src/reader.rs +4 -1
- data/core/services/azblob/src/writer.rs +48 -15
- data/core/services/azdls/Cargo.toml +3 -3
- data/core/services/azdls/src/backend.rs +45 -15
- data/core/services/azdls/src/config.rs +4 -1
- data/core/services/azdls/src/core.rs +276 -87
- data/core/services/azdls/src/deleter.rs +13 -4
- data/core/services/azdls/src/lister.rs +18 -12
- data/core/services/azdls/src/reader.rs +6 -2
- data/core/services/azdls/src/writer.rs +44 -23
- data/core/services/azfile/Cargo.toml +2 -2
- data/core/services/azfile/src/backend.rs +24 -8
- data/core/services/azfile/src/core.rs +195 -77
- data/core/services/azfile/src/deleter.rs +8 -1
- data/core/services/azfile/src/lister.rs +20 -14
- data/core/services/azfile/src/reader.rs +5 -2
- data/core/services/azfile/src/writer.rs +27 -15
- data/core/services/azure-common/Cargo.toml +1 -1
- data/core/services/b2/Cargo.toml +2 -2
- data/core/services/b2/src/backend.rs +30 -11
- data/core/services/b2/src/core.rs +134 -120
- data/core/services/b2/src/deleter.rs +1 -1
- data/core/services/b2/src/lister.rs +5 -2
- data/core/services/b2/src/reader.rs +5 -2
- data/core/services/b2/src/writer.rs +29 -12
- data/core/services/cacache/Cargo.toml +1 -1
- data/core/services/cacache/src/backend.rs +4 -5
- data/core/services/cacache/src/reader.rs +4 -1
- data/core/services/cacache/src/writer.rs +6 -2
- data/core/services/cloudflare-kv/Cargo.toml +1 -1
- data/core/services/cloudflare-kv/src/backend.rs +26 -15
- data/core/services/cloudflare-kv/src/config.rs +13 -2
- data/core/services/cloudflare-kv/src/core.rs +62 -56
- data/core/services/cloudflare-kv/src/deleter.rs +12 -3
- data/core/services/cloudflare-kv/src/lister.rs +66 -17
- data/core/services/cloudflare-kv/src/reader.rs +14 -9
- data/core/services/cloudflare-kv/src/writer.rs +10 -7
- data/core/services/compfs/Cargo.toml +1 -1
- data/core/services/compfs/src/backend.rs +11 -7
- data/core/services/compfs/src/lister.rs +7 -7
- data/core/services/compfs/src/writer.rs +1 -1
- data/core/services/cos/Cargo.toml +1 -1
- data/core/services/cos/src/backend.rs +129 -16
- data/core/services/cos/src/core.rs +198 -80
- data/core/services/cos/src/deleter.rs +4 -1
- data/core/services/cos/src/lister.rs +42 -27
- data/core/services/cos/src/reader.rs +6 -2
- data/core/services/cos/src/writer.rs +45 -21
- data/core/services/d1/Cargo.toml +1 -1
- data/core/services/d1/src/backend.rs +6 -5
- data/core/services/d1/src/core.rs +79 -63
- data/core/services/d1/src/reader.rs +4 -1
- data/core/services/d1/src/writer.rs +6 -2
- data/core/services/dashmap/Cargo.toml +1 -1
- data/core/services/dashmap/src/backend.rs +2 -2
- data/core/services/dashmap/src/reader.rs +4 -1
- data/core/services/dashmap/src/writer.rs +11 -8
- data/core/services/dbfs/Cargo.toml +1 -1
- data/core/services/dbfs/src/backend.rs +21 -15
- data/core/services/dbfs/src/core.rs +65 -60
- data/core/services/dbfs/src/deleter.rs +4 -1
- data/core/services/dbfs/src/lister.rs +10 -10
- data/core/services/dbfs/src/writer.rs +6 -3
- data/core/services/dropbox/Cargo.toml +2 -2
- data/core/services/dropbox/src/backend.rs +30 -23
- data/core/services/dropbox/src/core.rs +65 -63
- data/core/services/dropbox/src/deleter.rs +1 -1
- data/core/services/dropbox/src/lister.rs +17 -8
- data/core/services/dropbox/src/reader.rs +4 -1
- data/core/services/dropbox/src/writer.rs +13 -10
- data/core/services/etcd/Cargo.toml +1 -1
- data/core/services/etcd/src/backend.rs +8 -5
- data/core/services/etcd/src/core.rs +5 -11
- data/core/services/etcd/src/lister.rs +6 -1
- data/core/services/etcd/src/reader.rs +4 -1
- data/core/services/etcd/src/writer.rs +6 -3
- data/core/services/foundationdb/Cargo.toml +1 -1
- data/core/services/foundationdb/src/backend.rs +6 -5
- data/core/services/foundationdb/src/reader.rs +4 -1
- data/core/services/foundationdb/src/writer.rs +6 -2
- data/core/services/foyer/Cargo.toml +3 -3
- data/core/services/foyer/src/backend.rs +6 -5
- data/core/services/foyer/src/reader.rs +4 -1
- data/core/services/foyer/src/writer.rs +6 -2
- data/core/services/fs/Cargo.toml +1 -1
- data/core/services/fs/src/backend.rs +4 -3
- data/core/services/fs/src/core.rs +44 -26
- data/core/services/fs/src/lister.rs +9 -6
- data/core/services/fs/src/writer.rs +20 -13
- data/core/services/ftp/Cargo.toml +4 -6
- data/core/services/ftp/src/backend.rs +9 -5
- data/core/services/ftp/src/core.rs +9 -7
- data/core/services/ftp/src/lister.rs +7 -8
- data/core/services/ftp/src/reader.rs +3 -3
- data/core/services/ftp/src/writer.rs +3 -3
- data/core/services/gcs/Cargo.toml +4 -2
- data/core/services/gcs/src/backend.rs +53 -7
- data/core/services/gcs/src/composer.rs +705 -0
- data/core/services/gcs/src/copier.rs +14 -4
- data/core/services/gcs/src/core.rs +828 -124
- data/core/services/gcs/src/deleter.rs +33 -18
- data/core/services/gcs/src/lib.rs +1 -0
- data/core/services/gcs/src/lister.rs +21 -12
- data/core/services/gcs/src/reader.rs +12 -5
- data/core/services/gcs/src/writer.rs +632 -10
- data/core/services/gcs-grpc/Cargo.toml +58 -0
- data/core/services/gcs-grpc/README.md +19 -0
- data/core/services/gcs-grpc/proto/README.md +23 -0
- data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
- data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
- data/core/services/gcs-grpc/src/backend.rs +449 -0
- data/core/services/gcs-grpc/src/config.rs +116 -0
- data/core/services/gcs-grpc/src/copier.rs +98 -0
- data/core/services/gcs-grpc/src/core.rs +298 -0
- data/core/services/gcs-grpc/src/deleter.rs +65 -0
- data/core/services/gcs-grpc/src/docs.md +19 -0
- data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
- data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
- data/core/services/gcs-grpc/src/lib.rs +43 -0
- data/core/services/gcs-grpc/src/lister.rs +125 -0
- data/core/services/gcs-grpc/src/reader.rs +140 -0
- data/core/services/gcs-grpc/src/writer.rs +780 -0
- data/core/services/gdrive/Cargo.toml +2 -2
- data/core/services/gdrive/src/backend.rs +123 -69
- data/core/services/gdrive/src/core.rs +87 -69
- data/core/services/gdrive/src/deleter.rs +8 -2
- data/core/services/gdrive/src/lister.rs +38 -26
- data/core/services/gdrive/src/path_index.rs +2 -2
- data/core/services/gdrive/src/reader.rs +9 -3
- data/core/services/gdrive/src/writer.rs +13 -7
- data/core/services/ghac/Cargo.toml +2 -2
- data/core/services/ghac/src/backend.rs +5 -2
- data/core/services/ghac/src/core.rs +59 -37
- data/core/services/ghac/src/reader.rs +5 -2
- data/core/services/ghac/src/writer.rs +16 -6
- data/core/services/github/Cargo.toml +1 -1
- data/core/services/github/src/backend.rs +16 -8
- data/core/services/github/src/core.rs +88 -76
- data/core/services/github/src/lister.rs +17 -15
- data/core/services/github/src/reader.rs +5 -2
- data/core/services/github/src/writer.rs +12 -6
- data/core/services/goosefs/Cargo.toml +2 -2
- data/core/services/goosefs/src/backend.rs +2 -2
- data/core/services/goosefs/src/core.rs +291 -208
- data/core/services/goosefs/src/docs.md +3 -0
- data/core/services/goosefs/src/lister.rs +2 -3
- data/core/services/goosefs/src/reader.rs +7 -3
- data/core/services/goosefs/src/writer.rs +21 -10
- data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
- data/core/services/gridfs/Cargo.toml +2 -2
- data/core/services/gridfs/src/backend.rs +12 -8
- data/core/services/gridfs/src/core.rs +1 -1
- data/core/services/gridfs/src/reader.rs +4 -1
- data/core/services/gridfs/src/writer.rs +6 -2
- data/core/services/hdfs/Cargo.toml +1 -1
- data/core/services/hdfs/src/backend.rs +1 -1
- data/core/services/hdfs/src/core.rs +7 -4
- data/core/services/hdfs/src/lister.rs +7 -9
- data/core/services/hdfs/src/writer.rs +5 -1
- data/core/services/hdfs-native/Cargo.toml +1 -1
- data/core/services/hdfs-native/src/backend.rs +1 -1
- data/core/services/hdfs-native/src/core.rs +44 -50
- data/core/services/hdfs-native/src/lister.rs +8 -10
- data/core/services/hdfs-native/src/writer.rs +5 -1
- data/core/services/hf/Cargo.toml +4 -3
- data/core/services/hf/src/backend.rs +4 -3
- data/core/services/hf/src/core.rs +907 -153
- data/core/services/hf/src/lister.rs +16 -6
- data/core/services/hf/src/reader.rs +377 -57
- data/core/services/hf/src/writer.rs +6 -2
- data/core/services/http/Cargo.toml +3 -3
- data/core/services/http/src/backend.rs +8 -6
- data/core/services/http/src/core.rs +57 -35
- data/core/services/http/src/reader.rs +5 -2
- data/core/services/ipfs/Cargo.toml +1 -1
- data/core/services/ipfs/src/backend.rs +20 -9
- data/core/services/ipfs/src/core.rs +43 -34
- data/core/services/ipfs/src/reader.rs +5 -2
- data/core/services/ipmfs/Cargo.toml +1 -1
- data/core/services/ipmfs/src/backend.rs +17 -8
- data/core/services/ipmfs/src/core.rs +66 -60
- data/core/services/ipmfs/src/deleter.rs +5 -2
- data/core/services/ipmfs/src/lister.rs +15 -12
- data/core/services/ipmfs/src/reader.rs +5 -2
- data/core/services/ipmfs/src/writer.rs +6 -3
- data/core/services/koofr/Cargo.toml +2 -2
- data/core/services/koofr/src/backend.rs +46 -17
- data/core/services/koofr/src/core.rs +64 -48
- data/core/services/koofr/src/deleter.rs +4 -1
- data/core/services/koofr/src/lister.rs +11 -10
- data/core/services/koofr/src/reader.rs +5 -2
- data/core/services/koofr/src/writer.rs +13 -7
- data/core/services/lakefs/Cargo.toml +1 -1
- data/core/services/lakefs/src/backend.rs +25 -8
- data/core/services/lakefs/src/core.rs +79 -73
- data/core/services/lakefs/src/deleter.rs +4 -1
- data/core/services/lakefs/src/lister.rs +17 -14
- data/core/services/lakefs/src/reader.rs +5 -2
- data/core/services/lakefs/src/writer.rs +11 -10
- data/core/services/memcached/Cargo.toml +1 -1
- data/core/services/memcached/src/backend.rs +57 -7
- data/core/services/memcached/src/config.rs +13 -2
- data/core/services/memcached/src/core.rs +11 -1
- data/core/services/memcached/src/reader.rs +4 -1
- data/core/services/memcached/src/writer.rs +6 -2
- data/core/services/mini_moka/Cargo.toml +1 -1
- data/core/services/mini_moka/src/backend.rs +8 -8
- data/core/services/mini_moka/src/reader.rs +4 -1
- data/core/services/mini_moka/src/writer.rs +11 -7
- data/core/services/moka/Cargo.toml +1 -1
- data/core/services/moka/src/backend.rs +7 -7
- data/core/services/moka/src/reader.rs +4 -1
- data/core/services/moka/src/writer.rs +10 -6
- data/core/services/mongodb/Cargo.toml +2 -2
- data/core/services/mongodb/src/backend.rs +7 -6
- data/core/services/mongodb/src/core.rs +1 -1
- data/core/services/mongodb/src/reader.rs +4 -1
- data/core/services/mongodb/src/writer.rs +6 -2
- data/core/services/monoiofs/Cargo.toml +2 -2
- data/core/services/monoiofs/src/backend.rs +16 -10
- data/core/services/monoiofs/src/core.rs +8 -9
- data/core/services/monoiofs/src/reader.rs +8 -10
- data/core/services/monoiofs/src/writer.rs +19 -24
- data/core/services/mysql/Cargo.toml +2 -2
- data/core/services/mysql/src/backend.rs +7 -6
- data/core/services/mysql/src/core.rs +1 -1
- data/core/services/mysql/src/lister.rs +7 -3
- data/core/services/mysql/src/reader.rs +4 -1
- data/core/services/mysql/src/writer.rs +6 -2
- data/core/services/obs/Cargo.toml +1 -1
- data/core/services/obs/src/backend.rs +29 -11
- data/core/services/obs/src/core.rs +81 -82
- data/core/services/obs/src/deleter.rs +4 -1
- data/core/services/obs/src/lister.rs +12 -5
- data/core/services/obs/src/reader.rs +5 -2
- data/core/services/obs/src/writer.rs +40 -19
- data/core/services/onedrive/Cargo.toml +2 -2
- data/core/services/onedrive/src/backend.rs +30 -21
- data/core/services/onedrive/src/config.rs +2 -2
- data/core/services/onedrive/src/core.rs +191 -90
- data/core/services/onedrive/src/deleter.rs +5 -2
- data/core/services/onedrive/src/graph_model.rs +10 -51
- data/core/services/onedrive/src/lister.rs +60 -24
- data/core/services/onedrive/src/reader.rs +17 -2
- data/core/services/onedrive/src/writer.rs +26 -15
- data/core/services/opfs/Cargo.toml +1 -1
- data/core/services/opfs/src/backend.rs +6 -13
- data/core/services/opfs/src/core.rs +33 -37
- data/core/services/opfs/src/lister.rs +4 -4
- data/core/services/opfs/src/writer.rs +1 -3
- data/core/services/oss/Cargo.toml +1 -1
- data/core/services/oss/src/backend.rs +40 -8
- data/core/services/oss/src/core.rs +192 -92
- data/core/services/oss/src/deleter.rs +16 -6
- data/core/services/oss/src/lister.rs +43 -27
- data/core/services/oss/src/reader.rs +6 -2
- data/core/services/oss/src/writer.rs +40 -22
- data/core/services/pcloud/Cargo.toml +1 -1
- data/core/services/pcloud/src/backend.rs +40 -6
- data/core/services/pcloud/src/core.rs +80 -66
- data/core/services/pcloud/src/deleter.rs +8 -1
- data/core/services/pcloud/src/lister.rs +4 -1
- data/core/services/pcloud/src/reader.rs +5 -2
- data/core/services/pcloud/src/writer.rs +6 -3
- data/core/services/persy/Cargo.toml +1 -1
- data/core/services/persy/src/backend.rs +6 -5
- data/core/services/persy/src/core.rs +60 -15
- data/core/services/persy/src/reader.rs +4 -1
- data/core/services/persy/src/writer.rs +6 -2
- data/core/services/postgresql/Cargo.toml +2 -2
- data/core/services/postgresql/src/backend.rs +7 -6
- data/core/services/postgresql/src/core.rs +1 -1
- data/core/services/postgresql/src/reader.rs +4 -1
- data/core/services/postgresql/src/writer.rs +6 -2
- data/core/services/redb/Cargo.toml +1 -1
- data/core/services/redb/src/backend.rs +6 -5
- data/core/services/redb/src/reader.rs +4 -1
- data/core/services/redb/src/writer.rs +6 -2
- data/core/services/redis/Cargo.toml +1 -1
- data/core/services/redis/src/backend.rs +18 -8
- data/core/services/redis/src/config.rs +13 -2
- data/core/services/redis/src/reader.rs +8 -4
- data/core/services/redis/src/writer.rs +6 -2
- data/core/services/rocksdb/Cargo.toml +1 -1
- data/core/services/rocksdb/src/backend.rs +6 -5
- data/core/services/rocksdb/src/lister.rs +6 -5
- data/core/services/rocksdb/src/reader.rs +4 -1
- data/core/services/rocksdb/src/writer.rs +6 -2
- data/core/services/s3/Cargo.toml +5 -4
- data/core/services/s3/src/backend.rs +995 -39
- data/core/services/s3/src/compatible_services.md +57 -0
- data/core/services/s3/src/config.rs +16 -0
- data/core/services/s3/src/copier.rs +190 -55
- data/core/services/s3/src/core.rs +299 -190
- data/core/services/s3/src/deleter.rs +26 -7
- data/core/services/s3/src/docs.md +57 -0
- data/core/services/s3/src/lister.rs +55 -36
- data/core/services/s3/src/reader.rs +22 -2
- data/core/services/s3/src/writer.rs +154 -32
- data/core/services/seafile/Cargo.toml +2 -2
- data/core/services/seafile/src/backend.rs +3 -3
- data/core/services/seafile/src/core.rs +105 -79
- data/core/services/seafile/src/lister.rs +11 -8
- data/core/services/seafile/src/reader.rs +5 -2
- data/core/services/seafile/src/writer.rs +6 -3
- data/core/services/sftp/Cargo.toml +1 -1
- data/core/services/sftp/src/backend.rs +19 -4
- data/core/services/sftp/src/core.rs +51 -46
- data/core/services/sftp/src/lister.rs +4 -4
- data/core/services/sftp/src/writer.rs +1 -1
- data/core/services/sled/Cargo.toml +1 -1
- data/core/services/sled/src/backend.rs +6 -5
- data/core/services/sled/src/core.rs +28 -6
- data/core/services/sled/src/lister.rs +6 -5
- data/core/services/sled/src/reader.rs +4 -1
- data/core/services/sled/src/writer.rs +6 -2
- data/core/services/sqlite/Cargo.toml +2 -2
- data/core/services/sqlite/src/backend.rs +12 -7
- data/core/services/sqlite/src/core.rs +3 -3
- data/core/services/sqlite/src/reader.rs +4 -1
- data/core/services/sqlite/src/writer.rs +7 -3
- data/core/services/surrealdb/Cargo.toml +2 -2
- data/core/services/surrealdb/src/backend.rs +7 -6
- data/core/services/surrealdb/src/core.rs +1 -1
- data/core/services/surrealdb/src/reader.rs +4 -1
- data/core/services/surrealdb/src/writer.rs +6 -2
- data/core/services/swift/Cargo.toml +1 -1
- data/core/services/swift/src/backend.rs +38 -8
- data/core/services/swift/src/core.rs +63 -67
- data/core/services/swift/src/deleter.rs +8 -2
- data/core/services/swift/src/lister.rs +11 -8
- data/core/services/swift/src/reader.rs +5 -2
- data/core/services/swift/src/writer.rs +17 -8
- data/core/services/tikv/Cargo.toml +2 -2
- data/core/services/tikv/src/backend.rs +7 -6
- data/core/services/tikv/src/core.rs +31 -14
- data/core/services/tikv/src/reader.rs +4 -1
- data/core/services/tikv/src/writer.rs +6 -2
- data/core/services/tos/Cargo.toml +2 -2
- data/core/services/tos/src/backend.rs +45 -21
- data/core/services/tos/src/copier.rs +44 -17
- data/core/services/tos/src/core.rs +147 -117
- data/core/services/tos/src/deleter.rs +9 -3
- data/core/services/tos/src/docs.md +1 -1
- data/core/services/tos/src/lister.rs +43 -28
- data/core/services/tos/src/reader.rs +5 -2
- data/core/services/tos/src/writer.rs +37 -16
- data/core/services/upyun/Cargo.toml +1 -1
- data/core/services/upyun/src/backend.rs +30 -7
- data/core/services/upyun/src/core.rs +122 -87
- data/core/services/upyun/src/deleter.rs +4 -1
- data/core/services/upyun/src/lister.rs +10 -10
- data/core/services/upyun/src/reader.rs +5 -2
- data/core/services/upyun/src/writer.rs +19 -7
- data/core/services/vercel-artifacts/Cargo.toml +1 -1
- data/core/services/vercel-artifacts/src/backend.rs +6 -3
- data/core/services/vercel-artifacts/src/core.rs +32 -27
- data/core/services/vercel-artifacts/src/reader.rs +5 -2
- data/core/services/vercel-artifacts/src/writer.rs +6 -3
- data/core/services/vercel-blob/Cargo.toml +1 -1
- data/core/services/vercel-blob/src/backend.rs +23 -6
- data/core/services/vercel-blob/src/core.rs +92 -81
- data/core/services/vercel-blob/src/lister.rs +12 -3
- data/core/services/vercel-blob/src/reader.rs +5 -2
- data/core/services/vercel-blob/src/writer.rs +19 -7
- data/core/services/webdav/Cargo.toml +2 -2
- data/core/services/webdav/src/backend.rs +25 -5
- data/core/services/webdav/src/core.rs +116 -59
- data/core/services/webdav/src/deleter.rs +4 -1
- data/core/services/webdav/src/lister.rs +4 -1
- data/core/services/webdav/src/reader.rs +6 -2
- data/core/services/webdav/src/writer.rs +17 -7
- data/core/services/webhdfs/Cargo.toml +2 -2
- data/core/services/webhdfs/src/backend.rs +24 -11
- data/core/services/webhdfs/src/core.rs +99 -98
- data/core/services/webhdfs/src/deleter.rs +5 -2
- data/core/services/webhdfs/src/lister.rs +21 -9
- data/core/services/webhdfs/src/reader.rs +5 -2
- data/core/services/webhdfs/src/writer.rs +49 -14
- data/core/services/yandex-disk/Cargo.toml +1 -1
- data/core/services/yandex-disk/src/backend.rs +28 -6
- data/core/services/yandex-disk/src/core.rs +97 -79
- data/core/services/yandex-disk/src/deleter.rs +4 -1
- data/core/services/yandex-disk/src/lister.rs +25 -9
- data/core/services/yandex-disk/src/reader.rs +5 -2
- data/core/services/yandex-disk/src/writer.rs +6 -3
- data/core/src/lib.rs +5 -0
- data/core/testkit/Cargo.toml +4 -4
- data/core/testkit/src/utils.rs +53 -8
- data/core/tests/behavior/async_compose.rs +297 -0
- data/core/tests/behavior/async_copy.rs +162 -1
- data/core/tests/behavior/async_delete.rs +364 -3
- data/core/tests/behavior/async_read.rs +86 -0
- data/core/tests/behavior/async_restore.rs +162 -0
- data/core/tests/behavior/async_stat.rs +90 -0
- data/core/tests/behavior/async_write.rs +330 -2
- data/core/tests/behavior/main.rs +4 -0
- data/core/upgrade.md +114 -1
- data/src/capability.rs +0 -1
- data/src/io.rs +0 -1
- data/src/lister.rs +0 -1
- data/src/metadata.rs +0 -1
- data/src/operator_info.rs +0 -1
- metadata +44 -2
|
@@ -18,9 +18,8 @@
|
|
|
18
18
|
use std::sync::Arc;
|
|
19
19
|
|
|
20
20
|
use super::core::GoosefsCore;
|
|
21
|
-
use opendal_core::EntryMode;
|
|
22
21
|
use opendal_core::ErrorKind;
|
|
23
|
-
use opendal_core::
|
|
22
|
+
use opendal_core::MetadataBuilder;
|
|
24
23
|
use opendal_core::Result;
|
|
25
24
|
use opendal_core::raw::oio::Entry;
|
|
26
25
|
use opendal_core::raw::*;
|
|
@@ -65,7 +64,7 @@ impl oio::PageList for GoosefsLister {
|
|
|
65
64
|
// synthesis for it.
|
|
66
65
|
if !self.path.is_empty() && self.path.ends_with('/') {
|
|
67
66
|
ctx.entries
|
|
68
|
-
.push_back(Entry::new(&self.path,
|
|
67
|
+
.push_back(Entry::new(&self.path, MetadataBuilder::dir().build()));
|
|
69
68
|
}
|
|
70
69
|
|
|
71
70
|
for file_info in file_infos {
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
use super::backend::*;
|
|
19
19
|
use super::core::GoosefsCore;
|
|
20
|
-
use super::core::parse_error;
|
|
20
|
+
use super::core::{ErrorContext, parse_error};
|
|
21
21
|
use goosefs_sdk::io::GoosefsFileReader as SdkReader;
|
|
22
22
|
use opendal_core::raw::*;
|
|
23
23
|
use opendal_core::*;
|
|
@@ -103,7 +103,11 @@ impl oio::ReadStream for GoosefsReadStream {
|
|
|
103
103
|
}
|
|
104
104
|
let reader = self.inner.as_mut().expect("inner was just set");
|
|
105
105
|
|
|
106
|
-
match reader
|
|
106
|
+
match reader
|
|
107
|
+
.read_next_block()
|
|
108
|
+
.await
|
|
109
|
+
.map_err(|err| parse_error(ErrorContext::new(ServiceOperation("ReadNextBlock")), err))?
|
|
110
|
+
{
|
|
107
111
|
Some(block) => Ok(Buffer::from(block)),
|
|
108
112
|
None => {
|
|
109
113
|
self.done = true;
|
|
@@ -138,7 +142,7 @@ impl oio::StreamRead for GoosefsReader {
|
|
|
138
142
|
Some(
|
|
139
143
|
backend
|
|
140
144
|
.core
|
|
141
|
-
.file_info_to_metadata(&file_info)
|
|
145
|
+
.file_info_to_metadata(&file_info)?
|
|
142
146
|
.content_length(),
|
|
143
147
|
)
|
|
144
148
|
} else {
|
|
@@ -24,7 +24,7 @@ use std::time::UNIX_EPOCH;
|
|
|
24
24
|
use goosefs_sdk::io::GoosefsFileWriter as ClientWriter;
|
|
25
25
|
|
|
26
26
|
use super::core::GoosefsCore;
|
|
27
|
-
use super::core::parse_error;
|
|
27
|
+
use super::core::{ErrorContext, parse_error};
|
|
28
28
|
use opendal_core::raw::*;
|
|
29
29
|
use opendal_core::*;
|
|
30
30
|
|
|
@@ -226,7 +226,10 @@ impl oio::Write for GoosefsWriter {
|
|
|
226
226
|
// Iterator yields one `Bytes` per segment; the SDK writer
|
|
227
227
|
// takes `&[u8]`, so each chunk flows through without a copy.
|
|
228
228
|
for chunk in bs {
|
|
229
|
-
writer
|
|
229
|
+
writer
|
|
230
|
+
.write(&chunk)
|
|
231
|
+
.await
|
|
232
|
+
.map_err(|err| parse_error(ErrorContext::new(ServiceOperation("Write")), err))?;
|
|
230
233
|
}
|
|
231
234
|
Ok(())
|
|
232
235
|
}
|
|
@@ -235,7 +238,11 @@ impl oio::Write for GoosefsWriter {
|
|
|
235
238
|
if let Some(mut writer) = self.writer.take() {
|
|
236
239
|
// Commit the temp inode on the master, then publish it
|
|
237
240
|
// onto the caller's final path via `finalize_rename`.
|
|
238
|
-
if let Err(e) = writer
|
|
241
|
+
if let Err(e) = writer
|
|
242
|
+
.close()
|
|
243
|
+
.await
|
|
244
|
+
.map_err(|err| parse_error(ErrorContext::new(ServiceOperation("Close")), err))
|
|
245
|
+
{
|
|
239
246
|
// Close failed — the temp is in an indeterminate state
|
|
240
247
|
// on the master; best-effort sweep to avoid leaks.
|
|
241
248
|
if let Some(tmp) = self.tmp_path.take() {
|
|
@@ -244,12 +251,12 @@ impl oio::Write for GoosefsWriter {
|
|
|
244
251
|
return Err(e);
|
|
245
252
|
}
|
|
246
253
|
// Capture file_id before rename; Master rename keeps the same inode id.
|
|
247
|
-
let mut meta =
|
|
254
|
+
let mut meta = MetadataBuilder::unknown();
|
|
248
255
|
if let Some(fid) = writer.file_info().file_id {
|
|
249
|
-
meta.
|
|
256
|
+
meta.etag(fid.to_string());
|
|
250
257
|
}
|
|
251
258
|
self.finalize_rename().await?;
|
|
252
|
-
return Ok(meta);
|
|
259
|
+
return Ok(meta.build());
|
|
253
260
|
}
|
|
254
261
|
|
|
255
262
|
// Zero-write path: the caller closed without ever calling
|
|
@@ -267,17 +274,21 @@ impl oio::Write for GoosefsWriter {
|
|
|
267
274
|
|
|
268
275
|
let tmp = Self::make_tmp_path(&self.path);
|
|
269
276
|
let mut w = self.core.create_writer(&tmp).await?;
|
|
270
|
-
if let Err(e) = w
|
|
277
|
+
if let Err(e) = w
|
|
278
|
+
.close()
|
|
279
|
+
.await
|
|
280
|
+
.map_err(|err| parse_error(ErrorContext::new(ServiceOperation("Close")), err))
|
|
281
|
+
{
|
|
271
282
|
let _ = self.core.delete(&tmp).await;
|
|
272
283
|
return Err(e);
|
|
273
284
|
}
|
|
274
|
-
let mut meta =
|
|
285
|
+
let mut meta = MetadataBuilder::unknown();
|
|
275
286
|
if let Some(fid) = w.file_info().file_id {
|
|
276
|
-
meta.
|
|
287
|
+
meta.etag(fid.to_string());
|
|
277
288
|
}
|
|
278
289
|
self.tmp_path = Some(tmp);
|
|
279
290
|
self.finalize_rename().await?;
|
|
280
|
-
Ok(meta)
|
|
291
|
+
Ok(meta.build())
|
|
281
292
|
}
|
|
282
293
|
|
|
283
294
|
async fn abort(&mut self) -> Result<()> {
|
|
@@ -197,3 +197,40 @@ async fn concurrent_write_if_not_exists_exactly_one_wins() {
|
|
|
197
197
|
let _ = op.delete(&p).await;
|
|
198
198
|
}
|
|
199
199
|
}
|
|
200
|
+
|
|
201
|
+
/// Nested write: CreateFile(recursive) on the temp sibling creates the parent,
|
|
202
|
+
/// so publish rename must succeed without an eager CreateDirectory.
|
|
203
|
+
#[tokio::test]
|
|
204
|
+
async fn write_nested_path_creates_parents() {
|
|
205
|
+
let Some(op) = maybe_operator() else {
|
|
206
|
+
eprintln!("skip: OPENDAL_GOOSEFS_MASTER_ADDR unset");
|
|
207
|
+
return;
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
let path = format!("{}/nested/file", unique("nested-write"));
|
|
211
|
+
op.write(&path, "nested-body")
|
|
212
|
+
.await
|
|
213
|
+
.expect("write into missing parents");
|
|
214
|
+
let body = op.read(&path).await.expect("read").to_bytes();
|
|
215
|
+
assert_eq!(&body[..], b"nested-body");
|
|
216
|
+
let _ = op.delete(&path).await;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
/// Public rename still creates a destination parent that does not exist yet.
|
|
220
|
+
#[tokio::test]
|
|
221
|
+
async fn rename_creates_missing_destination_parent() {
|
|
222
|
+
let Some(op) = maybe_operator() else {
|
|
223
|
+
eprintln!("skip: OPENDAL_GOOSEFS_MASTER_ADDR unset");
|
|
224
|
+
return;
|
|
225
|
+
};
|
|
226
|
+
|
|
227
|
+
let src = unique("mkdir-src");
|
|
228
|
+
op.write(&src, "payload").await.expect("write src");
|
|
229
|
+
let dst = format!("{}/nested/file", unique("mkdir-dst"));
|
|
230
|
+
op.rename(&src, &dst)
|
|
231
|
+
.await
|
|
232
|
+
.expect("rename should create missing parent");
|
|
233
|
+
let body = op.read(&dst).await.expect("read dst").to_bytes();
|
|
234
|
+
assert_eq!(&body[..], b"payload");
|
|
235
|
+
let _ = op.delete(&dst).await;
|
|
236
|
+
}
|
|
@@ -32,10 +32,10 @@ version = { workspace = true }
|
|
|
32
32
|
all-features = true
|
|
33
33
|
|
|
34
34
|
[dependencies]
|
|
35
|
+
asyncband = { workspace = true }
|
|
35
36
|
futures = { workspace = true }
|
|
36
|
-
mea = { workspace = true }
|
|
37
37
|
mongodb = "3.3.0"
|
|
38
|
-
opendal-core = { path = "../../core", version = "0.
|
|
38
|
+
opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
|
|
39
39
|
serde = { workspace = true, features = ["derive"] }
|
|
40
40
|
|
|
41
41
|
[dev-dependencies]
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
use std::sync::Arc;
|
|
19
19
|
|
|
20
|
-
use
|
|
20
|
+
use asyncband::once::OnceCell;
|
|
21
21
|
use opendal_core::raw::*;
|
|
22
22
|
use opendal_core::*;
|
|
23
23
|
|
|
@@ -93,7 +93,7 @@ impl GridfsBuilder {
|
|
|
93
93
|
self
|
|
94
94
|
}
|
|
95
95
|
|
|
96
|
-
/// Set the chunk size
|
|
96
|
+
/// Set the chunk size in bytes for MongoDB GridFs service. We break the user file into chunks by size.
|
|
97
97
|
///
|
|
98
98
|
/// Default to `255 KiB` if not specified.
|
|
99
99
|
pub fn chunk_size(mut self, chunk_size: u32) -> Self {
|
|
@@ -104,6 +104,9 @@ impl GridfsBuilder {
|
|
|
104
104
|
}
|
|
105
105
|
}
|
|
106
106
|
|
|
107
|
+
/// Default GridFS chunk size in bytes.
|
|
108
|
+
const DEFAULT_CHUNK_SIZE_BYTES: u32 = 255 * 1024; // 255 KiB
|
|
109
|
+
|
|
107
110
|
impl Builder for GridfsBuilder {
|
|
108
111
|
type Config = GridfsConfig;
|
|
109
112
|
|
|
@@ -128,7 +131,7 @@ impl Builder for GridfsBuilder {
|
|
|
128
131
|
Some(v) => v.clone(),
|
|
129
132
|
None => "fs".to_string(),
|
|
130
133
|
};
|
|
131
|
-
let chunk_size = self.config.chunk_size.unwrap_or(
|
|
134
|
+
let chunk_size = self.config.chunk_size.unwrap_or(DEFAULT_CHUNK_SIZE_BYTES);
|
|
132
135
|
|
|
133
136
|
let root = normalize_root(
|
|
134
137
|
self.config
|
|
@@ -196,6 +199,7 @@ impl Service for GridfsBackend {
|
|
|
196
199
|
type Lister = ();
|
|
197
200
|
type Deleter = oio::OneShotDeleter<GridfsDeleter>;
|
|
198
201
|
type Copier = ();
|
|
202
|
+
type Composer = ();
|
|
199
203
|
|
|
200
204
|
fn info(&self) -> ServiceInfo {
|
|
201
205
|
self.info.clone()
|
|
@@ -221,12 +225,13 @@ impl Service for GridfsBackend {
|
|
|
221
225
|
let p = build_abs_path(&self.root, path);
|
|
222
226
|
|
|
223
227
|
if p == build_abs_path(&self.root, "") {
|
|
224
|
-
Ok(RpStat::new(
|
|
228
|
+
Ok(RpStat::new(MetadataBuilder::dir().build()))
|
|
225
229
|
} else {
|
|
226
230
|
match self.core.get_length(&p).await? {
|
|
227
|
-
Some(len) => Ok(RpStat::new(
|
|
228
|
-
|
|
229
|
-
|
|
231
|
+
Some(len) => Ok(RpStat::new({
|
|
232
|
+
let metadata = MetadataBuilder::file(len as u64);
|
|
233
|
+
metadata.build()
|
|
234
|
+
})),
|
|
230
235
|
None => Err(Error::new(ErrorKind::NotFound, "kv not found in gridfs")),
|
|
231
236
|
}
|
|
232
237
|
}
|
|
@@ -276,7 +281,6 @@ impl Service for GridfsBackend {
|
|
|
276
281
|
_from: &str,
|
|
277
282
|
_to: &str,
|
|
278
283
|
_args: OpCopy,
|
|
279
|
-
_opts: OpCopier,
|
|
280
284
|
) -> Result<Self::Copier> {
|
|
281
285
|
Err(Error::new(
|
|
282
286
|
ErrorKind::Unsupported,
|
|
@@ -17,9 +17,9 @@
|
|
|
17
17
|
|
|
18
18
|
use std::fmt::Debug;
|
|
19
19
|
|
|
20
|
+
use asyncband::once::OnceCell;
|
|
20
21
|
use futures::AsyncReadExt;
|
|
21
22
|
use futures::AsyncWriteExt;
|
|
22
|
-
use mea::once::OnceCell;
|
|
23
23
|
use mongodb::bson::doc;
|
|
24
24
|
use mongodb::gridfs::{FilesCollectionDocument, GridFsBucket};
|
|
25
25
|
use mongodb::options::ClientOptions;
|
|
@@ -46,7 +46,10 @@ impl oio::StreamRead for GridfsReader {
|
|
|
46
46
|
}
|
|
47
47
|
};
|
|
48
48
|
let content = bs.slice(range.to_content_range(bs.len())?);
|
|
49
|
-
let metadata =
|
|
49
|
+
let metadata = {
|
|
50
|
+
let metadata = MetadataBuilder::file(bs.len() as u64);
|
|
51
|
+
metadata.build()
|
|
52
|
+
};
|
|
50
53
|
Ok((
|
|
51
54
|
RpRead::new(metadata),
|
|
52
55
|
Box::new(content) as Box<dyn oio::ReadStreamDyn>,
|
|
@@ -49,8 +49,12 @@ impl oio::Write for GridfsWriter {
|
|
|
49
49
|
let length = buf.len() as u64;
|
|
50
50
|
self.core.set(&self.path, buf).await?;
|
|
51
51
|
|
|
52
|
-
let
|
|
53
|
-
|
|
52
|
+
let metadata = if self.path.ends_with('/') {
|
|
53
|
+
MetadataBuilder::dir()
|
|
54
|
+
} else {
|
|
55
|
+
MetadataBuilder::file(length)
|
|
56
|
+
};
|
|
57
|
+
Ok(metadata.build())
|
|
54
58
|
}
|
|
55
59
|
|
|
56
60
|
async fn abort(&mut self) -> Result<()> {
|
|
@@ -32,7 +32,7 @@ version = { workspace = true }
|
|
|
32
32
|
all-features = true
|
|
33
33
|
|
|
34
34
|
[dependencies]
|
|
35
|
-
opendal-core = { path = "../../core", version = "0.
|
|
35
|
+
opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
|
|
36
36
|
|
|
37
37
|
bytes = { workspace = true }
|
|
38
38
|
futures = { workspace = true }
|
|
@@ -196,6 +196,7 @@ impl Service for HdfsBackend {
|
|
|
196
196
|
type Lister = Option<HdfsLister>;
|
|
197
197
|
type Deleter = oio::OneShotDeleter<HdfsDeleter>;
|
|
198
198
|
type Copier = ();
|
|
199
|
+
type Composer = ();
|
|
199
200
|
|
|
200
201
|
fn info(&self) -> ServiceInfo {
|
|
201
202
|
self.core.info.clone()
|
|
@@ -260,7 +261,6 @@ impl Service for HdfsBackend {
|
|
|
260
261
|
_from: &str,
|
|
261
262
|
_to: &str,
|
|
262
263
|
_args: OpCopy,
|
|
263
|
-
_opts: OpCopier,
|
|
264
264
|
) -> Result<Self::Copier> {
|
|
265
265
|
Err(Error::new(
|
|
266
266
|
ErrorKind::Unsupported,
|
|
@@ -72,11 +72,14 @@ impl HdfsCore {
|
|
|
72
72
|
} else {
|
|
73
73
|
EntryMode::Unknown
|
|
74
74
|
};
|
|
75
|
-
let mut m =
|
|
76
|
-
|
|
77
|
-
|
|
75
|
+
let mut m = match mode {
|
|
76
|
+
EntryMode::FILE => MetadataBuilder::file(meta.len()),
|
|
77
|
+
EntryMode::DIR => MetadataBuilder::dir(),
|
|
78
|
+
EntryMode::Unknown => MetadataBuilder::unknown(),
|
|
79
|
+
};
|
|
80
|
+
m.last_modified(Timestamp::try_from(meta.modified())?);
|
|
78
81
|
|
|
79
|
-
Ok(m)
|
|
82
|
+
Ok(m.build())
|
|
80
83
|
}
|
|
81
84
|
|
|
82
85
|
pub async fn hdfs_open(&self, path: &str) -> Result<hdrs::File> {
|
|
@@ -15,8 +15,7 @@
|
|
|
15
15
|
// specific language governing permissions and limitations
|
|
16
16
|
// under the License.
|
|
17
17
|
|
|
18
|
-
use opendal_core::
|
|
19
|
-
use opendal_core::Metadata;
|
|
18
|
+
use opendal_core::MetadataBuilder;
|
|
20
19
|
use opendal_core::Result;
|
|
21
20
|
use opendal_core::raw::*;
|
|
22
21
|
|
|
@@ -43,7 +42,7 @@ impl HdfsLister {
|
|
|
43
42
|
impl oio::List for HdfsLister {
|
|
44
43
|
async fn next(&mut self) -> Result<Option<oio::Entry>> {
|
|
45
44
|
if let Some(path) = self.current_path.take() {
|
|
46
|
-
return Ok(Some(oio::Entry::new(&path,
|
|
45
|
+
return Ok(Some(oio::Entry::new(&path, MetadataBuilder::dir().build())));
|
|
47
46
|
}
|
|
48
47
|
|
|
49
48
|
let de = match self.rd.next() {
|
|
@@ -54,15 +53,14 @@ impl oio::List for HdfsLister {
|
|
|
54
53
|
let path = build_rel_path(&self.root, de.path());
|
|
55
54
|
|
|
56
55
|
let entry = if de.is_file() {
|
|
57
|
-
let meta =
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
oio::Entry::new(&path, meta)
|
|
56
|
+
let mut meta = MetadataBuilder::file(de.len());
|
|
57
|
+
meta.last_modified(Timestamp::try_from(de.modified())?);
|
|
58
|
+
oio::Entry::new(&path, meta.build())
|
|
61
59
|
} else if de.is_dir() {
|
|
62
60
|
// Make sure we are returning the correct path.
|
|
63
|
-
oio::Entry::new(&format!("{path}/"),
|
|
61
|
+
oio::Entry::new(&format!("{path}/"), MetadataBuilder::dir().build())
|
|
64
62
|
} else {
|
|
65
|
-
oio::Entry::new(&path,
|
|
63
|
+
oio::Entry::new(&path, MetadataBuilder::unknown().build())
|
|
66
64
|
};
|
|
67
65
|
|
|
68
66
|
Ok(Some(entry))
|
|
@@ -92,7 +92,11 @@ impl oio::Write for HdfsWriter<hdrs::AsyncFile> {
|
|
|
92
92
|
.map_err(new_std_io_error)?
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
-
Ok(
|
|
95
|
+
Ok({
|
|
96
|
+
let mut metadata = MetadataBuilder::unknown();
|
|
97
|
+
metadata.set_file(self.size);
|
|
98
|
+
metadata.build()
|
|
99
|
+
})
|
|
96
100
|
}
|
|
97
101
|
|
|
98
102
|
async fn abort(&mut self) -> Result<()> {
|
|
@@ -36,5 +36,5 @@ bytes = { workspace = true }
|
|
|
36
36
|
futures = { workspace = true }
|
|
37
37
|
hdfs-native = { version = "0.14" }
|
|
38
38
|
log = { workspace = true }
|
|
39
|
-
opendal-core = { path = "../../core", version = "0.
|
|
39
|
+
opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
|
|
40
40
|
serde = { workspace = true, features = ["derive"] }
|
|
@@ -167,6 +167,7 @@ impl Service for HdfsNativeBackend {
|
|
|
167
167
|
type Lister = HdfsNativeLazyLister;
|
|
168
168
|
type Deleter = oio::OneShotDeleter<HdfsNativeDeleter>;
|
|
169
169
|
type Copier = ();
|
|
170
|
+
type Composer = ();
|
|
170
171
|
|
|
171
172
|
fn info(&self) -> ServiceInfo {
|
|
172
173
|
self.core.info.clone()
|
|
@@ -221,7 +222,6 @@ impl Service for HdfsNativeBackend {
|
|
|
221
222
|
_from: &str,
|
|
222
223
|
_to: &str,
|
|
223
224
|
_args: OpCopy,
|
|
224
|
-
_opts: OpCopier,
|
|
225
225
|
) -> Result<Self::Copier> {
|
|
226
226
|
Err(Error::new(
|
|
227
227
|
ErrorKind::Unsupported,
|
|
@@ -68,14 +68,16 @@ impl HdfsNativeCore {
|
|
|
68
68
|
EntryMode::FILE
|
|
69
69
|
};
|
|
70
70
|
|
|
71
|
-
let mut metadata =
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
71
|
+
let mut metadata = if mode == EntryMode::FILE {
|
|
72
|
+
MetadataBuilder::file(status.length as u64)
|
|
73
|
+
} else {
|
|
74
|
+
MetadataBuilder::dir()
|
|
75
|
+
};
|
|
76
|
+
metadata.last_modified(Timestamp::from_millisecond(
|
|
77
|
+
status.modification_time as i64,
|
|
78
|
+
)?);
|
|
79
|
+
|
|
80
|
+
Ok(metadata.build())
|
|
79
81
|
}
|
|
80
82
|
|
|
81
83
|
pub async fn hdfs_open(&self, path: &str) -> Result<hdfs_native::file::FileReader> {
|
|
@@ -200,49 +202,41 @@ impl HdfsNativeCore {
|
|
|
200
202
|
}
|
|
201
203
|
}
|
|
202
204
|
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
)
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
HdfsError::RPCError(msg0, msg1) => {
|
|
222
|
-
if msg0.contains("java.io.FileNotFoundException") {
|
|
223
|
-
(ErrorKind::NotFound, false, msg1.clone())
|
|
224
|
-
} else {
|
|
225
|
-
(ErrorKind::Unexpected, false, msg1.clone())
|
|
226
|
-
}
|
|
205
|
+
/// Parse hdfs-native error into opendal::Error.
|
|
206
|
+
pub fn parse_hdfs_error(hdfs_error: HdfsError) -> Error {
|
|
207
|
+
let (kind, retryable, msg) = match &hdfs_error {
|
|
208
|
+
HdfsError::IOError(err) => (ErrorKind::Unexpected, false, err.to_string()),
|
|
209
|
+
HdfsError::DataTransferError(msg) => (ErrorKind::Unexpected, false, msg.clone()),
|
|
210
|
+
HdfsError::ChecksumError => (
|
|
211
|
+
ErrorKind::Unexpected,
|
|
212
|
+
false,
|
|
213
|
+
"checksums didn't match".to_string(),
|
|
214
|
+
),
|
|
215
|
+
HdfsError::UrlParseError(err) => (ErrorKind::Unexpected, false, err.to_string()),
|
|
216
|
+
HdfsError::AlreadyExists(msg) => (ErrorKind::AlreadyExists, false, msg.clone()),
|
|
217
|
+
HdfsError::OperationFailed(msg) => (ErrorKind::Unexpected, false, msg.clone()),
|
|
218
|
+
HdfsError::RPCError(msg0, msg1) => {
|
|
219
|
+
if msg0.contains("java.io.FileNotFoundException") {
|
|
220
|
+
(ErrorKind::NotFound, false, msg1.clone())
|
|
221
|
+
} else {
|
|
222
|
+
(ErrorKind::Unexpected, false, msg1.clone())
|
|
227
223
|
}
|
|
228
|
-
HdfsError::FileNotFound(msg) => (ErrorKind::NotFound, false, msg.clone()),
|
|
229
|
-
HdfsError::BlocksNotFound(msg) => (ErrorKind::NotFound, false, msg.clone()),
|
|
230
|
-
HdfsError::IsADirectoryError(msg) => (ErrorKind::IsADirectory, false, msg.clone()),
|
|
231
|
-
_ => (
|
|
232
|
-
ErrorKind::Unexpected,
|
|
233
|
-
false,
|
|
234
|
-
"unexpected error from hdfs".to_string(),
|
|
235
|
-
),
|
|
236
|
-
};
|
|
237
|
-
|
|
238
|
-
let mut err = Error::new(kind, msg).set_source(hdfs_error);
|
|
239
|
-
|
|
240
|
-
if retryable {
|
|
241
|
-
err = err.set_temporary();
|
|
242
224
|
}
|
|
243
|
-
|
|
244
|
-
|
|
225
|
+
HdfsError::FileNotFound(msg) => (ErrorKind::NotFound, false, msg.clone()),
|
|
226
|
+
HdfsError::BlocksNotFound(msg) => (ErrorKind::NotFound, false, msg.clone()),
|
|
227
|
+
HdfsError::IsADirectoryError(msg) => (ErrorKind::IsADirectory, false, msg.clone()),
|
|
228
|
+
_ => (
|
|
229
|
+
ErrorKind::Unexpected,
|
|
230
|
+
false,
|
|
231
|
+
"unexpected error from hdfs".to_string(),
|
|
232
|
+
),
|
|
233
|
+
};
|
|
234
|
+
|
|
235
|
+
let mut err = Error::new(kind, msg).set_source(hdfs_error);
|
|
236
|
+
|
|
237
|
+
if retryable {
|
|
238
|
+
err = err.set_temporary();
|
|
245
239
|
}
|
|
246
|
-
}
|
|
247
240
|
|
|
248
|
-
|
|
241
|
+
err
|
|
242
|
+
}
|
|
@@ -17,8 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
use hdfs_native::client::ListStatusIterator;
|
|
19
19
|
|
|
20
|
-
use opendal_core::
|
|
21
|
-
use opendal_core::Metadata;
|
|
20
|
+
use opendal_core::MetadataBuilder;
|
|
22
21
|
use opendal_core::Result;
|
|
23
22
|
use opendal_core::raw::*;
|
|
24
23
|
|
|
@@ -54,7 +53,7 @@ impl oio::List for HdfsNativeLister {
|
|
|
54
53
|
}
|
|
55
54
|
|
|
56
55
|
if let Some(path) = self.current_path.take() {
|
|
57
|
-
return Ok(Some(oio::Entry::new(&path,
|
|
56
|
+
return Ok(Some(oio::Entry::new(&path, MetadataBuilder::dir().build())));
|
|
58
57
|
}
|
|
59
58
|
|
|
60
59
|
match self.iter.next().await {
|
|
@@ -62,14 +61,13 @@ impl oio::List for HdfsNativeLister {
|
|
|
62
61
|
let path = build_rel_path(&self.root, &status.path);
|
|
63
62
|
|
|
64
63
|
let entry = if status.isdir {
|
|
65
|
-
oio::Entry::new(&format!("{path}/"),
|
|
64
|
+
oio::Entry::new(&format!("{path}/"), MetadataBuilder::dir().build())
|
|
66
65
|
} else {
|
|
67
|
-
let meta =
|
|
68
|
-
|
|
69
|
-
.
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
oio::Entry::new(&path, meta)
|
|
66
|
+
let mut meta = MetadataBuilder::file(status.length as u64);
|
|
67
|
+
meta.last_modified(Timestamp::from_millisecond(
|
|
68
|
+
status.modification_time as i64,
|
|
69
|
+
)?);
|
|
70
|
+
oio::Entry::new(&path, meta.build())
|
|
73
71
|
};
|
|
74
72
|
|
|
75
73
|
Ok(Some(entry))
|
|
@@ -49,7 +49,11 @@ impl oio::Write for HdfsNativeWriter {
|
|
|
49
49
|
async fn close(&mut self) -> Result<Metadata> {
|
|
50
50
|
self.f.close().await.map_err(parse_hdfs_error)?;
|
|
51
51
|
|
|
52
|
-
Ok(
|
|
52
|
+
Ok({
|
|
53
|
+
let mut metadata = MetadataBuilder::unknown();
|
|
54
|
+
metadata.set_file(self.size);
|
|
55
|
+
metadata.build()
|
|
56
|
+
})
|
|
53
57
|
}
|
|
54
58
|
|
|
55
59
|
async fn abort(&mut self) -> Result<()> {
|
data/core/services/hf/Cargo.toml
CHANGED
|
@@ -32,11 +32,12 @@ version = { workspace = true }
|
|
|
32
32
|
all-features = true
|
|
33
33
|
|
|
34
34
|
[dependencies]
|
|
35
|
+
asyncband = { workspace = true }
|
|
35
36
|
bytes = { workspace = true }
|
|
36
|
-
hf-xet = "1.
|
|
37
|
+
hf-xet = "1.6.0"
|
|
37
38
|
http = { workspace = true }
|
|
38
39
|
log = { workspace = true }
|
|
39
|
-
opendal-core = { path = "../../core", version = "0.
|
|
40
|
+
opendal-core = { path = "../../core", version = "0.59.0", default-features = false }
|
|
40
41
|
percent-encoding = "2"
|
|
41
42
|
serde = { workspace = true, features = ["derive"] }
|
|
42
43
|
serde_json = { workspace = true }
|
|
@@ -44,6 +45,6 @@ serde_json = { workspace = true }
|
|
|
44
45
|
[dev-dependencies]
|
|
45
46
|
base64 = { workspace = true }
|
|
46
47
|
futures = { workspace = true }
|
|
47
|
-
opendal-http-transport-reqwest = { path = "../../http-transports/reqwest", version = "0.
|
|
48
|
+
opendal-http-transport-reqwest = { path = "../../http-transports/reqwest", version = "0.59.0" }
|
|
48
49
|
serde_json = { workspace = true }
|
|
49
50
|
tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
|
|
@@ -248,6 +248,7 @@ impl Builder for HfBuilder {
|
|
|
248
248
|
stat: true,
|
|
249
249
|
read: true,
|
|
250
250
|
write: token.is_some(),
|
|
251
|
+
write_can_multi: token.is_some(),
|
|
251
252
|
delete: token.is_some(),
|
|
252
253
|
delete_max_size: Some(100),
|
|
253
254
|
list: true,
|
|
@@ -285,6 +286,7 @@ impl Service for HfBackend {
|
|
|
285
286
|
type Lister = oio::PageLister<HfLister>;
|
|
286
287
|
type Deleter = oio::BatchDeleter<HfDeleter>;
|
|
287
288
|
type Copier = ();
|
|
289
|
+
type Composer = ();
|
|
288
290
|
|
|
289
291
|
fn info(&self) -> ServiceInfo {
|
|
290
292
|
self.core.info.clone()
|
|
@@ -309,12 +311,12 @@ impl Service for HfBackend {
|
|
|
309
311
|
async fn stat(&self, ctx: &OperationContext, path: &str, _: OpStat) -> Result<RpStat> {
|
|
310
312
|
// Stat root always returns a DIR.
|
|
311
313
|
if path == "/" {
|
|
312
|
-
return Ok(RpStat::new(
|
|
314
|
+
return Ok(RpStat::new(MetadataBuilder::dir().build()));
|
|
313
315
|
}
|
|
314
316
|
|
|
315
317
|
// Buckets have no git directory entries; treat any trailing-slash path as a virtual dir.
|
|
316
318
|
if self.core.repo.is_bucket() && path.ends_with('/') {
|
|
317
|
-
return Ok(RpStat::new(
|
|
319
|
+
return Ok(RpStat::new(MetadataBuilder::dir().build()));
|
|
318
320
|
}
|
|
319
321
|
|
|
320
322
|
let info = self.core.path_info(ctx, path).await?;
|
|
@@ -371,7 +373,6 @@ impl Service for HfBackend {
|
|
|
371
373
|
_from: &str,
|
|
372
374
|
_to: &str,
|
|
373
375
|
_args: OpCopy,
|
|
374
|
-
_opts: OpCopier,
|
|
375
376
|
) -> Result<Self::Copier> {
|
|
376
377
|
Err(Error::new(
|
|
377
378
|
ErrorKind::Unsupported,
|