opendal 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +89 -56
- data/Cargo.toml +1 -1
- data/DEPENDENCIES.rust.tsv +55 -41
- data/LICENSE +201 -0
- data/LICENSE-MPL-2.0.txt +373 -0
- data/NOTICE +5 -0
- data/core/CHANGELOG.md +175 -0
- data/core/Cargo.lock +225 -298
- data/core/Cargo.toml +100 -94
- data/core/DEPENDENCIES.rust.tsv +44 -73
- data/core/benches/metadata/README.md +21 -0
- data/core/benches/metadata/main.rs +541 -0
- data/core/benches/vs_fs/Cargo.toml +1 -1
- data/core/benches/vs_s3/Cargo.toml +1 -1
- data/core/core/Cargo.toml +7 -2
- data/core/core/src/blocking/delete.rs +5 -1
- data/core/core/src/blocking/operator.rs +12 -0
- data/core/core/src/blocking/read/std_reader.rs +25 -0
- data/core/core/src/docs/concepts.rs +70 -94
- data/core/core/src/docs/internals/accessor.rs +62 -293
- data/core/core/src/docs/internals/layer.rs +43 -59
- data/core/core/src/docs/internals/mod.rs +21 -23
- data/core/core/src/docs/mod.rs +10 -6
- data/core/core/src/docs/performance/concurrent_write.md +92 -55
- data/core/core/src/docs/performance/http_optimization.md +120 -104
- data/core/core/src/docs/performance/mod.rs +8 -6
- data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
- data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
- data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
- data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
- data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
- data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
- data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
- data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
- data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
- data/core/core/src/docs/rfcs/README.md +14 -3
- data/core/core/src/docs/rfcs/mod.rs +46 -22
- data/core/core/src/docs/specs/README.md +34 -0
- data/core/core/src/docs/specs/conditional-operations.md +156 -0
- data/core/core/src/docs/specs/metadata.md +63 -0
- data/core/core/src/docs/specs/mod.rs +26 -0
- data/core/core/src/docs/upgrade.md +114 -1
- data/core/core/src/layers/capability_override.rs +15 -2
- data/core/core/src/layers/complete.rs +347 -43
- data/core/core/src/layers/correctness_check.rs +718 -51
- data/core/core/src/layers/error_context.rs +76 -2
- data/core/core/src/layers/simulate.rs +91 -28
- data/core/core/src/lib.rs +1 -0
- data/core/core/src/raw/accessor.rs +91 -15
- data/core/core/src/raw/enum_utils.rs +7 -0
- data/core/core/src/raw/http_util/header.rs +22 -17
- data/core/core/src/raw/oio/compose/api.rs +92 -0
- data/core/core/src/raw/oio/compose/mod.rs +19 -0
- data/core/core/src/raw/oio/copy/api.rs +3 -3
- data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
- data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
- data/core/core/src/raw/oio/entry.rs +48 -22
- data/core/core/src/raw/oio/list/flat_list.rs +1 -1
- data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
- data/core/core/src/raw/oio/mod.rs +3 -0
- data/core/core/src/raw/oio/read/position_read.rs +1 -1
- data/core/core/src/raw/oio/write/api.rs +40 -0
- data/core/core/src/raw/oio/write/append_write.rs +4 -4
- data/core/core/src/raw/oio/write/block_write.rs +2 -2
- data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
- data/core/core/src/raw/oio/write/position_write.rs +1 -1
- data/core/core/src/raw/operation.rs +6 -0
- data/core/core/src/raw/ops.rs +1191 -456
- data/core/core/src/raw/rps.rs +11 -0
- data/core/core/src/raw/time.rs +14 -3
- data/core/core/src/services/memory/backend.rs +5 -10
- data/core/core/src/services/memory/lister.rs +4 -4
- data/core/core/src/services/memory/writer.rs +7 -6
- data/core/core/src/types/buffer.rs +102 -12
- data/core/core/src/types/bytes_range.rs +20 -1
- data/core/core/src/types/capability.rs +65 -0
- data/core/core/src/types/compact.rs +329 -0
- data/core/core/src/types/compose/composer.rs +183 -0
- data/core/core/src/types/compose/input.rs +57 -0
- data/core/core/src/types/compose/mod.rs +23 -0
- data/core/core/src/types/context/read.rs +16 -11
- data/core/core/src/types/context/write.rs +150 -2
- data/core/core/src/types/copy.rs +5 -3
- data/core/core/src/types/delete/deleter.rs +20 -12
- data/core/core/src/types/delete/input.rs +22 -64
- data/core/core/src/types/delete/mod.rs +0 -1
- data/core/core/src/types/entry.rs +8 -14
- data/core/core/src/types/error.rs +28 -11
- data/core/core/src/types/metadata.rs +709 -322
- data/core/core/src/types/mod.rs +9 -1
- data/core/core/src/types/operator/operator.rs +278 -29
- data/core/core/src/types/operator/operator_futures.rs +466 -105
- data/core/core/src/types/options.rs +631 -158
- data/core/core/src/types/read/futures_async_reader.rs +663 -50
- data/core/core/src/types/read/reader.rs +23 -5
- data/core/core/src/types/write/futures_async_writer.rs +27 -0
- data/core/core/src/types/write/writer.rs +47 -5
- data/core/http-transports/reqwest/Cargo.toml +1 -1
- data/core/layers/async-backtrace/Cargo.toml +2 -2
- data/core/layers/async-backtrace/src/lib.rs +21 -2
- data/core/layers/await-tree/Cargo.toml +2 -2
- data/core/layers/await-tree/src/lib.rs +25 -2
- data/core/layers/capability-check/Cargo.toml +2 -2
- data/core/layers/capability-check/src/lib.rs +98 -10
- data/core/layers/chaos/Cargo.toml +2 -2
- data/core/layers/chaos/src/lib.rs +18 -3
- data/core/layers/concurrent-limit/Cargo.toml +3 -3
- data/core/layers/concurrent-limit/src/lib.rs +53 -13
- data/core/layers/dtrace/Cargo.toml +2 -2
- data/core/layers/dtrace/src/lib.rs +29 -2
- data/core/layers/fastmetrics/Cargo.toml +3 -3
- data/core/layers/fastrace/Cargo.toml +2 -2
- data/core/layers/fastrace/src/lib.rs +28 -2
- data/core/layers/foyer/Cargo.toml +2 -2
- data/core/layers/foyer/src/full.rs +16 -19
- data/core/layers/foyer/src/lib.rs +67 -14
- data/core/layers/hotpath/Cargo.toml +2 -2
- data/core/layers/hotpath/src/lib.rs +20 -2
- data/core/layers/immutable-index/Cargo.toml +2 -2
- data/core/layers/immutable-index/src/lib.rs +48 -24
- data/core/layers/logging/Cargo.toml +2 -2
- data/core/layers/logging/src/lib.rs +46 -2
- data/core/layers/metrics/Cargo.toml +3 -3
- data/core/layers/mime-guess/Cargo.toml +2 -2
- data/core/layers/mime-guess/src/lib.rs +31 -4
- data/core/layers/observe-metrics-common/Cargo.toml +2 -2
- data/core/layers/observe-metrics-common/src/lib.rs +55 -2
- data/core/layers/otelmetrics/Cargo.toml +3 -3
- data/core/layers/oteltrace/Cargo.toml +2 -2
- data/core/layers/oteltrace/src/lib.rs +27 -2
- data/core/layers/prometheus/Cargo.toml +3 -3
- data/core/layers/prometheus-client/Cargo.toml +3 -3
- data/core/layers/retry/Cargo.toml +4 -4
- data/core/layers/retry/src/lib.rs +76 -19
- data/core/layers/route/Cargo.toml +3 -3
- data/core/layers/route/src/lib.rs +34 -7
- data/core/layers/tail-cut/Cargo.toml +2 -2
- data/core/layers/tail-cut/src/lib.rs +28 -2
- data/core/layers/throttle/Cargo.toml +2 -2
- data/core/layers/throttle/src/lib.rs +19 -2
- data/core/layers/timeout/Cargo.toml +3 -3
- data/core/layers/timeout/src/lib.rs +38 -13
- data/core/layers/tracing/Cargo.toml +2 -2
- data/core/layers/tracing/src/lib.rs +24 -3
- data/core/services/aliyun-drive/Cargo.toml +2 -2
- data/core/services/aliyun-drive/src/backend.rs +34 -24
- data/core/services/aliyun-drive/src/core.rs +143 -54
- data/core/services/aliyun-drive/src/lister.rs +19 -18
- data/core/services/aliyun-drive/src/reader.rs +4 -1
- data/core/services/aliyun-drive/src/writer.rs +3 -3
- data/core/services/alluxio/Cargo.toml +1 -1
- data/core/services/alluxio/src/backend.rs +1 -1
- data/core/services/alluxio/src/core.rs +113 -94
- data/core/services/alluxio/src/reader.rs +7 -3
- data/core/services/alluxio/src/writer.rs +2 -2
- data/core/services/azblob/Cargo.toml +2 -2
- data/core/services/azblob/src/backend.rs +17 -9
- data/core/services/azblob/src/copier.rs +41 -15
- data/core/services/azblob/src/core.rs +372 -135
- data/core/services/azblob/src/deleter.rs +41 -18
- data/core/services/azblob/src/lister.rs +16 -10
- data/core/services/azblob/src/reader.rs +4 -1
- data/core/services/azblob/src/writer.rs +48 -15
- data/core/services/azdls/Cargo.toml +3 -3
- data/core/services/azdls/src/backend.rs +45 -15
- data/core/services/azdls/src/config.rs +4 -1
- data/core/services/azdls/src/core.rs +276 -87
- data/core/services/azdls/src/deleter.rs +13 -4
- data/core/services/azdls/src/lister.rs +18 -12
- data/core/services/azdls/src/reader.rs +6 -2
- data/core/services/azdls/src/writer.rs +44 -23
- data/core/services/azfile/Cargo.toml +2 -2
- data/core/services/azfile/src/backend.rs +24 -8
- data/core/services/azfile/src/core.rs +195 -77
- data/core/services/azfile/src/deleter.rs +8 -1
- data/core/services/azfile/src/lister.rs +20 -14
- data/core/services/azfile/src/reader.rs +5 -2
- data/core/services/azfile/src/writer.rs +27 -15
- data/core/services/azure-common/Cargo.toml +1 -1
- data/core/services/b2/Cargo.toml +2 -2
- data/core/services/b2/src/backend.rs +30 -11
- data/core/services/b2/src/core.rs +134 -120
- data/core/services/b2/src/deleter.rs +1 -1
- data/core/services/b2/src/lister.rs +5 -2
- data/core/services/b2/src/reader.rs +5 -2
- data/core/services/b2/src/writer.rs +29 -12
- data/core/services/cacache/Cargo.toml +1 -1
- data/core/services/cacache/src/backend.rs +4 -5
- data/core/services/cacache/src/reader.rs +4 -1
- data/core/services/cacache/src/writer.rs +6 -2
- data/core/services/cloudflare-kv/Cargo.toml +1 -1
- data/core/services/cloudflare-kv/src/backend.rs +26 -15
- data/core/services/cloudflare-kv/src/config.rs +13 -2
- data/core/services/cloudflare-kv/src/core.rs +62 -56
- data/core/services/cloudflare-kv/src/deleter.rs +12 -3
- data/core/services/cloudflare-kv/src/lister.rs +66 -17
- data/core/services/cloudflare-kv/src/reader.rs +14 -9
- data/core/services/cloudflare-kv/src/writer.rs +10 -7
- data/core/services/compfs/Cargo.toml +1 -1
- data/core/services/compfs/src/backend.rs +11 -7
- data/core/services/compfs/src/lister.rs +7 -7
- data/core/services/compfs/src/writer.rs +1 -1
- data/core/services/cos/Cargo.toml +1 -1
- data/core/services/cos/src/backend.rs +129 -16
- data/core/services/cos/src/core.rs +198 -80
- data/core/services/cos/src/deleter.rs +4 -1
- data/core/services/cos/src/lister.rs +42 -27
- data/core/services/cos/src/reader.rs +6 -2
- data/core/services/cos/src/writer.rs +45 -21
- data/core/services/d1/Cargo.toml +1 -1
- data/core/services/d1/src/backend.rs +6 -5
- data/core/services/d1/src/core.rs +79 -63
- data/core/services/d1/src/reader.rs +4 -1
- data/core/services/d1/src/writer.rs +6 -2
- data/core/services/dashmap/Cargo.toml +1 -1
- data/core/services/dashmap/src/backend.rs +2 -2
- data/core/services/dashmap/src/reader.rs +4 -1
- data/core/services/dashmap/src/writer.rs +11 -8
- data/core/services/dbfs/Cargo.toml +1 -1
- data/core/services/dbfs/src/backend.rs +21 -15
- data/core/services/dbfs/src/core.rs +65 -60
- data/core/services/dbfs/src/deleter.rs +4 -1
- data/core/services/dbfs/src/lister.rs +10 -10
- data/core/services/dbfs/src/writer.rs +6 -3
- data/core/services/dropbox/Cargo.toml +2 -2
- data/core/services/dropbox/src/backend.rs +30 -23
- data/core/services/dropbox/src/core.rs +65 -63
- data/core/services/dropbox/src/deleter.rs +1 -1
- data/core/services/dropbox/src/lister.rs +17 -8
- data/core/services/dropbox/src/reader.rs +4 -1
- data/core/services/dropbox/src/writer.rs +13 -10
- data/core/services/etcd/Cargo.toml +1 -1
- data/core/services/etcd/src/backend.rs +8 -5
- data/core/services/etcd/src/core.rs +5 -11
- data/core/services/etcd/src/lister.rs +6 -1
- data/core/services/etcd/src/reader.rs +4 -1
- data/core/services/etcd/src/writer.rs +6 -3
- data/core/services/foundationdb/Cargo.toml +1 -1
- data/core/services/foundationdb/src/backend.rs +6 -5
- data/core/services/foundationdb/src/reader.rs +4 -1
- data/core/services/foundationdb/src/writer.rs +6 -2
- data/core/services/foyer/Cargo.toml +3 -3
- data/core/services/foyer/src/backend.rs +6 -5
- data/core/services/foyer/src/reader.rs +4 -1
- data/core/services/foyer/src/writer.rs +6 -2
- data/core/services/fs/Cargo.toml +1 -1
- data/core/services/fs/src/backend.rs +4 -3
- data/core/services/fs/src/core.rs +44 -26
- data/core/services/fs/src/lister.rs +9 -6
- data/core/services/fs/src/writer.rs +20 -13
- data/core/services/ftp/Cargo.toml +4 -6
- data/core/services/ftp/src/backend.rs +9 -5
- data/core/services/ftp/src/core.rs +9 -7
- data/core/services/ftp/src/lister.rs +7 -8
- data/core/services/ftp/src/reader.rs +3 -3
- data/core/services/ftp/src/writer.rs +3 -3
- data/core/services/gcs/Cargo.toml +4 -2
- data/core/services/gcs/src/backend.rs +53 -7
- data/core/services/gcs/src/composer.rs +705 -0
- data/core/services/gcs/src/copier.rs +14 -4
- data/core/services/gcs/src/core.rs +828 -124
- data/core/services/gcs/src/deleter.rs +33 -18
- data/core/services/gcs/src/lib.rs +1 -0
- data/core/services/gcs/src/lister.rs +21 -12
- data/core/services/gcs/src/reader.rs +12 -5
- data/core/services/gcs/src/writer.rs +632 -10
- data/core/services/gcs-grpc/Cargo.toml +58 -0
- data/core/services/gcs-grpc/README.md +19 -0
- data/core/services/gcs-grpc/proto/README.md +23 -0
- data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
- data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
- data/core/services/gcs-grpc/src/backend.rs +449 -0
- data/core/services/gcs-grpc/src/config.rs +116 -0
- data/core/services/gcs-grpc/src/copier.rs +98 -0
- data/core/services/gcs-grpc/src/core.rs +298 -0
- data/core/services/gcs-grpc/src/deleter.rs +65 -0
- data/core/services/gcs-grpc/src/docs.md +19 -0
- data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
- data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
- data/core/services/gcs-grpc/src/lib.rs +43 -0
- data/core/services/gcs-grpc/src/lister.rs +125 -0
- data/core/services/gcs-grpc/src/reader.rs +140 -0
- data/core/services/gcs-grpc/src/writer.rs +780 -0
- data/core/services/gdrive/Cargo.toml +2 -2
- data/core/services/gdrive/src/backend.rs +123 -69
- data/core/services/gdrive/src/core.rs +87 -69
- data/core/services/gdrive/src/deleter.rs +8 -2
- data/core/services/gdrive/src/lister.rs +38 -26
- data/core/services/gdrive/src/path_index.rs +2 -2
- data/core/services/gdrive/src/reader.rs +9 -3
- data/core/services/gdrive/src/writer.rs +13 -7
- data/core/services/ghac/Cargo.toml +2 -2
- data/core/services/ghac/src/backend.rs +5 -2
- data/core/services/ghac/src/core.rs +59 -37
- data/core/services/ghac/src/reader.rs +5 -2
- data/core/services/ghac/src/writer.rs +16 -6
- data/core/services/github/Cargo.toml +1 -1
- data/core/services/github/src/backend.rs +16 -8
- data/core/services/github/src/core.rs +88 -76
- data/core/services/github/src/lister.rs +17 -15
- data/core/services/github/src/reader.rs +5 -2
- data/core/services/github/src/writer.rs +12 -6
- data/core/services/goosefs/Cargo.toml +2 -2
- data/core/services/goosefs/src/backend.rs +2 -2
- data/core/services/goosefs/src/core.rs +291 -208
- data/core/services/goosefs/src/docs.md +3 -0
- data/core/services/goosefs/src/lister.rs +2 -3
- data/core/services/goosefs/src/reader.rs +7 -3
- data/core/services/goosefs/src/writer.rs +21 -10
- data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
- data/core/services/gridfs/Cargo.toml +2 -2
- data/core/services/gridfs/src/backend.rs +12 -8
- data/core/services/gridfs/src/core.rs +1 -1
- data/core/services/gridfs/src/reader.rs +4 -1
- data/core/services/gridfs/src/writer.rs +6 -2
- data/core/services/hdfs/Cargo.toml +1 -1
- data/core/services/hdfs/src/backend.rs +1 -1
- data/core/services/hdfs/src/core.rs +7 -4
- data/core/services/hdfs/src/lister.rs +7 -9
- data/core/services/hdfs/src/writer.rs +5 -1
- data/core/services/hdfs-native/Cargo.toml +1 -1
- data/core/services/hdfs-native/src/backend.rs +1 -1
- data/core/services/hdfs-native/src/core.rs +44 -50
- data/core/services/hdfs-native/src/lister.rs +8 -10
- data/core/services/hdfs-native/src/writer.rs +5 -1
- data/core/services/hf/Cargo.toml +4 -3
- data/core/services/hf/src/backend.rs +4 -3
- data/core/services/hf/src/core.rs +907 -153
- data/core/services/hf/src/lister.rs +16 -6
- data/core/services/hf/src/reader.rs +377 -57
- data/core/services/hf/src/writer.rs +6 -2
- data/core/services/http/Cargo.toml +3 -3
- data/core/services/http/src/backend.rs +8 -6
- data/core/services/http/src/core.rs +57 -35
- data/core/services/http/src/reader.rs +5 -2
- data/core/services/ipfs/Cargo.toml +1 -1
- data/core/services/ipfs/src/backend.rs +20 -9
- data/core/services/ipfs/src/core.rs +43 -34
- data/core/services/ipfs/src/reader.rs +5 -2
- data/core/services/ipmfs/Cargo.toml +1 -1
- data/core/services/ipmfs/src/backend.rs +17 -8
- data/core/services/ipmfs/src/core.rs +66 -60
- data/core/services/ipmfs/src/deleter.rs +5 -2
- data/core/services/ipmfs/src/lister.rs +15 -12
- data/core/services/ipmfs/src/reader.rs +5 -2
- data/core/services/ipmfs/src/writer.rs +6 -3
- data/core/services/koofr/Cargo.toml +2 -2
- data/core/services/koofr/src/backend.rs +46 -17
- data/core/services/koofr/src/core.rs +64 -48
- data/core/services/koofr/src/deleter.rs +4 -1
- data/core/services/koofr/src/lister.rs +11 -10
- data/core/services/koofr/src/reader.rs +5 -2
- data/core/services/koofr/src/writer.rs +13 -7
- data/core/services/lakefs/Cargo.toml +1 -1
- data/core/services/lakefs/src/backend.rs +25 -8
- data/core/services/lakefs/src/core.rs +79 -73
- data/core/services/lakefs/src/deleter.rs +4 -1
- data/core/services/lakefs/src/lister.rs +17 -14
- data/core/services/lakefs/src/reader.rs +5 -2
- data/core/services/lakefs/src/writer.rs +11 -10
- data/core/services/memcached/Cargo.toml +1 -1
- data/core/services/memcached/src/backend.rs +57 -7
- data/core/services/memcached/src/config.rs +13 -2
- data/core/services/memcached/src/core.rs +11 -1
- data/core/services/memcached/src/reader.rs +4 -1
- data/core/services/memcached/src/writer.rs +6 -2
- data/core/services/mini_moka/Cargo.toml +1 -1
- data/core/services/mini_moka/src/backend.rs +8 -8
- data/core/services/mini_moka/src/reader.rs +4 -1
- data/core/services/mini_moka/src/writer.rs +11 -7
- data/core/services/moka/Cargo.toml +1 -1
- data/core/services/moka/src/backend.rs +7 -7
- data/core/services/moka/src/reader.rs +4 -1
- data/core/services/moka/src/writer.rs +10 -6
- data/core/services/mongodb/Cargo.toml +2 -2
- data/core/services/mongodb/src/backend.rs +7 -6
- data/core/services/mongodb/src/core.rs +1 -1
- data/core/services/mongodb/src/reader.rs +4 -1
- data/core/services/mongodb/src/writer.rs +6 -2
- data/core/services/monoiofs/Cargo.toml +2 -2
- data/core/services/monoiofs/src/backend.rs +16 -10
- data/core/services/monoiofs/src/core.rs +8 -9
- data/core/services/monoiofs/src/reader.rs +8 -10
- data/core/services/monoiofs/src/writer.rs +19 -24
- data/core/services/mysql/Cargo.toml +2 -2
- data/core/services/mysql/src/backend.rs +7 -6
- data/core/services/mysql/src/core.rs +1 -1
- data/core/services/mysql/src/lister.rs +7 -3
- data/core/services/mysql/src/reader.rs +4 -1
- data/core/services/mysql/src/writer.rs +6 -2
- data/core/services/obs/Cargo.toml +1 -1
- data/core/services/obs/src/backend.rs +29 -11
- data/core/services/obs/src/core.rs +81 -82
- data/core/services/obs/src/deleter.rs +4 -1
- data/core/services/obs/src/lister.rs +12 -5
- data/core/services/obs/src/reader.rs +5 -2
- data/core/services/obs/src/writer.rs +40 -19
- data/core/services/onedrive/Cargo.toml +2 -2
- data/core/services/onedrive/src/backend.rs +30 -21
- data/core/services/onedrive/src/config.rs +2 -2
- data/core/services/onedrive/src/core.rs +191 -90
- data/core/services/onedrive/src/deleter.rs +5 -2
- data/core/services/onedrive/src/graph_model.rs +10 -51
- data/core/services/onedrive/src/lister.rs +60 -24
- data/core/services/onedrive/src/reader.rs +17 -2
- data/core/services/onedrive/src/writer.rs +26 -15
- data/core/services/opfs/Cargo.toml +1 -1
- data/core/services/opfs/src/backend.rs +6 -13
- data/core/services/opfs/src/core.rs +33 -37
- data/core/services/opfs/src/lister.rs +4 -4
- data/core/services/opfs/src/writer.rs +1 -3
- data/core/services/oss/Cargo.toml +1 -1
- data/core/services/oss/src/backend.rs +40 -8
- data/core/services/oss/src/core.rs +192 -92
- data/core/services/oss/src/deleter.rs +16 -6
- data/core/services/oss/src/lister.rs +43 -27
- data/core/services/oss/src/reader.rs +6 -2
- data/core/services/oss/src/writer.rs +40 -22
- data/core/services/pcloud/Cargo.toml +1 -1
- data/core/services/pcloud/src/backend.rs +40 -6
- data/core/services/pcloud/src/core.rs +80 -66
- data/core/services/pcloud/src/deleter.rs +8 -1
- data/core/services/pcloud/src/lister.rs +4 -1
- data/core/services/pcloud/src/reader.rs +5 -2
- data/core/services/pcloud/src/writer.rs +6 -3
- data/core/services/persy/Cargo.toml +1 -1
- data/core/services/persy/src/backend.rs +6 -5
- data/core/services/persy/src/core.rs +60 -15
- data/core/services/persy/src/reader.rs +4 -1
- data/core/services/persy/src/writer.rs +6 -2
- data/core/services/postgresql/Cargo.toml +2 -2
- data/core/services/postgresql/src/backend.rs +7 -6
- data/core/services/postgresql/src/core.rs +1 -1
- data/core/services/postgresql/src/reader.rs +4 -1
- data/core/services/postgresql/src/writer.rs +6 -2
- data/core/services/redb/Cargo.toml +1 -1
- data/core/services/redb/src/backend.rs +6 -5
- data/core/services/redb/src/reader.rs +4 -1
- data/core/services/redb/src/writer.rs +6 -2
- data/core/services/redis/Cargo.toml +1 -1
- data/core/services/redis/src/backend.rs +18 -8
- data/core/services/redis/src/config.rs +13 -2
- data/core/services/redis/src/reader.rs +8 -4
- data/core/services/redis/src/writer.rs +6 -2
- data/core/services/rocksdb/Cargo.toml +1 -1
- data/core/services/rocksdb/src/backend.rs +6 -5
- data/core/services/rocksdb/src/lister.rs +6 -5
- data/core/services/rocksdb/src/reader.rs +4 -1
- data/core/services/rocksdb/src/writer.rs +6 -2
- data/core/services/s3/Cargo.toml +5 -4
- data/core/services/s3/src/backend.rs +995 -39
- data/core/services/s3/src/compatible_services.md +57 -0
- data/core/services/s3/src/config.rs +16 -0
- data/core/services/s3/src/copier.rs +190 -55
- data/core/services/s3/src/core.rs +299 -190
- data/core/services/s3/src/deleter.rs +26 -7
- data/core/services/s3/src/docs.md +57 -0
- data/core/services/s3/src/lister.rs +55 -36
- data/core/services/s3/src/reader.rs +22 -2
- data/core/services/s3/src/writer.rs +154 -32
- data/core/services/seafile/Cargo.toml +2 -2
- data/core/services/seafile/src/backend.rs +3 -3
- data/core/services/seafile/src/core.rs +105 -79
- data/core/services/seafile/src/lister.rs +11 -8
- data/core/services/seafile/src/reader.rs +5 -2
- data/core/services/seafile/src/writer.rs +6 -3
- data/core/services/sftp/Cargo.toml +1 -1
- data/core/services/sftp/src/backend.rs +19 -4
- data/core/services/sftp/src/core.rs +51 -46
- data/core/services/sftp/src/lister.rs +4 -4
- data/core/services/sftp/src/writer.rs +1 -1
- data/core/services/sled/Cargo.toml +1 -1
- data/core/services/sled/src/backend.rs +6 -5
- data/core/services/sled/src/core.rs +28 -6
- data/core/services/sled/src/lister.rs +6 -5
- data/core/services/sled/src/reader.rs +4 -1
- data/core/services/sled/src/writer.rs +6 -2
- data/core/services/sqlite/Cargo.toml +2 -2
- data/core/services/sqlite/src/backend.rs +12 -7
- data/core/services/sqlite/src/core.rs +3 -3
- data/core/services/sqlite/src/reader.rs +4 -1
- data/core/services/sqlite/src/writer.rs +7 -3
- data/core/services/surrealdb/Cargo.toml +2 -2
- data/core/services/surrealdb/src/backend.rs +7 -6
- data/core/services/surrealdb/src/core.rs +1 -1
- data/core/services/surrealdb/src/reader.rs +4 -1
- data/core/services/surrealdb/src/writer.rs +6 -2
- data/core/services/swift/Cargo.toml +1 -1
- data/core/services/swift/src/backend.rs +38 -8
- data/core/services/swift/src/core.rs +63 -67
- data/core/services/swift/src/deleter.rs +8 -2
- data/core/services/swift/src/lister.rs +11 -8
- data/core/services/swift/src/reader.rs +5 -2
- data/core/services/swift/src/writer.rs +17 -8
- data/core/services/tikv/Cargo.toml +2 -2
- data/core/services/tikv/src/backend.rs +7 -6
- data/core/services/tikv/src/core.rs +31 -14
- data/core/services/tikv/src/reader.rs +4 -1
- data/core/services/tikv/src/writer.rs +6 -2
- data/core/services/tos/Cargo.toml +2 -2
- data/core/services/tos/src/backend.rs +45 -21
- data/core/services/tos/src/copier.rs +44 -17
- data/core/services/tos/src/core.rs +147 -117
- data/core/services/tos/src/deleter.rs +9 -3
- data/core/services/tos/src/docs.md +1 -1
- data/core/services/tos/src/lister.rs +43 -28
- data/core/services/tos/src/reader.rs +5 -2
- data/core/services/tos/src/writer.rs +37 -16
- data/core/services/upyun/Cargo.toml +1 -1
- data/core/services/upyun/src/backend.rs +30 -7
- data/core/services/upyun/src/core.rs +122 -87
- data/core/services/upyun/src/deleter.rs +4 -1
- data/core/services/upyun/src/lister.rs +10 -10
- data/core/services/upyun/src/reader.rs +5 -2
- data/core/services/upyun/src/writer.rs +19 -7
- data/core/services/vercel-artifacts/Cargo.toml +1 -1
- data/core/services/vercel-artifacts/src/backend.rs +6 -3
- data/core/services/vercel-artifacts/src/core.rs +32 -27
- data/core/services/vercel-artifacts/src/reader.rs +5 -2
- data/core/services/vercel-artifacts/src/writer.rs +6 -3
- data/core/services/vercel-blob/Cargo.toml +1 -1
- data/core/services/vercel-blob/src/backend.rs +23 -6
- data/core/services/vercel-blob/src/core.rs +92 -81
- data/core/services/vercel-blob/src/lister.rs +12 -3
- data/core/services/vercel-blob/src/reader.rs +5 -2
- data/core/services/vercel-blob/src/writer.rs +19 -7
- data/core/services/webdav/Cargo.toml +2 -2
- data/core/services/webdav/src/backend.rs +25 -5
- data/core/services/webdav/src/core.rs +116 -59
- data/core/services/webdav/src/deleter.rs +4 -1
- data/core/services/webdav/src/lister.rs +4 -1
- data/core/services/webdav/src/reader.rs +6 -2
- data/core/services/webdav/src/writer.rs +17 -7
- data/core/services/webhdfs/Cargo.toml +2 -2
- data/core/services/webhdfs/src/backend.rs +24 -11
- data/core/services/webhdfs/src/core.rs +99 -98
- data/core/services/webhdfs/src/deleter.rs +5 -2
- data/core/services/webhdfs/src/lister.rs +21 -9
- data/core/services/webhdfs/src/reader.rs +5 -2
- data/core/services/webhdfs/src/writer.rs +49 -14
- data/core/services/yandex-disk/Cargo.toml +1 -1
- data/core/services/yandex-disk/src/backend.rs +28 -6
- data/core/services/yandex-disk/src/core.rs +97 -79
- data/core/services/yandex-disk/src/deleter.rs +4 -1
- data/core/services/yandex-disk/src/lister.rs +25 -9
- data/core/services/yandex-disk/src/reader.rs +5 -2
- data/core/services/yandex-disk/src/writer.rs +6 -3
- data/core/src/lib.rs +5 -0
- data/core/testkit/Cargo.toml +4 -4
- data/core/testkit/src/utils.rs +53 -8
- data/core/tests/behavior/async_compose.rs +297 -0
- data/core/tests/behavior/async_copy.rs +162 -1
- data/core/tests/behavior/async_delete.rs +364 -3
- data/core/tests/behavior/async_read.rs +86 -0
- data/core/tests/behavior/async_restore.rs +162 -0
- data/core/tests/behavior/async_stat.rs +90 -0
- data/core/tests/behavior/async_write.rs +330 -2
- data/core/tests/behavior/main.rs +4 -0
- data/core/upgrade.md +114 -1
- data/src/capability.rs +0 -1
- data/src/io.rs +0 -1
- data/src/lister.rs +0 -1
- data/src/metadata.rs +0 -1
- data/src/operator_info.rs +0 -1
- metadata +44 -2
|
@@ -39,6 +39,7 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
|
|
|
39
39
|
test_stat_not_exist,
|
|
40
40
|
test_stat_with_if_match,
|
|
41
41
|
test_stat_with_if_none_match,
|
|
42
|
+
test_stat_with_version_conditions,
|
|
42
43
|
test_stat_with_if_modified_since,
|
|
43
44
|
test_stat_with_if_unmodified_since,
|
|
44
45
|
test_stat_with_override_cache_control,
|
|
@@ -200,6 +201,14 @@ pub async fn test_stat_with_if_match(op: Operator) -> Result<()> {
|
|
|
200
201
|
.await;
|
|
201
202
|
assert!(result.is_ok());
|
|
202
203
|
|
|
204
|
+
let missing = TEST_FIXTURE.new_file_path();
|
|
205
|
+
let err = op
|
|
206
|
+
.stat_with(&missing)
|
|
207
|
+
.if_match(meta.etag().expect("etag must exist"))
|
|
208
|
+
.await
|
|
209
|
+
.expect_err("missing target must fail");
|
|
210
|
+
assert_eq!(err.kind(), ErrorKind::NotFound);
|
|
211
|
+
|
|
203
212
|
Ok(())
|
|
204
213
|
}
|
|
205
214
|
|
|
@@ -233,6 +242,71 @@ pub async fn test_stat_with_if_none_match(op: Operator) -> Result<()> {
|
|
|
233
242
|
assert_eq!(res.mode(), meta.mode());
|
|
234
243
|
assert_eq!(res.content_length(), meta.content_length());
|
|
235
244
|
|
|
245
|
+
let missing = TEST_FIXTURE.new_file_path();
|
|
246
|
+
let err = op
|
|
247
|
+
.stat_with(&missing)
|
|
248
|
+
.if_none_match(meta.etag().expect("etag must exist"))
|
|
249
|
+
.await
|
|
250
|
+
.expect_err("missing target must fail");
|
|
251
|
+
assert_eq!(err.kind(), ErrorKind::NotFound);
|
|
252
|
+
|
|
253
|
+
Ok(())
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/// Version preconditions should compare against the current live object version.
|
|
257
|
+
pub async fn test_stat_with_version_conditions(op: Operator) -> Result<()> {
|
|
258
|
+
let cap = op.info().capability();
|
|
259
|
+
if !cap.stat_with_if_version_match || !cap.stat_with_if_version_not_match {
|
|
260
|
+
return Ok(());
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
let path = TEST_FIXTURE.new_file_path();
|
|
264
|
+
let (first, _) = gen_bytes(cap);
|
|
265
|
+
let (second, _) = gen_bytes(cap);
|
|
266
|
+
assert_ne!(first, second);
|
|
267
|
+
|
|
268
|
+
op.write(&path, first).await?;
|
|
269
|
+
let stale = op
|
|
270
|
+
.stat(&path)
|
|
271
|
+
.await?
|
|
272
|
+
.version()
|
|
273
|
+
.expect("version must exist")
|
|
274
|
+
.to_string();
|
|
275
|
+
op.write(&path, second).await?;
|
|
276
|
+
let current = op
|
|
277
|
+
.stat(&path)
|
|
278
|
+
.await?
|
|
279
|
+
.version()
|
|
280
|
+
.expect("version must exist")
|
|
281
|
+
.to_string();
|
|
282
|
+
|
|
283
|
+
op.stat_with(&path).if_version_match(¤t).await?;
|
|
284
|
+
let err = op
|
|
285
|
+
.stat_with(&path)
|
|
286
|
+
.if_version_match(&stale)
|
|
287
|
+
.await
|
|
288
|
+
.expect_err("stale version match must fail");
|
|
289
|
+
assert_eq!(err.kind(), ErrorKind::ConditionNotMatch);
|
|
290
|
+
|
|
291
|
+
op.stat_with(&path).if_version_not_match(&stale).await?;
|
|
292
|
+
let err = op
|
|
293
|
+
.stat_with(&path)
|
|
294
|
+
.if_version_not_match(¤t)
|
|
295
|
+
.await
|
|
296
|
+
.expect_err("equal version non-match must fail");
|
|
297
|
+
assert_eq!(err.kind(), ErrorKind::ConditionNotMatch);
|
|
298
|
+
|
|
299
|
+
let missing = TEST_FIXTURE.new_file_path();
|
|
300
|
+
for result in [
|
|
301
|
+
op.stat_with(&missing).if_version_match(¤t).await,
|
|
302
|
+
op.stat_with(&missing).if_version_not_match(¤t).await,
|
|
303
|
+
] {
|
|
304
|
+
assert_eq!(
|
|
305
|
+
result.expect_err("missing target must fail").kind(),
|
|
306
|
+
ErrorKind::NotFound
|
|
307
|
+
);
|
|
308
|
+
}
|
|
309
|
+
|
|
236
310
|
Ok(())
|
|
237
311
|
}
|
|
238
312
|
|
|
@@ -263,6 +337,14 @@ pub async fn test_stat_with_if_modified_since(op: Operator) -> Result<()> {
|
|
|
263
337
|
assert!(res.is_err());
|
|
264
338
|
assert_eq!(res.err().unwrap().kind(), ErrorKind::ConditionNotMatch);
|
|
265
339
|
|
|
340
|
+
let missing = TEST_FIXTURE.new_file_path();
|
|
341
|
+
let err = op
|
|
342
|
+
.stat_with(&missing)
|
|
343
|
+
.if_modified_since(since)
|
|
344
|
+
.await
|
|
345
|
+
.expect_err("missing target must fail");
|
|
346
|
+
assert_eq!(err.kind(), ErrorKind::NotFound);
|
|
347
|
+
|
|
266
348
|
Ok(())
|
|
267
349
|
}
|
|
268
350
|
|
|
@@ -293,6 +375,14 @@ pub async fn test_stat_with_if_unmodified_since(op: Operator) -> Result<()> {
|
|
|
293
375
|
let res = op.stat_with(&path).if_unmodified_since(since).await?;
|
|
294
376
|
assert_eq!(res.last_modified(), meta.last_modified());
|
|
295
377
|
|
|
378
|
+
let missing = TEST_FIXTURE.new_file_path();
|
|
379
|
+
let err = op
|
|
380
|
+
.stat_with(&missing)
|
|
381
|
+
.if_unmodified_since(since)
|
|
382
|
+
.await
|
|
383
|
+
.expect_err("missing target must fail");
|
|
384
|
+
assert_eq!(err.kind(), ErrorKind::NotFound);
|
|
385
|
+
|
|
296
386
|
Ok(())
|
|
297
387
|
}
|
|
298
388
|
|
|
@@ -45,6 +45,8 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
|
|
|
45
45
|
test_write_with_if_none_match,
|
|
46
46
|
test_write_with_if_not_exists,
|
|
47
47
|
test_write_with_if_match,
|
|
48
|
+
test_write_with_version_conditions,
|
|
49
|
+
test_write_if_not_changed,
|
|
48
50
|
test_write_with_user_metadata,
|
|
49
51
|
test_write_returns_metadata,
|
|
50
52
|
test_writer_write,
|
|
@@ -57,7 +59,12 @@ pub fn tests(op: &Operator, tests: &mut Vec<Trial>) {
|
|
|
57
59
|
test_writer_futures_copy,
|
|
58
60
|
test_writer_futures_copy_with_concurrent,
|
|
59
61
|
test_writer_return_metadata,
|
|
60
|
-
|
|
62
|
+
test_writer_copy_from_interleaved,
|
|
63
|
+
test_writer_write_non_contiguous_data,
|
|
64
|
+
test_writer_write_with_if_not_exists,
|
|
65
|
+
test_writer_write_with_if_none_match,
|
|
66
|
+
test_writer_write_with_if_match,
|
|
67
|
+
test_writer_write_with_version_conditions
|
|
61
68
|
))
|
|
62
69
|
}
|
|
63
70
|
|
|
@@ -242,7 +249,10 @@ pub async fn test_write_with_user_metadata(op: Operator) -> Result<()> {
|
|
|
242
249
|
let resp_meta = meta.user_metadata().expect("meta data must exist");
|
|
243
250
|
|
|
244
251
|
assert_eq!(
|
|
245
|
-
|
|
252
|
+
resp_meta
|
|
253
|
+
.into_iter()
|
|
254
|
+
.map(|(key, value)| (key.to_owned(), value.to_owned()))
|
|
255
|
+
.collect::<HashMap<_, _>>(),
|
|
246
256
|
target_user_metadata.into_iter().collect::<HashMap<_, _>>()
|
|
247
257
|
);
|
|
248
258
|
|
|
@@ -349,6 +359,59 @@ pub async fn test_writer_write(op: Operator) -> Result<()> {
|
|
|
349
359
|
Ok(())
|
|
350
360
|
}
|
|
351
361
|
|
|
362
|
+
/// Assemble a destination from local bytes and source ranges in call order.
|
|
363
|
+
pub async fn test_writer_copy_from_interleaved(op: Operator) -> Result<()> {
|
|
364
|
+
if !op.info().capability().write_can_multi {
|
|
365
|
+
return Ok(());
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
let source_path = TEST_FIXTURE.new_file_path();
|
|
369
|
+
let target_path = TEST_FIXTURE.new_file_path();
|
|
370
|
+
let source = gen_fixed_bytes(18 * 1024 * 1024);
|
|
371
|
+
op.write(&source_path, source.clone()).await?;
|
|
372
|
+
let source_meta = op.stat(&source_path).await?;
|
|
373
|
+
let source_if_match =
|
|
374
|
+
if op.info().capability().read_with_if_match && op.info().capability().stat_with_if_match {
|
|
375
|
+
source_meta.etag().map(str::to_string)
|
|
376
|
+
} else {
|
|
377
|
+
None
|
|
378
|
+
};
|
|
379
|
+
|
|
380
|
+
let mut writer = op.writer(&target_path).await?;
|
|
381
|
+
writer.write("header").await?;
|
|
382
|
+
writer.copy_from(&source_path, 1024_u64..2048).await?;
|
|
383
|
+
writer
|
|
384
|
+
.copy_from_options(
|
|
385
|
+
&source_path,
|
|
386
|
+
options::ReadOptions {
|
|
387
|
+
range: (2048_u64..14 * 1024 * 1024).into(),
|
|
388
|
+
if_match: source_if_match.clone(),
|
|
389
|
+
..Default::default()
|
|
390
|
+
},
|
|
391
|
+
)
|
|
392
|
+
.await?;
|
|
393
|
+
writer.write("footer").await?;
|
|
394
|
+
writer
|
|
395
|
+
.copy_from_options(
|
|
396
|
+
&source_path,
|
|
397
|
+
options::ReadOptions {
|
|
398
|
+
range: (15 * 1024 * 1024_u64..).into(),
|
|
399
|
+
if_match: source_if_match,
|
|
400
|
+
..Default::default()
|
|
401
|
+
},
|
|
402
|
+
)
|
|
403
|
+
.await?;
|
|
404
|
+
writer.close().await?;
|
|
405
|
+
|
|
406
|
+
let mut expected = Vec::new();
|
|
407
|
+
expected.extend_from_slice(b"header");
|
|
408
|
+
expected.extend_from_slice(&source[1024..14 * 1024 * 1024]);
|
|
409
|
+
expected.extend_from_slice(b"footer");
|
|
410
|
+
expected.extend_from_slice(&source[15 * 1024 * 1024..]);
|
|
411
|
+
assert_eq!(op.read(&target_path).await?.to_bytes(), expected);
|
|
412
|
+
Ok(())
|
|
413
|
+
}
|
|
414
|
+
|
|
352
415
|
/// Append data into writer
|
|
353
416
|
pub async fn test_writer_write_with_concurrent(op: Operator) -> Result<()> {
|
|
354
417
|
if !(op.info().capability().write_can_multi) {
|
|
@@ -810,6 +873,271 @@ pub async fn test_write_with_if_match(op: Operator) -> Result<()> {
|
|
|
810
873
|
Ok(())
|
|
811
874
|
}
|
|
812
875
|
|
|
876
|
+
/// Version preconditions should compare against the current live object version.
|
|
877
|
+
pub async fn test_write_with_version_conditions(op: Operator) -> Result<()> {
|
|
878
|
+
let cap = op.info().capability();
|
|
879
|
+
if !cap.write_with_if_version_match || !cap.write_with_if_version_not_match {
|
|
880
|
+
return Ok(());
|
|
881
|
+
}
|
|
882
|
+
|
|
883
|
+
let path = TEST_FIXTURE.new_file_path();
|
|
884
|
+
let (initial, _) = gen_bytes(cap);
|
|
885
|
+
let (replacement, _) = gen_bytes(cap);
|
|
886
|
+
assert_ne!(initial, replacement);
|
|
887
|
+
|
|
888
|
+
op.write(&path, initial).await?;
|
|
889
|
+
let first_version = op
|
|
890
|
+
.stat(&path)
|
|
891
|
+
.await?
|
|
892
|
+
.version()
|
|
893
|
+
.expect("version must exist")
|
|
894
|
+
.to_string();
|
|
895
|
+
|
|
896
|
+
op.write_with(&path, replacement.clone())
|
|
897
|
+
.if_version_match(&first_version)
|
|
898
|
+
.await?;
|
|
899
|
+
|
|
900
|
+
let current_version = op
|
|
901
|
+
.stat(&path)
|
|
902
|
+
.await?
|
|
903
|
+
.version()
|
|
904
|
+
.expect("version must exist")
|
|
905
|
+
.to_string();
|
|
906
|
+
let err = op
|
|
907
|
+
.write_with(&path, replacement.clone())
|
|
908
|
+
.if_version_match(&first_version)
|
|
909
|
+
.await
|
|
910
|
+
.expect_err("stale version match must fail");
|
|
911
|
+
assert_eq!(err.kind(), ErrorKind::ConditionNotMatch);
|
|
912
|
+
|
|
913
|
+
let err = op
|
|
914
|
+
.write_with(&path, replacement.clone())
|
|
915
|
+
.if_version_not_match(¤t_version)
|
|
916
|
+
.await
|
|
917
|
+
.expect_err("equal version non-match must fail");
|
|
918
|
+
assert_eq!(err.kind(), ErrorKind::ConditionNotMatch);
|
|
919
|
+
op.write_with(&path, replacement)
|
|
920
|
+
.if_version_not_match(&first_version)
|
|
921
|
+
.await?;
|
|
922
|
+
|
|
923
|
+
let missing = TEST_FIXTURE.new_file_path();
|
|
924
|
+
for result in [
|
|
925
|
+
op.write_with(&missing, Vec::<u8>::new())
|
|
926
|
+
.if_version_match(¤t_version)
|
|
927
|
+
.await,
|
|
928
|
+
op.write_with(&missing, Vec::<u8>::new())
|
|
929
|
+
.if_version_not_match(¤t_version)
|
|
930
|
+
.await,
|
|
931
|
+
] {
|
|
932
|
+
assert_eq!(
|
|
933
|
+
result.expect_err("missing target must fail").kind(),
|
|
934
|
+
ErrorKind::ConditionNotMatch
|
|
935
|
+
);
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
Ok(())
|
|
939
|
+
}
|
|
940
|
+
|
|
941
|
+
/// `if_not_changed` should accept the observed state once and reject it after replacement.
|
|
942
|
+
pub async fn test_write_if_not_changed(op: Operator) -> Result<()> {
|
|
943
|
+
let cap = op.info().capability();
|
|
944
|
+
if !cap.write_with_if_version_match && !cap.write_with_if_match {
|
|
945
|
+
return Ok(());
|
|
946
|
+
}
|
|
947
|
+
|
|
948
|
+
let path = TEST_FIXTURE.new_file_path();
|
|
949
|
+
let (initial, _) = gen_bytes(cap);
|
|
950
|
+
let (replacement, _) = gen_bytes(cap);
|
|
951
|
+
assert_ne!(initial, replacement);
|
|
952
|
+
|
|
953
|
+
op.write(&path, initial).await?;
|
|
954
|
+
let expected = op.stat(&path).await?;
|
|
955
|
+
|
|
956
|
+
let mut conflicting = options::WriteOptions {
|
|
957
|
+
if_not_changed: Some(expected.clone()),
|
|
958
|
+
..Default::default()
|
|
959
|
+
};
|
|
960
|
+
let mut matching = conflicting.clone();
|
|
961
|
+
if cap.write_with_if_version_match {
|
|
962
|
+
let version = expected.version().expect("version must exist");
|
|
963
|
+
conflicting.if_version_match = Some(format!("different-{version}"));
|
|
964
|
+
matching.if_version_match = Some(version.to_string());
|
|
965
|
+
} else {
|
|
966
|
+
let etag = expected.etag().expect("etag must exist");
|
|
967
|
+
conflicting.if_match = Some(format!("different-{etag}"));
|
|
968
|
+
matching.if_match = Some(etag.to_string());
|
|
969
|
+
}
|
|
970
|
+
|
|
971
|
+
let err = op
|
|
972
|
+
.write_options(&path, replacement.clone(), conflicting)
|
|
973
|
+
.await
|
|
974
|
+
.expect_err("conflicting explicit condition must fail");
|
|
975
|
+
assert_eq!(err.kind(), ErrorKind::ConditionNotMatch);
|
|
976
|
+
|
|
977
|
+
op.write_options(&path, replacement.clone(), matching)
|
|
978
|
+
.await?;
|
|
979
|
+
let err = op
|
|
980
|
+
.write_with(&path, replacement)
|
|
981
|
+
.if_not_changed(&expected)
|
|
982
|
+
.await
|
|
983
|
+
.expect_err("stale metadata must fail");
|
|
984
|
+
assert_eq!(err.kind(), ErrorKind::ConditionNotMatch);
|
|
985
|
+
|
|
986
|
+
Ok(())
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
/// Write an existing file through a chunked writer with if_not_exists should get a
|
|
990
|
+
/// ConditionNotMatch error.
|
|
991
|
+
pub async fn test_writer_write_with_if_not_exists(op: Operator) -> Result<()> {
|
|
992
|
+
let cap = op.info().capability();
|
|
993
|
+
if !cap.write_with_if_not_exists || !cap.write_can_multi {
|
|
994
|
+
return Ok(());
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
let path = TEST_FIXTURE.new_file_path();
|
|
998
|
+
let content = gen_fixed_bytes(cap.write_multi_min_size.unwrap_or(1));
|
|
999
|
+
|
|
1000
|
+
op.write(&path, content.clone())
|
|
1001
|
+
.await
|
|
1002
|
+
.expect("write must succeed");
|
|
1003
|
+
|
|
1004
|
+
// Some services reject the precondition when the writer is created or on an early
|
|
1005
|
+
// write rather than at commit time
|
|
1006
|
+
let res: opendal::Result<()> = async {
|
|
1007
|
+
let mut w = op.writer_with(&path).if_not_exists(true).await?;
|
|
1008
|
+
w.write(content.clone()).await?;
|
|
1009
|
+
w.write(content.clone()).await?;
|
|
1010
|
+
w.close().await?;
|
|
1011
|
+
Ok(())
|
|
1012
|
+
}
|
|
1013
|
+
.await;
|
|
1014
|
+
assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
|
|
1015
|
+
|
|
1016
|
+
Ok(())
|
|
1017
|
+
}
|
|
1018
|
+
|
|
1019
|
+
/// Write an existing file through a chunked writer with its own etag as if_none_match
|
|
1020
|
+
/// should get a ConditionNotMatch error.
|
|
1021
|
+
pub async fn test_writer_write_with_if_none_match(op: Operator) -> Result<()> {
|
|
1022
|
+
let cap = op.info().capability();
|
|
1023
|
+
if !cap.write_with_if_none_match || !cap.write_can_multi {
|
|
1024
|
+
return Ok(());
|
|
1025
|
+
}
|
|
1026
|
+
|
|
1027
|
+
let path = TEST_FIXTURE.new_file_path();
|
|
1028
|
+
let content = gen_fixed_bytes(cap.write_multi_min_size.unwrap_or(1));
|
|
1029
|
+
|
|
1030
|
+
op.write(&path, content.clone())
|
|
1031
|
+
.await
|
|
1032
|
+
.expect("write must succeed");
|
|
1033
|
+
|
|
1034
|
+
let meta = op.stat(&path).await?;
|
|
1035
|
+
let etag = meta.etag().expect("etag must exist");
|
|
1036
|
+
|
|
1037
|
+
let res: opendal::Result<()> = async {
|
|
1038
|
+
let mut w = op.writer_with(&path).if_none_match(etag).await?;
|
|
1039
|
+
w.write(content.clone()).await?;
|
|
1040
|
+
w.write(content.clone()).await?;
|
|
1041
|
+
w.close().await?;
|
|
1042
|
+
Ok(())
|
|
1043
|
+
}
|
|
1044
|
+
.await;
|
|
1045
|
+
assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
|
|
1046
|
+
|
|
1047
|
+
Ok(())
|
|
1048
|
+
}
|
|
1049
|
+
|
|
1050
|
+
/// Write a file through a chunked writer with if_match should succeed with the file's own
|
|
1051
|
+
/// etag and get a ConditionNotMatch error with a stale one.
|
|
1052
|
+
pub async fn test_writer_write_with_if_match(op: Operator) -> Result<()> {
|
|
1053
|
+
let cap = op.info().capability();
|
|
1054
|
+
if !cap.write_with_if_match || !cap.write_can_multi {
|
|
1055
|
+
return Ok(());
|
|
1056
|
+
}
|
|
1057
|
+
|
|
1058
|
+
let path_a = TEST_FIXTURE.new_file_path();
|
|
1059
|
+
let content_a = gen_fixed_bytes(cap.write_multi_min_size.unwrap_or(1));
|
|
1060
|
+
let (path_b, content_b, _) = TEST_FIXTURE.new_file(op.clone());
|
|
1061
|
+
|
|
1062
|
+
op.write(&path_a, content_a.clone()).await?;
|
|
1063
|
+
op.write(&path_b, content_b.clone()).await?;
|
|
1064
|
+
|
|
1065
|
+
let etag_a = op
|
|
1066
|
+
.stat(&path_a)
|
|
1067
|
+
.await?
|
|
1068
|
+
.etag()
|
|
1069
|
+
.expect("etag must exist")
|
|
1070
|
+
.to_string();
|
|
1071
|
+
let etag_b = op
|
|
1072
|
+
.stat(&path_b)
|
|
1073
|
+
.await?
|
|
1074
|
+
.etag()
|
|
1075
|
+
.expect("etag must exist")
|
|
1076
|
+
.to_string();
|
|
1077
|
+
|
|
1078
|
+
let mut w = op.writer_with(&path_a).if_match(&etag_a).await?;
|
|
1079
|
+
w.write(content_a.clone()).await?;
|
|
1080
|
+
w.write(content_a.clone()).await?;
|
|
1081
|
+
w.close().await.expect("close with own etag must succeed");
|
|
1082
|
+
|
|
1083
|
+
// Should fail: writing to path_a with path_b's etag.
|
|
1084
|
+
let res: opendal::Result<()> = async {
|
|
1085
|
+
let mut w = op.writer_with(&path_a).if_match(&etag_b).await?;
|
|
1086
|
+
w.write(content_a.clone()).await?;
|
|
1087
|
+
w.write(content_a.clone()).await?;
|
|
1088
|
+
w.close().await?;
|
|
1089
|
+
Ok(())
|
|
1090
|
+
}
|
|
1091
|
+
.await;
|
|
1092
|
+
assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
|
|
1093
|
+
|
|
1094
|
+
Ok(())
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
/// Chunked writers should preserve version preconditions through final commit.
|
|
1098
|
+
pub async fn test_writer_write_with_version_conditions(op: Operator) -> Result<()> {
|
|
1099
|
+
let cap = op.info().capability();
|
|
1100
|
+
if !cap.write_can_multi
|
|
1101
|
+
|| !cap.write_with_if_version_match
|
|
1102
|
+
|| !cap.write_with_if_version_not_match
|
|
1103
|
+
{
|
|
1104
|
+
return Ok(());
|
|
1105
|
+
}
|
|
1106
|
+
|
|
1107
|
+
let path = TEST_FIXTURE.new_file_path();
|
|
1108
|
+
let body = gen_fixed_bytes(cap.write_multi_min_size.unwrap_or(1));
|
|
1109
|
+
op.write(&path, body.clone()).await?;
|
|
1110
|
+
let expected = op
|
|
1111
|
+
.stat(&path)
|
|
1112
|
+
.await?
|
|
1113
|
+
.version()
|
|
1114
|
+
.expect("version must exist")
|
|
1115
|
+
.to_string();
|
|
1116
|
+
|
|
1117
|
+
let mut writer = op.writer_with(&path).if_version_match(&expected).await?;
|
|
1118
|
+
writer.write(body.clone()).await?;
|
|
1119
|
+
writer.write(body.clone()).await?;
|
|
1120
|
+
writer.close().await?;
|
|
1121
|
+
|
|
1122
|
+
let current = op
|
|
1123
|
+
.stat(&path)
|
|
1124
|
+
.await?
|
|
1125
|
+
.version()
|
|
1126
|
+
.expect("version must exist")
|
|
1127
|
+
.to_string();
|
|
1128
|
+
let result: opendal::Result<()> = async {
|
|
1129
|
+
let mut writer = op.writer_with(&path).if_version_not_match(¤t).await?;
|
|
1130
|
+
writer.write(body.clone()).await?;
|
|
1131
|
+
writer.write(body).await?;
|
|
1132
|
+
writer.close().await?;
|
|
1133
|
+
Ok(())
|
|
1134
|
+
}
|
|
1135
|
+
.await;
|
|
1136
|
+
assert_eq!(result.unwrap_err().kind(), ErrorKind::ConditionNotMatch);
|
|
1137
|
+
|
|
1138
|
+
Ok(())
|
|
1139
|
+
}
|
|
1140
|
+
|
|
813
1141
|
pub async fn test_writer_write_non_contiguous_data(op: Operator) -> Result<()> {
|
|
814
1142
|
let path = TEST_FIXTURE.new_file_path();
|
|
815
1143
|
let size = 1024 * 1024; // write file with 1 MiB
|
data/core/tests/behavior/main.rs
CHANGED
|
@@ -21,6 +21,7 @@ mod utils;
|
|
|
21
21
|
|
|
22
22
|
pub use utils::*;
|
|
23
23
|
|
|
24
|
+
mod async_compose;
|
|
24
25
|
mod async_copy;
|
|
25
26
|
mod async_create_dir;
|
|
26
27
|
mod async_delete;
|
|
@@ -28,6 +29,7 @@ mod async_list;
|
|
|
28
29
|
mod async_presign;
|
|
29
30
|
mod async_read;
|
|
30
31
|
mod async_rename;
|
|
32
|
+
mod async_restore;
|
|
31
33
|
mod async_stat;
|
|
32
34
|
mod async_write;
|
|
33
35
|
|
|
@@ -55,12 +57,14 @@ fn main() -> anyhow::Result<()> {
|
|
|
55
57
|
let mut tests = Vec::new();
|
|
56
58
|
|
|
57
59
|
async_copy::tests(&op, &mut tests);
|
|
60
|
+
async_compose::tests(&op, &mut tests);
|
|
58
61
|
async_create_dir::tests(&op, &mut tests);
|
|
59
62
|
async_delete::tests(&op, &mut tests);
|
|
60
63
|
async_list::tests(&op, &mut tests);
|
|
61
64
|
async_presign::tests(&op, &mut tests);
|
|
62
65
|
async_read::tests(&op, &mut tests);
|
|
63
66
|
async_rename::tests(&op, &mut tests);
|
|
67
|
+
async_restore::tests(&op, &mut tests);
|
|
64
68
|
async_stat::tests(&op, &mut tests);
|
|
65
69
|
async_write::tests(&op, &mut tests);
|
|
66
70
|
|
data/core/upgrade.md
CHANGED
|
@@ -1,3 +1,116 @@
|
|
|
1
|
+
# Upgrade to v0.59
|
|
2
|
+
|
|
3
|
+
## Public API
|
|
4
|
+
|
|
5
|
+
### Construct `Metadata` with `MetadataBuilder`
|
|
6
|
+
|
|
7
|
+
`Metadata::new`, `Default`, direct metadata setters, and `with_*` methods have been removed. Construct metadata with `MetadataBuilder::file(content_length)`, `MetadataBuilder::dir()`, or `MetadataBuilder::unknown()`, set optional fields on the builder, then call `build()`:
|
|
8
|
+
|
|
9
|
+
```diff
|
|
10
|
+
-let mut metadata = Metadata::new(EntryMode::FILE);
|
|
11
|
+
-metadata.set_content_length(size);
|
|
12
|
+
-metadata.set_content_type("text/plain");
|
|
13
|
+
+let mut builder = MetadataBuilder::file(size);
|
|
14
|
+
+builder.content_type("text/plain");
|
|
15
|
+
+let metadata = builder.build();
|
|
16
|
+
```
|
|
17
|
+
|
|
18
|
+
Consume existing metadata with `into_builder()` when creating a modified value. Use `set_file(content_length)`, `set_dir()`, or `set_unknown()` on the builder to change its mode.
|
|
19
|
+
|
|
20
|
+
`Metadata::user_metadata()` now returns a borrowed `UserMetadata` view instead of `&HashMap<String, String>`. The view supports `get`, `len`, `is_empty`, and `IntoIterator`; collect owned string pairs at boundaries that require a map.
|
|
21
|
+
|
|
22
|
+
Every file `Metadata` contains an explicit full-object content length by construction. Raw services use `Unknown` for incomplete metadata and promote it with `set_file(content_length)` only when the full length is authoritative.
|
|
23
|
+
|
|
24
|
+
Compact metadata and finalized raw-operation value blocks are limited to `u16::MAX` encoded bytes, including indexes. `MetadataBuilder::build`, raw option freezing, and raw listed-entry construction panic when a block exceeds this bound.
|
|
25
|
+
|
|
26
|
+
### Delete inputs use `DeleteOptions`
|
|
27
|
+
|
|
28
|
+
`DeleteInput` has been removed. Pass `(String, DeleteOptions)` to `Deleter`, delete iterators, streams, and sinks when an entry needs options. Plain paths and `Entry` values remain supported through `IntoDeleteInput`. Custom `IntoDeleteInput` implementations must now return `(String, DeleteOptions)` from `into_delete_input`. `(String, OpDelete)` is no longer accepted by the public delete APIs because raw arguments are already capability-resolved.
|
|
29
|
+
|
|
30
|
+
```diff
|
|
31
|
+
-deleter.delete(DeleteInput {
|
|
32
|
+
- path,
|
|
33
|
+
- version,
|
|
34
|
+
- recursive: false,
|
|
35
|
+
-});
|
|
36
|
+
+deleter.delete((
|
|
37
|
+
+ path,
|
|
38
|
+
+ DeleteOptions {
|
|
39
|
+
+ version,
|
|
40
|
+
+ ..Default::default()
|
|
41
|
+
+ },
|
|
42
|
+
+));
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
### Existing option and capability literals require defaults
|
|
46
|
+
|
|
47
|
+
OpenDAL added fields to `Capability`, `ReadOptions`, `ReaderOptions`, `StatOptions`, `WriteOptions`, `DeleteOptions`, and `CopyOptions`. Code that constructs any of these types with an exhaustive struct literal must initialize the new fields. Prefer update syntax so later additions remain source-compatible:
|
|
48
|
+
|
|
49
|
+
```rust
|
|
50
|
+
use opendal_core::Capability;
|
|
51
|
+
use opendal_core::options::DeleteOptions;
|
|
52
|
+
|
|
53
|
+
let options = DeleteOptions {
|
|
54
|
+
recursive: true,
|
|
55
|
+
..Default::default()
|
|
56
|
+
};
|
|
57
|
+
|
|
58
|
+
let capability = Capability {
|
|
59
|
+
read: true,
|
|
60
|
+
..Default::default()
|
|
61
|
+
};
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
### Conditional errors distinguish conflicts
|
|
65
|
+
|
|
66
|
+
Failed conditions supplied through operation options return `ErrorKind::ConditionNotMatch`. Conflicts reported independently by a service return the new `ErrorKind::Conflict`; inspect `Error::is_temporary()` to decide whether the receiving OpenDAL layer can replay the same operation. Unsupported conditions return `ErrorKind::Unsupported`, and a missing target observed by `stat` or `read` remains `ErrorKind::NotFound` even when the request was conditional. Update error recovery code that previously treated every service conflict as `ConditionNotMatch` or `Unexpected`.
|
|
67
|
+
|
|
68
|
+
## Raw API
|
|
69
|
+
|
|
70
|
+
### `OpCopier` merged into `OpCopy`
|
|
71
|
+
|
|
72
|
+
`OpCopier` has been removed. `Service::copy` and `ServiceDyn::copy_dyn` now receive a single `OpCopy` argument, and raw services read `chunk`, `concurrent`, and `source_content_length_hint` from `OpCopy`.
|
|
73
|
+
|
|
74
|
+
### Raw operation arguments are frozen from public options
|
|
75
|
+
|
|
76
|
+
Public `with_*` mutation methods have been removed from `OpRead`, `OpStat`, `OpWrite`, `OpDelete`, `OpCopy`, and `OpList`. Construct non-empty read, stat, list, and restore arguments by converting the corresponding public `*Options`. Use the capability-aware `from_options(&capability, options)` constructors for `OpWrite`, `OpDelete`, `OpCopy`, and `OpCompose`; `new` and `Default` remain available for empty arguments.
|
|
77
|
+
|
|
78
|
+
The direct `From<WriteOptions>`, `From<DeleteOptions>`, and `From<CopyOptions>` conversions have been removed. The capability-aware constructors lower `if_not_changed` before freezing, so raw arguments contain only service-ready primitive conditions. `OpWrite::from_options` still returns `(OpWrite, OpWriter)`, while copy execution settings now live directly in `OpCopy`. `OpWrite::user_metadata()` returns the same borrowed `UserMetadata` view as `Metadata`.
|
|
79
|
+
|
|
80
|
+
### Listed entries contain finalized metadata
|
|
81
|
+
|
|
82
|
+
`raw::oio::Entry::set_mode` and `metadata_mut` have been removed. Build the final `Metadata` first and pass it to `raw::oio::Entry::new` or `with`. To modify an existing raw entry, consume it with `into_parts`, rebuild its metadata, and construct a replacement entry:
|
|
83
|
+
|
|
84
|
+
```rust
|
|
85
|
+
use opendal_core::MetadataBuilder;
|
|
86
|
+
use opendal_core::raw;
|
|
87
|
+
|
|
88
|
+
let entry = raw::oio::Entry::new("path", MetadataBuilder::file(0).build());
|
|
89
|
+
let (path, metadata) = entry.into_parts();
|
|
90
|
+
let mut builder = metadata.into_builder();
|
|
91
|
+
builder.content_type("text/plain");
|
|
92
|
+
let entry = raw::oio::Entry::new(&path, builder.build());
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
The metadata mode must continue to match the path: directory paths end in `/`, while file paths do not.
|
|
96
|
+
|
|
97
|
+
### Do not use numeric `Operation` discriminants
|
|
98
|
+
|
|
99
|
+
The new `Compose` and `Restore` variants changed the implicit numeric discriminants of later `Operation` variants. Code that casts `Operation` with `as usize` must migrate to `Operation::into_static()` or `Display` for stable operation names. `Operation` has no numeric representation contract.
|
|
100
|
+
|
|
101
|
+
## Services
|
|
102
|
+
|
|
103
|
+
### Duration config fields use `SignedDuration`
|
|
104
|
+
|
|
105
|
+
The public `default_ttl` fields on `CloudflareKvConfig`, `MemcachedConfig`, and `RedisConfig` now use `Option<raw::SignedDuration>` instead of `Option<std::time::Duration>`. This allows configuration maps and serialized configuration to parse duration strings. Direct config construction can use the re-exported duration type:
|
|
106
|
+
|
|
107
|
+
```diff
|
|
108
|
+
-default_ttl: Some(std::time::Duration::from_secs(30)),
|
|
109
|
+
+default_ttl: Some(opendal::raw::SignedDuration::from_secs(30)),
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The corresponding builder methods continue to accept `std::time::Duration`.
|
|
113
|
+
|
|
1
114
|
# Upgrade to v0.58
|
|
2
115
|
|
|
3
116
|
## Public API
|
|
@@ -388,7 +501,7 @@ The following new APIs have been added:
|
|
|
388
501
|
- [`Deleter::flush`]
|
|
389
502
|
- [`Deleter::close`]
|
|
390
503
|
- [`Deleter::into_sink`]
|
|
391
|
-
-
|
|
504
|
+
- `DeleteInput`
|
|
392
505
|
- [`IntoDeleteInput`]
|
|
393
506
|
- [`FuturesDeleteSink`]
|
|
394
507
|
|
data/src/capability.rs
CHANGED
data/src/io.rs
CHANGED
data/src/lister.rs
CHANGED
data/src/metadata.rs
CHANGED