opendal 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +89 -56
- data/Cargo.toml +1 -1
- data/DEPENDENCIES.rust.tsv +55 -41
- data/LICENSE +201 -0
- data/LICENSE-MPL-2.0.txt +373 -0
- data/NOTICE +5 -0
- data/core/CHANGELOG.md +175 -0
- data/core/Cargo.lock +225 -298
- data/core/Cargo.toml +100 -94
- data/core/DEPENDENCIES.rust.tsv +44 -73
- data/core/benches/metadata/README.md +21 -0
- data/core/benches/metadata/main.rs +541 -0
- data/core/benches/vs_fs/Cargo.toml +1 -1
- data/core/benches/vs_s3/Cargo.toml +1 -1
- data/core/core/Cargo.toml +7 -2
- data/core/core/src/blocking/delete.rs +5 -1
- data/core/core/src/blocking/operator.rs +12 -0
- data/core/core/src/blocking/read/std_reader.rs +25 -0
- data/core/core/src/docs/concepts.rs +70 -94
- data/core/core/src/docs/internals/accessor.rs +62 -293
- data/core/core/src/docs/internals/layer.rs +43 -59
- data/core/core/src/docs/internals/mod.rs +21 -23
- data/core/core/src/docs/mod.rs +10 -6
- data/core/core/src/docs/performance/concurrent_write.md +92 -55
- data/core/core/src/docs/performance/http_optimization.md +120 -104
- data/core/core/src/docs/performance/mod.rs +8 -6
- data/core/core/src/docs/rfcs/5485_conditional_reader.md +4 -0
- data/core/core/src/docs/rfcs/7182_restore_api.md +139 -557
- data/core/core/src/docs/rfcs/7945_limited_read.md +210 -0
- data/core/core/src/docs/rfcs/7948_s3_provider_presets.md +254 -0
- data/core/core/src/docs/rfcs/8109_writer_copy_from.md +330 -0
- data/core/core/src/docs/rfcs/8145_conflict_error_kind.md +210 -0
- data/core/core/src/docs/rfcs/8147_write_if_not_changed.md +233 -0
- data/core/core/src/docs/rfcs/8188_compose_api.md +292 -0
- data/core/core/src/docs/rfcs/8194_compact_metadata_and_operation_args.md +444 -0
- data/core/core/src/docs/rfcs/README.md +14 -3
- data/core/core/src/docs/rfcs/mod.rs +46 -22
- data/core/core/src/docs/specs/README.md +34 -0
- data/core/core/src/docs/specs/conditional-operations.md +156 -0
- data/core/core/src/docs/specs/metadata.md +63 -0
- data/core/core/src/docs/specs/mod.rs +26 -0
- data/core/core/src/docs/upgrade.md +114 -1
- data/core/core/src/layers/capability_override.rs +15 -2
- data/core/core/src/layers/complete.rs +347 -43
- data/core/core/src/layers/correctness_check.rs +718 -51
- data/core/core/src/layers/error_context.rs +76 -2
- data/core/core/src/layers/simulate.rs +91 -28
- data/core/core/src/lib.rs +1 -0
- data/core/core/src/raw/accessor.rs +91 -15
- data/core/core/src/raw/enum_utils.rs +7 -0
- data/core/core/src/raw/http_util/header.rs +22 -17
- data/core/core/src/raw/oio/compose/api.rs +92 -0
- data/core/core/src/raw/oio/compose/mod.rs +19 -0
- data/core/core/src/raw/oio/copy/api.rs +3 -3
- data/core/core/src/raw/oio/copy/block_copy.rs +32 -4
- data/core/core/src/raw/oio/copy/multipart_copy.rs +36 -7
- data/core/core/src/raw/oio/entry.rs +48 -22
- data/core/core/src/raw/oio/list/flat_list.rs +1 -1
- data/core/core/src/raw/oio/list/hierarchy_list.rs +3 -2
- data/core/core/src/raw/oio/mod.rs +3 -0
- data/core/core/src/raw/oio/read/position_read.rs +1 -1
- data/core/core/src/raw/oio/write/api.rs +40 -0
- data/core/core/src/raw/oio/write/append_write.rs +4 -4
- data/core/core/src/raw/oio/write/block_write.rs +2 -2
- data/core/core/src/raw/oio/write/multipart_write.rs +181 -48
- data/core/core/src/raw/oio/write/position_write.rs +1 -1
- data/core/core/src/raw/operation.rs +6 -0
- data/core/core/src/raw/ops.rs +1191 -456
- data/core/core/src/raw/rps.rs +11 -0
- data/core/core/src/raw/time.rs +14 -3
- data/core/core/src/services/memory/backend.rs +5 -10
- data/core/core/src/services/memory/lister.rs +4 -4
- data/core/core/src/services/memory/writer.rs +7 -6
- data/core/core/src/types/buffer.rs +102 -12
- data/core/core/src/types/bytes_range.rs +20 -1
- data/core/core/src/types/capability.rs +65 -0
- data/core/core/src/types/compact.rs +329 -0
- data/core/core/src/types/compose/composer.rs +183 -0
- data/core/core/src/types/compose/input.rs +57 -0
- data/core/core/src/types/compose/mod.rs +23 -0
- data/core/core/src/types/context/read.rs +16 -11
- data/core/core/src/types/context/write.rs +150 -2
- data/core/core/src/types/copy.rs +5 -3
- data/core/core/src/types/delete/deleter.rs +20 -12
- data/core/core/src/types/delete/input.rs +22 -64
- data/core/core/src/types/delete/mod.rs +0 -1
- data/core/core/src/types/entry.rs +8 -14
- data/core/core/src/types/error.rs +28 -11
- data/core/core/src/types/metadata.rs +709 -322
- data/core/core/src/types/mod.rs +9 -1
- data/core/core/src/types/operator/operator.rs +278 -29
- data/core/core/src/types/operator/operator_futures.rs +466 -105
- data/core/core/src/types/options.rs +631 -158
- data/core/core/src/types/read/futures_async_reader.rs +663 -50
- data/core/core/src/types/read/reader.rs +23 -5
- data/core/core/src/types/write/futures_async_writer.rs +27 -0
- data/core/core/src/types/write/writer.rs +47 -5
- data/core/http-transports/reqwest/Cargo.toml +1 -1
- data/core/layers/async-backtrace/Cargo.toml +2 -2
- data/core/layers/async-backtrace/src/lib.rs +21 -2
- data/core/layers/await-tree/Cargo.toml +2 -2
- data/core/layers/await-tree/src/lib.rs +25 -2
- data/core/layers/capability-check/Cargo.toml +2 -2
- data/core/layers/capability-check/src/lib.rs +98 -10
- data/core/layers/chaos/Cargo.toml +2 -2
- data/core/layers/chaos/src/lib.rs +18 -3
- data/core/layers/concurrent-limit/Cargo.toml +3 -3
- data/core/layers/concurrent-limit/src/lib.rs +53 -13
- data/core/layers/dtrace/Cargo.toml +2 -2
- data/core/layers/dtrace/src/lib.rs +29 -2
- data/core/layers/fastmetrics/Cargo.toml +3 -3
- data/core/layers/fastrace/Cargo.toml +2 -2
- data/core/layers/fastrace/src/lib.rs +28 -2
- data/core/layers/foyer/Cargo.toml +2 -2
- data/core/layers/foyer/src/full.rs +16 -19
- data/core/layers/foyer/src/lib.rs +67 -14
- data/core/layers/hotpath/Cargo.toml +2 -2
- data/core/layers/hotpath/src/lib.rs +20 -2
- data/core/layers/immutable-index/Cargo.toml +2 -2
- data/core/layers/immutable-index/src/lib.rs +48 -24
- data/core/layers/logging/Cargo.toml +2 -2
- data/core/layers/logging/src/lib.rs +46 -2
- data/core/layers/metrics/Cargo.toml +3 -3
- data/core/layers/mime-guess/Cargo.toml +2 -2
- data/core/layers/mime-guess/src/lib.rs +31 -4
- data/core/layers/observe-metrics-common/Cargo.toml +2 -2
- data/core/layers/observe-metrics-common/src/lib.rs +55 -2
- data/core/layers/otelmetrics/Cargo.toml +3 -3
- data/core/layers/oteltrace/Cargo.toml +2 -2
- data/core/layers/oteltrace/src/lib.rs +27 -2
- data/core/layers/prometheus/Cargo.toml +3 -3
- data/core/layers/prometheus-client/Cargo.toml +3 -3
- data/core/layers/retry/Cargo.toml +4 -4
- data/core/layers/retry/src/lib.rs +76 -19
- data/core/layers/route/Cargo.toml +3 -3
- data/core/layers/route/src/lib.rs +34 -7
- data/core/layers/tail-cut/Cargo.toml +2 -2
- data/core/layers/tail-cut/src/lib.rs +28 -2
- data/core/layers/throttle/Cargo.toml +2 -2
- data/core/layers/throttle/src/lib.rs +19 -2
- data/core/layers/timeout/Cargo.toml +3 -3
- data/core/layers/timeout/src/lib.rs +38 -13
- data/core/layers/tracing/Cargo.toml +2 -2
- data/core/layers/tracing/src/lib.rs +24 -3
- data/core/services/aliyun-drive/Cargo.toml +2 -2
- data/core/services/aliyun-drive/src/backend.rs +34 -24
- data/core/services/aliyun-drive/src/core.rs +143 -54
- data/core/services/aliyun-drive/src/lister.rs +19 -18
- data/core/services/aliyun-drive/src/reader.rs +4 -1
- data/core/services/aliyun-drive/src/writer.rs +3 -3
- data/core/services/alluxio/Cargo.toml +1 -1
- data/core/services/alluxio/src/backend.rs +1 -1
- data/core/services/alluxio/src/core.rs +113 -94
- data/core/services/alluxio/src/reader.rs +7 -3
- data/core/services/alluxio/src/writer.rs +2 -2
- data/core/services/azblob/Cargo.toml +2 -2
- data/core/services/azblob/src/backend.rs +17 -9
- data/core/services/azblob/src/copier.rs +41 -15
- data/core/services/azblob/src/core.rs +372 -135
- data/core/services/azblob/src/deleter.rs +41 -18
- data/core/services/azblob/src/lister.rs +16 -10
- data/core/services/azblob/src/reader.rs +4 -1
- data/core/services/azblob/src/writer.rs +48 -15
- data/core/services/azdls/Cargo.toml +3 -3
- data/core/services/azdls/src/backend.rs +45 -15
- data/core/services/azdls/src/config.rs +4 -1
- data/core/services/azdls/src/core.rs +276 -87
- data/core/services/azdls/src/deleter.rs +13 -4
- data/core/services/azdls/src/lister.rs +18 -12
- data/core/services/azdls/src/reader.rs +6 -2
- data/core/services/azdls/src/writer.rs +44 -23
- data/core/services/azfile/Cargo.toml +2 -2
- data/core/services/azfile/src/backend.rs +24 -8
- data/core/services/azfile/src/core.rs +195 -77
- data/core/services/azfile/src/deleter.rs +8 -1
- data/core/services/azfile/src/lister.rs +20 -14
- data/core/services/azfile/src/reader.rs +5 -2
- data/core/services/azfile/src/writer.rs +27 -15
- data/core/services/azure-common/Cargo.toml +1 -1
- data/core/services/b2/Cargo.toml +2 -2
- data/core/services/b2/src/backend.rs +30 -11
- data/core/services/b2/src/core.rs +134 -120
- data/core/services/b2/src/deleter.rs +1 -1
- data/core/services/b2/src/lister.rs +5 -2
- data/core/services/b2/src/reader.rs +5 -2
- data/core/services/b2/src/writer.rs +29 -12
- data/core/services/cacache/Cargo.toml +1 -1
- data/core/services/cacache/src/backend.rs +4 -5
- data/core/services/cacache/src/reader.rs +4 -1
- data/core/services/cacache/src/writer.rs +6 -2
- data/core/services/cloudflare-kv/Cargo.toml +1 -1
- data/core/services/cloudflare-kv/src/backend.rs +26 -15
- data/core/services/cloudflare-kv/src/config.rs +13 -2
- data/core/services/cloudflare-kv/src/core.rs +62 -56
- data/core/services/cloudflare-kv/src/deleter.rs +12 -3
- data/core/services/cloudflare-kv/src/lister.rs +66 -17
- data/core/services/cloudflare-kv/src/reader.rs +14 -9
- data/core/services/cloudflare-kv/src/writer.rs +10 -7
- data/core/services/compfs/Cargo.toml +1 -1
- data/core/services/compfs/src/backend.rs +11 -7
- data/core/services/compfs/src/lister.rs +7 -7
- data/core/services/compfs/src/writer.rs +1 -1
- data/core/services/cos/Cargo.toml +1 -1
- data/core/services/cos/src/backend.rs +129 -16
- data/core/services/cos/src/core.rs +198 -80
- data/core/services/cos/src/deleter.rs +4 -1
- data/core/services/cos/src/lister.rs +42 -27
- data/core/services/cos/src/reader.rs +6 -2
- data/core/services/cos/src/writer.rs +45 -21
- data/core/services/d1/Cargo.toml +1 -1
- data/core/services/d1/src/backend.rs +6 -5
- data/core/services/d1/src/core.rs +79 -63
- data/core/services/d1/src/reader.rs +4 -1
- data/core/services/d1/src/writer.rs +6 -2
- data/core/services/dashmap/Cargo.toml +1 -1
- data/core/services/dashmap/src/backend.rs +2 -2
- data/core/services/dashmap/src/reader.rs +4 -1
- data/core/services/dashmap/src/writer.rs +11 -8
- data/core/services/dbfs/Cargo.toml +1 -1
- data/core/services/dbfs/src/backend.rs +21 -15
- data/core/services/dbfs/src/core.rs +65 -60
- data/core/services/dbfs/src/deleter.rs +4 -1
- data/core/services/dbfs/src/lister.rs +10 -10
- data/core/services/dbfs/src/writer.rs +6 -3
- data/core/services/dropbox/Cargo.toml +2 -2
- data/core/services/dropbox/src/backend.rs +30 -23
- data/core/services/dropbox/src/core.rs +65 -63
- data/core/services/dropbox/src/deleter.rs +1 -1
- data/core/services/dropbox/src/lister.rs +17 -8
- data/core/services/dropbox/src/reader.rs +4 -1
- data/core/services/dropbox/src/writer.rs +13 -10
- data/core/services/etcd/Cargo.toml +1 -1
- data/core/services/etcd/src/backend.rs +8 -5
- data/core/services/etcd/src/core.rs +5 -11
- data/core/services/etcd/src/lister.rs +6 -1
- data/core/services/etcd/src/reader.rs +4 -1
- data/core/services/etcd/src/writer.rs +6 -3
- data/core/services/foundationdb/Cargo.toml +1 -1
- data/core/services/foundationdb/src/backend.rs +6 -5
- data/core/services/foundationdb/src/reader.rs +4 -1
- data/core/services/foundationdb/src/writer.rs +6 -2
- data/core/services/foyer/Cargo.toml +3 -3
- data/core/services/foyer/src/backend.rs +6 -5
- data/core/services/foyer/src/reader.rs +4 -1
- data/core/services/foyer/src/writer.rs +6 -2
- data/core/services/fs/Cargo.toml +1 -1
- data/core/services/fs/src/backend.rs +4 -3
- data/core/services/fs/src/core.rs +44 -26
- data/core/services/fs/src/lister.rs +9 -6
- data/core/services/fs/src/writer.rs +20 -13
- data/core/services/ftp/Cargo.toml +4 -6
- data/core/services/ftp/src/backend.rs +9 -5
- data/core/services/ftp/src/core.rs +9 -7
- data/core/services/ftp/src/lister.rs +7 -8
- data/core/services/ftp/src/reader.rs +3 -3
- data/core/services/ftp/src/writer.rs +3 -3
- data/core/services/gcs/Cargo.toml +4 -2
- data/core/services/gcs/src/backend.rs +53 -7
- data/core/services/gcs/src/composer.rs +705 -0
- data/core/services/gcs/src/copier.rs +14 -4
- data/core/services/gcs/src/core.rs +828 -124
- data/core/services/gcs/src/deleter.rs +33 -18
- data/core/services/gcs/src/lib.rs +1 -0
- data/core/services/gcs/src/lister.rs +21 -12
- data/core/services/gcs/src/reader.rs +12 -5
- data/core/services/gcs/src/writer.rs +632 -10
- data/core/services/gcs-grpc/Cargo.toml +58 -0
- data/core/services/gcs-grpc/README.md +19 -0
- data/core/services/gcs-grpc/proto/README.md +23 -0
- data/core/services/gcs-grpc/proto/google/storage/v2/storage.proto +171 -0
- data/core/services/gcs-grpc/proto/upstream-descriptor.bin +0 -0
- data/core/services/gcs-grpc/src/backend.rs +449 -0
- data/core/services/gcs-grpc/src/config.rs +116 -0
- data/core/services/gcs-grpc/src/copier.rs +98 -0
- data/core/services/gcs-grpc/src/core.rs +298 -0
- data/core/services/gcs-grpc/src/deleter.rs +65 -0
- data/core/services/gcs-grpc/src/docs.md +19 -0
- data/core/services/gcs-grpc/src/generated/google.storage.v2.rs +541 -0
- data/core/services/gcs-grpc/src/generated/mod.rs +24 -0
- data/core/services/gcs-grpc/src/lib.rs +43 -0
- data/core/services/gcs-grpc/src/lister.rs +125 -0
- data/core/services/gcs-grpc/src/reader.rs +140 -0
- data/core/services/gcs-grpc/src/writer.rs +780 -0
- data/core/services/gdrive/Cargo.toml +2 -2
- data/core/services/gdrive/src/backend.rs +123 -69
- data/core/services/gdrive/src/core.rs +87 -69
- data/core/services/gdrive/src/deleter.rs +8 -2
- data/core/services/gdrive/src/lister.rs +38 -26
- data/core/services/gdrive/src/path_index.rs +2 -2
- data/core/services/gdrive/src/reader.rs +9 -3
- data/core/services/gdrive/src/writer.rs +13 -7
- data/core/services/ghac/Cargo.toml +2 -2
- data/core/services/ghac/src/backend.rs +5 -2
- data/core/services/ghac/src/core.rs +59 -37
- data/core/services/ghac/src/reader.rs +5 -2
- data/core/services/ghac/src/writer.rs +16 -6
- data/core/services/github/Cargo.toml +1 -1
- data/core/services/github/src/backend.rs +16 -8
- data/core/services/github/src/core.rs +88 -76
- data/core/services/github/src/lister.rs +17 -15
- data/core/services/github/src/reader.rs +5 -2
- data/core/services/github/src/writer.rs +12 -6
- data/core/services/goosefs/Cargo.toml +2 -2
- data/core/services/goosefs/src/backend.rs +2 -2
- data/core/services/goosefs/src/core.rs +291 -208
- data/core/services/goosefs/src/docs.md +3 -0
- data/core/services/goosefs/src/lister.rs +2 -3
- data/core/services/goosefs/src/reader.rs +7 -3
- data/core/services/goosefs/src/writer.rs +21 -10
- data/core/services/goosefs/tests/safe_conditional_put.rs +37 -0
- data/core/services/gridfs/Cargo.toml +2 -2
- data/core/services/gridfs/src/backend.rs +12 -8
- data/core/services/gridfs/src/core.rs +1 -1
- data/core/services/gridfs/src/reader.rs +4 -1
- data/core/services/gridfs/src/writer.rs +6 -2
- data/core/services/hdfs/Cargo.toml +1 -1
- data/core/services/hdfs/src/backend.rs +1 -1
- data/core/services/hdfs/src/core.rs +7 -4
- data/core/services/hdfs/src/lister.rs +7 -9
- data/core/services/hdfs/src/writer.rs +5 -1
- data/core/services/hdfs-native/Cargo.toml +1 -1
- data/core/services/hdfs-native/src/backend.rs +1 -1
- data/core/services/hdfs-native/src/core.rs +44 -50
- data/core/services/hdfs-native/src/lister.rs +8 -10
- data/core/services/hdfs-native/src/writer.rs +5 -1
- data/core/services/hf/Cargo.toml +4 -3
- data/core/services/hf/src/backend.rs +4 -3
- data/core/services/hf/src/core.rs +907 -153
- data/core/services/hf/src/lister.rs +16 -6
- data/core/services/hf/src/reader.rs +377 -57
- data/core/services/hf/src/writer.rs +6 -2
- data/core/services/http/Cargo.toml +3 -3
- data/core/services/http/src/backend.rs +8 -6
- data/core/services/http/src/core.rs +57 -35
- data/core/services/http/src/reader.rs +5 -2
- data/core/services/ipfs/Cargo.toml +1 -1
- data/core/services/ipfs/src/backend.rs +20 -9
- data/core/services/ipfs/src/core.rs +43 -34
- data/core/services/ipfs/src/reader.rs +5 -2
- data/core/services/ipmfs/Cargo.toml +1 -1
- data/core/services/ipmfs/src/backend.rs +17 -8
- data/core/services/ipmfs/src/core.rs +66 -60
- data/core/services/ipmfs/src/deleter.rs +5 -2
- data/core/services/ipmfs/src/lister.rs +15 -12
- data/core/services/ipmfs/src/reader.rs +5 -2
- data/core/services/ipmfs/src/writer.rs +6 -3
- data/core/services/koofr/Cargo.toml +2 -2
- data/core/services/koofr/src/backend.rs +46 -17
- data/core/services/koofr/src/core.rs +64 -48
- data/core/services/koofr/src/deleter.rs +4 -1
- data/core/services/koofr/src/lister.rs +11 -10
- data/core/services/koofr/src/reader.rs +5 -2
- data/core/services/koofr/src/writer.rs +13 -7
- data/core/services/lakefs/Cargo.toml +1 -1
- data/core/services/lakefs/src/backend.rs +25 -8
- data/core/services/lakefs/src/core.rs +79 -73
- data/core/services/lakefs/src/deleter.rs +4 -1
- data/core/services/lakefs/src/lister.rs +17 -14
- data/core/services/lakefs/src/reader.rs +5 -2
- data/core/services/lakefs/src/writer.rs +11 -10
- data/core/services/memcached/Cargo.toml +1 -1
- data/core/services/memcached/src/backend.rs +57 -7
- data/core/services/memcached/src/config.rs +13 -2
- data/core/services/memcached/src/core.rs +11 -1
- data/core/services/memcached/src/reader.rs +4 -1
- data/core/services/memcached/src/writer.rs +6 -2
- data/core/services/mini_moka/Cargo.toml +1 -1
- data/core/services/mini_moka/src/backend.rs +8 -8
- data/core/services/mini_moka/src/reader.rs +4 -1
- data/core/services/mini_moka/src/writer.rs +11 -7
- data/core/services/moka/Cargo.toml +1 -1
- data/core/services/moka/src/backend.rs +7 -7
- data/core/services/moka/src/reader.rs +4 -1
- data/core/services/moka/src/writer.rs +10 -6
- data/core/services/mongodb/Cargo.toml +2 -2
- data/core/services/mongodb/src/backend.rs +7 -6
- data/core/services/mongodb/src/core.rs +1 -1
- data/core/services/mongodb/src/reader.rs +4 -1
- data/core/services/mongodb/src/writer.rs +6 -2
- data/core/services/monoiofs/Cargo.toml +2 -2
- data/core/services/monoiofs/src/backend.rs +16 -10
- data/core/services/monoiofs/src/core.rs +8 -9
- data/core/services/monoiofs/src/reader.rs +8 -10
- data/core/services/monoiofs/src/writer.rs +19 -24
- data/core/services/mysql/Cargo.toml +2 -2
- data/core/services/mysql/src/backend.rs +7 -6
- data/core/services/mysql/src/core.rs +1 -1
- data/core/services/mysql/src/lister.rs +7 -3
- data/core/services/mysql/src/reader.rs +4 -1
- data/core/services/mysql/src/writer.rs +6 -2
- data/core/services/obs/Cargo.toml +1 -1
- data/core/services/obs/src/backend.rs +29 -11
- data/core/services/obs/src/core.rs +81 -82
- data/core/services/obs/src/deleter.rs +4 -1
- data/core/services/obs/src/lister.rs +12 -5
- data/core/services/obs/src/reader.rs +5 -2
- data/core/services/obs/src/writer.rs +40 -19
- data/core/services/onedrive/Cargo.toml +2 -2
- data/core/services/onedrive/src/backend.rs +30 -21
- data/core/services/onedrive/src/config.rs +2 -2
- data/core/services/onedrive/src/core.rs +191 -90
- data/core/services/onedrive/src/deleter.rs +5 -2
- data/core/services/onedrive/src/graph_model.rs +10 -51
- data/core/services/onedrive/src/lister.rs +60 -24
- data/core/services/onedrive/src/reader.rs +17 -2
- data/core/services/onedrive/src/writer.rs +26 -15
- data/core/services/opfs/Cargo.toml +1 -1
- data/core/services/opfs/src/backend.rs +6 -13
- data/core/services/opfs/src/core.rs +33 -37
- data/core/services/opfs/src/lister.rs +4 -4
- data/core/services/opfs/src/writer.rs +1 -3
- data/core/services/oss/Cargo.toml +1 -1
- data/core/services/oss/src/backend.rs +40 -8
- data/core/services/oss/src/core.rs +192 -92
- data/core/services/oss/src/deleter.rs +16 -6
- data/core/services/oss/src/lister.rs +43 -27
- data/core/services/oss/src/reader.rs +6 -2
- data/core/services/oss/src/writer.rs +40 -22
- data/core/services/pcloud/Cargo.toml +1 -1
- data/core/services/pcloud/src/backend.rs +40 -6
- data/core/services/pcloud/src/core.rs +80 -66
- data/core/services/pcloud/src/deleter.rs +8 -1
- data/core/services/pcloud/src/lister.rs +4 -1
- data/core/services/pcloud/src/reader.rs +5 -2
- data/core/services/pcloud/src/writer.rs +6 -3
- data/core/services/persy/Cargo.toml +1 -1
- data/core/services/persy/src/backend.rs +6 -5
- data/core/services/persy/src/core.rs +60 -15
- data/core/services/persy/src/reader.rs +4 -1
- data/core/services/persy/src/writer.rs +6 -2
- data/core/services/postgresql/Cargo.toml +2 -2
- data/core/services/postgresql/src/backend.rs +7 -6
- data/core/services/postgresql/src/core.rs +1 -1
- data/core/services/postgresql/src/reader.rs +4 -1
- data/core/services/postgresql/src/writer.rs +6 -2
- data/core/services/redb/Cargo.toml +1 -1
- data/core/services/redb/src/backend.rs +6 -5
- data/core/services/redb/src/reader.rs +4 -1
- data/core/services/redb/src/writer.rs +6 -2
- data/core/services/redis/Cargo.toml +1 -1
- data/core/services/redis/src/backend.rs +18 -8
- data/core/services/redis/src/config.rs +13 -2
- data/core/services/redis/src/reader.rs +8 -4
- data/core/services/redis/src/writer.rs +6 -2
- data/core/services/rocksdb/Cargo.toml +1 -1
- data/core/services/rocksdb/src/backend.rs +6 -5
- data/core/services/rocksdb/src/lister.rs +6 -5
- data/core/services/rocksdb/src/reader.rs +4 -1
- data/core/services/rocksdb/src/writer.rs +6 -2
- data/core/services/s3/Cargo.toml +5 -4
- data/core/services/s3/src/backend.rs +995 -39
- data/core/services/s3/src/compatible_services.md +57 -0
- data/core/services/s3/src/config.rs +16 -0
- data/core/services/s3/src/copier.rs +190 -55
- data/core/services/s3/src/core.rs +299 -190
- data/core/services/s3/src/deleter.rs +26 -7
- data/core/services/s3/src/docs.md +57 -0
- data/core/services/s3/src/lister.rs +55 -36
- data/core/services/s3/src/reader.rs +22 -2
- data/core/services/s3/src/writer.rs +154 -32
- data/core/services/seafile/Cargo.toml +2 -2
- data/core/services/seafile/src/backend.rs +3 -3
- data/core/services/seafile/src/core.rs +105 -79
- data/core/services/seafile/src/lister.rs +11 -8
- data/core/services/seafile/src/reader.rs +5 -2
- data/core/services/seafile/src/writer.rs +6 -3
- data/core/services/sftp/Cargo.toml +1 -1
- data/core/services/sftp/src/backend.rs +19 -4
- data/core/services/sftp/src/core.rs +51 -46
- data/core/services/sftp/src/lister.rs +4 -4
- data/core/services/sftp/src/writer.rs +1 -1
- data/core/services/sled/Cargo.toml +1 -1
- data/core/services/sled/src/backend.rs +6 -5
- data/core/services/sled/src/core.rs +28 -6
- data/core/services/sled/src/lister.rs +6 -5
- data/core/services/sled/src/reader.rs +4 -1
- data/core/services/sled/src/writer.rs +6 -2
- data/core/services/sqlite/Cargo.toml +2 -2
- data/core/services/sqlite/src/backend.rs +12 -7
- data/core/services/sqlite/src/core.rs +3 -3
- data/core/services/sqlite/src/reader.rs +4 -1
- data/core/services/sqlite/src/writer.rs +7 -3
- data/core/services/surrealdb/Cargo.toml +2 -2
- data/core/services/surrealdb/src/backend.rs +7 -6
- data/core/services/surrealdb/src/core.rs +1 -1
- data/core/services/surrealdb/src/reader.rs +4 -1
- data/core/services/surrealdb/src/writer.rs +6 -2
- data/core/services/swift/Cargo.toml +1 -1
- data/core/services/swift/src/backend.rs +38 -8
- data/core/services/swift/src/core.rs +63 -67
- data/core/services/swift/src/deleter.rs +8 -2
- data/core/services/swift/src/lister.rs +11 -8
- data/core/services/swift/src/reader.rs +5 -2
- data/core/services/swift/src/writer.rs +17 -8
- data/core/services/tikv/Cargo.toml +2 -2
- data/core/services/tikv/src/backend.rs +7 -6
- data/core/services/tikv/src/core.rs +31 -14
- data/core/services/tikv/src/reader.rs +4 -1
- data/core/services/tikv/src/writer.rs +6 -2
- data/core/services/tos/Cargo.toml +2 -2
- data/core/services/tos/src/backend.rs +45 -21
- data/core/services/tos/src/copier.rs +44 -17
- data/core/services/tos/src/core.rs +147 -117
- data/core/services/tos/src/deleter.rs +9 -3
- data/core/services/tos/src/docs.md +1 -1
- data/core/services/tos/src/lister.rs +43 -28
- data/core/services/tos/src/reader.rs +5 -2
- data/core/services/tos/src/writer.rs +37 -16
- data/core/services/upyun/Cargo.toml +1 -1
- data/core/services/upyun/src/backend.rs +30 -7
- data/core/services/upyun/src/core.rs +122 -87
- data/core/services/upyun/src/deleter.rs +4 -1
- data/core/services/upyun/src/lister.rs +10 -10
- data/core/services/upyun/src/reader.rs +5 -2
- data/core/services/upyun/src/writer.rs +19 -7
- data/core/services/vercel-artifacts/Cargo.toml +1 -1
- data/core/services/vercel-artifacts/src/backend.rs +6 -3
- data/core/services/vercel-artifacts/src/core.rs +32 -27
- data/core/services/vercel-artifacts/src/reader.rs +5 -2
- data/core/services/vercel-artifacts/src/writer.rs +6 -3
- data/core/services/vercel-blob/Cargo.toml +1 -1
- data/core/services/vercel-blob/src/backend.rs +23 -6
- data/core/services/vercel-blob/src/core.rs +92 -81
- data/core/services/vercel-blob/src/lister.rs +12 -3
- data/core/services/vercel-blob/src/reader.rs +5 -2
- data/core/services/vercel-blob/src/writer.rs +19 -7
- data/core/services/webdav/Cargo.toml +2 -2
- data/core/services/webdav/src/backend.rs +25 -5
- data/core/services/webdav/src/core.rs +116 -59
- data/core/services/webdav/src/deleter.rs +4 -1
- data/core/services/webdav/src/lister.rs +4 -1
- data/core/services/webdav/src/reader.rs +6 -2
- data/core/services/webdav/src/writer.rs +17 -7
- data/core/services/webhdfs/Cargo.toml +2 -2
- data/core/services/webhdfs/src/backend.rs +24 -11
- data/core/services/webhdfs/src/core.rs +99 -98
- data/core/services/webhdfs/src/deleter.rs +5 -2
- data/core/services/webhdfs/src/lister.rs +21 -9
- data/core/services/webhdfs/src/reader.rs +5 -2
- data/core/services/webhdfs/src/writer.rs +49 -14
- data/core/services/yandex-disk/Cargo.toml +1 -1
- data/core/services/yandex-disk/src/backend.rs +28 -6
- data/core/services/yandex-disk/src/core.rs +97 -79
- data/core/services/yandex-disk/src/deleter.rs +4 -1
- data/core/services/yandex-disk/src/lister.rs +25 -9
- data/core/services/yandex-disk/src/reader.rs +5 -2
- data/core/services/yandex-disk/src/writer.rs +6 -3
- data/core/src/lib.rs +5 -0
- data/core/testkit/Cargo.toml +4 -4
- data/core/testkit/src/utils.rs +53 -8
- data/core/tests/behavior/async_compose.rs +297 -0
- data/core/tests/behavior/async_copy.rs +162 -1
- data/core/tests/behavior/async_delete.rs +364 -3
- data/core/tests/behavior/async_read.rs +86 -0
- data/core/tests/behavior/async_restore.rs +162 -0
- data/core/tests/behavior/async_stat.rs +90 -0
- data/core/tests/behavior/async_write.rs +330 -2
- data/core/tests/behavior/main.rs +4 -0
- data/core/upgrade.md +114 -1
- data/src/capability.rs +0 -1
- data/src/io.rs +0 -1
- data/src/lister.rs +0 -1
- data/src/metadata.rs +0 -1
- data/src/operator_info.rs +0 -1
- metadata +44 -2
|
@@ -0,0 +1,705 @@
|
|
|
1
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
// or more contributor license agreements. See the NOTICE file
|
|
3
|
+
// distributed with this work for additional information
|
|
4
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
// to you under the Apache License, Version 2.0 (the
|
|
6
|
+
// "License"); you may not use this file except in compliance
|
|
7
|
+
// with the License. You may obtain a copy of the License at
|
|
8
|
+
//
|
|
9
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
//
|
|
11
|
+
// Unless required by applicable law or agreed to in writing,
|
|
12
|
+
// software distributed under the License is distributed on an
|
|
13
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
// KIND, either express or implied. See the License for the
|
|
15
|
+
// specific language governing permissions and limitations
|
|
16
|
+
// under the License.
|
|
17
|
+
|
|
18
|
+
use std::collections::HashMap;
|
|
19
|
+
use std::sync::Arc;
|
|
20
|
+
|
|
21
|
+
use super::core::ErrorContext;
|
|
22
|
+
use super::core::GcsComposeSource;
|
|
23
|
+
use super::core::GcsCore;
|
|
24
|
+
use super::core::parse_error;
|
|
25
|
+
use super::deleter::GcsDeleter;
|
|
26
|
+
use opendal_core::raw::oio::Delete;
|
|
27
|
+
use opendal_core::raw::*;
|
|
28
|
+
use opendal_core::*;
|
|
29
|
+
|
|
30
|
+
const GCS_COMPOSE_MAX_SOURCES: usize = 32;
|
|
31
|
+
const COMPOSE_TOKEN_KEY: &str = "opendal-compose-token";
|
|
32
|
+
|
|
33
|
+
pub struct GcsComposer {
|
|
34
|
+
core: Arc<GcsCore>,
|
|
35
|
+
ctx: OperationContext,
|
|
36
|
+
to: String,
|
|
37
|
+
args: OpCompose,
|
|
38
|
+
buffered: Vec<GcsComposeSource>,
|
|
39
|
+
tasks: ConcurrentTasks<GcsComposeTask, GcsComposeSource>,
|
|
40
|
+
input_count: usize,
|
|
41
|
+
intermediates: Vec<GcsComposeSource>,
|
|
42
|
+
metadata: Option<Metadata>,
|
|
43
|
+
errored: bool,
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
impl GcsComposer {
|
|
47
|
+
pub fn new(core: Arc<GcsCore>, ctx: OperationContext, to: &str, args: OpCompose) -> Self {
|
|
48
|
+
let tasks = new_tasks(ctx.executor().clone(), args.concurrent());
|
|
49
|
+
Self {
|
|
50
|
+
core,
|
|
51
|
+
ctx,
|
|
52
|
+
to: to.to_string(),
|
|
53
|
+
args,
|
|
54
|
+
buffered: Vec::with_capacity(GCS_COMPOSE_MAX_SOURCES + 1),
|
|
55
|
+
tasks,
|
|
56
|
+
input_count: 0,
|
|
57
|
+
intermediates: Vec::new(),
|
|
58
|
+
metadata: None,
|
|
59
|
+
errored: false,
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
fn new_intermediate_task(&self, sources: Vec<GcsComposeSource>) -> Result<GcsComposeTask> {
|
|
64
|
+
let token = uuid::Uuid::new_v4().to_string();
|
|
65
|
+
let to = format!("__opendal/compose/{token}");
|
|
66
|
+
let mut metadata = HashMap::new();
|
|
67
|
+
metadata.insert(COMPOSE_TOKEN_KEY.to_string(), token.clone());
|
|
68
|
+
let args = OpCompose::from_options(
|
|
69
|
+
&self.core.capability,
|
|
70
|
+
options::ComposeOptions {
|
|
71
|
+
if_not_exists: true,
|
|
72
|
+
content_type: Some("application/octet-stream".to_string()),
|
|
73
|
+
user_metadata: Some(metadata),
|
|
74
|
+
..Default::default()
|
|
75
|
+
},
|
|
76
|
+
)?;
|
|
77
|
+
|
|
78
|
+
Ok(GcsComposeTask {
|
|
79
|
+
core: self.core.clone(),
|
|
80
|
+
ctx: self.ctx.clone(),
|
|
81
|
+
sources,
|
|
82
|
+
to,
|
|
83
|
+
args,
|
|
84
|
+
token: Some(token),
|
|
85
|
+
})
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
async fn collect_tasks(
|
|
89
|
+
tasks: &mut ConcurrentTasks<GcsComposeTask, GcsComposeSource>,
|
|
90
|
+
) -> Result<Vec<GcsComposeSource>> {
|
|
91
|
+
let mut outputs = Vec::new();
|
|
92
|
+
while let Some(output) = tasks.next().await {
|
|
93
|
+
match output {
|
|
94
|
+
Ok(output) => outputs.push(output),
|
|
95
|
+
Err(err) if err.is_temporary() => continue,
|
|
96
|
+
Err(err) => return Err(err),
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
Ok(outputs)
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
async fn execute_task(
|
|
103
|
+
tasks: &mut ConcurrentTasks<GcsComposeTask, GcsComposeSource>,
|
|
104
|
+
task: GcsComposeTask,
|
|
105
|
+
) -> Result<()> {
|
|
106
|
+
loop {
|
|
107
|
+
match tasks.execute(task.clone()).await {
|
|
108
|
+
Ok(()) => return Ok(()),
|
|
109
|
+
Err(err) if err.is_temporary() => continue,
|
|
110
|
+
Err(err) => return Err(err),
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
async fn reduce_level(
|
|
116
|
+
&mut self,
|
|
117
|
+
sources: Vec<GcsComposeSource>,
|
|
118
|
+
) -> Result<Vec<GcsComposeSource>> {
|
|
119
|
+
let mut tasks = new_tasks(self.ctx.executor().clone(), self.args.concurrent());
|
|
120
|
+
let mut carried = None;
|
|
121
|
+
|
|
122
|
+
for chunk in sources.chunks(GCS_COMPOSE_MAX_SOURCES) {
|
|
123
|
+
if chunk.len() == 1 {
|
|
124
|
+
carried = Some(chunk[0].clone());
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
Self::execute_task(&mut tasks, self.new_intermediate_task(chunk.to_vec())?).await?;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
let mut outputs = Self::collect_tasks(&mut tasks).await?;
|
|
131
|
+
self.intermediates.extend(outputs.iter().cloned());
|
|
132
|
+
if let Some(source) = carried {
|
|
133
|
+
outputs.push(source);
|
|
134
|
+
}
|
|
135
|
+
Ok(outputs)
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async fn compose_final(&self, sources: &[GcsComposeSource]) -> Result<Metadata> {
|
|
139
|
+
let resp = self
|
|
140
|
+
.core
|
|
141
|
+
.gcs_compose_object(&self.ctx, sources, &self.to, &self.args)
|
|
142
|
+
.await?;
|
|
143
|
+
if !resp.status().is_success() {
|
|
144
|
+
return Err(parse_error(
|
|
145
|
+
ErrorContext::new(ServiceOperation("ComposeObject"))
|
|
146
|
+
.with_caller_condition(self.args.is_conditional()),
|
|
147
|
+
resp,
|
|
148
|
+
));
|
|
149
|
+
}
|
|
150
|
+
GcsCore::build_metadata_from_object_response(&self.to, resp.into_body())
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
async fn cleanup_intermediates(&self) -> Result<()> {
|
|
154
|
+
let mut deleter = oio::BatchDeleter::new(
|
|
155
|
+
GcsDeleter::new(self.core.clone(), self.ctx.clone()),
|
|
156
|
+
self.core.capability.delete_max_size,
|
|
157
|
+
);
|
|
158
|
+
for source in &self.intermediates {
|
|
159
|
+
let Some(version) = source.version.as_deref() else {
|
|
160
|
+
continue;
|
|
161
|
+
};
|
|
162
|
+
let args = OpDelete::from_options(
|
|
163
|
+
&self.core.capability,
|
|
164
|
+
options::DeleteOptions {
|
|
165
|
+
if_version_match: Some(version.to_string()),
|
|
166
|
+
..Default::default()
|
|
167
|
+
},
|
|
168
|
+
)?;
|
|
169
|
+
deleter.delete(&source.path, args).await?;
|
|
170
|
+
}
|
|
171
|
+
deleter.close().await
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
async fn close_inner(&mut self) -> Result<Metadata> {
|
|
175
|
+
if self.input_count == 0 {
|
|
176
|
+
return Err(Error::new(
|
|
177
|
+
ErrorKind::ConfigInvalid,
|
|
178
|
+
"compose requires at least one source object",
|
|
179
|
+
));
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
let mut sources = Self::collect_tasks(&mut self.tasks).await?;
|
|
183
|
+
self.intermediates.extend(sources.iter().cloned());
|
|
184
|
+
sources.append(&mut self.buffered);
|
|
185
|
+
|
|
186
|
+
while sources.len() > GCS_COMPOSE_MAX_SOURCES {
|
|
187
|
+
sources = self.reduce_level(sources).await?;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
let metadata = self.compose_final(&sources).await?;
|
|
191
|
+
if let Err(err) = self.cleanup_intermediates().await {
|
|
192
|
+
log::warn!("failed to clean GCS compose intermediate objects: {err}");
|
|
193
|
+
}
|
|
194
|
+
Ok(metadata)
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
impl oio::Compose for GcsComposer {
|
|
199
|
+
async fn compose(&mut self, path: &str, args: OpRead) -> Result<()> {
|
|
200
|
+
if self.errored {
|
|
201
|
+
return Err(Error::new(
|
|
202
|
+
ErrorKind::Unexpected,
|
|
203
|
+
"GCS composer has already failed",
|
|
204
|
+
));
|
|
205
|
+
}
|
|
206
|
+
if self.metadata.is_some() {
|
|
207
|
+
return Err(Error::new(
|
|
208
|
+
ErrorKind::Unexpected,
|
|
209
|
+
"GCS composer is already closed",
|
|
210
|
+
));
|
|
211
|
+
}
|
|
212
|
+
if path == self.to {
|
|
213
|
+
return Err(Error::new(
|
|
214
|
+
ErrorKind::IsSameFile,
|
|
215
|
+
"source and destination paths are same",
|
|
216
|
+
));
|
|
217
|
+
}
|
|
218
|
+
if args.if_match().is_some()
|
|
219
|
+
|| args.if_none_match().is_some()
|
|
220
|
+
|| args.if_version_not_match().is_some()
|
|
221
|
+
{
|
|
222
|
+
return Err(Error::new(
|
|
223
|
+
ErrorKind::Unsupported,
|
|
224
|
+
"GCS compose supports source generation conditions only",
|
|
225
|
+
));
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
self.buffered.push(GcsComposeSource {
|
|
229
|
+
path: path.to_string(),
|
|
230
|
+
version: args.version().map(str::to_string),
|
|
231
|
+
if_version_match: args.if_version_match().map(str::to_string),
|
|
232
|
+
});
|
|
233
|
+
self.input_count += 1;
|
|
234
|
+
|
|
235
|
+
if self.buffered.len() > GCS_COMPOSE_MAX_SOURCES {
|
|
236
|
+
let remaining = self.buffered.split_off(GCS_COMPOSE_MAX_SOURCES);
|
|
237
|
+
let sources = std::mem::replace(&mut self.buffered, remaining);
|
|
238
|
+
let task = match self.new_intermediate_task(sources) {
|
|
239
|
+
Ok(task) => task,
|
|
240
|
+
Err(err) => {
|
|
241
|
+
self.errored = true;
|
|
242
|
+
return Err(err);
|
|
243
|
+
}
|
|
244
|
+
};
|
|
245
|
+
if let Err(err) = Self::execute_task(&mut self.tasks, task).await {
|
|
246
|
+
self.errored = true;
|
|
247
|
+
return Err(err);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
Ok(())
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
async fn close(&mut self) -> Result<Metadata> {
|
|
254
|
+
if self.errored {
|
|
255
|
+
return Err(Error::new(
|
|
256
|
+
ErrorKind::Unexpected,
|
|
257
|
+
"GCS composer has already failed and can't be closed",
|
|
258
|
+
));
|
|
259
|
+
}
|
|
260
|
+
if let Some(metadata) = self.metadata.clone() {
|
|
261
|
+
return Ok(metadata);
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
match self.close_inner().await {
|
|
265
|
+
Ok(metadata) => {
|
|
266
|
+
self.metadata = Some(metadata.clone());
|
|
267
|
+
Ok(metadata)
|
|
268
|
+
}
|
|
269
|
+
Err(err) => {
|
|
270
|
+
self.errored = true;
|
|
271
|
+
Err(err)
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
#[derive(Clone)]
|
|
278
|
+
struct GcsComposeTask {
|
|
279
|
+
core: Arc<GcsCore>,
|
|
280
|
+
ctx: OperationContext,
|
|
281
|
+
sources: Vec<GcsComposeSource>,
|
|
282
|
+
to: String,
|
|
283
|
+
args: OpCompose,
|
|
284
|
+
token: Option<String>,
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
impl GcsComposeTask {
|
|
288
|
+
async fn run(&self) -> Result<GcsComposeSource> {
|
|
289
|
+
let resp = self
|
|
290
|
+
.core
|
|
291
|
+
.gcs_compose_object(&self.ctx, &self.sources, &self.to, &self.args)
|
|
292
|
+
.await?;
|
|
293
|
+
if !resp.status().is_success() {
|
|
294
|
+
let err = parse_error(
|
|
295
|
+
ErrorContext::new(ServiceOperation("ComposeObject"))
|
|
296
|
+
.with_caller_condition(self.args.is_conditional())
|
|
297
|
+
.with_internal_condition(self.token.is_some()),
|
|
298
|
+
resp,
|
|
299
|
+
);
|
|
300
|
+
let recoverable_condition = if self.token.is_some() {
|
|
301
|
+
ErrorKind::Conflict
|
|
302
|
+
} else {
|
|
303
|
+
ErrorKind::ConditionNotMatch
|
|
304
|
+
};
|
|
305
|
+
if err.kind() == recoverable_condition
|
|
306
|
+
&& let Some(source) = self.recover_completed_intermediate().await?
|
|
307
|
+
{
|
|
308
|
+
return Ok(source);
|
|
309
|
+
}
|
|
310
|
+
return Err(err);
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
let metadata = GcsCore::build_metadata_from_object_response(&self.to, resp.into_body())?;
|
|
314
|
+
self.output_from_metadata(metadata)
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
async fn recover_completed_intermediate(&self) -> Result<Option<GcsComposeSource>> {
|
|
318
|
+
let Some(token) = self.token.as_deref() else {
|
|
319
|
+
return Ok(None);
|
|
320
|
+
};
|
|
321
|
+
let resp = self
|
|
322
|
+
.core
|
|
323
|
+
.gcs_get_object_metadata(&self.ctx, &self.to, &OpStat::new())
|
|
324
|
+
.await?;
|
|
325
|
+
if !resp.status().is_success() {
|
|
326
|
+
return Ok(None);
|
|
327
|
+
}
|
|
328
|
+
let metadata = GcsCore::build_metadata_from_object_response(&self.to, resp.into_body())?;
|
|
329
|
+
if metadata
|
|
330
|
+
.user_metadata()
|
|
331
|
+
.and_then(|values| values.get(COMPOSE_TOKEN_KEY))
|
|
332
|
+
.is_none_or(|value| value != token)
|
|
333
|
+
{
|
|
334
|
+
return Ok(None);
|
|
335
|
+
}
|
|
336
|
+
self.output_from_metadata(metadata).map(Some)
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
fn output_from_metadata(&self, metadata: Metadata) -> Result<GcsComposeSource> {
|
|
340
|
+
let version = metadata.version().ok_or_else(|| {
|
|
341
|
+
Error::new(
|
|
342
|
+
ErrorKind::Unexpected,
|
|
343
|
+
"GCS compose response is missing object generation",
|
|
344
|
+
)
|
|
345
|
+
})?;
|
|
346
|
+
Ok(GcsComposeSource {
|
|
347
|
+
path: self.to.clone(),
|
|
348
|
+
version: Some(version.to_string()),
|
|
349
|
+
if_version_match: None,
|
|
350
|
+
})
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
fn new_tasks(
|
|
355
|
+
executor: Executor,
|
|
356
|
+
concurrent: usize,
|
|
357
|
+
) -> ConcurrentTasks<GcsComposeTask, GcsComposeSource> {
|
|
358
|
+
ConcurrentTasks::new(executor, concurrent.max(1), concurrent.max(1), |task| {
|
|
359
|
+
Box::pin(async move {
|
|
360
|
+
let result = task.run().await;
|
|
361
|
+
(task, result)
|
|
362
|
+
})
|
|
363
|
+
})
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
#[cfg(test)]
|
|
367
|
+
mod tests {
|
|
368
|
+
use std::sync::Mutex;
|
|
369
|
+
use std::sync::atomic::AtomicUsize;
|
|
370
|
+
use std::sync::atomic::Ordering;
|
|
371
|
+
use std::time::Duration;
|
|
372
|
+
|
|
373
|
+
use futures::stream;
|
|
374
|
+
use http::Method;
|
|
375
|
+
use http::Response;
|
|
376
|
+
use reqsign_core::Context;
|
|
377
|
+
use reqsign_core::ProvideCredentialChain;
|
|
378
|
+
use reqsign_core::Signer;
|
|
379
|
+
use reqsign_google::RequestSigner;
|
|
380
|
+
use reqsign_google::TokenCredentialProvider;
|
|
381
|
+
use tokio::sync::Notify;
|
|
382
|
+
|
|
383
|
+
use super::*;
|
|
384
|
+
use opendal_core::raw::oio::Compose;
|
|
385
|
+
|
|
386
|
+
struct ComposeTransport {
|
|
387
|
+
started: AtomicUsize,
|
|
388
|
+
active: AtomicUsize,
|
|
389
|
+
max_active: AtomicUsize,
|
|
390
|
+
generation: AtomicUsize,
|
|
391
|
+
notify: Notify,
|
|
392
|
+
requests: Mutex<Vec<serde_json::Value>>,
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
impl ComposeTransport {
|
|
396
|
+
fn new() -> Self {
|
|
397
|
+
Self {
|
|
398
|
+
started: AtomicUsize::new(0),
|
|
399
|
+
active: AtomicUsize::new(0),
|
|
400
|
+
max_active: AtomicUsize::new(0),
|
|
401
|
+
generation: AtomicUsize::new(1),
|
|
402
|
+
notify: Notify::new(),
|
|
403
|
+
requests: Mutex::new(Vec::new()),
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
|
|
407
|
+
fn response(status: http::StatusCode, body: Buffer) -> Result<Response<HttpBody>> {
|
|
408
|
+
let size = body.len() as u64;
|
|
409
|
+
Response::builder()
|
|
410
|
+
.status(status)
|
|
411
|
+
.body(HttpBody::new(stream::iter([Ok(body)]), Some(size)))
|
|
412
|
+
.map_err(new_request_build_error)
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
#[derive(Clone)]
|
|
417
|
+
struct SharedComposeTransport(Arc<ComposeTransport>);
|
|
418
|
+
|
|
419
|
+
struct TokioTestExecutor;
|
|
420
|
+
|
|
421
|
+
impl Execute for TokioTestExecutor {
|
|
422
|
+
fn execute(&self, future: BoxedStaticFuture<()>) {
|
|
423
|
+
tokio::spawn(future);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
impl HttpTransport for SharedComposeTransport {
|
|
428
|
+
async fn fetch(&self, req: http::Request<Buffer>) -> Result<Response<HttpBody>> {
|
|
429
|
+
if req.method() == Method::DELETE {
|
|
430
|
+
return ComposeTransport::response(http::StatusCode::NO_CONTENT, Buffer::new());
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
let value: serde_json::Value = serde_json::from_slice(&req.body().to_bytes())
|
|
434
|
+
.expect("compose request body must be valid JSON");
|
|
435
|
+
self.0.requests.lock().unwrap().push(value.clone());
|
|
436
|
+
|
|
437
|
+
if req.uri().path().contains("__opendal%2Fcompose%2F") {
|
|
438
|
+
let active = self.0.active.fetch_add(1, Ordering::SeqCst) + 1;
|
|
439
|
+
self.0.max_active.fetch_max(active, Ordering::SeqCst);
|
|
440
|
+
self.0.started.fetch_add(1, Ordering::SeqCst);
|
|
441
|
+
self.0.notify.notify_waiters();
|
|
442
|
+
|
|
443
|
+
loop {
|
|
444
|
+
let notified = self.0.notify.notified();
|
|
445
|
+
if self.0.started.load(Ordering::SeqCst) >= 2 {
|
|
446
|
+
break;
|
|
447
|
+
}
|
|
448
|
+
notified.await;
|
|
449
|
+
}
|
|
450
|
+
self.0.active.fetch_sub(1, Ordering::SeqCst);
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
let generation = self.0.generation.fetch_add(1, Ordering::SeqCst);
|
|
454
|
+
let metadata = value
|
|
455
|
+
.get("destination")
|
|
456
|
+
.and_then(|destination| destination.get("metadata"))
|
|
457
|
+
.cloned()
|
|
458
|
+
.unwrap_or_else(|| serde_json::json!({}));
|
|
459
|
+
let body = Buffer::from(
|
|
460
|
+
serde_json::to_vec(&serde_json::json!({
|
|
461
|
+
"size": "1",
|
|
462
|
+
"generation": generation.to_string(),
|
|
463
|
+
"metadata": metadata,
|
|
464
|
+
}))
|
|
465
|
+
.expect("response JSON must serialize"),
|
|
466
|
+
);
|
|
467
|
+
ComposeTransport::response(http::StatusCode::OK, body)
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
struct RecoveryTransport {
|
|
472
|
+
intermediate_posts: AtomicUsize,
|
|
473
|
+
metadata_reads: AtomicUsize,
|
|
474
|
+
token: Mutex<Option<String>>,
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
#[derive(Clone)]
|
|
478
|
+
struct SharedRecoveryTransport(Arc<RecoveryTransport>);
|
|
479
|
+
|
|
480
|
+
impl HttpTransport for SharedRecoveryTransport {
|
|
481
|
+
async fn fetch(&self, req: http::Request<Buffer>) -> Result<Response<HttpBody>> {
|
|
482
|
+
if req.method() == Method::DELETE {
|
|
483
|
+
return ComposeTransport::response(http::StatusCode::NO_CONTENT, Buffer::new());
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
if req.method() == Method::GET {
|
|
487
|
+
self.0.metadata_reads.fetch_add(1, Ordering::SeqCst);
|
|
488
|
+
let token = self.0.token.lock().unwrap().clone().unwrap();
|
|
489
|
+
let body = Buffer::from(
|
|
490
|
+
serde_json::to_vec(&serde_json::json!({
|
|
491
|
+
"size": "32",
|
|
492
|
+
"generation": "7",
|
|
493
|
+
"metadata": {(COMPOSE_TOKEN_KEY): token},
|
|
494
|
+
}))
|
|
495
|
+
.unwrap(),
|
|
496
|
+
);
|
|
497
|
+
return ComposeTransport::response(http::StatusCode::OK, body);
|
|
498
|
+
}
|
|
499
|
+
|
|
500
|
+
if req.uri().path().contains("__opendal%2Fcompose%2F") {
|
|
501
|
+
let value: serde_json::Value = serde_json::from_slice(&req.body().to_bytes())
|
|
502
|
+
.expect("compose request body must be valid JSON");
|
|
503
|
+
let token = value["destination"]["metadata"][COMPOSE_TOKEN_KEY]
|
|
504
|
+
.as_str()
|
|
505
|
+
.unwrap()
|
|
506
|
+
.to_string();
|
|
507
|
+
*self.0.token.lock().unwrap() = Some(token);
|
|
508
|
+
|
|
509
|
+
if self.0.intermediate_posts.fetch_add(1, Ordering::SeqCst) == 0 {
|
|
510
|
+
return Err(Error::new(
|
|
511
|
+
ErrorKind::Unexpected,
|
|
512
|
+
"compose response was lost after the object was created",
|
|
513
|
+
)
|
|
514
|
+
.set_temporary());
|
|
515
|
+
}
|
|
516
|
+
return ComposeTransport::response(
|
|
517
|
+
http::StatusCode::PRECONDITION_FAILED,
|
|
518
|
+
Buffer::new(),
|
|
519
|
+
);
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
let body = Buffer::from(
|
|
523
|
+
serde_json::to_vec(&serde_json::json!({
|
|
524
|
+
"size": "33",
|
|
525
|
+
"generation": "8",
|
|
526
|
+
}))
|
|
527
|
+
.unwrap(),
|
|
528
|
+
);
|
|
529
|
+
ComposeTransport::response(http::StatusCode::OK, body)
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
fn test_core() -> Arc<GcsCore> {
|
|
534
|
+
let sign_ctx = Context::new();
|
|
535
|
+
Arc::new(GcsCore {
|
|
536
|
+
info: ServiceInfo::new("gcs", "/", "test-bucket"),
|
|
537
|
+
capability: Capability {
|
|
538
|
+
delete_max_size: Some(1),
|
|
539
|
+
..Default::default()
|
|
540
|
+
},
|
|
541
|
+
endpoint: "https://storage.googleapis.com".to_string(),
|
|
542
|
+
bucket: "test-bucket".to_string(),
|
|
543
|
+
root: "/".to_string(),
|
|
544
|
+
signer: Signer::new(
|
|
545
|
+
sign_ctx.clone(),
|
|
546
|
+
ProvideCredentialChain::new().push(TokenCredentialProvider::new("test-token")),
|
|
547
|
+
RequestSigner::new("storage"),
|
|
548
|
+
),
|
|
549
|
+
sign_ctx,
|
|
550
|
+
predefined_acl: None,
|
|
551
|
+
default_storage_class: None,
|
|
552
|
+
skip_signature: true,
|
|
553
|
+
})
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
#[tokio::test]
|
|
557
|
+
async fn test_compose_builds_concurrent_ordered_tree() {
|
|
558
|
+
let transport = Arc::new(ComposeTransport::new());
|
|
559
|
+
let ctx = OperationContext::default().with_http_transport(HttpTransporter::new(
|
|
560
|
+
SharedComposeTransport(transport.clone()),
|
|
561
|
+
));
|
|
562
|
+
let ctx = ctx.with_executor(Executor::with(TokioTestExecutor));
|
|
563
|
+
let mut composer = GcsComposer::new(
|
|
564
|
+
test_core(),
|
|
565
|
+
ctx,
|
|
566
|
+
"target",
|
|
567
|
+
OpCompose::from_options(
|
|
568
|
+
&Capability::default(),
|
|
569
|
+
options::ComposeOptions {
|
|
570
|
+
concurrent: 2,
|
|
571
|
+
..Default::default()
|
|
572
|
+
},
|
|
573
|
+
)
|
|
574
|
+
.unwrap(),
|
|
575
|
+
);
|
|
576
|
+
|
|
577
|
+
for index in 0..65 {
|
|
578
|
+
composer
|
|
579
|
+
.compose(&format!("source-{index:02}"), OpRead::new())
|
|
580
|
+
.await
|
|
581
|
+
.expect("source must be accepted");
|
|
582
|
+
}
|
|
583
|
+
tokio::time::timeout(Duration::from_secs(5), composer.close())
|
|
584
|
+
.await
|
|
585
|
+
.expect("compose tasks must make concurrent progress")
|
|
586
|
+
.expect("composition must succeed");
|
|
587
|
+
|
|
588
|
+
assert_eq!(transport.max_active.load(Ordering::SeqCst), 2);
|
|
589
|
+
let requests = transport.requests.lock().unwrap();
|
|
590
|
+
assert_eq!(requests.len(), 3);
|
|
591
|
+
|
|
592
|
+
let source_lists: Vec<&Vec<serde_json::Value>> = requests
|
|
593
|
+
.iter()
|
|
594
|
+
.map(|request| request["sourceObjects"].as_array().unwrap())
|
|
595
|
+
.collect();
|
|
596
|
+
assert_eq!(
|
|
597
|
+
source_lists
|
|
598
|
+
.iter()
|
|
599
|
+
.filter(|sources| sources.len() == 32)
|
|
600
|
+
.count(),
|
|
601
|
+
2
|
|
602
|
+
);
|
|
603
|
+
let final_sources = source_lists
|
|
604
|
+
.iter()
|
|
605
|
+
.find(|sources| sources.len() == 3)
|
|
606
|
+
.expect("final composition must contain the reduced source sequence");
|
|
607
|
+
assert!(
|
|
608
|
+
final_sources[0]["name"]
|
|
609
|
+
.as_str()
|
|
610
|
+
.unwrap()
|
|
611
|
+
.starts_with("__opendal/compose/")
|
|
612
|
+
);
|
|
613
|
+
assert!(
|
|
614
|
+
final_sources[1]["name"]
|
|
615
|
+
.as_str()
|
|
616
|
+
.unwrap()
|
|
617
|
+
.starts_with("__opendal/compose/")
|
|
618
|
+
);
|
|
619
|
+
assert_eq!(final_sources[2]["name"], "source-64");
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
#[tokio::test]
|
|
623
|
+
async fn test_compose_recovers_ambiguous_intermediate_completion() {
|
|
624
|
+
let transport = Arc::new(RecoveryTransport {
|
|
625
|
+
intermediate_posts: AtomicUsize::new(0),
|
|
626
|
+
metadata_reads: AtomicUsize::new(0),
|
|
627
|
+
token: Mutex::new(None),
|
|
628
|
+
});
|
|
629
|
+
let ctx = OperationContext::default().with_http_transport(HttpTransporter::new(
|
|
630
|
+
SharedRecoveryTransport(transport.clone()),
|
|
631
|
+
));
|
|
632
|
+
let mut composer = GcsComposer::new(test_core(), ctx, "target", OpCompose::new());
|
|
633
|
+
|
|
634
|
+
for index in 0..33 {
|
|
635
|
+
composer
|
|
636
|
+
.compose(&format!("source-{index:02}"), OpRead::new())
|
|
637
|
+
.await
|
|
638
|
+
.expect("source must be accepted");
|
|
639
|
+
}
|
|
640
|
+
composer.close().await.expect("composition must recover");
|
|
641
|
+
|
|
642
|
+
assert_eq!(transport.intermediate_posts.load(Ordering::SeqCst), 2);
|
|
643
|
+
assert_eq!(transport.metadata_reads.load(Ordering::SeqCst), 1);
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
#[tokio::test]
|
|
647
|
+
async fn test_compose_hides_request_limit_across_tree_levels() {
|
|
648
|
+
let transport = Arc::new(ComposeTransport::new());
|
|
649
|
+
let ctx = OperationContext::default().with_http_transport(HttpTransporter::new(
|
|
650
|
+
SharedComposeTransport(transport.clone()),
|
|
651
|
+
));
|
|
652
|
+
let ctx = ctx.with_executor(Executor::with(TokioTestExecutor));
|
|
653
|
+
let mut composer = GcsComposer::new(
|
|
654
|
+
test_core(),
|
|
655
|
+
ctx,
|
|
656
|
+
"target",
|
|
657
|
+
OpCompose::from_options(
|
|
658
|
+
&Capability::default(),
|
|
659
|
+
options::ComposeOptions {
|
|
660
|
+
concurrent: 4,
|
|
661
|
+
..Default::default()
|
|
662
|
+
},
|
|
663
|
+
)
|
|
664
|
+
.unwrap(),
|
|
665
|
+
);
|
|
666
|
+
|
|
667
|
+
for index in 0..1025 {
|
|
668
|
+
composer
|
|
669
|
+
.compose(&format!("source-{index:04}"), OpRead::new())
|
|
670
|
+
.await
|
|
671
|
+
.expect("source must be accepted");
|
|
672
|
+
}
|
|
673
|
+
composer.close().await.expect("composition must succeed");
|
|
674
|
+
|
|
675
|
+
let requests = transport.requests.lock().unwrap();
|
|
676
|
+
assert_eq!(requests.len(), 34);
|
|
677
|
+
let source_lists: Vec<&Vec<serde_json::Value>> = requests
|
|
678
|
+
.iter()
|
|
679
|
+
.map(|request| request["sourceObjects"].as_array().unwrap())
|
|
680
|
+
.collect();
|
|
681
|
+
assert!(
|
|
682
|
+
source_lists
|
|
683
|
+
.iter()
|
|
684
|
+
.all(|sources| !sources.is_empty() && sources.len() <= GCS_COMPOSE_MAX_SOURCES)
|
|
685
|
+
);
|
|
686
|
+
assert_eq!(
|
|
687
|
+
source_lists
|
|
688
|
+
.iter()
|
|
689
|
+
.filter(|sources| sources.len() == 32)
|
|
690
|
+
.count(),
|
|
691
|
+
33
|
|
692
|
+
);
|
|
693
|
+
let final_sources = source_lists
|
|
694
|
+
.iter()
|
|
695
|
+
.find(|sources| sources.len() == 2)
|
|
696
|
+
.expect("final composition must contain the reduced source sequence");
|
|
697
|
+
assert!(
|
|
698
|
+
final_sources[0]["name"]
|
|
699
|
+
.as_str()
|
|
700
|
+
.unwrap()
|
|
701
|
+
.starts_with("__opendal/compose/")
|
|
702
|
+
);
|
|
703
|
+
assert_eq!(final_sources[1]["name"], "source-1024");
|
|
704
|
+
}
|
|
705
|
+
}
|