opendal 0.1.6.pre.rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.standard.yml +20 -0
- data/.tool-versions +1 -0
- data/.yardopts +1 -0
- data/Cargo.toml +65 -0
- data/DEPENDENCIES.md +9 -0
- data/DEPENDENCIES.rust.tsv +277 -0
- data/Gemfile +35 -0
- data/README.md +159 -0
- data/Rakefile +149 -0
- data/build.rs +22 -0
- data/core/CHANGELOG.md +4929 -0
- data/core/CONTRIBUTING.md +61 -0
- data/core/Cargo.lock +10259 -0
- data/core/Cargo.toml +437 -0
- data/core/DEPENDENCIES.md +3 -0
- data/core/DEPENDENCIES.rust.tsv +185 -0
- data/core/LICENSE +201 -0
- data/core/README.md +228 -0
- data/core/benches/README.md +18 -0
- data/core/benches/ops/README.md +26 -0
- data/core/benches/ops/main.rs +25 -0
- data/core/benches/ops/read.rs +100 -0
- data/core/benches/ops/utils.rs +59 -0
- data/core/benches/ops/write.rs +106 -0
- data/core/benches/types/README.md +9 -0
- data/core/benches/types/buffer.rs +114 -0
- data/core/benches/types/main.rs +23 -0
- data/core/benches/types/tasks.rs +64 -0
- data/core/benches/vs_fs/Cargo.toml +32 -0
- data/core/benches/vs_fs/README.md +35 -0
- data/core/benches/vs_fs/src/main.rs +83 -0
- data/core/benches/vs_s3/Cargo.toml +38 -0
- data/core/benches/vs_s3/README.md +55 -0
- data/core/benches/vs_s3/src/main.rs +123 -0
- data/core/edge/README.md +3 -0
- data/core/edge/file_write_on_full_disk/Cargo.toml +31 -0
- data/core/edge/file_write_on_full_disk/README.md +14 -0
- data/core/edge/file_write_on_full_disk/src/main.rs +43 -0
- data/core/edge/s3_aws_assume_role_with_web_identity/Cargo.toml +30 -0
- data/core/edge/s3_aws_assume_role_with_web_identity/README.md +18 -0
- data/core/edge/s3_aws_assume_role_with_web_identity/src/main.rs +34 -0
- data/core/edge/s3_read_on_wasm/.gitignore +3 -0
- data/core/edge/s3_read_on_wasm/Cargo.toml +38 -0
- data/core/edge/s3_read_on_wasm/README.md +42 -0
- data/core/edge/s3_read_on_wasm/src/lib.rs +60 -0
- data/core/edge/s3_read_on_wasm/webdriver.json +15 -0
- data/core/examples/README.md +23 -0
- data/core/examples/basic/Cargo.toml +29 -0
- data/core/examples/basic/README.md +15 -0
- data/core/examples/basic/src/main.rs +51 -0
- data/core/examples/concurrent-upload/Cargo.toml +29 -0
- data/core/examples/concurrent-upload/README.md +15 -0
- data/core/examples/concurrent-upload/src/main.rs +68 -0
- data/core/examples/multipart-upload/Cargo.toml +29 -0
- data/core/examples/multipart-upload/README.md +15 -0
- data/core/examples/multipart-upload/src/main.rs +56 -0
- data/core/fuzz/.gitignore +5 -0
- data/core/fuzz/Cargo.toml +92 -0
- data/core/fuzz/README.md +68 -0
- data/core/fuzz/fuzz_reader.rs +102 -0
- data/core/fuzz/fuzz_writer.rs +123 -0
- data/core/src/blocking/delete.rs +74 -0
- data/core/src/blocking/list.rs +71 -0
- data/core/src/blocking/mod.rs +33 -0
- data/core/src/blocking/operator.rs +729 -0
- data/core/src/blocking/read/buffer_iterator.rs +66 -0
- data/core/src/blocking/read/mod.rs +27 -0
- data/core/src/blocking/read/reader.rs +124 -0
- data/core/src/blocking/read/std_bytes_iterator.rs +69 -0
- data/core/src/blocking/read/std_reader.rs +95 -0
- data/core/src/blocking/write/mod.rs +22 -0
- data/core/src/blocking/write/std_writer.rs +82 -0
- data/core/src/blocking/write/writer.rs +109 -0
- data/core/src/docs/comparisons/mod.rs +30 -0
- data/core/src/docs/comparisons/vs_object_store.md +183 -0
- data/core/src/docs/concepts.rs +135 -0
- data/core/src/docs/internals/accessor.rs +306 -0
- data/core/src/docs/internals/layer.rs +42 -0
- data/core/src/docs/internals/mod.rs +62 -0
- data/core/src/docs/mod.rs +43 -0
- data/core/src/docs/performance/concurrent_write.md +101 -0
- data/core/src/docs/performance/http_optimization.md +124 -0
- data/core/src/docs/performance/mod.rs +32 -0
- data/core/src/docs/rfcs/0000_example.md +74 -0
- data/core/src/docs/rfcs/0000_foyer_integration.md +111 -0
- data/core/src/docs/rfcs/0041_object_native_api.md +185 -0
- data/core/src/docs/rfcs/0044_error_handle.md +198 -0
- data/core/src/docs/rfcs/0057_auto_region.md +160 -0
- data/core/src/docs/rfcs/0069_object_stream.md +145 -0
- data/core/src/docs/rfcs/0090_limited_reader.md +155 -0
- data/core/src/docs/rfcs/0112_path_normalization.md +79 -0
- data/core/src/docs/rfcs/0191_async_streaming_io.md +328 -0
- data/core/src/docs/rfcs/0203_remove_credential.md +96 -0
- data/core/src/docs/rfcs/0221_create_dir.md +89 -0
- data/core/src/docs/rfcs/0247_retryable_error.md +87 -0
- data/core/src/docs/rfcs/0293_object_id.md +67 -0
- data/core/src/docs/rfcs/0337_dir_entry.md +191 -0
- data/core/src/docs/rfcs/0409_accessor_capabilities.md +67 -0
- data/core/src/docs/rfcs/0413_presign.md +154 -0
- data/core/src/docs/rfcs/0423_command_line_interface.md +268 -0
- data/core/src/docs/rfcs/0429_init_from_iter.md +107 -0
- data/core/src/docs/rfcs/0438_multipart.md +163 -0
- data/core/src/docs/rfcs/0443_gateway.md +73 -0
- data/core/src/docs/rfcs/0501_new_builder.md +111 -0
- data/core/src/docs/rfcs/0554_write_refactor.md +96 -0
- data/core/src/docs/rfcs/0561_list_metadata_reuse.md +210 -0
- data/core/src/docs/rfcs/0599_blocking_api.md +157 -0
- data/core/src/docs/rfcs/0623_redis_service.md +300 -0
- data/core/src/docs/rfcs/0627_split_capabilities.md +89 -0
- data/core/src/docs/rfcs/0661_path_in_accessor.md +126 -0
- data/core/src/docs/rfcs/0793_generic_kv_services.md +209 -0
- data/core/src/docs/rfcs/0926_object_reader.md +93 -0
- data/core/src/docs/rfcs/0977_refactor_error.md +151 -0
- data/core/src/docs/rfcs/1085_object_handler.md +73 -0
- data/core/src/docs/rfcs/1391_object_metadataer.md +110 -0
- data/core/src/docs/rfcs/1398_query_based_metadata.md +125 -0
- data/core/src/docs/rfcs/1420_object_writer.md +147 -0
- data/core/src/docs/rfcs/1477_remove_object_concept.md +159 -0
- data/core/src/docs/rfcs/1735_operation_extension.md +117 -0
- data/core/src/docs/rfcs/2083_writer_sink_api.md +106 -0
- data/core/src/docs/rfcs/2133_append_api.md +88 -0
- data/core/src/docs/rfcs/2299_chain_based_operator_api.md +99 -0
- data/core/src/docs/rfcs/2602_object_versioning.md +138 -0
- data/core/src/docs/rfcs/2758_merge_append_into_write.md +79 -0
- data/core/src/docs/rfcs/2774_lister_api.md +66 -0
- data/core/src/docs/rfcs/2779_list_with_metakey.md +143 -0
- data/core/src/docs/rfcs/2852_native_capability.md +58 -0
- data/core/src/docs/rfcs/2884_merge_range_read_into_read.md +80 -0
- data/core/src/docs/rfcs/3017_remove_write_copy_from.md +94 -0
- data/core/src/docs/rfcs/3197_config.md +237 -0
- data/core/src/docs/rfcs/3232_align_list_api.md +69 -0
- data/core/src/docs/rfcs/3243_list_prefix.md +128 -0
- data/core/src/docs/rfcs/3356_lazy_reader.md +111 -0
- data/core/src/docs/rfcs/3526_list_recursive.md +59 -0
- data/core/src/docs/rfcs/3574_concurrent_stat_in_list.md +80 -0
- data/core/src/docs/rfcs/3734_buffered_reader.md +64 -0
- data/core/src/docs/rfcs/3898_concurrent_writer.md +66 -0
- data/core/src/docs/rfcs/3911_deleter_api.md +165 -0
- data/core/src/docs/rfcs/4382_range_based_read.md +213 -0
- data/core/src/docs/rfcs/4638_executor.md +215 -0
- data/core/src/docs/rfcs/5314_remove_metakey.md +120 -0
- data/core/src/docs/rfcs/5444_operator_from_uri.md +162 -0
- data/core/src/docs/rfcs/5479_context.md +140 -0
- data/core/src/docs/rfcs/5485_conditional_reader.md +112 -0
- data/core/src/docs/rfcs/5495_list_with_deleted.md +81 -0
- data/core/src/docs/rfcs/5556_write_returns_metadata.md +121 -0
- data/core/src/docs/rfcs/5871_read_returns_metadata.md +112 -0
- data/core/src/docs/rfcs/6189_remove_native_blocking.md +106 -0
- data/core/src/docs/rfcs/6209_glob_support.md +132 -0
- data/core/src/docs/rfcs/6213_options_api.md +142 -0
- data/core/src/docs/rfcs/README.md +62 -0
- data/core/src/docs/rfcs/mod.rs +278 -0
- data/core/src/docs/upgrade.md +1556 -0
- data/core/src/layers/async_backtrace.rs +174 -0
- data/core/src/layers/await_tree.rs +202 -0
- data/core/src/layers/capability_check.rs +239 -0
- data/core/src/layers/chaos.rs +170 -0
- data/core/src/layers/complete.rs +385 -0
- data/core/src/layers/concurrent_limit.rs +322 -0
- data/core/src/layers/correctness_check.rs +440 -0
- data/core/src/layers/dtrace.rs +294 -0
- data/core/src/layers/error_context.rs +310 -0
- data/core/src/layers/fastmetrics.rs +525 -0
- data/core/src/layers/fastrace.rs +271 -0
- data/core/src/layers/http_client.rs +206 -0
- data/core/src/layers/immutable_index.rs +408 -0
- data/core/src/layers/logging.rs +842 -0
- data/core/src/layers/metrics.rs +182 -0
- data/core/src/layers/mime_guess.rs +199 -0
- data/core/src/layers/mod.rs +130 -0
- data/core/src/layers/observe/metrics.rs +936 -0
- data/core/src/layers/observe/mod.rs +93 -0
- data/core/src/layers/otelmetrics.rs +496 -0
- data/core/src/layers/oteltrace.rs +203 -0
- data/core/src/layers/prometheus.rs +686 -0
- data/core/src/layers/prometheus_client.rs +519 -0
- data/core/src/layers/retry.rs +933 -0
- data/core/src/layers/throttle.rs +204 -0
- data/core/src/layers/timeout.rs +513 -0
- data/core/src/layers/tracing.rs +349 -0
- data/core/src/layers/type_eraser.rs +91 -0
- data/core/src/lib.rs +204 -0
- data/core/src/raw/accessor.rs +856 -0
- data/core/src/raw/adapters/kv/api.rs +164 -0
- data/core/src/raw/adapters/kv/backend.rs +253 -0
- data/core/src/raw/adapters/kv/mod.rs +31 -0
- data/core/src/raw/adapters/mod.rs +50 -0
- data/core/src/raw/adapters/typed_kv/api.rs +171 -0
- data/core/src/raw/adapters/typed_kv/backend.rs +279 -0
- data/core/src/raw/adapters/typed_kv/mod.rs +29 -0
- data/core/src/raw/atomic_util.rs +57 -0
- data/core/src/raw/azure.rs +570 -0
- data/core/src/raw/chrono_util.rs +109 -0
- data/core/src/raw/enum_utils.rs +201 -0
- data/core/src/raw/futures_util.rs +470 -0
- data/core/src/raw/http_util/body.rs +144 -0
- data/core/src/raw/http_util/bytes_content_range.rs +239 -0
- data/core/src/raw/http_util/bytes_range.rs +260 -0
- data/core/src/raw/http_util/client.rs +276 -0
- data/core/src/raw/http_util/error.rs +68 -0
- data/core/src/raw/http_util/header.rs +356 -0
- data/core/src/raw/http_util/mod.rs +78 -0
- data/core/src/raw/http_util/multipart.rs +1180 -0
- data/core/src/raw/http_util/uri.rs +190 -0
- data/core/src/raw/layer.rs +295 -0
- data/core/src/raw/mod.rs +101 -0
- data/core/src/raw/oio/buf/flex_buf.rs +118 -0
- data/core/src/raw/oio/buf/mod.rs +25 -0
- data/core/src/raw/oio/buf/pooled_buf.rs +126 -0
- data/core/src/raw/oio/buf/queue_buf.rs +117 -0
- data/core/src/raw/oio/delete/api.rs +102 -0
- data/core/src/raw/oio/delete/batch_delete.rs +127 -0
- data/core/src/raw/oio/delete/mod.rs +30 -0
- data/core/src/raw/oio/delete/one_shot_delete.rs +79 -0
- data/core/src/raw/oio/entry.rs +89 -0
- data/core/src/raw/oio/list/api.rs +69 -0
- data/core/src/raw/oio/list/flat_list.rs +137 -0
- data/core/src/raw/oio/list/hierarchy_list.rs +135 -0
- data/core/src/raw/oio/list/mod.rs +35 -0
- data/core/src/raw/oio/list/page_list.rs +105 -0
- data/core/src/raw/oio/list/prefix_list.rs +64 -0
- data/core/src/raw/oio/mod.rs +40 -0
- data/core/src/raw/oio/read/api.rs +119 -0
- data/core/src/raw/oio/read/mod.rs +21 -0
- data/core/src/raw/oio/write/api.rs +103 -0
- data/core/src/raw/oio/write/append_write.rs +111 -0
- data/core/src/raw/oio/write/block_write.rs +405 -0
- data/core/src/raw/oio/write/mod.rs +42 -0
- data/core/src/raw/oio/write/multipart_write.rs +518 -0
- data/core/src/raw/oio/write/one_shot_write.rs +77 -0
- data/core/src/raw/oio/write/position_write.rs +284 -0
- data/core/src/raw/operation.rs +88 -0
- data/core/src/raw/ops.rs +917 -0
- data/core/src/raw/path.rs +451 -0
- data/core/src/raw/path_cache.rs +244 -0
- data/core/src/raw/rps.rs +249 -0
- data/core/src/raw/serde_util.rs +423 -0
- data/core/src/raw/std_io_util.rs +65 -0
- data/core/src/raw/tests/mod.rs +30 -0
- data/core/src/raw/tests/read.rs +116 -0
- data/core/src/raw/tests/utils.rs +80 -0
- data/core/src/raw/tests/write.rs +79 -0
- data/core/src/raw/tokio_util.rs +24 -0
- data/core/src/raw/version.rs +19 -0
- data/core/src/services/aliyun_drive/backend.rs +421 -0
- data/core/src/services/aliyun_drive/config.rs +72 -0
- data/core/src/services/aliyun_drive/core.rs +651 -0
- data/core/src/services/aliyun_drive/delete.rs +51 -0
- data/core/src/services/aliyun_drive/docs.md +61 -0
- data/core/src/services/aliyun_drive/error.rs +56 -0
- data/core/src/services/aliyun_drive/lister.rs +134 -0
- data/core/src/services/aliyun_drive/mod.rs +39 -0
- data/core/src/services/aliyun_drive/writer.rs +114 -0
- data/core/src/services/alluxio/backend.rs +257 -0
- data/core/src/services/alluxio/config.rs +50 -0
- data/core/src/services/alluxio/core.rs +367 -0
- data/core/src/services/alluxio/delete.rs +38 -0
- data/core/src/services/alluxio/docs.md +45 -0
- data/core/src/services/alluxio/error.rs +99 -0
- data/core/src/services/alluxio/lister.rs +73 -0
- data/core/src/services/alluxio/mod.rs +39 -0
- data/core/src/services/alluxio/writer.rs +74 -0
- data/core/src/services/azblob/backend.rs +594 -0
- data/core/src/services/azblob/config.rs +220 -0
- data/core/src/services/azblob/core.rs +937 -0
- data/core/src/services/azblob/delete.rs +108 -0
- data/core/src/services/azblob/docs.md +77 -0
- data/core/src/services/azblob/error.rs +164 -0
- data/core/src/services/azblob/lister.rs +107 -0
- data/core/src/services/azblob/mod.rs +38 -0
- data/core/src/services/azblob/writer.rs +177 -0
- data/core/src/services/azdls/backend.rs +435 -0
- data/core/src/services/azdls/config.rs +89 -0
- data/core/src/services/azdls/core.rs +388 -0
- data/core/src/services/azdls/delete.rs +48 -0
- data/core/src/services/azdls/docs.md +73 -0
- data/core/src/services/azdls/error.rs +107 -0
- data/core/src/services/azdls/lister.rs +165 -0
- data/core/src/services/azdls/mod.rs +38 -0
- data/core/src/services/azdls/writer.rs +129 -0
- data/core/src/services/azfile/backend.rs +373 -0
- data/core/src/services/azfile/config.rs +61 -0
- data/core/src/services/azfile/core.rs +435 -0
- data/core/src/services/azfile/delete.rs +51 -0
- data/core/src/services/azfile/docs.md +65 -0
- data/core/src/services/azfile/error.rs +108 -0
- data/core/src/services/azfile/lister.rs +217 -0
- data/core/src/services/azfile/mod.rs +39 -0
- data/core/src/services/azfile/writer.rs +92 -0
- data/core/src/services/b2/backend.rs +434 -0
- data/core/src/services/b2/config.rs +64 -0
- data/core/src/services/b2/core.rs +742 -0
- data/core/src/services/b2/delete.rs +56 -0
- data/core/src/services/b2/docs.md +54 -0
- data/core/src/services/b2/error.rs +132 -0
- data/core/src/services/b2/lister.rs +110 -0
- data/core/src/services/b2/mod.rs +39 -0
- data/core/src/services/b2/writer.rs +189 -0
- data/core/src/services/cacache/backend.rs +160 -0
- data/core/src/services/cacache/config.rs +28 -0
- data/core/src/services/cacache/core.rs +96 -0
- data/core/src/services/cacache/delete.rs +39 -0
- data/core/src/services/cacache/docs.md +38 -0
- data/core/src/services/cacache/mod.rs +34 -0
- data/core/src/services/cacache/writer.rs +61 -0
- data/core/src/services/cloudflare_kv/backend.rs +513 -0
- data/core/src/services/cloudflare_kv/config.rs +55 -0
- data/core/src/services/cloudflare_kv/core.rs +168 -0
- data/core/src/services/cloudflare_kv/delete.rs +119 -0
- data/core/src/services/cloudflare_kv/docs.md +21 -0
- data/core/src/services/cloudflare_kv/error.rs +79 -0
- data/core/src/services/cloudflare_kv/lister.rs +170 -0
- data/core/src/services/cloudflare_kv/mod.rs +39 -0
- data/core/src/services/cloudflare_kv/model.rs +76 -0
- data/core/src/services/cloudflare_kv/writer.rs +68 -0
- data/core/src/services/compfs/backend.rs +290 -0
- data/core/src/services/compfs/config.rs +30 -0
- data/core/src/services/compfs/core.rs +159 -0
- data/core/src/services/compfs/delete.rs +53 -0
- data/core/src/services/compfs/lister.rs +98 -0
- data/core/src/services/compfs/mod.rs +38 -0
- data/core/src/services/compfs/reader.rs +79 -0
- data/core/src/services/compfs/writer.rs +90 -0
- data/core/src/services/cos/backend.rs +442 -0
- data/core/src/services/cos/config.rs +54 -0
- data/core/src/services/cos/core.rs +761 -0
- data/core/src/services/cos/delete.rs +48 -0
- data/core/src/services/cos/docs.md +55 -0
- data/core/src/services/cos/error.rs +105 -0
- data/core/src/services/cos/lister.rs +237 -0
- data/core/src/services/cos/mod.rs +39 -0
- data/core/src/services/cos/writer.rs +234 -0
- data/core/src/services/d1/backend.rs +330 -0
- data/core/src/services/d1/config.rs +55 -0
- data/core/src/services/d1/docs.md +48 -0
- data/core/src/services/d1/error.rs +79 -0
- data/core/src/services/d1/mod.rs +29 -0
- data/core/src/services/d1/model.rs +125 -0
- data/core/src/services/dashmap/backend.rs +203 -0
- data/core/src/services/dashmap/config.rs +37 -0
- data/core/src/services/dashmap/core.rs +61 -0
- data/core/src/services/dashmap/delete.rs +40 -0
- data/core/src/services/dashmap/docs.md +38 -0
- data/core/src/services/dashmap/lister.rs +63 -0
- data/core/src/services/dashmap/mod.rs +36 -0
- data/core/src/services/dashmap/writer.rs +87 -0
- data/core/src/services/dbfs/backend.rs +258 -0
- data/core/src/services/dbfs/config.rs +48 -0
- data/core/src/services/dbfs/core.rs +191 -0
- data/core/src/services/dbfs/delete.rs +49 -0
- data/core/src/services/dbfs/docs.md +57 -0
- data/core/src/services/dbfs/error.rs +74 -0
- data/core/src/services/dbfs/lister.rs +96 -0
- data/core/src/services/dbfs/mod.rs +38 -0
- data/core/src/services/dbfs/writer.rs +64 -0
- data/core/src/services/dropbox/backend.rs +187 -0
- data/core/src/services/dropbox/builder.rs +222 -0
- data/core/src/services/dropbox/config.rs +47 -0
- data/core/src/services/dropbox/core.rs +496 -0
- data/core/src/services/dropbox/delete.rs +54 -0
- data/core/src/services/dropbox/docs.md +64 -0
- data/core/src/services/dropbox/error.rs +85 -0
- data/core/src/services/dropbox/lister.rs +117 -0
- data/core/src/services/dropbox/mod.rs +40 -0
- data/core/src/services/dropbox/writer.rs +51 -0
- data/core/src/services/etcd/backend.rs +345 -0
- data/core/src/services/etcd/config.rs +86 -0
- data/core/src/services/etcd/core.rs +143 -0
- data/core/src/services/etcd/deleter.rs +41 -0
- data/core/src/services/etcd/docs.md +45 -0
- data/core/src/services/etcd/error.rs +26 -0
- data/core/src/services/etcd/lister.rs +79 -0
- data/core/src/services/etcd/mod.rs +36 -0
- data/core/src/services/etcd/writer.rs +61 -0
- data/core/src/services/foundationdb/backend.rs +171 -0
- data/core/src/services/foundationdb/config.rs +45 -0
- data/core/src/services/foundationdb/docs.md +42 -0
- data/core/src/services/foundationdb/mod.rs +24 -0
- data/core/src/services/fs/backend.rs +299 -0
- data/core/src/services/fs/config.rs +33 -0
- data/core/src/services/fs/core.rs +227 -0
- data/core/src/services/fs/delete.rs +53 -0
- data/core/src/services/fs/docs.md +49 -0
- data/core/src/services/fs/error.rs +31 -0
- data/core/src/services/fs/lister.rs +81 -0
- data/core/src/services/fs/mod.rs +40 -0
- data/core/src/services/fs/reader.rs +83 -0
- data/core/src/services/fs/writer.rs +212 -0
- data/core/src/services/ftp/backend.rs +388 -0
- data/core/src/services/ftp/config.rs +46 -0
- data/core/src/services/ftp/core.rs +136 -0
- data/core/src/services/ftp/delete.rs +62 -0
- data/core/src/services/ftp/docs.md +42 -0
- data/core/src/services/ftp/err.rs +47 -0
- data/core/src/services/ftp/lister.rs +72 -0
- data/core/src/services/ftp/mod.rs +41 -0
- data/core/src/services/ftp/reader.rs +84 -0
- data/core/src/services/ftp/writer.rs +122 -0
- data/core/src/services/gcs/backend.rs +499 -0
- data/core/src/services/gcs/config.rs +168 -0
- data/core/src/services/gcs/core.rs +1079 -0
- data/core/src/services/gcs/delete.rs +98 -0
- data/core/src/services/gcs/docs.md +76 -0
- data/core/src/services/gcs/error.rs +122 -0
- data/core/src/services/gcs/lister.rs +136 -0
- data/core/src/services/gcs/mod.rs +40 -0
- data/core/src/services/gcs/uri.rs +75 -0
- data/core/src/services/gcs/writer.rs +163 -0
- data/core/src/services/gdrive/backend.rs +176 -0
- data/core/src/services/gdrive/builder.rs +228 -0
- data/core/src/services/gdrive/config.rs +47 -0
- data/core/src/services/gdrive/core.rs +499 -0
- data/core/src/services/gdrive/delete.rs +57 -0
- data/core/src/services/gdrive/docs.md +65 -0
- data/core/src/services/gdrive/error.rs +80 -0
- data/core/src/services/gdrive/lister.rs +110 -0
- data/core/src/services/gdrive/mod.rs +40 -0
- data/core/src/services/gdrive/writer.rs +77 -0
- data/core/src/services/ghac/backend.rs +285 -0
- data/core/src/services/ghac/config.rs +36 -0
- data/core/src/services/ghac/core.rs +459 -0
- data/core/src/services/ghac/docs.md +84 -0
- data/core/src/services/ghac/error.rs +52 -0
- data/core/src/services/ghac/mod.rs +35 -0
- data/core/src/services/ghac/writer.rs +201 -0
- data/core/src/services/github/backend.rs +285 -0
- data/core/src/services/github/config.rs +59 -0
- data/core/src/services/github/core.rs +351 -0
- data/core/src/services/github/delete.rs +41 -0
- data/core/src/services/github/docs.md +52 -0
- data/core/src/services/github/error.rs +101 -0
- data/core/src/services/github/lister.rs +112 -0
- data/core/src/services/github/mod.rs +38 -0
- data/core/src/services/github/writer.rs +51 -0
- data/core/src/services/gridfs/backend.rs +166 -0
- data/core/src/services/gridfs/config.rs +50 -0
- data/core/src/services/gridfs/core.rs +154 -0
- data/core/src/services/gridfs/docs.md +46 -0
- data/core/src/services/gridfs/mod.rs +26 -0
- data/core/src/services/hdfs/backend.rs +413 -0
- data/core/src/services/hdfs/config.rs +59 -0
- data/core/src/services/hdfs/delete.rs +62 -0
- data/core/src/services/hdfs/docs.md +140 -0
- data/core/src/services/hdfs/lister.rs +70 -0
- data/core/src/services/hdfs/mod.rs +36 -0
- data/core/src/services/hdfs/reader.rs +79 -0
- data/core/src/services/hdfs/writer.rs +104 -0
- data/core/src/services/hdfs_native/backend.rs +340 -0
- data/core/src/services/hdfs_native/config.rs +45 -0
- data/core/src/services/hdfs_native/delete.rs +47 -0
- data/core/src/services/hdfs_native/docs.md +35 -0
- data/core/src/services/hdfs_native/error.rs +59 -0
- data/core/src/services/hdfs_native/lister.rs +85 -0
- data/core/src/services/hdfs_native/mod.rs +39 -0
- data/core/src/services/hdfs_native/reader.rs +62 -0
- data/core/src/services/hdfs_native/writer.rs +61 -0
- data/core/src/services/http/backend.rs +291 -0
- data/core/src/services/http/config.rs +49 -0
- data/core/src/services/http/core.rs +125 -0
- data/core/src/services/http/docs.md +45 -0
- data/core/src/services/http/error.rs +53 -0
- data/core/src/services/http/mod.rs +32 -0
- data/core/src/services/huggingface/backend.rs +289 -0
- data/core/src/services/huggingface/config.rs +75 -0
- data/core/src/services/huggingface/core.rs +406 -0
- data/core/src/services/huggingface/docs.md +61 -0
- data/core/src/services/huggingface/error.rs +93 -0
- data/core/src/services/huggingface/lister.rs +91 -0
- data/core/src/services/huggingface/mod.rs +34 -0
- data/core/src/services/ipfs/backend.rs +257 -0
- data/core/src/services/ipfs/config.rs +32 -0
- data/core/src/services/ipfs/core.rs +239 -0
- data/core/src/services/ipfs/docs.md +45 -0
- data/core/src/services/ipfs/error.rs +52 -0
- data/core/src/services/ipfs/ipld.rs +162 -0
- data/core/src/services/ipfs/mod.rs +34 -0
- data/core/src/services/ipmfs/backend.rs +147 -0
- data/core/src/services/ipmfs/builder.rs +166 -0
- data/core/src/services/ipmfs/config.rs +32 -0
- data/core/src/services/ipmfs/core.rs +142 -0
- data/core/src/services/ipmfs/delete.rs +48 -0
- data/core/src/services/ipmfs/docs.md +14 -0
- data/core/src/services/ipmfs/error.rs +83 -0
- data/core/src/services/ipmfs/lister.rs +135 -0
- data/core/src/services/ipmfs/mod.rs +40 -0
- data/core/src/services/ipmfs/writer.rs +49 -0
- data/core/src/services/koofr/backend.rs +361 -0
- data/core/src/services/koofr/config.rs +50 -0
- data/core/src/services/koofr/core.rs +458 -0
- data/core/src/services/koofr/delete.rs +50 -0
- data/core/src/services/koofr/docs.md +51 -0
- data/core/src/services/koofr/error.rs +72 -0
- data/core/src/services/koofr/lister.rs +88 -0
- data/core/src/services/koofr/mod.rs +38 -0
- data/core/src/services/koofr/writer.rs +53 -0
- data/core/src/services/lakefs/backend.rs +309 -0
- data/core/src/services/lakefs/config.rs +81 -0
- data/core/src/services/lakefs/core.rs +261 -0
- data/core/src/services/lakefs/delete.rs +54 -0
- data/core/src/services/lakefs/docs.md +62 -0
- data/core/src/services/lakefs/error.rs +93 -0
- data/core/src/services/lakefs/lister.rs +120 -0
- data/core/src/services/lakefs/mod.rs +38 -0
- data/core/src/services/lakefs/writer.rs +50 -0
- data/core/src/services/memcached/backend.rs +284 -0
- data/core/src/services/memcached/binary.rs +289 -0
- data/core/src/services/memcached/config.rs +43 -0
- data/core/src/services/memcached/docs.md +47 -0
- data/core/src/services/memcached/mod.rs +27 -0
- data/core/src/services/memory/backend.rs +205 -0
- data/core/src/services/memory/config.rs +30 -0
- data/core/src/services/memory/core.rs +80 -0
- data/core/src/services/memory/delete.rs +42 -0
- data/core/src/services/memory/docs.md +36 -0
- data/core/src/services/memory/lister.rs +56 -0
- data/core/src/services/memory/mod.rs +36 -0
- data/core/src/services/memory/writer.rs +85 -0
- data/core/src/services/mini_moka/backend.rs +260 -0
- data/core/src/services/mini_moka/config.rs +56 -0
- data/core/src/services/mini_moka/core.rs +52 -0
- data/core/src/services/mini_moka/delete.rs +42 -0
- data/core/src/services/mini_moka/docs.md +19 -0
- data/core/src/services/mini_moka/lister.rs +68 -0
- data/core/src/services/mini_moka/mod.rs +36 -0
- data/core/src/services/mini_moka/writer.rs +84 -0
- data/core/src/services/mod.rs +206 -0
- data/core/src/services/moka/backend.rs +326 -0
- data/core/src/services/moka/config.rs +59 -0
- data/core/src/services/moka/core.rs +62 -0
- data/core/src/services/moka/delete.rs +42 -0
- data/core/src/services/moka/docs.md +42 -0
- data/core/src/services/moka/lister.rs +65 -0
- data/core/src/services/moka/mod.rs +41 -0
- data/core/src/services/moka/writer.rs +83 -0
- data/core/src/services/mongodb/backend.rs +291 -0
- data/core/src/services/mongodb/config.rs +54 -0
- data/core/src/services/mongodb/docs.md +49 -0
- data/core/src/services/mongodb/mod.rs +24 -0
- data/core/src/services/monoiofs/backend.rs +238 -0
- data/core/src/services/monoiofs/config.rs +34 -0
- data/core/src/services/monoiofs/core.rs +313 -0
- data/core/src/services/monoiofs/delete.rs +64 -0
- data/core/src/services/monoiofs/docs.md +46 -0
- data/core/src/services/monoiofs/mod.rs +36 -0
- data/core/src/services/monoiofs/reader.rs +147 -0
- data/core/src/services/monoiofs/writer.rs +189 -0
- data/core/src/services/mysql/backend.rs +256 -0
- data/core/src/services/mysql/config.rs +66 -0
- data/core/src/services/mysql/docs.md +47 -0
- data/core/src/services/mysql/mod.rs +24 -0
- data/core/src/services/obs/backend.rs +442 -0
- data/core/src/services/obs/config.rs +53 -0
- data/core/src/services/obs/core.rs +608 -0
- data/core/src/services/obs/delete.rs +48 -0
- data/core/src/services/obs/docs.md +54 -0
- data/core/src/services/obs/error.rs +106 -0
- data/core/src/services/obs/lister.rs +101 -0
- data/core/src/services/obs/mod.rs +38 -0
- data/core/src/services/obs/writer.rs +235 -0
- data/core/src/services/onedrive/backend.rs +127 -0
- data/core/src/services/onedrive/builder.rs +236 -0
- data/core/src/services/onedrive/config.rs +49 -0
- data/core/src/services/onedrive/core.rs +691 -0
- data/core/src/services/onedrive/delete.rs +47 -0
- data/core/src/services/onedrive/docs.md +115 -0
- data/core/src/services/onedrive/error.rs +61 -0
- data/core/src/services/onedrive/graph_model.rs +425 -0
- data/core/src/services/onedrive/lister.rs +150 -0
- data/core/src/services/onedrive/mod.rs +42 -0
- data/core/src/services/onedrive/writer.rs +168 -0
- data/core/src/services/opfs/backend.rs +50 -0
- data/core/src/services/opfs/config.rs +25 -0
- data/core/src/services/opfs/core.rs +74 -0
- data/core/src/services/opfs/docs.md +18 -0
- data/core/src/services/opfs/error.rs +27 -0
- data/core/src/services/opfs/mod.rs +30 -0
- data/core/src/services/opfs/utils.rs +70 -0
- data/core/src/services/oss/backend.rs +734 -0
- data/core/src/services/oss/config.rs +113 -0
- data/core/src/services/oss/core.rs +1088 -0
- data/core/src/services/oss/delete.rs +109 -0
- data/core/src/services/oss/docs.md +74 -0
- data/core/src/services/oss/error.rs +109 -0
- data/core/src/services/oss/lister.rs +256 -0
- data/core/src/services/oss/mod.rs +38 -0
- data/core/src/services/oss/writer.rs +228 -0
- data/core/src/services/pcloud/backend.rs +358 -0
- data/core/src/services/pcloud/config.rs +51 -0
- data/core/src/services/pcloud/core.rs +461 -0
- data/core/src/services/pcloud/delete.rs +66 -0
- data/core/src/services/pcloud/docs.md +51 -0
- data/core/src/services/pcloud/error.rs +88 -0
- data/core/src/services/pcloud/lister.rs +95 -0
- data/core/src/services/pcloud/mod.rs +38 -0
- data/core/src/services/pcloud/writer.rs +66 -0
- data/core/src/services/persy/backend.rs +226 -0
- data/core/src/services/persy/config.rs +32 -0
- data/core/src/services/persy/docs.md +43 -0
- data/core/src/services/persy/mod.rs +24 -0
- data/core/src/services/postgresql/backend.rs +258 -0
- data/core/src/services/postgresql/config.rs +66 -0
- data/core/src/services/postgresql/docs.md +47 -0
- data/core/src/services/postgresql/mod.rs +24 -0
- data/core/src/services/redb/backend.rs +280 -0
- data/core/src/services/redb/config.rs +34 -0
- data/core/src/services/redb/docs.md +41 -0
- data/core/src/services/redb/mod.rs +24 -0
- data/core/src/services/redis/backend.rs +442 -0
- data/core/src/services/redis/config.rs +79 -0
- data/core/src/services/redis/core.rs +209 -0
- data/core/src/services/redis/delete.rs +40 -0
- data/core/src/services/redis/docs.md +43 -0
- data/core/src/services/redis/mod.rs +34 -0
- data/core/src/services/redis/writer.rs +57 -0
- data/core/src/services/rocksdb/backend.rs +159 -0
- data/core/src/services/rocksdb/config.rs +34 -0
- data/core/src/services/rocksdb/docs.md +54 -0
- data/core/src/services/rocksdb/mod.rs +24 -0
- data/core/src/services/s3/backend.rs +1293 -0
- data/core/src/services/s3/compatible_services.md +126 -0
- data/core/src/services/s3/config.rs +327 -0
- data/core/src/services/s3/core.rs +1741 -0
- data/core/src/services/s3/delete.rs +109 -0
- data/core/src/services/s3/docs.md +244 -0
- data/core/src/services/s3/error.rs +171 -0
- data/core/src/services/s3/lister.rs +405 -0
- data/core/src/services/s3/mod.rs +38 -0
- data/core/src/services/s3/writer.rs +262 -0
- data/core/src/services/seafile/backend.rs +297 -0
- data/core/src/services/seafile/config.rs +56 -0
- data/core/src/services/seafile/core.rs +475 -0
- data/core/src/services/seafile/delete.rs +40 -0
- data/core/src/services/seafile/docs.md +54 -0
- data/core/src/services/seafile/error.rs +86 -0
- data/core/src/services/seafile/lister.rs +83 -0
- data/core/src/services/seafile/mod.rs +38 -0
- data/core/src/services/seafile/writer.rs +55 -0
- data/core/src/services/sftp/backend.rs +397 -0
- data/core/src/services/sftp/config.rs +50 -0
- data/core/src/services/sftp/core.rs +154 -0
- data/core/src/services/sftp/delete.rs +55 -0
- data/core/src/services/sftp/docs.md +49 -0
- data/core/src/services/sftp/error.rs +57 -0
- data/core/src/services/sftp/lister.rs +88 -0
- data/core/src/services/sftp/mod.rs +42 -0
- data/core/src/services/sftp/reader.rs +78 -0
- data/core/src/services/sftp/utils.rs +51 -0
- data/core/src/services/sftp/writer.rs +67 -0
- data/core/src/services/sled/backend.rs +194 -0
- data/core/src/services/sled/config.rs +45 -0
- data/core/src/services/sled/docs.md +39 -0
- data/core/src/services/sled/mod.rs +24 -0
- data/core/src/services/sqlite/backend.rs +326 -0
- data/core/src/services/sqlite/config.rs +70 -0
- data/core/src/services/sqlite/docs.md +46 -0
- data/core/src/services/sqlite/mod.rs +24 -0
- data/core/src/services/surrealdb/backend.rs +365 -0
- data/core/src/services/surrealdb/config.rs +64 -0
- data/core/src/services/surrealdb/docs.md +54 -0
- data/core/src/services/surrealdb/mod.rs +24 -0
- data/core/src/services/swift/backend.rs +275 -0
- data/core/src/services/swift/compatible_services.md +53 -0
- data/core/src/services/swift/config.rs +53 -0
- data/core/src/services/swift/core.rs +310 -0
- data/core/src/services/swift/delete.rs +49 -0
- data/core/src/services/swift/docs.md +52 -0
- data/core/src/services/swift/error.rs +90 -0
- data/core/src/services/swift/lister.rs +119 -0
- data/core/src/services/swift/mod.rs +38 -0
- data/core/src/services/swift/writer.rs +53 -0
- data/core/src/services/tikv/backend.rs +237 -0
- data/core/src/services/tikv/config.rs +52 -0
- data/core/src/services/tikv/docs.md +43 -0
- data/core/src/services/tikv/mod.rs +24 -0
- data/core/src/services/upyun/backend.rs +317 -0
- data/core/src/services/upyun/config.rs +51 -0
- data/core/src/services/upyun/core.rs +521 -0
- data/core/src/services/upyun/delete.rs +50 -0
- data/core/src/services/upyun/docs.md +51 -0
- data/core/src/services/upyun/error.rs +97 -0
- data/core/src/services/upyun/lister.rs +101 -0
- data/core/src/services/upyun/mod.rs +38 -0
- data/core/src/services/upyun/writer.rs +127 -0
- data/core/src/services/vercel_artifacts/backend.rs +99 -0
- data/core/src/services/vercel_artifacts/builder.rs +117 -0
- data/core/src/services/vercel_artifacts/config.rs +39 -0
- data/core/src/services/vercel_artifacts/core.rs +112 -0
- data/core/src/services/vercel_artifacts/docs.md +40 -0
- data/core/src/services/vercel_artifacts/error.rs +50 -0
- data/core/src/services/vercel_artifacts/mod.rs +36 -0
- data/core/src/services/vercel_artifacts/writer.rs +58 -0
- data/core/src/services/vercel_blob/backend.rs +251 -0
- data/core/src/services/vercel_blob/config.rs +45 -0
- data/core/src/services/vercel_blob/core.rs +449 -0
- data/core/src/services/vercel_blob/delete.rs +38 -0
- data/core/src/services/vercel_blob/docs.md +45 -0
- data/core/src/services/vercel_blob/error.rs +110 -0
- data/core/src/services/vercel_blob/lister.rs +69 -0
- data/core/src/services/vercel_blob/mod.rs +38 -0
- data/core/src/services/vercel_blob/writer.rs +143 -0
- data/core/src/services/webdav/backend.rs +318 -0
- data/core/src/services/webdav/config.rs +53 -0
- data/core/src/services/webdav/core.rs +859 -0
- data/core/src/services/webdav/delete.rs +47 -0
- data/core/src/services/webdav/docs.md +49 -0
- data/core/src/services/webdav/error.rs +53 -0
- data/core/src/services/webdav/lister.rs +106 -0
- data/core/src/services/webdav/mod.rs +38 -0
- data/core/src/services/webdav/writer.rs +56 -0
- data/core/src/services/webhdfs/backend.rs +376 -0
- data/core/src/services/webhdfs/config.rs +52 -0
- data/core/src/services/webhdfs/core.rs +398 -0
- data/core/src/services/webhdfs/delete.rs +46 -0
- data/core/src/services/webhdfs/docs.md +90 -0
- data/core/src/services/webhdfs/error.rs +126 -0
- data/core/src/services/webhdfs/lister.rs +130 -0
- data/core/src/services/webhdfs/message.rs +249 -0
- data/core/src/services/webhdfs/mod.rs +41 -0
- data/core/src/services/webhdfs/writer.rs +177 -0
- data/core/src/services/yandex_disk/backend.rs +267 -0
- data/core/src/services/yandex_disk/config.rs +45 -0
- data/core/src/services/yandex_disk/core.rs +340 -0
- data/core/src/services/yandex_disk/delete.rs +54 -0
- data/core/src/services/yandex_disk/docs.md +45 -0
- data/core/src/services/yandex_disk/error.rs +104 -0
- data/core/src/services/yandex_disk/lister.rs +113 -0
- data/core/src/services/yandex_disk/mod.rs +38 -0
- data/core/src/services/yandex_disk/writer.rs +52 -0
- data/core/src/types/buffer.rs +991 -0
- data/core/src/types/builder.rs +152 -0
- data/core/src/types/capability.rs +209 -0
- data/core/src/types/context/mod.rs +22 -0
- data/core/src/types/context/read.rs +231 -0
- data/core/src/types/context/write.rs +441 -0
- data/core/src/types/delete/deleter.rs +220 -0
- data/core/src/types/delete/futures_delete_sink.rs +176 -0
- data/core/src/types/delete/input.rs +97 -0
- data/core/src/types/delete/mod.rs +26 -0
- data/core/src/types/entry.rs +69 -0
- data/core/src/types/error.rs +570 -0
- data/core/src/types/execute/api.rs +110 -0
- data/core/src/types/execute/executor.rs +96 -0
- data/core/src/types/execute/executors/mod.rs +27 -0
- data/core/src/types/execute/executors/tokio_executor.rs +60 -0
- data/core/src/types/execute/mod.rs +25 -0
- data/core/src/types/list.rs +137 -0
- data/core/src/types/metadata.rs +436 -0
- data/core/src/types/mod.rs +72 -0
- data/core/src/types/mode.rs +68 -0
- data/core/src/types/operator/builder.rs +535 -0
- data/core/src/types/operator/info.rs +63 -0
- data/core/src/types/operator/mod.rs +33 -0
- data/core/src/types/operator/operator.rs +2236 -0
- data/core/src/types/operator/operator_futures.rs +1430 -0
- data/core/src/types/operator/registry.rs +129 -0
- data/core/src/types/options.rs +548 -0
- data/core/src/types/read/buffer_stream.rs +273 -0
- data/core/src/types/read/futures_async_reader.rs +289 -0
- data/core/src/types/read/futures_bytes_stream.rs +157 -0
- data/core/src/types/read/mod.rs +29 -0
- data/core/src/types/read/reader.rs +604 -0
- data/core/src/types/scheme.rs +475 -0
- data/core/src/types/write/buffer_sink.rs +188 -0
- data/core/src/types/write/futures_async_writer.rs +136 -0
- data/core/src/types/write/futures_bytes_sink.rs +103 -0
- data/core/src/types/write/mod.rs +26 -0
- data/core/src/types/write/writer.rs +411 -0
- data/core/tests/behavior/README.md +77 -0
- data/core/tests/behavior/async_copy.rs +314 -0
- data/core/tests/behavior/async_create_dir.rs +53 -0
- data/core/tests/behavior/async_delete.rs +354 -0
- data/core/tests/behavior/async_list.rs +739 -0
- data/core/tests/behavior/async_presign.rs +175 -0
- data/core/tests/behavior/async_read.rs +871 -0
- data/core/tests/behavior/async_rename.rs +210 -0
- data/core/tests/behavior/async_stat.rs +628 -0
- data/core/tests/behavior/async_write.rs +819 -0
- data/core/tests/behavior/main.rs +78 -0
- data/core/tests/behavior/utils.rs +187 -0
- data/core/tests/data/normal_dir/.gitkeep +0 -0
- data/core/tests/data/normal_file.txt +1041 -0
- data/core/tests/data/special_dir !@#$%^&()_+-=;',/.gitkeep +0 -0
- data/core/tests/data/special_file !@#$%^&()_+-=;',.txt +1041 -0
- data/core/users.md +13 -0
- data/extconf.rb +24 -0
- data/lib/opendal.rb +25 -0
- data/lib/opendal_ruby/entry.rb +35 -0
- data/lib/opendal_ruby/io.rb +70 -0
- data/lib/opendal_ruby/metadata.rb +44 -0
- data/lib/opendal_ruby/operator.rb +29 -0
- data/lib/opendal_ruby/operator_info.rb +26 -0
- data/src/capability.rs +146 -0
- data/src/io.rs +464 -0
- data/src/lib.rs +63 -0
- data/src/lister.rs +141 -0
- data/src/metadata.rs +111 -0
- data/src/middlewares.rs +174 -0
- data/src/operator.rs +310 -0
- data/src/operator_info.rs +83 -0
- data/test/blocking_op_test.rb +112 -0
- data/test/capability_test.rb +42 -0
- data/test/io_test.rb +172 -0
- data/test/lister_test.rb +77 -0
- data/test/metadata_test.rb +78 -0
- data/test/middlewares_test.rb +46 -0
- data/test/operator_info_test.rb +35 -0
- data/test/test_helper.rb +36 -0
- metadata +857 -0
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
// or more contributor license agreements. See the NOTICE file
|
|
3
|
+
// distributed with this work for additional information
|
|
4
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
// to you under the Apache License, Version 2.0 (the
|
|
6
|
+
// "License"); you may not use this file except in compliance
|
|
7
|
+
// with the License. You may obtain a copy of the License at
|
|
8
|
+
//
|
|
9
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
//
|
|
11
|
+
// Unless required by applicable law or agreed to in writing,
|
|
12
|
+
// software distributed under the License is distributed on an
|
|
13
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
// KIND, either express or implied. See the License for the
|
|
15
|
+
// specific language governing permissions and limitations
|
|
16
|
+
// under the License.
|
|
17
|
+
|
|
18
|
+
use divan::counter::BytesCount;
|
|
19
|
+
use divan::Bencher;
|
|
20
|
+
use opendal::raw::tests::init_test_service;
|
|
21
|
+
use opendal::raw::tests::TEST_RUNTIME;
|
|
22
|
+
use rand::prelude::*;
|
|
23
|
+
use size::Size;
|
|
24
|
+
|
|
25
|
+
use super::utils::*;
|
|
26
|
+
|
|
27
|
+
#[divan::bench(
|
|
28
|
+
args = [
|
|
29
|
+
Size::from_kib(4),
|
|
30
|
+
Size::from_kib(64),
|
|
31
|
+
Size::from_mib(1),
|
|
32
|
+
Size::from_mib(16),
|
|
33
|
+
],
|
|
34
|
+
ignore = std::env::var("OPENDAL_TEST").is_err()
|
|
35
|
+
)]
|
|
36
|
+
fn whole(b: Bencher, size: Size) {
|
|
37
|
+
let op = init_test_service().unwrap().unwrap();
|
|
38
|
+
let path = uuid::Uuid::new_v4().to_string();
|
|
39
|
+
let _temp_data = TempData::existing(op.clone(), &path);
|
|
40
|
+
|
|
41
|
+
b.counter(BytesCount::from(size.bytes() as u64))
|
|
42
|
+
.with_inputs(|| {
|
|
43
|
+
let mut rng = thread_rng();
|
|
44
|
+
gen_bytes(&mut rng, size.bytes() as usize)
|
|
45
|
+
})
|
|
46
|
+
.bench_values(|content| {
|
|
47
|
+
let op = op.clone();
|
|
48
|
+
let path = path.clone();
|
|
49
|
+
TEST_RUNTIME.block_on(async move {
|
|
50
|
+
let _ = op.write(&path, content).await.unwrap();
|
|
51
|
+
})
|
|
52
|
+
})
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
mod concurrent {
|
|
56
|
+
use super::*;
|
|
57
|
+
|
|
58
|
+
/// Size of the data to be written.
|
|
59
|
+
const SIZE: usize = 64 * 1024 * 1024;
|
|
60
|
+
|
|
61
|
+
#[divan::bench(
|
|
62
|
+
ignore = std::env::var("OPENDAL_TEST").is_err()
|
|
63
|
+
)]
|
|
64
|
+
fn baseline(b: Bencher) {
|
|
65
|
+
let op = init_test_service().unwrap().unwrap();
|
|
66
|
+
let path = uuid::Uuid::new_v4().to_string();
|
|
67
|
+
let _temp_data = TempData::existing(op.clone(), &path);
|
|
68
|
+
let mut rng = thread_rng();
|
|
69
|
+
let content = gen_bytes(&mut rng, SIZE);
|
|
70
|
+
|
|
71
|
+
b.counter(BytesCount::from(SIZE)).bench(|| {
|
|
72
|
+
let op = op.clone();
|
|
73
|
+
let path = path.clone();
|
|
74
|
+
let content = content.clone();
|
|
75
|
+
TEST_RUNTIME.block_on(async move {
|
|
76
|
+
let _ = op.write(&path, content).await.unwrap();
|
|
77
|
+
})
|
|
78
|
+
})
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
#[divan::bench(
|
|
82
|
+
args = [1, 2, 4, 8],
|
|
83
|
+
ignore = std::env::var("OPENDAL_TEST").is_err()
|
|
84
|
+
)]
|
|
85
|
+
fn concurrent(b: Bencher, concurrent: usize) {
|
|
86
|
+
let op = init_test_service().unwrap().unwrap();
|
|
87
|
+
let path = uuid::Uuid::new_v4().to_string();
|
|
88
|
+
let _temp_data = TempData::existing(op.clone(), &path);
|
|
89
|
+
let mut rng = thread_rng();
|
|
90
|
+
let content = gen_bytes(&mut rng, SIZE);
|
|
91
|
+
|
|
92
|
+
b.counter(BytesCount::from(SIZE)).bench(|| {
|
|
93
|
+
let op = op.clone();
|
|
94
|
+
let path = path.clone();
|
|
95
|
+
let content = content.clone();
|
|
96
|
+
TEST_RUNTIME.block_on(async move {
|
|
97
|
+
let _ = op
|
|
98
|
+
.write_with(&path, content)
|
|
99
|
+
.chunk(8 * 1024 * 1024)
|
|
100
|
+
.concurrent(concurrent)
|
|
101
|
+
.await
|
|
102
|
+
.unwrap();
|
|
103
|
+
})
|
|
104
|
+
})
|
|
105
|
+
}
|
|
106
|
+
}
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
// or more contributor license agreements. See the NOTICE file
|
|
3
|
+
// distributed with this work for additional information
|
|
4
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
// to you under the Apache License, Version 2.0 (the
|
|
6
|
+
// "License"); you may not use this file except in compliance
|
|
7
|
+
// with the License. You may obtain a copy of the License at
|
|
8
|
+
//
|
|
9
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
//
|
|
11
|
+
// Unless required by applicable law or agreed to in writing,
|
|
12
|
+
// software distributed under the License is distributed on an
|
|
13
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
// KIND, either express or implied. See the License for the
|
|
15
|
+
// specific language governing permissions and limitations
|
|
16
|
+
// under the License.
|
|
17
|
+
|
|
18
|
+
use bytes::Buf;
|
|
19
|
+
use bytes::Bytes;
|
|
20
|
+
use divan::black_box;
|
|
21
|
+
use divan::Bencher;
|
|
22
|
+
use opendal::Buffer;
|
|
23
|
+
|
|
24
|
+
mod chunk {
|
|
25
|
+
use super::*;
|
|
26
|
+
|
|
27
|
+
#[divan::bench]
|
|
28
|
+
fn bytes(b: Bencher) {
|
|
29
|
+
b.with_inputs(|| Bytes::from(vec![1; 10]))
|
|
30
|
+
.bench_refs(|buffer| {
|
|
31
|
+
black_box(buffer.chunk());
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
#[divan::bench]
|
|
36
|
+
fn contiguous(b: Bencher) {
|
|
37
|
+
b.with_inputs(|| Buffer::from(vec![1; 10]))
|
|
38
|
+
.bench_refs(|buffer| {
|
|
39
|
+
black_box(buffer.chunk());
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
#[divan::bench(args = [10, 1_000, 1_000_000])]
|
|
44
|
+
fn non_contiguous(b: Bencher, parts: usize) {
|
|
45
|
+
b.with_inputs(|| Buffer::from([1; 1].repeat(parts)))
|
|
46
|
+
.bench_refs(|buffer| {
|
|
47
|
+
black_box(buffer.chunk());
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
mod advance {
|
|
53
|
+
use super::*;
|
|
54
|
+
|
|
55
|
+
#[divan::bench]
|
|
56
|
+
fn bytes(b: Bencher) {
|
|
57
|
+
b.with_inputs(|| Bytes::from(vec![1; 10]))
|
|
58
|
+
.bench_refs(|buffer| buffer.advance(4));
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
#[divan::bench]
|
|
62
|
+
fn contiguous(b: Bencher) {
|
|
63
|
+
b.with_inputs(|| Buffer::from(vec![1; 10]))
|
|
64
|
+
.bench_refs(|buffer| buffer.advance(4));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
#[divan::bench(args = [10, 1_000, 1_000_000])]
|
|
68
|
+
fn non_contiguous(b: Bencher, parts: usize) {
|
|
69
|
+
b.with_inputs(|| Buffer::from([1; 1].repeat(parts)))
|
|
70
|
+
.bench_refs(|buffer| buffer.advance(4));
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
mod truncate {
|
|
75
|
+
use super::*;
|
|
76
|
+
|
|
77
|
+
#[divan::bench]
|
|
78
|
+
fn bytes(b: Bencher) {
|
|
79
|
+
b.with_inputs(|| Bytes::from(vec![1; 10]))
|
|
80
|
+
.bench_refs(|buffer| buffer.truncate(5));
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
#[divan::bench]
|
|
84
|
+
fn contiguous(b: Bencher) {
|
|
85
|
+
b.with_inputs(|| Buffer::from(vec![1; 10]))
|
|
86
|
+
.bench_refs(|buffer| buffer.truncate(5));
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
#[divan::bench(args = [10, 1_000, 1_000_000])]
|
|
90
|
+
fn non_contiguous(b: Bencher, parts: usize) {
|
|
91
|
+
b.with_inputs(|| Buffer::from([1; 1].repeat(parts)))
|
|
92
|
+
.bench_refs(|buffer| buffer.truncate(4));
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
mod iterator {
|
|
97
|
+
use super::*;
|
|
98
|
+
|
|
99
|
+
#[divan::bench]
|
|
100
|
+
fn contiguous(b: Bencher) {
|
|
101
|
+
b.with_inputs(|| Buffer::from(vec![1; 1_000_000]))
|
|
102
|
+
.bench_refs(|buffer| loop {
|
|
103
|
+
let Some(_) = buffer.next() else { break };
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
#[divan::bench(args = [10, 1_000, 1_000_000])]
|
|
108
|
+
fn non_contiguous(b: Bencher, parts: usize) {
|
|
109
|
+
b.with_inputs(|| Buffer::from(vec![1; 1_000_000 / parts].repeat(parts)))
|
|
110
|
+
.bench_refs(|buffer| loop {
|
|
111
|
+
let Some(_) = buffer.next() else { break };
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
// or more contributor license agreements. See the NOTICE file
|
|
3
|
+
// distributed with this work for additional information
|
|
4
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
// to you under the Apache License, Version 2.0 (the
|
|
6
|
+
// "License"); you may not use this file except in compliance
|
|
7
|
+
// with the License. You may obtain a copy of the License at
|
|
8
|
+
//
|
|
9
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
//
|
|
11
|
+
// Unless required by applicable law or agreed to in writing,
|
|
12
|
+
// software distributed under the License is distributed on an
|
|
13
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
// KIND, either express or implied. See the License for the
|
|
15
|
+
// specific language governing permissions and limitations
|
|
16
|
+
// under the License.
|
|
17
|
+
|
|
18
|
+
mod buffer;
|
|
19
|
+
mod tasks;
|
|
20
|
+
|
|
21
|
+
fn main() {
|
|
22
|
+
divan::main();
|
|
23
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
// or more contributor license agreements. See the NOTICE file
|
|
3
|
+
// distributed with this work for additional information
|
|
4
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
// to you under the Apache License, Version 2.0 (the
|
|
6
|
+
// "License"); you may not use this file except in compliance
|
|
7
|
+
// with the License. You may obtain a copy of the License at
|
|
8
|
+
//
|
|
9
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
//
|
|
11
|
+
// Unless required by applicable law or agreed to in writing,
|
|
12
|
+
// software distributed under the License is distributed on an
|
|
13
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
// KIND, either express or implied. See the License for the
|
|
15
|
+
// specific language governing permissions and limitations
|
|
16
|
+
// under the License.
|
|
17
|
+
|
|
18
|
+
use std::sync::LazyLock;
|
|
19
|
+
use std::time::Duration;
|
|
20
|
+
|
|
21
|
+
use divan::Bencher;
|
|
22
|
+
use opendal::raw::ConcurrentTasks;
|
|
23
|
+
use opendal::Executor;
|
|
24
|
+
|
|
25
|
+
pub static TOKIO: LazyLock<tokio::runtime::Runtime> =
|
|
26
|
+
LazyLock::new(|| tokio::runtime::Runtime::new().expect("build tokio runtime"));
|
|
27
|
+
|
|
28
|
+
pub static TOTAL_TASKS: usize = 128;
|
|
29
|
+
|
|
30
|
+
#[divan::bench]
|
|
31
|
+
fn baseline(b: Bencher) {
|
|
32
|
+
b.bench(|| {
|
|
33
|
+
TOKIO.block_on(async move {
|
|
34
|
+
for _ in 0..TOTAL_TASKS {
|
|
35
|
+
tokio::time::sleep(Duration::from_millis(1)).await;
|
|
36
|
+
}
|
|
37
|
+
})
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
#[divan::bench(args = [1, 2, 8, 32])]
|
|
42
|
+
fn concurrent(b: Bencher, concurrent: usize) {
|
|
43
|
+
b.with_inputs(|| {
|
|
44
|
+
ConcurrentTasks::new(Executor::new(), concurrent, 8, |()| {
|
|
45
|
+
Box::pin(async {
|
|
46
|
+
tokio::time::sleep(Duration::from_millis(1)).await;
|
|
47
|
+
((), Ok(()))
|
|
48
|
+
})
|
|
49
|
+
})
|
|
50
|
+
})
|
|
51
|
+
.bench_refs(|tasks| {
|
|
52
|
+
TOKIO.block_on(async move {
|
|
53
|
+
for _ in 0..TOTAL_TASKS {
|
|
54
|
+
let _ = tasks.execute(()).await;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
loop {
|
|
58
|
+
if tasks.next().await.is_none() {
|
|
59
|
+
break;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
})
|
|
63
|
+
});
|
|
64
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
|
|
18
|
+
[package]
|
|
19
|
+
description = "OpenDAL Benchmark vs fs"
|
|
20
|
+
edition = "2021"
|
|
21
|
+
license = "Apache-2.0"
|
|
22
|
+
name = "opendal-benchmark-vs-fs"
|
|
23
|
+
publish = false
|
|
24
|
+
rust-version = "1.82"
|
|
25
|
+
version = "0.0.0"
|
|
26
|
+
|
|
27
|
+
[dependencies]
|
|
28
|
+
criterion = { version = "0.7", features = ["async", "async_tokio"] }
|
|
29
|
+
opendal = { path = "../..", features = ["tests"] }
|
|
30
|
+
rand = "0.8"
|
|
31
|
+
tokio = { version = "1", features = ["full"] }
|
|
32
|
+
uuid = { version = "1", features = ["v4"] }
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# OpenDAL Benchmark VS Fs
|
|
2
|
+
|
|
3
|
+
This benchmark compares the performance of OpenDAL with the performance of the `std::fs`.
|
|
4
|
+
|
|
5
|
+
## Goal
|
|
6
|
+
|
|
7
|
+
We expect OpenDAL to match `std::fs` in speed: the throughput of OpenDAL should be within a `5%` range of `std::fs`.
|
|
8
|
+
|
|
9
|
+
## Usage
|
|
10
|
+
|
|
11
|
+
For test: `cargo run`
|
|
12
|
+
|
|
13
|
+
```shell
|
|
14
|
+
Testing vs_fs/std_fs_read
|
|
15
|
+
Success
|
|
16
|
+
Testing vs_fs/opendal_fs_read
|
|
17
|
+
Success
|
|
18
|
+
Testing vs_fs/opendal_fs_read_with_range
|
|
19
|
+
Success
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
For bench: `cargo run --release -- --bench`
|
|
23
|
+
|
|
24
|
+
```shell
|
|
25
|
+
read/std_fs time: [749.57 µs 762.69 µs 777.07 µs]
|
|
26
|
+
thrpt: [20.108 GiB/s 20.487 GiB/s 20.845 GiB/s]
|
|
27
|
+
|
|
28
|
+
read/opendal_fs time: [750.90 µs 755.39 µs 760.49 µs]
|
|
29
|
+
thrpt: [20.546 GiB/s 20.685 GiB/s 20.808 GiB/s]
|
|
30
|
+
|
|
31
|
+
read/opendal_fs_with_range
|
|
32
|
+
time: [684.02 µs 690.77 µs 697.99 µs]
|
|
33
|
+
thrpt: [22.386 GiB/s 22.620 GiB/s 22.843 GiB/s]
|
|
34
|
+
|
|
35
|
+
```
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
// or more contributor license agreements. See the NOTICE file
|
|
3
|
+
// distributed with this work for additional information
|
|
4
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
// to you under the Apache License, Version 2.0 (the
|
|
6
|
+
// "License"); you may not use this file except in compliance
|
|
7
|
+
// with the License. You may obtain a copy of the License at
|
|
8
|
+
//
|
|
9
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
//
|
|
11
|
+
// Unless required by applicable law or agreed to in writing,
|
|
12
|
+
// software distributed under the License is distributed on an
|
|
13
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
// KIND, either express or implied. See the License for the
|
|
15
|
+
// specific language governing permissions and limitations
|
|
16
|
+
// under the License.
|
|
17
|
+
|
|
18
|
+
use criterion::Criterion;
|
|
19
|
+
use opendal::blocking;
|
|
20
|
+
use opendal::options;
|
|
21
|
+
use opendal::services;
|
|
22
|
+
use opendal::Operator;
|
|
23
|
+
use rand::prelude::*;
|
|
24
|
+
|
|
25
|
+
fn main() {
|
|
26
|
+
let mut c = Criterion::default().configure_from_args();
|
|
27
|
+
bench_vs_fs(&mut c);
|
|
28
|
+
|
|
29
|
+
c.final_summary();
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
fn bench_vs_fs(c: &mut Criterion) {
|
|
33
|
+
let cfg = services::Fs::default().root("/tmp/opendal/");
|
|
34
|
+
let op = Operator::new(cfg).unwrap().finish();
|
|
35
|
+
|
|
36
|
+
let runtime = tokio::runtime::Runtime::new().unwrap();
|
|
37
|
+
let _guard = runtime.enter();
|
|
38
|
+
let op = blocking::Operator::new(op).unwrap();
|
|
39
|
+
|
|
40
|
+
let mut group = c.benchmark_group("read");
|
|
41
|
+
group.throughput(criterion::Throughput::Bytes(16 * 1024 * 1024));
|
|
42
|
+
|
|
43
|
+
group.bench_function("std_fs", |b| {
|
|
44
|
+
let path = format!("/tmp/opendal/{}", prepare());
|
|
45
|
+
b.iter(|| {
|
|
46
|
+
let _ = std::fs::read(&path).unwrap();
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
group.bench_function("opendal_fs", |b| {
|
|
50
|
+
let path = prepare();
|
|
51
|
+
b.iter(|| {
|
|
52
|
+
let _ = op.read(&path).unwrap();
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
group.bench_function("opendal_fs_with_range", |b| {
|
|
56
|
+
let path = prepare();
|
|
57
|
+
b.iter(|| {
|
|
58
|
+
let _ = op
|
|
59
|
+
.read_options(
|
|
60
|
+
&path,
|
|
61
|
+
options::ReadOptions {
|
|
62
|
+
range: (0..16 * 1024 * 1024).into(),
|
|
63
|
+
..Default::default()
|
|
64
|
+
},
|
|
65
|
+
)
|
|
66
|
+
.unwrap();
|
|
67
|
+
});
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
group.finish()
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
fn prepare() -> String {
|
|
74
|
+
let mut rng = thread_rng();
|
|
75
|
+
let mut content = vec![0; 16 * 1024 * 1024];
|
|
76
|
+
rng.fill_bytes(&mut content);
|
|
77
|
+
|
|
78
|
+
let name = uuid::Uuid::new_v4();
|
|
79
|
+
let path = format!("/tmp/opendal/{name}");
|
|
80
|
+
let _ = std::fs::write(path, content);
|
|
81
|
+
|
|
82
|
+
name.to_string()
|
|
83
|
+
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
|
3
|
+
# distributed with this work for additional information
|
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
|
6
|
+
# "License"); you may not use this file except in compliance
|
|
7
|
+
# with the License. You may obtain a copy of the License at
|
|
8
|
+
#
|
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
#
|
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
|
12
|
+
# software distributed under the License is distributed on an
|
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
# KIND, either express or implied. See the License for the
|
|
15
|
+
# specific language governing permissions and limitations
|
|
16
|
+
# under the License.
|
|
17
|
+
|
|
18
|
+
[package]
|
|
19
|
+
description = "OpenDAL Benchmark vs s3"
|
|
20
|
+
edition = "2021"
|
|
21
|
+
license = "Apache-2.0"
|
|
22
|
+
name = "opendal-benchmark-vs-s3"
|
|
23
|
+
publish = false
|
|
24
|
+
rust-version = "1.82"
|
|
25
|
+
version = "0.0.0"
|
|
26
|
+
|
|
27
|
+
[dependencies]
|
|
28
|
+
aws-config = { version = "1.0.1", features = ["behavior-version-latest"] }
|
|
29
|
+
aws-credential-types = { version = "1.2.4", features = [
|
|
30
|
+
"hardcoded-credentials",
|
|
31
|
+
] }
|
|
32
|
+
aws-sdk-s3 = "1.91.0"
|
|
33
|
+
criterion = { version = "0.7", features = ["async", "async_tokio"] }
|
|
34
|
+
dotenvy = "0.15"
|
|
35
|
+
logforth = { version = "0.23.1", default-features = false }
|
|
36
|
+
opendal = { path = "../..", features = ["tests"] }
|
|
37
|
+
rand = "0.8"
|
|
38
|
+
tokio = { version = "1", features = ["full"] }
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# OpenDAL Benchmark VS AWS SDK S3
|
|
2
|
+
|
|
3
|
+
This benchmark compares the performance of OpenDAL with the performance of the `aws_sdk_s3`.
|
|
4
|
+
|
|
5
|
+
## Goal
|
|
6
|
+
|
|
7
|
+
We expect OpenDAL to match `aws_sdk_s3` in speed: the throughput of OpenDAL should be within a `5%` range of `aws_sdk_s3`.
|
|
8
|
+
|
|
9
|
+
## Notes
|
|
10
|
+
|
|
11
|
+
Please run bench case separately, otherwise the result will be affected by each other since we are sharing the same runtime.
|
|
12
|
+
|
|
13
|
+
## Usage
|
|
14
|
+
|
|
15
|
+
For test: `cargo run`
|
|
16
|
+
|
|
17
|
+
```shell
|
|
18
|
+
> cargo run
|
|
19
|
+
Testing read/opendal_s3_reader
|
|
20
|
+
Success
|
|
21
|
+
Testing read/aws_s3_sdk_into_async_read
|
|
22
|
+
Success
|
|
23
|
+
Testing read/opendal_s3_reader_with_capacity
|
|
24
|
+
Success
|
|
25
|
+
Testing read/aws_s3_sdk_into_async_read_with_capacity
|
|
26
|
+
Success
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
For bench case: `cargo run --release -- opendal_s3_reader --bench`
|
|
30
|
+
|
|
31
|
+
```shell
|
|
32
|
+
> cargo run --release -- opendal_s3_reader --bench
|
|
33
|
+
read/opendal_s3_reader time: [12.773 ms 13.004 ms 13.232 ms]
|
|
34
|
+
thrpt: [1.1809 GiB/s 1.2016 GiB/s 1.2232 GiB/s]
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
For bench: `cargo run --release -- --bench`
|
|
38
|
+
|
|
39
|
+
```shell
|
|
40
|
+
> cargo run --release -- --bench
|
|
41
|
+
read/opendal_s3_reader time: [12.773 ms 13.004 ms 13.232 ms]
|
|
42
|
+
thrpt: [1.1809 GiB/s 1.2016 GiB/s 1.2232 GiB/s]
|
|
43
|
+
|
|
44
|
+
read/aws_s3_sdk_into_async_read
|
|
45
|
+
time: [12.527 ms 12.842 ms 13.158 ms]
|
|
46
|
+
thrpt: [1.1875 GiB/s 1.2168 GiB/s 1.2473 GiB/s]
|
|
47
|
+
|
|
48
|
+
read/opendal_s3_reader_with_capacity
|
|
49
|
+
time: [9.6098 ms 9.8133 ms 10.017 ms]
|
|
50
|
+
thrpt: [1.5599 GiB/s 1.5922 GiB/s 1.6259 GiB/s]
|
|
51
|
+
|
|
52
|
+
read/aws_s3_sdk_into_async_read_with_capacity
|
|
53
|
+
time: [9.8970 ms 10.113 ms 10.329 ms]
|
|
54
|
+
thrpt: [1.5128 GiB/s 1.5451 GiB/s 1.5788 GiB/s]
|
|
55
|
+
```
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
// Licensed to the Apache Software Foundation (ASF) under one
|
|
2
|
+
// or more contributor license agreements. See the NOTICE file
|
|
3
|
+
// distributed with this work for additional information
|
|
4
|
+
// regarding copyright ownership. The ASF licenses this file
|
|
5
|
+
// to you under the Apache License, Version 2.0 (the
|
|
6
|
+
// "License"); you may not use this file except in compliance
|
|
7
|
+
// with the License. You may obtain a copy of the License at
|
|
8
|
+
//
|
|
9
|
+
// http://www.apache.org/licenses/LICENSE-2.0
|
|
10
|
+
//
|
|
11
|
+
// Unless required by applicable law or agreed to in writing,
|
|
12
|
+
// software distributed under the License is distributed on an
|
|
13
|
+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
14
|
+
// KIND, either express or implied. See the License for the
|
|
15
|
+
// specific language governing permissions and limitations
|
|
16
|
+
// under the License.
|
|
17
|
+
|
|
18
|
+
use std::env;
|
|
19
|
+
|
|
20
|
+
use aws_config::BehaviorVersion;
|
|
21
|
+
use aws_config::Region;
|
|
22
|
+
use aws_credential_types::Credentials;
|
|
23
|
+
use criterion::Criterion;
|
|
24
|
+
use opendal::raw::tests::TEST_RUNTIME;
|
|
25
|
+
use opendal::services;
|
|
26
|
+
use opendal::Operator;
|
|
27
|
+
use rand::prelude::*;
|
|
28
|
+
use tokio::io::AsyncReadExt;
|
|
29
|
+
|
|
30
|
+
fn main() {
|
|
31
|
+
let _ = dotenvy::dotenv();
|
|
32
|
+
let _ = logforth::stderr().try_apply();
|
|
33
|
+
|
|
34
|
+
let endpoint = env::var("OPENDAL_S3_ENDPOINT").unwrap();
|
|
35
|
+
let access_key = env::var("OPENDAL_S3_ACCESS_KEY_ID").unwrap();
|
|
36
|
+
let secret_key = env::var("OPENDAL_S3_SECRET_ACCESS_KEY").unwrap();
|
|
37
|
+
let bucket = env::var("OPENDAL_S3_BUCKET").unwrap();
|
|
38
|
+
let region = env::var("OPENDAL_S3_REGION").unwrap();
|
|
39
|
+
|
|
40
|
+
// Init OpenDAL Operator.
|
|
41
|
+
let cfg = services::S3::default()
|
|
42
|
+
.endpoint(&endpoint)
|
|
43
|
+
.access_key_id(&access_key)
|
|
44
|
+
.secret_access_key(&secret_key)
|
|
45
|
+
.bucket(&bucket)
|
|
46
|
+
.region(®ion);
|
|
47
|
+
let op = Operator::new(cfg).unwrap().finish();
|
|
48
|
+
|
|
49
|
+
// Init AWS S3 SDK.
|
|
50
|
+
let mut config_loader = aws_config::defaults(BehaviorVersion::latest());
|
|
51
|
+
config_loader = config_loader.endpoint_url(&endpoint);
|
|
52
|
+
config_loader = config_loader.region(Region::new(region.to_string()));
|
|
53
|
+
config_loader =
|
|
54
|
+
config_loader.credentials_provider(Credentials::from_keys(&access_key, &secret_key, None));
|
|
55
|
+
let config = TEST_RUNTIME.block_on(config_loader.load());
|
|
56
|
+
let s3_client = aws_sdk_s3::Client::new(&config);
|
|
57
|
+
|
|
58
|
+
let mut c = Criterion::default().configure_from_args();
|
|
59
|
+
bench_read(&mut c, op, s3_client, &bucket);
|
|
60
|
+
|
|
61
|
+
c.final_summary();
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
fn bench_read(c: &mut Criterion, op: Operator, s3_client: aws_sdk_s3::Client, bucket: &str) {
|
|
65
|
+
let mut group = c.benchmark_group("read");
|
|
66
|
+
group.throughput(criterion::Throughput::Bytes(16 * 1024 * 1024));
|
|
67
|
+
|
|
68
|
+
TEST_RUNTIME.block_on(prepare(&op));
|
|
69
|
+
|
|
70
|
+
group.bench_function("opendal_s3_reader", |b| {
|
|
71
|
+
b.to_async(&*TEST_RUNTIME).iter(|| async {
|
|
72
|
+
let r = op.reader("file").await.unwrap();
|
|
73
|
+
let _ = r.read(..).await.unwrap();
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
group.bench_function("aws_s3_sdk_into_async_read", |b| {
|
|
77
|
+
b.to_async(&*TEST_RUNTIME).iter(|| async {
|
|
78
|
+
let mut r = s3_client
|
|
79
|
+
.get_object()
|
|
80
|
+
.bucket(bucket)
|
|
81
|
+
.key("file")
|
|
82
|
+
.send()
|
|
83
|
+
.await
|
|
84
|
+
.unwrap()
|
|
85
|
+
.body
|
|
86
|
+
.into_async_read();
|
|
87
|
+
let mut bs = Vec::new();
|
|
88
|
+
let _ = r.read_to_end(&mut bs).await.unwrap();
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
group.bench_function("opendal_s3_reader_with_capacity", |b| {
|
|
93
|
+
b.to_async(&*TEST_RUNTIME).iter(|| async {
|
|
94
|
+
let r = op.reader("file").await.unwrap();
|
|
95
|
+
let _ = r.read(..16 * 1024 * 1024).await.unwrap();
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
group.bench_function("aws_s3_sdk_into_async_read_with_capacity", |b| {
|
|
99
|
+
b.to_async(&*TEST_RUNTIME).iter(|| async {
|
|
100
|
+
let mut r = s3_client
|
|
101
|
+
.get_object()
|
|
102
|
+
.bucket(bucket)
|
|
103
|
+
.key("file")
|
|
104
|
+
.send()
|
|
105
|
+
.await
|
|
106
|
+
.unwrap()
|
|
107
|
+
.body
|
|
108
|
+
.into_async_read();
|
|
109
|
+
let mut bs = Vec::with_capacity(16 * 1024 * 1024);
|
|
110
|
+
let _ = r.read_to_end(&mut bs).await.unwrap();
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
group.finish()
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async fn prepare(op: &Operator) {
|
|
118
|
+
let mut rng = thread_rng();
|
|
119
|
+
let mut content = vec![0; 16 * 1024 * 1024];
|
|
120
|
+
rng.fill_bytes(&mut content);
|
|
121
|
+
|
|
122
|
+
op.write("file", content).await.unwrap();
|
|
123
|
+
}
|
data/core/edge/README.md
ADDED